From f63bb510b3dd372394d5d380ea75259a6d58ddbb Mon Sep 17 00:00:00 2001 From: Daviw Williams Date: Tue, 1 Oct 2013 15:51:23 +0200 Subject: [PATCH] Merged some code from MinizCompressor into MinizBlockCompressor. We don't really need two separate classes for this stuff. --- .../PolyVoxCore/MinizBlockCompressor.h | 16 ++- .../PolyVoxCore/MinizBlockCompressor.inl | 114 ++++++++++++++++-- 2 files changed, 120 insertions(+), 10 deletions(-) diff --git a/library/PolyVoxCore/include/PolyVoxCore/MinizBlockCompressor.h b/library/PolyVoxCore/include/PolyVoxCore/MinizBlockCompressor.h index e4b57cf6..57698a34 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/MinizBlockCompressor.h +++ b/library/PolyVoxCore/include/PolyVoxCore/MinizBlockCompressor.h @@ -26,7 +26,7 @@ freely, subject to the following restrictions: #include "PolyVoxCore/BlockCompressor.h" -#include "PolyVoxCore/MinizCompressor.h" +#include "PolyVoxCore/Impl/MinizWrapper.h" namespace PolyVox { @@ -37,13 +37,23 @@ namespace PolyVox class MinizBlockCompressor : public BlockCompressor { public: - MinizBlockCompressor(); + MinizBlockCompressor(int iCompressionLevel = 6); // Miniz defines MZ_DEFAULT_LEVEL = 6 so we use the same here ~MinizBlockCompressor(); void compress(UncompressedBlock* pSrcBlock, CompressedBlock* pDstBlock); void decompress(CompressedBlock* pSrcBlock, UncompressedBlock* pDstBlock); - MinizCompressor* m_pCompressor; + private: + uint32_t getMaxCompressedSize(uint32_t uUncompressedInputSize); + uint32_t compress(const void* pSrcData, uint32_t uSrcLength, void* pDstData, uint32_t uDstLength); + uint32_t decompress(const void* pSrcData, uint32_t uSrcLength, void* pDstData, uint32_t uDstLength); + + unsigned int m_uCompressionFlags; + + // tdefl_compressor contains all the state needed by the low-level compressor so it's a pretty big struct (~300k). + // We're storing it by void* because miniz does not supply a header and we don't want to include the .c file from + // here as it will cause linker problems. + tdefl_compressor* m_pDeflator; }; } diff --git a/library/PolyVoxCore/include/PolyVoxCore/MinizBlockCompressor.inl b/library/PolyVoxCore/include/PolyVoxCore/MinizBlockCompressor.inl index 39afa12f..59ad7708 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/MinizBlockCompressor.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/MinizBlockCompressor.inl @@ -21,18 +21,36 @@ freely, subject to the following restrictions: distribution. *******************************************************************************/ +#include + namespace PolyVox { template - MinizBlockCompressor::MinizBlockCompressor() + MinizBlockCompressor::MinizBlockCompressor(int iCompressionLevel) + :m_pDeflator(0) { - m_pCompressor = new MinizCompressor; + // Create and store the deflator. + tdefl_compressor* pDeflator = new tdefl_compressor; + m_pDeflator = /*reinterpret_cast*/(pDeflator); + + // The number of dictionary probes to use at each compression level (0-10). 0=implies fastest/minimal possible probing. + // The discontinuity is unsettling but may be explained by the 'iCompressionLevel <= 3' check later? + static const mz_uint s_tdefl_num_probes[11] = { 0, 1, 6, 32, 16, 32, 128, 256, 512, 768, 1500 }; + + // Create tdefl() compatible flags (we have to compose the low-level flags ourselves, or use tdefl_create_comp_flags_from_zip_params() but that means MINIZ_NO_ZLIB_APIS can't be defined). + m_uCompressionFlags = TDEFL_WRITE_ZLIB_HEADER | s_tdefl_num_probes[(std::min)(10, iCompressionLevel)] | ((iCompressionLevel <= 3) ? TDEFL_GREEDY_PARSING_FLAG : 0); + if (!iCompressionLevel) + { + m_uCompressionFlags |= TDEFL_FORCE_ALL_RAW_BLOCKS; + } } template MinizBlockCompressor::~MinizBlockCompressor() { - delete m_pCompressor; + // Delete the deflator + tdefl_compressor* pDeflator = /*reinterpret_cast*/(m_pDeflator); + delete pDeflator; } template @@ -50,7 +68,7 @@ namespace PolyVox try { // Perform the compression - uCompressedLength = m_pCompressor->compress(pSrcData, uSrcLength, pDstData, uDstLength); + uCompressedLength = compress(pSrcData, uSrcLength, pDstData, uDstLength); // Copy the resulting compressed data into the compressed block pDstBlock->setData(pDstData, uCompressedLength); @@ -62,7 +80,7 @@ namespace PolyVox // Note that ideally we will choose our earlier buffer size so that this almost never happens. logWarning() << "The compressor failed to compress the block, probabaly due to the buffer being too small."; logWarning() << "The compression will be tried again with a larger buffer"; - uint32_t uMaxCompressedSize = m_pCompressor->getMaxCompressedSize(uSrcLength); + uint32_t uMaxCompressedSize = getMaxCompressedSize(uSrcLength); uint8_t* buffer = new uint8_t[ uMaxCompressedSize ]; pDstData = reinterpret_cast( buffer ); @@ -71,7 +89,7 @@ namespace PolyVox try { // Perform the compression - uCompressedLength = m_pCompressor->compress(pSrcData, uSrcLength, pDstData, uDstLength); + uCompressedLength = compress(pSrcData, uSrcLength, pDstData, uDstLength); // Copy the resulting compressed data into the compressed block pDstBlock->setData(pDstData, uCompressedLength); @@ -98,8 +116,90 @@ namespace PolyVox //RLECompressor compressor; - uint32_t uUncompressedLength = m_pCompressor->decompress(pSrcData, uSrcLength, pDstData, uDstLength); + uint32_t uUncompressedLength = decompress(pSrcData, uSrcLength, pDstData, uDstLength); POLYVOX_ASSERT(uUncompressedLength == pDstBlock->getDataSizeInBytes(), "Destination length has changed."); } + + template + uint32_t MinizBlockCompressor::getMaxCompressedSize(uint32_t uUncompressedInputSize) + { + // The contents of this function are copied from miniz's 'mz_deflateBound()' + // (which we can't use because it is part of the zlib-style higher level API). + unsigned long source_len = uUncompressedInputSize; + + // This is really over conservative. (And lame, but it's actually pretty tricky to compute a true upper bound given the way tdefl's blocking works.) + return (std::max)(128 + (source_len * 110) / 100, 128 + source_len + ((source_len / (31 * 1024)) + 1) * 5); + } + + template + uint32_t MinizBlockCompressor::compress(const void* pSrcData, uint32_t uSrcLength, void* pDstData, uint32_t uDstLength) + { + //Get the deflator + tdefl_compressor* pDeflator = reinterpret_cast(m_pDeflator); + + // It seems we have to reinitialise the deflator for each fresh dataset (it's probably intended for streaming, which we're not doing here) + tdefl_status status = tdefl_init(pDeflator, NULL, NULL, m_uCompressionFlags); + if (status != TDEFL_STATUS_OKAY) + { + std::stringstream ss; + ss << "tdefl_init() failed with return code '" << status << "'"; + POLYVOX_THROW(std::runtime_error, ss.str()); + } + + // Change the type to avoid compiler warnings + size_t ulSrcLength = uSrcLength; + size_t ulDstLength = uDstLength; + + // Compress as much of the input as possible (or all of it) to the output buffer. + status = tdefl_compress(pDeflator, pSrcData, &ulSrcLength, pDstData, &ulDstLength, TDEFL_FINISH); + + //Check whther the compression was successful. + if (status != TDEFL_STATUS_DONE) + { + std::stringstream ss; + ss << "tdefl_compress() failed with return code '" << status << "'"; + POLYVOX_THROW(std::runtime_error, ss.str()); + } + + // The compression modifies 'ulDstLength' to hold the new length. + return ulDstLength; + } + + template + uint32_t MinizBlockCompressor::decompress(const void* pSrcData, uint32_t uSrcLength, void* pDstData, uint32_t uDstLength) + { + // I don't know exactly why this limitation exists but it's an implementation detail of miniz. It shouldn't matter for our purposes + // as our detination is a Block and those are always a power of two. If you need to use this class for other purposes then you'll + // probably have to scale up your dst buffer to the nearest appropriate size. Alternatively you can use the mz_uncompress function, + // but that means enabling parts of the miniz API which are #defined out at the top of this file. + POLYVOX_ASSERT(isPowerOf2(uDstLength), "Miniz decompressor requires the destination buffer to have a size which is a power of two."); + if(isPowerOf2(uDstLength) == false) + { + POLYVOX_THROW(std::invalid_argument, "Miniz decompressor requires the destination buffer to have a size which is a power of two."); + } + + // Change the type to avoid compiler warnings + size_t ulSrcLength = uSrcLength; + size_t ulDstLength = uDstLength; + + // Create and initialise the decompressor (I believe this is much small than the compressor). + tinfl_decompressor inflator; + tinfl_init(&inflator); + + // Do the decompression. In some scenarios 'tinfl_decompress' would be called multiple times with the same dest buffer but + // different locations within it. In our scenario it's only called once so the start and the location are the same (both pDstData). + tinfl_status status = tinfl_decompress(&inflator, (const mz_uint8 *)pSrcData, &ulSrcLength, (mz_uint8 *)pDstData, (mz_uint8 *)pDstData, &ulDstLength, TINFL_FLAG_PARSE_ZLIB_HEADER); + + //Check whther the decompression was successful. + if (status != TINFL_STATUS_DONE) + { + std::stringstream ss; + ss << "tinfl_decompress() failed with return code '" << status << "'"; + POLYVOX_THROW(std::runtime_error, ss.str()); + } + + // The decompression modifies 'ulDstLength' to hold the new length. + return ulDstLength; + } } \ No newline at end of file