AnotherFoxGuy/polyvox
1
0
Fork 0
Code Issues Pull Requests Activity

Merge branch 'develop' into feature/cubiquity-version

Browse Source
This commit is contained in:
Daviw Williams
2013-10-04 13:57:03 +02:00
parent 886c66d7c3 29bded222f
commit 8a07d93b54
14 changed files with 239 additions and 349 deletions
Download Patch File Download Diff File Expand all files Collapse all files

100
library/PolyVoxCore/include/PolyVoxCore/Compressor.h
View File

@@ -1,100 +0,0 @@
/*******************************************************************************
Copyright (c) 2005-2009 David Williams
This software is provided 'as-is', without any express or implied
warranty. In no event will the authors be held liable for any damages
arising from the use of this software.
Permission is granted to anyone to use this software for any purpose,
including commercial applications, and to alter it and redistribute it
freely, subject to the following restrictions:
1. The origin of this software must not be misrepresented; you must not
claim that you wrote the original software. If you use this software
in a product, an acknowledgment in the product documentation would be
appreciated but is not required.
2. Altered source versions must be plainly marked as such, and must not be
misrepresented as being the original software.
3. This notice may not be removed or altered from any source
distribution.
*******************************************************************************/
#ifndef __PolyVox_Compressor_H__
#define __PolyVox_Compressor_H__
#include "PolyVoxCore/Impl/TypeDef.h"
namespace PolyVox
{
/**
* Provides an interface for performing compression of data.
*
* This class provides an interface which can be implemented by derived classes which perform data compression.
* The main purpose of this is to allow the user to change the compression algorithm which is used by a LargeVolume,
* based on the kind of voxel data it is storing. Users may also wish to use Compressor subclasses in more general
* compression-related scenarios but this is not well tested.
*
* If you wish to implement your own compression algorithms for use in PolyVox then you should begin by subclassing this class.
*
* \sa MinizCompressor, RLECompressor
*/
class Compressor
{
public:
/// Constructor
Compressor() {};
/// Destructor
virtual ~Compressor() {};
/**
* Computes a worst-case scenario for how big the output can be for a given input size.
*
* If necessary you can use this as a destination buffer size, though it may be somewhat
* wasteful. It is not guarenteed that compression actually shrinks the data, so the
* worst-case value returned by this function may be bigger than the input size.
*
* \param uUncompressedInputSize The size of the uncompressed input data
* \return The largest possible size of the compressed output data.
*/
virtual uint32_t getMaxCompressedSize(uint32_t uUncompressedInputSize) = 0;
/**
* Compresses the data.
*
* Performs compression of the data pointed to by pSrcData and stores the result in pDstData.
* The user is responsible for allocating both buffers and for making sure that the destination
* buffer is large enough to hold the result. If you don't know how big the compressed data
* will be (and you probably won't know this) then you can call getMaxCompressedSize() to get
* an upper bound. The *actual* compressed size is then returned by this function and you can
* use this to copy your compressed data to a more suitably size buffer.
*
* \param pSrcData A pointer to the data to be compressed.
* \param uSrcLength The length of the data to be compressed.
* \param pDstData A pointer to the memory where the result should be stored.
* \param uDstLength The length of the destination buffer (compression will fail if this isn't big enough).
* \return The size of the resulting compressed data.
*/
virtual uint32_t compress(const void* pSrcData, uint32_t uSrcLength, void* pDstData, uint32_t uDstLength) = 0;
/**
* Decompresses the data.
*
* Performs decompression of the data pointed to by pSrcData and stores the result in pDstData.
* The user is responsible for allocating both buffers and for making sure that the destination
* buffer is large enough to hold the result. This means you need to know how large the resulting
* data might be, so before you compress the data it may be worth storing this information somewhere.
* The *actual* decompressed size is then returned by this function
*
* \param pSrcData A pointer to the data to be decompressed.
* \param uSrcLength The length of the data to be decompressed.
* \param pDstData A pointer to the memory where the result should be stored.
* \param uDstLength The length of the destination buffer (decompression will fail if this isn't big enough).
* \return The size of the resulting uncompressed data.
*/
virtual uint32_t decompress(const void* pSrcData, uint32_t uSrcLength, void* pDstData, uint32_t uDstLength) = 0;
};
}
#endif //__PolyVox_Compressor_H__

49
library/PolyVoxCore/include/PolyVoxCore/Impl/MinizWrapper.h Normal file
View File

@@ -0,0 +1,49 @@
/*******************************************************************************
Copyright (c) 2005-2013 David Williams
This software is provided 'as-is', without any express or implied
warranty. In no event will the authors be held liable for any damages
arising from the use of this software.
Permission is granted to anyone to use this software for any purpose,
including commercial applications, and to alter it and redistribute it
freely, subject to the following restrictions:
1. The origin of this software must not be misrepresented; you must not
claim that you wrote the original software. If you use this software
in a product, an acknowledgment in the product documentation would be
appreciated but is not required.
2. Altered source versions must be plainly marked as such, and must not be
misrepresented as being the original software.
3. This notice may not be removed or altered from any source
distribution.
*******************************************************************************/
#ifndef __PolyVox_MinizWrapper_H__
#define __PolyVox_MinizWrapper_H__
// The miniz library is suplied as a single '.c' file, but this is messy for a project like PolyVox
// because it consists mostly of headers. Many of our headers may want to make use of Miniz code which
// means we need to be careful about #including 'miniz.c' multiple times and getting linker errors.
// We simplify this situation with this 'MinizWrapper.h' and 'MinizWrapper.c' which include 'miniz.c'
// with only declarations and definitions respectively. 'MinizWrapper.cpp' only gets compiled once,
// and all other parts of PolyVox can simply include 'MinizWrapper.h' to get the declarations.
// Diable things we don't need, and in particular the zlib compatible names which
// would cause conflicts if a user application is using both PolyVox and zlib.
#define MINIZ_NO_STDIO
#define MINIZ_NO_ARCHIVE_APIS
#define MINIZ_NO_TIME
#define MINIZ_NO_ZLIB_APIS
#define MINIZ_NO_ZLIB_COMPATIBLE_NAMES
#define MINIZ_NO_MALLOC
// Include only the declarations of the functions in miniz.c. Don't include
// the actual definitions, as this 'MinizWrapper.h' may be included from multiple
// locations and we want to avoid linker errors regarding multiple definitions.
#define MINIZ_HEADER_FILE_ONLY
#include "PolyVoxCore/Impl/miniz.c"
#endif //__PolyVox_MinizWrapper_H__

24
library/PolyVoxCore/include/PolyVoxCore/Impl/Timer.h
View File

@@ -28,37 +28,31 @@ freely, subject to the following restrictions:
#ifdef _MSC_VER // Don't worry about the exact version, as long as this is defied.
#include <Windows.h>
#else
#include <chrono>
#endif //_MSC_VER
namespace PolyVox
{
#if defined(_MSC_VER)
class Timer
{
public:
Timer(bool bAutoStart = true);
void start(void);
float elapsedTimeInSeconds(void);
uint32_t elapsedTimeInMilliSeconds(void);
private:
#if defined(_MSC_VER)
double m_fPCFreq;
__int64 m_iStartTime;
};
#else //_MSC_VER
class Timer
{
public:
Timer(bool bAutoStart = true);
void start(void);
float elapsedTimeInSeconds(void);
uint32_t elapsedTimeInMilliSeconds(void);
};
typedef std::chrono::system_clock clock;
std::chrono::time_point<clock> m_start;
#endif //_MSC_VER
};
}
#endif //__PolyVox_Timer_H__

11
library/PolyVoxCore/include/PolyVoxCore/Interpolation.h
View File

@@ -31,9 +31,6 @@ namespace PolyVox
const Type& v0,const Type& v1,
const float x)
{
//This function is called frequently and is very short, so assert rather than exceptions.
POLYVOX_ASSERT((x >= 0.0f) && (x <= 1.0f), "Interpolation input out of 0.0 to 1.0 range.");
//Interpolate along X
Type v0_1 = (v1 - v0) * x + v0;
@@ -45,10 +42,6 @@ namespace PolyVox
const Type& v00,const Type& v10,const Type& v01,const Type& v11,
const float x, const float y)
{
//This function is called frequently and is very short, so assert rather than exceptions.
POLYVOX_ASSERT((x >= 0.0f) && (y >= 0.0f) &&
(x <= 1.0f) && (y <= 1.0f), "Interpolation input out of 0.0 to 1.0 range.");
// Linearly interpolate along x
Type v00_10 = lerp(v00, v10, x);
Type v01_11 = lerp(v01, v11, x);
@@ -65,10 +58,6 @@ namespace PolyVox
const Type& v001,const Type& v101,const Type& v011,const Type& v111,
const float x, const float y, const float z)
{
//This function is called frequently and is very short, so assert rather than exceptions.
POLYVOX_ASSERT((x >= 0.0f) && (y >= 0.0f) && (z >= 0.0f) &&
(x <= 1.0f) && (y <= 1.0f) && (z <= 1.0f), "Interpolation input out of 0.0 to 1.0 range.");
// Bilinearly interpolate along Y
Type v000_v100__v010_v110 = bilerp(v000, v100, v010, v110, x, y);
Type v001_v101__v011_v111 = bilerp(v001, v101, v011, v111, x, y);

19
library/PolyVoxCore/include/PolyVoxCore/MinizBlockCompressor.h
View File

@@ -26,7 +26,7 @@ freely, subject to the following restrictions:
#include "PolyVoxCore/BlockCompressor.h"
#include "PolyVoxCore/MinizCompressor.h"
#include "PolyVoxCore/Impl/MinizWrapper.h"
namespace PolyVox
{
@@ -37,13 +37,26 @@ namespace PolyVox
class MinizBlockCompressor : public BlockCompressor<VoxelType>
{
public:
MinizBlockCompressor();
MinizBlockCompressor(int iCompressionLevel = 6); // Miniz defines MZ_DEFAULT_LEVEL = 6 so we use the same here
~MinizBlockCompressor();
void compress(UncompressedBlock<VoxelType>* pSrcBlock, CompressedBlock<VoxelType>* pDstBlock);
void decompress(CompressedBlock<VoxelType>* pSrcBlock, UncompressedBlock<VoxelType>* pDstBlock);
MinizCompressor* m_pCompressor;
private:
uint32_t getExpectedCompressedSize(uint32_t uUncompressedInputSize);
uint32_t getMaxCompressedSize(uint32_t uUncompressedInputSize);
uint32_t compressWithMiniz(const void* pSrcData, size_t uSrcLength, void* pDstData, size_t uDstLength);
uint32_t decompressWithMiniz(const void* pSrcData, size_t uSrcLength, void* pDstData, size_t uDstLength);
unsigned int m_uCompressionFlags;
// Data gets compressed into this, then we check how big the result
// is and copy the required number of bytes to the destination block.
std::vector<uint8_t> m_vecTempBuffer;
// tdefl_compressor contains all the state needed by the low-level compressor so it's a pretty big struct (~300k).
tdefl_compressor* m_pDeflator;
};
}

153
library/PolyVoxCore/include/PolyVoxCore/MinizBlockCompressor.inl
View File

@@ -21,36 +21,66 @@ freely, subject to the following restrictions:
distribution.
*******************************************************************************/
#include <sstream>
namespace PolyVox
{
/**
* You can specify a compression level when constructing this compressor. This controls the tradeoff between speed and compression
* rate. Levels 0-9 are the standard zlib-style levels, 10 is best possible compression (not zlib compatible, and may be very slow).
* \param iCompressionLevel The desired compression level.
*/
template <typename VoxelType>
MinizBlockCompressor<VoxelType>::MinizBlockCompressor()
MinizBlockCompressor<VoxelType>::MinizBlockCompressor(int iCompressionLevel)
:m_pDeflator(0)
{
m_pCompressor = new MinizCompressor;
// Create and store the deflator.
m_pDeflator = new tdefl_compressor;
// The number of dictionary probes to use at each compression level (0-10). 0=implies fastest/minimal possible probing.
// The discontinuity is unsettling but may be explained by the 'iCompressionLevel <= 3' check later?
static const mz_uint s_tdefl_num_probes[11] = { 0, 1, 6, 32, 16, 32, 128, 256, 512, 768, 1500 };
// Create tdefl() compatible flags (we have to compose the low-level flags ourselves, or use tdefl_create_comp_flags_from_zip_params() but that means MINIZ_NO_ZLIB_APIS can't be defined).
m_uCompressionFlags = TDEFL_WRITE_ZLIB_HEADER | s_tdefl_num_probes[(std::min)(10, iCompressionLevel)] | ((iCompressionLevel <= 3) ? TDEFL_GREEDY_PARSING_FLAG : 0);
if (!iCompressionLevel)
{
m_uCompressionFlags |= TDEFL_FORCE_ALL_RAW_BLOCKS;
}
}
template <typename VoxelType>
MinizBlockCompressor<VoxelType>::~MinizBlockCompressor()
{
delete m_pCompressor;
// Delete the deflator
delete m_pDeflator;
}
template <typename VoxelType>
void MinizBlockCompressor<VoxelType>::compress(UncompressedBlock<VoxelType>* pSrcBlock, CompressedBlock<VoxelType>* pDstBlock)
{
// The uncompressed data will be read straight out of the block
void* pSrcData = reinterpret_cast<void*>(pSrcBlock->getData());
uint32_t uSrcLength = pSrcBlock->getDataSizeInBytes();
size_t uSrcLength = pSrcBlock->getDataSizeInBytes();
uint8_t tempBuffer[10000];
uint8_t* pDstData = reinterpret_cast<uint8_t*>( tempBuffer );
uint32_t uDstLength = 10000;
// This compressor is expected to be used many times to compress a large number of blocks, but they are all
// expected to have the same size. Therefore the resize() function below should only perform allocation once.
uint32_t expectedCompressedSize = getExpectedCompressedSize(uSrcLength);
if(m_vecTempBuffer.size() != expectedCompressedSize)
{
logInfo() << "Resizing temp buffer to " << expectedCompressedSize << "bytes. "
<< "This should only happen the first time the MinizBlockCompressor is used";
m_vecTempBuffer.resize(expectedCompressedSize);
}
uint32_t uCompressedLength = 0;
// The compressed data will be written into this temporary buffer.
uint8_t* pDstData = &(m_vecTempBuffer[0]);
size_t uDstLength = m_vecTempBuffer.size();
try
{
// Perform the compression
uCompressedLength = m_pCompressor->compress(pSrcData, uSrcLength, pDstData, uDstLength);
uint32_t uCompressedLength = compressWithMiniz(pSrcData, uSrcLength, pDstData, uDstLength);
// Copy the resulting compressed data into the compressed block
pDstBlock->setData(pDstData, uCompressedLength);
@@ -61,17 +91,18 @@ namespace PolyVox
// buffer is not big enough. So now we try again using a buffer that is definitely big enough.
// Note that ideally we will choose our earlier buffer size so that this almost never happens.
logWarning() << "The compressor failed to compress the block, probabaly due to the buffer being too small.";
logWarning() << "The compression will be tried again with a larger buffer";
uint32_t uMaxCompressedSize = m_pCompressor->getMaxCompressedSize(uSrcLength);
uint8_t* buffer = new uint8_t[ uMaxCompressedSize ];
logWarning() << "The compression will be tried again with a larger buffer.";
pDstData = reinterpret_cast<uint8_t*>( buffer );
uDstLength = uMaxCompressedSize;
std::vector<uint8_t> vecExtraBigBuffer;
vecExtraBigBuffer.resize(getMaxCompressedSize(uSrcLength));
uint8_t* pDstData = &(vecExtraBigBuffer[0]);
size_t uDstLength = vecExtraBigBuffer.size();
try
{
// Perform the compression
uCompressedLength = m_pCompressor->compress(pSrcData, uSrcLength, pDstData, uDstLength);
uint32_t uCompressedLength = compressWithMiniz(pSrcData, uSrcLength, pDstData, uDstLength);
// Copy the resulting compressed data into the compressed block
pDstBlock->setData(pDstData, uCompressedLength);
@@ -80,26 +111,100 @@ namespace PolyVox
{
// At this point it didn't work even with a bigger buffer.
// Not much more we can do so just rethrow the exception.
delete[] buffer;
POLYVOX_THROW(std::runtime_error, "Failed to compress block data");
}
delete[] buffer;
}
}
template <typename VoxelType>
void MinizBlockCompressor<VoxelType>::decompress(CompressedBlock<VoxelType>* pSrcBlock, UncompressedBlock<VoxelType>* pDstBlock)
{
// Get raw pointers so that the data can be decompressed directly into the destination block.
const void* pSrcData = reinterpret_cast<const void*>(pSrcBlock->getData());
void* pDstData = reinterpret_cast<void*>(pDstBlock->getData());
uint32_t uSrcLength = pSrcBlock->getDataSizeInBytes();
uint32_t uDstLength = pDstBlock->getDataSizeInBytes();
size_t uSrcLength = pSrcBlock->getDataSizeInBytes();
size_t uDstLength = pDstBlock->getDataSizeInBytes();
//RLECompressor<VoxelType, uint16_t> compressor;
uint32_t uUncompressedLength = m_pCompressor->decompress(pSrcData, uSrcLength, pDstData, uDstLength);
// Perform the decompression
uint32_t uUncompressedLength = decompressWithMiniz(pSrcData, uSrcLength, pDstData, uDstLength);
POLYVOX_ASSERT(uUncompressedLength == pDstBlock->getDataSizeInBytes(), "Destination length has changed.");
// We know we should have received exactly one block's worth of data. If not then something went wrong.
POLYVOX_THROW_IF(uUncompressedLength != pDstBlock->getDataSizeInBytes(), std::runtime_error, "Decompressed data does not have the expected length");
}
template <typename VoxelType>
uint32_t MinizBlockCompressor<VoxelType>::getExpectedCompressedSize(uint32_t uUncompressedInputSize)
{
// We expect this block compressor will be used for smoothly changing volume data such as density fields and so
// the compression rate might not be great. The value beloew is basically a guess based on previous experience.
uint32_t uExpectedCompressionRate = 4;
return uUncompressedInputSize / uExpectedCompressionRate;
}
template <typename VoxelType>
uint32_t MinizBlockCompressor<VoxelType>::getMaxCompressedSize(uint32_t uUncompressedInputSize)
{
// The contents of this function are copied from miniz's 'mz_deflateBound()'
// (which we can't use because it is part of the zlib-style higher level API).
unsigned long source_len = uUncompressedInputSize;
// This is really over conservative. (And lame, but it's actually pretty tricky to compute a true upper bound given the way tdefl's blocking works.)
return (std::max)(128 + (source_len * 110) / 100, 128 + source_len + ((source_len / (31 * 1024)) + 1) * 5);
}
template <typename VoxelType>
uint32_t MinizBlockCompressor<VoxelType>::compressWithMiniz(const void* pSrcData, size_t uSrcLength, void* pDstData, size_t uDstLength)
{
// It seems we have to reinitialise the deflator for each fresh dataset (it's probably intended for streaming, which we're not doing here)
tdefl_status status = tdefl_init(m_pDeflator, NULL, NULL, m_uCompressionFlags);
if (status != TDEFL_STATUS_OKAY)
{
std::stringstream ss;
ss << "tdefl_init() failed with return code '" << status << "'";
POLYVOX_THROW(std::runtime_error, ss.str());
}
// Compress as much of the input as possible (or all of it) to the output buffer.
status = tdefl_compress(m_pDeflator, pSrcData, &uSrcLength, pDstData, &uDstLength, TDEFL_FINISH);
//Check whether the compression was successful.
if (status != TDEFL_STATUS_DONE)
{
std::stringstream ss;
ss << "tdefl_compress() failed with return code '" << status << "'";
POLYVOX_THROW(std::runtime_error, ss.str());
}
// The compression modifies 'ulDstLength' to hold the new length.
return uDstLength;
}
template <typename VoxelType>
uint32_t MinizBlockCompressor<VoxelType>::decompressWithMiniz(const void* pSrcData, size_t uSrcLength, void* pDstData, size_t uDstLength)
{
// I don't know exactly why this limitation exists but it's an implementation detail of miniz. It shouldn't matter for our purposes
// as our destination is a Block and those are always a power of two. If you need to use this code for other purposes then you'll
// probably have to scale up your dst buffer to the nearest appropriate size. Alternatively you can use the mz_uncompress function,
// but that means enabling parts of the miniz API which are #defined out at the top of this file.
POLYVOX_THROW_IF(isPowerOf2(uDstLength) == false, std::invalid_argument, "Miniz decompressor requires the destination buffer to have a size which is a power of two.");
// Create and initialise the decompressor (I believe this is much small than the compressor).
tinfl_decompressor inflator;
tinfl_init(&inflator);
// Do the decompression. In some scenarios 'tinfl_decompress' would be called multiple times with the same dest buffer but
// different locations within it. In our scenario it's only called once so the start and the location are the same (both pDstData).
tinfl_status status = tinfl_decompress(&inflator, (const mz_uint8 *)pSrcData, &uSrcLength, (mz_uint8 *)pDstData, (mz_uint8 *)pDstData, &uDstLength, TINFL_FLAG_PARSE_ZLIB_HEADER);
//Check whether the decompression was successful.
if (status != TINFL_STATUS_DONE)
{
std::stringstream ss;
ss << "tinfl_decompress() failed with return code '" << status << "'";
POLYVOX_THROW(std::runtime_error, ss.str());
}
// The decompression modifies 'ulDstLength' to hold the new length.
return uDstLength;
}
}

41
library/PolyVoxCore/include/PolyVoxCore/MinizCompressor.h
View File

@@ -1,41 +0,0 @@
#ifndef __PolyVox_MinizCompressor_H__
#define __PolyVox_MinizCompressor_H__
#include "PolyVoxCore/Compressor.h"
namespace PolyVox
{
/**
* Performs compression of data using the miniz library.
*
* This compressor implements the DEFLATE (http://en.wikipedia.org/wiki/Deflate) compression algorithm via the pubic domain
* 'miniz' library (https://code.google.com/p/miniz/). This is a general purpose compression algorithm, and within PolyVox it
* is intended for situations in which the alternative RLECompressor is not appropriate. It is a good default choice if you
* are not sure which compressor is best for your needs.
*
* \sa RLECompressor
*/
class MinizCompressor : public Compressor
{
public:
/// Constructor
MinizCompressor(int iCompressionLevel = 6); // Miniz defines MZ_DEFAULT_LEVEL = 6 so we use the same here
/// Destructor
~MinizCompressor();
// API documentation is in base class and gets inherited by Doxygen.
uint32_t getMaxCompressedSize(uint32_t uUncompressedInputSize);
uint32_t compress(const void* pSrcData, uint32_t uSrcLength, void* pDstData, uint32_t uDstLength);
uint32_t decompress(const void* pSrcData, uint32_t uSrcLength, void* pDstData, uint32_t uDstLength);
private:
unsigned int m_uCompressionFlags;
// tdefl_compressor contains all the state needed by the low-level compressor so it's a pretty big struct (~300k).
// We're storing it by void* because miniz does not supply a header and we don't want to include the .c file from
// here as it will cause linker problems.
void* m_pDeflator;
};
}
#endif //__PolyVox_MinizCompressor_H__

2
library/PolyVoxCore/include/PolyVoxCore/RLEBlockCompressor.h
View File

@@ -26,8 +26,6 @@ freely, subject to the following restrictions:
#include "PolyVoxCore/BlockCompressor.h"
#include "PolyVoxCore/MinizCompressor.h"
namespace PolyVox
{
template <typename VoxelType>