From 26eb865897fd5f67ebd88df6d855110e7aff598f Mon Sep 17 00:00:00 2001 From: David Williams Date: Fri, 29 Apr 2011 23:04:33 +0100 Subject: [PATCH] Pulling compression out of SimpleVolume. --- library/PolyVoxCore/include/SimpleVolume.h | 15 --- library/PolyVoxCore/include/SimpleVolume.inl | 122 +----------------- .../PolyVoxCore/include/SimpleVolumeBlock.inl | 77 +---------- 3 files changed, 4 insertions(+), 210 deletions(-) diff --git a/library/PolyVoxCore/include/SimpleVolume.h b/library/PolyVoxCore/include/SimpleVolume.h index eb468533..45e36484 100644 --- a/library/PolyVoxCore/include/SimpleVolume.h +++ b/library/PolyVoxCore/include/SimpleVolume.h @@ -69,9 +69,6 @@ namespace PolyVox uint32_t calculateSizeInBytes(void); public: - void compress(void); - void uncompress(void); - std::vector< RunlengthEntry > m_vecCompressedData; VoxelType* m_tUncompressedData; uint16_t m_uSideLength; @@ -199,10 +196,6 @@ namespace PolyVox /// Gets a voxel at the position given by a 3D vector VoxelType getVoxelAt(const Vector3DInt32& v3dPos) const; - //Sets whether or not blocks are compressed in memory - void setCompressionEnabled(bool bCompressionEnabled); - /// Sets the number of blocks for which uncompressed data is stored - void setMaxNumberOfUncompressedBlocks(uint16_t uMaxNumberOfUncompressedBlocks); /// Sets the value used for voxels which are outside the volume void setBorderValue(const VoxelType& tBorder); /// Sets the voxel at the position given by x,y,z coordinates @@ -239,15 +232,9 @@ private: //The block data mutable std::map m_pBlocks; - //The cache of uncompressed blocks. The uncompressed block data and the timestamps are stored here rather - //than in the Block class. This is so that in the future each VolumeIterator might to maintain its own cache - //of blocks. However, this could mean the same block data is uncompressed and modified in more than one - //location in memory... could be messy with threading. - mutable std::vector< LoadedBlock* > m_vecUncompressedBlockCache; mutable uint32_t m_uTimestamper; mutable Vector3DInt32 m_v3dLastAccessedBlockPos; mutable Block* m_pLastAccessedBlock; - uint32_t m_uMaxNumberOfUncompressedBlocks; //We don't store an actual Block for the border, just the uncompressed data. This is partly because the border //block does not have a position (so can't be passed to getUncompressedBlock()) and partly because there's a @@ -267,8 +254,6 @@ private: int32_t m_uLongestSideLength; int32_t m_uShortestSideLength; float m_fDiagonalLength; - - bool m_bCompressionEnabled; }; } diff --git a/library/PolyVoxCore/include/SimpleVolume.inl b/library/PolyVoxCore/include/SimpleVolume.inl index 159b258f..39165c49 100644 --- a/library/PolyVoxCore/include/SimpleVolume.inl +++ b/library/PolyVoxCore/include/SimpleVolume.inl @@ -207,44 +207,6 @@ namespace PolyVox return getVoxelAt(v3dPos.getX(), v3dPos.getY(), v3dPos.getZ()); } - //////////////////////////////////////////////////////////////////////////////// - /// Enabling compression allows significantly more data to be stored in memory. - /// \param bCompressionEnabled Specifies whether compression is enabled. - //////////////////////////////////////////////////////////////////////////////// - template - void SimpleVolume::setCompressionEnabled(bool bCompressionEnabled) - { - //Early out - nothing to do - if(m_bCompressionEnabled == bCompressionEnabled) - { - return; - } - - m_bCompressionEnabled = bCompressionEnabled; - - if(m_bCompressionEnabled) - { - //If compression has been enabled then we need to start honouring the max number of - //uncompressed blocks. Because compression has been disabled for a while we might have - //gone above that limit. Easiest solution is just to clear the cache and start again. - clearBlockCache(); - } - } - - //////////////////////////////////////////////////////////////////////////////// - /// Increasing the size of the block cache will increase memory but may improve performance. - /// You may want to set this to a large value (e.g. 1024) when you are first loading your - /// volume data and then set it to a smaller value (e.g.64) for general processing. - /// \param uBlockCacheSize The number of blocks for which uncompressed data can be cached. - //////////////////////////////////////////////////////////////////////////////// - template - void SimpleVolume::setMaxNumberOfUncompressedBlocks(uint16_t uMaxNumberOfUncompressedBlocks) - { - clearBlockCache(); - - m_uMaxNumberOfUncompressedBlocks = uMaxNumberOfUncompressedBlocks; - } - //////////////////////////////////////////////////////////////////////////////// /// \param tBorder The value to use for voxels outside the volume. //////////////////////////////////////////////////////////////////////////////// @@ -301,11 +263,6 @@ namespace PolyVox template void SimpleVolume::clearBlockCache(void) { - for(uint32_t ct = 0; ct < m_vecUncompressedBlockCache.size(); ct++) - { - m_vecUncompressedBlockCache[ct]->block.compress(); - } - m_vecUncompressedBlockCache.clear(); } //////////////////////////////////////////////////////////////////////////////// @@ -328,20 +285,16 @@ namespace PolyVox } m_uTimestamper = 0; - m_uMaxNumberOfUncompressedBlocks = 16; m_uBlockSideLength = uBlockSideLength; m_pUncompressedBorderData = 0; m_v3dLastAccessedBlockPos = Vector3DInt32(0,0,0); //There are no invalid positions, but initially the m_pLastAccessedBlock pointer will be null; m_pLastAccessedBlock = 0; - m_bCompressionEnabled = true; m_regValidRegion = regValidRegion; m_regValidRegionInBlocks.setLowerCorner(m_regValidRegion.getLowerCorner() / static_cast(uBlockSideLength)); m_regValidRegionInBlocks.setUpperCorner(m_regValidRegion.getUpperCorner() / static_cast(uBlockSideLength)); - setMaxNumberOfUncompressedBlocks(m_uMaxNumberOfUncompressedBlocks); - //Clear the previous data m_pBlocks.clear(); @@ -365,33 +318,6 @@ namespace PolyVox template void SimpleVolume::eraseBlock(typename std::map::iterator itBlock) const { - /*if(m_funcDataOverflowHandler) - { - Vector3DInt32 v3dPos = itBlock->first; - Vector3DInt32 v3dLower(v3dPos.getX() << m_uBlockSideLengthPower, v3dPos.getY() << m_uBlockSideLengthPower, v3dPos.getZ() << m_uBlockSideLengthPower); - Vector3DInt32 v3dUpper = v3dLower + Vector3DInt32(m_uBlockSideLength-1, m_uBlockSideLength-1, m_uBlockSideLength-1); - - Region reg(v3dLower, v3dUpper); - ConstVolumeProxy ConstVolumeProxy(*this, reg); - - m_funcDataOverflowHandler(ConstVolumeProxy, reg); - }*/ - if(m_bCompressionEnabled) { - for(uint32_t ct = 0; ct < m_vecUncompressedBlockCache.size(); ct++) - { - // find the block in the uncompressed cache - if(m_vecUncompressedBlockCache[ct] == &(itBlock->second)) - { - // TODO: compression is unneccessary? or will not compressing this cause a memleak? - itBlock->second.block.compress(); - // put last object in cache here - m_vecUncompressedBlockCache[ct] = m_vecUncompressedBlockCache.back(); - // decrease cache size by one since last element is now in here twice - m_vecUncompressedBlockCache.resize(m_vecUncompressedBlockCache.size()-1); - break; - } - } - } m_pBlocks.erase(itBlock); } @@ -422,17 +348,7 @@ namespace PolyVox template typename SimpleVolume::Block* SimpleVolume::getUncompressedBlock(int32_t uBlockX, int32_t uBlockY, int32_t uBlockZ) const { - Vector3DInt32 v3dBlockPos(uBlockX, uBlockY, uBlockZ); - - //Check if we have the same block as last time, if so there's no need to even update - //the time stamp. If we updated it everytime then that would be every time we touched - //a voxel, which would overflow a uint32_t and require us to use a uint64_t instead. - //This check should also provide a significant speed boost as usually it is true. - if((v3dBlockPos == m_v3dLastAccessedBlockPos) && (m_pLastAccessedBlock != 0)) - { - assert(m_pLastAccessedBlock->m_tUncompressedData); - return m_pLastAccessedBlock; - } + Vector3DInt32 v3dBlockPos(uBlockX, uBlockY, uBlockZ); typename std::map::iterator itBlock = m_pBlocks.find(v3dBlockPos); // check whether the block is already loaded @@ -454,38 +370,8 @@ namespace PolyVox assert(m_pLastAccessedBlock->m_tUncompressedData); return m_pLastAccessedBlock; } - - //If we are allowed to compress then check whether we need to - if((m_bCompressionEnabled) && (m_vecUncompressedBlockCache.size() == m_uMaxNumberOfUncompressedBlocks)) - { - int32_t leastRecentlyUsedBlockIndex = -1; - uint32_t uLeastRecentTimestamp = (std::numeric_limits::max)(); - - //Currently we find the oldest block by iterating over the whole array. Of course we could store the blocks sorted by - //timestamp (set, priority_queue, etc) but then we'll need to move them around as the timestamp changes. Can come back - //to this if it proves to be a bottleneck (compraed to the cost of actually doing the compression/decompression). - for(uint32_t ct = 0; ct < m_vecUncompressedBlockCache.size(); ct++) - { - if(m_vecUncompressedBlockCache[ct]->timestamp < uLeastRecentTimestamp) - { - uLeastRecentTimestamp = m_vecUncompressedBlockCache[ct]->timestamp; - leastRecentlyUsedBlockIndex = ct; - } - } - - //Compress the least recently used block. - m_vecUncompressedBlockCache[leastRecentlyUsedBlockIndex]->block.compress(); - - //We don't actually remove any elements from this vector, we - //simply change the pointer to point at the new uncompressed bloack. - m_vecUncompressedBlockCache[leastRecentlyUsedBlockIndex] = &loadedBlock; - } - else - { - m_vecUncompressedBlockCache.push_back(&loadedBlock); - } - loadedBlock.block.uncompress(); + //loadedBlock.block.uncompress(); m_pLastAccessedBlock = &(loadedBlock.block); assert(m_pLastAccessedBlock->m_tUncompressedData); @@ -519,10 +405,6 @@ namespace PolyVox uSizeInBytes += i->second.block.calculateSizeInBytes(); } - //Memory used by the block cache. - uSizeInBytes += m_vecUncompressedBlockCache.capacity() * sizeof(LoadedBlock); - uSizeInBytes += m_vecUncompressedBlockCache.size() * m_uBlockSideLength * m_uBlockSideLength * m_uBlockSideLength * sizeof(VoxelType); - //Memory used by border data. if(m_pUncompressedBorderData) { diff --git a/library/PolyVoxCore/include/SimpleVolumeBlock.inl b/library/PolyVoxCore/include/SimpleVolumeBlock.inl index 753518a6..ffb24974 100644 --- a/library/PolyVoxCore/include/SimpleVolumeBlock.inl +++ b/library/PolyVoxCore/include/SimpleVolumeBlock.inl @@ -102,23 +102,12 @@ namespace PolyVox template void SimpleVolume::Block::fill(VoxelType tValue) { - if(!m_bIsCompressed) - { //The memset *may* be faster than the std::fill(), but it doesn't compile nicely //in 64-bit mode as casting the pointer to an int causes a loss of precision. const uint32_t uNoOfVoxels = m_uSideLength * m_uSideLength * m_uSideLength; std::fill(m_tUncompressedData, m_tUncompressedData + uNoOfVoxels, tValue); m_bIsUncompressedDataModified = true; - } - else - { - RunlengthEntry rle; - rle.length = m_uSideLength*m_uSideLength*m_uSideLength; - rle.value = tValue; - m_vecCompressedData.clear(); - m_vecCompressedData.push_back(rle); - } } template @@ -137,6 +126,8 @@ namespace PolyVox m_uSideLength = uSideLength; m_uSideLengthPower = logBase2(uSideLength); + m_tUncompressedData = new VoxelType[m_uSideLength * m_uSideLength * m_uSideLength]; + SimpleVolume::Block::fill(VoxelType()); } @@ -147,68 +138,4 @@ namespace PolyVox uSizeInBytes += m_vecCompressedData.capacity() * sizeof(RunlengthEntry); return uSizeInBytes; } - - template - void SimpleVolume::Block::compress(void) - { - assert(m_bIsCompressed == false); - assert(m_tUncompressedData != 0); - - //If the uncompressed data hasn't actually been - //modified then we don't need to redo the compression. - if(m_bIsUncompressedDataModified) - { - uint32_t uNoOfVoxels = m_uSideLength * m_uSideLength * m_uSideLength; - m_vecCompressedData.clear(); - - RunlengthEntry entry; - entry.length = 1; - entry.value = m_tUncompressedData[0]; - - for(uint32_t ct = 1; ct < uNoOfVoxels; ++ct) - { - VoxelType value = m_tUncompressedData[ct]; - if((value == entry.value) && (entry.length < entry.maxRunlength())) - { - entry.length++; - } - else - { - m_vecCompressedData.push_back(entry); - entry.value = value; - entry.length = 1; - } - } - - m_vecCompressedData.push_back(entry); - - //Shrink the vectors to their contents (maybe slow?): - //http://stackoverflow.com/questions/1111078/reduce-the-capacity-of-an-stl-vector - //C++0x may have a shrink_to_fit() function? - std::vector< RunlengthEntry >(m_vecCompressedData).swap(m_vecCompressedData); - } - - //Flag the uncompressed data as no longer being used. - delete[] m_tUncompressedData; - m_tUncompressedData = 0; - m_bIsCompressed = true; - } - - template - void SimpleVolume::Block::uncompress(void) - { - assert(m_bIsCompressed == true); - assert(m_tUncompressedData == 0); - m_tUncompressedData = new VoxelType[m_uSideLength * m_uSideLength * m_uSideLength]; - - VoxelType* pUncompressedData = m_tUncompressedData; - for(uint32_t ct = 0; ct < m_vecCompressedData.size(); ++ct) - { - std::fill(pUncompressedData, pUncompressedData + m_vecCompressedData[ct].length, m_vecCompressedData[ct].value); - pUncompressedData += m_vecCompressedData[ct].length; - } - - m_bIsCompressed = false; - m_bIsUncompressedDataModified = false; - } }