From a589c6e4acd4a215e37f36cca99c9524758e6751 Mon Sep 17 00:00:00 2001 From: David Williams Date: Thu, 11 Jul 2013 14:26:38 +0200 Subject: [PATCH 01/38] Split come code into getCompressedBlock() function. --- .../include/PolyVoxCore/LargeVolume.h | 1 + .../include/PolyVoxCore/LargeVolume.inl | 45 ++++++++++++------- 2 files changed, 29 insertions(+), 17 deletions(-) diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h index cd773156..934336bc 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h @@ -329,6 +329,7 @@ namespace PolyVox VoxelType getVoxelImpl(int32_t uXPos, int32_t uYPos, int32_t uZPos, WrapModeType, VoxelType tBorder) const; VoxelType getVoxelImpl(int32_t uXPos, int32_t uYPos, int32_t uZPos, WrapModeType, VoxelType tBorder) const; + Block* getCompressedBlock(int32_t uBlockX, int32_t uBlockY, int32_t uBlockZ) const; Block* getUncompressedBlock(int32_t uBlockX, int32_t uBlockY, int32_t uBlockZ) const; void eraseBlock(typename std::map, BlockPositionCompare>::iterator itBlock) const; diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl index 662d10db..fac37e2a 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl @@ -589,20 +589,10 @@ namespace PolyVox } template - Block* LargeVolume::getUncompressedBlock(int32_t uBlockX, int32_t uBlockY, int32_t uBlockZ) const + Block* LargeVolume::getCompressedBlock(int32_t uBlockX, int32_t uBlockY, int32_t uBlockZ) const { Vector3DInt32 v3dBlockPos(uBlockX, uBlockY, uBlockZ); - //Check if we have the same block as last time, if so there's no need to even update - //the time stamp. If we updated it everytime then that would be every time we touched - //a voxel, which would overflow a uint32_t and require us to use a uint64_t instead. - //This check should also provide a significant speed boost as usually it is true. - if((v3dBlockPos == m_v3dLastAccessedBlockPos) && (m_pLastAccessedBlock != 0)) - { - POLYVOX_ASSERT(m_pLastAccessedBlock->hasUncompressedData(), "Last accessed block has no uncompressed data."); - return m_pLastAccessedBlock; - } - typename std::map, BlockPositionCompare>::iterator itBlock = m_pBlocks.find(v3dBlockPos); // check whether the block is already loaded if(itBlock == m_pBlocks.end()) @@ -627,9 +617,30 @@ namespace PolyVox m_v3dLastAccessedBlockPos = v3dBlockPos; m_pLastAccessedBlock = █ - if(block.hasUncompressedData()) - { + return m_pLastAccessedBlock; + } + + template + Block* LargeVolume::getUncompressedBlock(int32_t uBlockX, int32_t uBlockY, int32_t uBlockZ) const + { + Vector3DInt32 v3dBlockPos(uBlockX, uBlockY, uBlockZ); + + //Check if we have the same block as last time, if so there's no need to even update + //the time stamp. If we updated it everytime then that would be every time we touched + //a voxel, which would overflow a uint32_t and require us to use a uint64_t instead. + //This check should also provide a significant speed boost as usually it is true. + if((v3dBlockPos == m_v3dLastAccessedBlockPos) && (m_pLastAccessedBlock != 0)) + { + POLYVOX_ASSERT(m_pLastAccessedBlock->hasUncompressedData(), "Last accessed block has no uncompressed data."); return m_pLastAccessedBlock; + } + + //Get the block and mark that we accessed it + Block* block = getCompressedBlock(uBlockX, uBlockY, uBlockZ); + + if(block->hasUncompressedData()) + { + return block; } //If we are allowed to compress then check whether we need to @@ -655,16 +666,16 @@ namespace PolyVox //We don't actually remove any elements from this vector, we //simply change the pointer to point at the new uncompressed bloack. - m_vecBlocksWithUncompressedData[leastRecentlyUsedBlockIndex] = █ + m_vecBlocksWithUncompressedData[leastRecentlyUsedBlockIndex] = block; } else { - m_vecBlocksWithUncompressedData.push_back(&block); + m_vecBlocksWithUncompressedData.push_back(block); } - block.createUncompressedData(); + block->createUncompressedData(); - m_pLastAccessedBlock = &(block); + m_pLastAccessedBlock = block; POLYVOX_ASSERT(m_pLastAccessedBlock->m_tUncompressedData, "Block has no uncompressed data"); return m_pLastAccessedBlock; } From abd1920d80c32e25795a1973de841a492d834840 Mon Sep 17 00:00:00 2001 From: David Williams Date: Thu, 11 Jul 2013 14:50:06 +0200 Subject: [PATCH 02/38] Adding new caching mechanism. --- .../include/PolyVoxCore/LargeVolume.h | 1 + .../include/PolyVoxCore/LargeVolume.inl | 21 +++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h index 934336bc..a87f2796 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h @@ -334,6 +334,7 @@ namespace PolyVox void eraseBlock(typename std::map, BlockPositionCompare>::iterator itBlock) const; // The block data + mutable std::map m_pUncompressedBlockCache; mutable std::map, BlockPositionCompare> m_pBlocks; // The cache of uncompressed blocks. The uncompressed block data and the timestamps are stored here rather diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl index fac37e2a..db604532 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl @@ -638,6 +638,27 @@ namespace PolyVox //Get the block and mark that we accessed it Block* block = getCompressedBlock(uBlockX, uBlockY, uBlockZ); + + typename std::map::iterator itUncompressedBlock = m_pUncompressedBlockCache.find(v3dBlockPos); + // check whether the block is already loaded + if(itUncompressedBlock == m_pUncompressedBlockCache.end()) + { + VoxelType* pUncompressedBlock = new VoxelType[m_uBlockSideLength * m_uBlockSideLength * m_uBlockSideLength]; + + void* pSrcData = reinterpret_cast(block->m_pCompressedData); + void* pDstData = reinterpret_cast(pUncompressedBlock); + uint32_t uSrcLength = block->m_uCompressedDataLength; + uint32_t uDstLength = m_uBlockSideLength * m_uBlockSideLength * m_uBlockSideLength * sizeof(VoxelType); + + //MinizCompressor compressor; + //RLECompressor compressor; + uint32_t uUncompressedLength = m_pCompressor->decompress(pSrcData, uSrcLength, pDstData, uDstLength); + + POLYVOX_ASSERT(uUncompressedLength == m_uBlockSideLength * m_uBlockSideLength * m_uBlockSideLength * sizeof(VoxelType), "Destination length has changed."); + + itUncompressedBlock = m_pUncompressedBlockCache.insert(std::make_pair(v3dBlockPos, pUncompressedBlock)).first; + } + if(block->hasUncompressedData()) { return block; From 884fe04c1224b513f9f6aa66748252ab90d5fd4c Mon Sep 17 00:00:00 2001 From: David Williams Date: Thu, 11 Jul 2013 15:01:18 +0200 Subject: [PATCH 03/38] Tidying up... --- .../include/PolyVoxCore/LargeVolume.inl | 36 ++++++++++++++----- .../PolyVoxCore/LargeVolumeSampler.inl | 4 +-- 2 files changed, 30 insertions(+), 10 deletions(-) diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl index db604532..502ac706 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl @@ -223,9 +223,14 @@ namespace PolyVox const uint16_t yOffset = static_cast(uYPos - (blockY << m_uBlockSideLengthPower)); const uint16_t zOffset = static_cast(uZPos - (blockZ << m_uBlockSideLengthPower)); - Block* pUncompressedBlock = getUncompressedBlock(blockX, blockY, blockZ); + VoxelType* pUncompressedBlock = getUncompressedBlock(blockX, blockY, blockZ)->m_tUncompressedData; - return pUncompressedBlock->getVoxel(xOffset,yOffset,zOffset); + return pUncompressedBlock + [ + xOffset + + yOffset * m_uBlockSideLength + + zOffset * m_uBlockSideLength * m_uBlockSideLength + ]; } else { @@ -312,9 +317,14 @@ namespace PolyVox const uint16_t yOffset = static_cast(uYPos - (blockY << m_uBlockSideLengthPower)); const uint16_t zOffset = static_cast(uZPos - (blockZ << m_uBlockSideLengthPower)); - Block* pUncompressedBlock = getUncompressedBlock(blockX, blockY, blockZ); + VoxelType* pUncompressedBlock = getUncompressedBlock(blockX, blockY, blockZ)->m_tUncompressedData; - pUncompressedBlock->setVoxelAt(xOffset,yOffset,zOffset, tValue); + pUncompressedBlock + [ + xOffset + + yOffset * m_uBlockSideLength + + zOffset * m_uBlockSideLength * m_uBlockSideLength + ] = tValue; } //////////////////////////////////////////////////////////////////////////////// @@ -350,9 +360,14 @@ namespace PolyVox const uint16_t yOffset = static_cast(uYPos - (blockY << m_uBlockSideLengthPower)); const uint16_t zOffset = static_cast(uZPos - (blockZ << m_uBlockSideLengthPower)); - Block* pUncompressedBlock = getUncompressedBlock(blockX, blockY, blockZ); + VoxelType* pUncompressedBlock = getUncompressedBlock(blockX, blockY, blockZ)->m_tUncompressedData; - pUncompressedBlock->setVoxelAt(xOffset,yOffset,zOffset, tValue); + pUncompressedBlock + [ + xOffset + + yOffset * m_uBlockSideLength + + zOffset * m_uBlockSideLength * m_uBlockSideLength + ] = tValue; //Return true to indicate that we modified a voxel. return true; @@ -840,9 +855,14 @@ namespace PolyVox const uint16_t yOffset = static_cast(uYPos - (blockY << m_uBlockSideLengthPower)); const uint16_t zOffset = static_cast(uZPos - (blockZ << m_uBlockSideLengthPower)); - Block* pUncompressedBlock = getUncompressedBlock(blockX, blockY, blockZ); + VoxelType* pUncompressedBlock = getUncompressedBlock(blockX, blockY, blockZ)->m_tUncompressedData; - return pUncompressedBlock->getVoxel(xOffset,yOffset,zOffset); + return pUncompressedBlock + [ + xOffset + + yOffset * m_uBlockSideLength + + zOffset * m_uBlockSideLength * m_uBlockSideLength + ]; } } diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolumeSampler.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolumeSampler.inl index 0d71a68f..dd61fcf0 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolumeSampler.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolumeSampler.inl @@ -119,9 +119,9 @@ namespace PolyVox uYPosInBlock * this->mVolume->m_uBlockSideLength + uZPosInBlock * this->mVolume->m_uBlockSideLength * this->mVolume->m_uBlockSideLength; - Block* pUncompressedCurrentBlock = this->mVolume->getUncompressedBlock(uXBlock, uYBlock, uZBlock); + VoxelType* pUncompressedCurrentBlock = this->mVolume->getUncompressedBlock(uXBlock, uYBlock, uZBlock)->m_tUncompressedData; - mCurrentVoxel = pUncompressedCurrentBlock->m_tUncompressedData + uVoxelIndexInBlock; + mCurrentVoxel = pUncompressedCurrentBlock + uVoxelIndexInBlock; } else { From e6cbc09e83ed36390e08b56334b4d6dbf77f3ca7 Mon Sep 17 00:00:00 2001 From: David Williams Date: Thu, 11 Jul 2013 15:30:15 +0200 Subject: [PATCH 04/38] getUncompressedBlock now return raw pointer instead of block pointer. --- .../include/PolyVoxCore/LargeVolume.h | 4 +-- .../include/PolyVoxCore/LargeVolume.inl | 25 ++++++++++--------- .../PolyVoxCore/LargeVolumeSampler.inl | 2 +- 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h index a87f2796..03b4ac01 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h @@ -330,7 +330,7 @@ namespace PolyVox VoxelType getVoxelImpl(int32_t uXPos, int32_t uYPos, int32_t uZPos, WrapModeType, VoxelType tBorder) const; Block* getCompressedBlock(int32_t uBlockX, int32_t uBlockY, int32_t uBlockZ) const; - Block* getUncompressedBlock(int32_t uBlockX, int32_t uBlockY, int32_t uBlockZ) const; + VoxelType* getUncompressedBlock(int32_t uBlockX, int32_t uBlockY, int32_t uBlockZ) const; void eraseBlock(typename std::map, BlockPositionCompare>::iterator itBlock) const; // The block data @@ -344,7 +344,7 @@ namespace PolyVox mutable std::vector< Block* > m_vecBlocksWithUncompressedData; mutable uint32_t m_uTimestamper; mutable Vector3DInt32 m_v3dLastAccessedBlockPos; - mutable Block* m_pLastAccessedBlock; + mutable VoxelType* m_pLastAccessedBlock; uint32_t m_uMaxNumberOfUncompressedBlocks; uint32_t m_uCompressedBlockMemoryLimitInBytes; diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl index 502ac706..794b283f 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl @@ -223,7 +223,7 @@ namespace PolyVox const uint16_t yOffset = static_cast(uYPos - (blockY << m_uBlockSideLengthPower)); const uint16_t zOffset = static_cast(uZPos - (blockZ << m_uBlockSideLengthPower)); - VoxelType* pUncompressedBlock = getUncompressedBlock(blockX, blockY, blockZ)->m_tUncompressedData; + VoxelType* pUncompressedBlock = getUncompressedBlock(blockX, blockY, blockZ); return pUncompressedBlock [ @@ -317,7 +317,7 @@ namespace PolyVox const uint16_t yOffset = static_cast(uYPos - (blockY << m_uBlockSideLengthPower)); const uint16_t zOffset = static_cast(uZPos - (blockZ << m_uBlockSideLengthPower)); - VoxelType* pUncompressedBlock = getUncompressedBlock(blockX, blockY, blockZ)->m_tUncompressedData; + VoxelType* pUncompressedBlock = getUncompressedBlock(blockX, blockY, blockZ); pUncompressedBlock [ @@ -360,7 +360,7 @@ namespace PolyVox const uint16_t yOffset = static_cast(uYPos - (blockY << m_uBlockSideLengthPower)); const uint16_t zOffset = static_cast(uZPos - (blockZ << m_uBlockSideLengthPower)); - VoxelType* pUncompressedBlock = getUncompressedBlock(blockX, blockY, blockZ)->m_tUncompressedData; + VoxelType* pUncompressedBlock = getUncompressedBlock(blockX, blockY, blockZ); pUncompressedBlock [ @@ -629,14 +629,13 @@ namespace PolyVox //Get the block and mark that we accessed it Block& block = itBlock->second; block.timestamp = ++m_uTimestamper; - m_v3dLastAccessedBlockPos = v3dBlockPos; - m_pLastAccessedBlock = █ + //m_v3dLastAccessedBlockPos = v3dBlockPos; - return m_pLastAccessedBlock; + return █ } template - Block* LargeVolume::getUncompressedBlock(int32_t uBlockX, int32_t uBlockY, int32_t uBlockZ) const + VoxelType* LargeVolume::getUncompressedBlock(int32_t uBlockX, int32_t uBlockY, int32_t uBlockZ) const { Vector3DInt32 v3dBlockPos(uBlockX, uBlockY, uBlockZ); @@ -646,7 +645,6 @@ namespace PolyVox //This check should also provide a significant speed boost as usually it is true. if((v3dBlockPos == m_v3dLastAccessedBlockPos) && (m_pLastAccessedBlock != 0)) { - POLYVOX_ASSERT(m_pLastAccessedBlock->hasUncompressedData(), "Last accessed block has no uncompressed data."); return m_pLastAccessedBlock; } @@ -674,9 +672,12 @@ namespace PolyVox itUncompressedBlock = m_pUncompressedBlockCache.insert(std::make_pair(v3dBlockPos, pUncompressedBlock)).first; } + m_pLastAccessedBlock = block->m_tUncompressedData; + m_v3dLastAccessedBlockPos = v3dBlockPos; + if(block->hasUncompressedData()) { - return block; + return block->m_tUncompressedData; } //If we are allowed to compress then check whether we need to @@ -711,8 +712,8 @@ namespace PolyVox block->createUncompressedData(); - m_pLastAccessedBlock = block; - POLYVOX_ASSERT(m_pLastAccessedBlock->m_tUncompressedData, "Block has no uncompressed data"); + m_pLastAccessedBlock = block->m_tUncompressedData; + POLYVOX_ASSERT(m_pLastAccessedBlock, "Block has no uncompressed data"); return m_pLastAccessedBlock; } @@ -855,7 +856,7 @@ namespace PolyVox const uint16_t yOffset = static_cast(uYPos - (blockY << m_uBlockSideLengthPower)); const uint16_t zOffset = static_cast(uZPos - (blockZ << m_uBlockSideLengthPower)); - VoxelType* pUncompressedBlock = getUncompressedBlock(blockX, blockY, blockZ)->m_tUncompressedData; + VoxelType* pUncompressedBlock = getUncompressedBlock(blockX, blockY, blockZ); return pUncompressedBlock [ diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolumeSampler.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolumeSampler.inl index dd61fcf0..d90ea5f4 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolumeSampler.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolumeSampler.inl @@ -119,7 +119,7 @@ namespace PolyVox uYPosInBlock * this->mVolume->m_uBlockSideLength + uZPosInBlock * this->mVolume->m_uBlockSideLength * this->mVolume->m_uBlockSideLength; - VoxelType* pUncompressedCurrentBlock = this->mVolume->getUncompressedBlock(uXBlock, uYBlock, uZBlock)->m_tUncompressedData; + VoxelType* pUncompressedCurrentBlock = this->mVolume->getUncompressedBlock(uXBlock, uYBlock, uZBlock); mCurrentVoxel = pUncompressedCurrentBlock + uVoxelIndexInBlock; } From 9ff95f7061eebe00e29db6607d5f00804d3eb1dd Mon Sep 17 00:00:00 2001 From: David Williams Date: Thu, 11 Jul 2013 15:55:34 +0200 Subject: [PATCH 05/38] Minor fixes. --- library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl index 794b283f..e8194b2f 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl @@ -646,7 +646,7 @@ namespace PolyVox if((v3dBlockPos == m_v3dLastAccessedBlockPos) && (m_pLastAccessedBlock != 0)) { return m_pLastAccessedBlock; - } + } //Get the block and mark that we accessed it Block* block = getCompressedBlock(uBlockX, uBlockY, uBlockZ); @@ -672,9 +672,11 @@ namespace PolyVox itUncompressedBlock = m_pUncompressedBlockCache.insert(std::make_pair(v3dBlockPos, pUncompressedBlock)).first; } - m_pLastAccessedBlock = block->m_tUncompressedData; + m_pLastAccessedBlock = (*itUncompressedBlock).second; m_v3dLastAccessedBlockPos = v3dBlockPos; + return m_pLastAccessedBlock; + if(block->hasUncompressedData()) { return block->m_tUncompressedData; From 0d92bc6c8cfb1c0db967d31581510455fbf423e1 Mon Sep 17 00:00:00 2001 From: David Williams Date: Thu, 11 Jul 2013 16:17:48 +0200 Subject: [PATCH 06/38] Starting to split Block class into CompressedBlock and UncompressedBlock. --- examples/Paging/main.cpp | 4 +- .../include/PolyVoxCore/Impl/Block.h | 3 - .../include/PolyVoxCore/Impl/Block.inl | 109 +----------------- .../include/PolyVoxCore/LargeVolume.h | 5 - .../include/PolyVoxCore/LargeVolume.inl | 64 ++-------- 5 files changed, 16 insertions(+), 169 deletions(-) diff --git a/examples/Paging/main.cpp b/examples/Paging/main.cpp index f172a214..c497a255 100644 --- a/examples/Paging/main.cpp +++ b/examples/Paging/main.cpp @@ -93,7 +93,7 @@ public: virtual void pageIn(const PolyVox::Region& region, Block* pBlockData) { - pBlockData->createUncompressedData(); + /*pBlockData->createUncompressedData(); Perlin perlin(2,2,1,234); @@ -134,7 +134,7 @@ public: pBlockData->setVoxelAt(x - region.getLowerX(), y - region.getLowerY(), z - region.getLowerZ(), voxel); } } - } + }*/ } virtual void pageOut(const PolyVox::Region& region, Block* /*pBlockData*/) diff --git a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h index 06963f7c..509b2dce 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h +++ b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h @@ -52,9 +52,6 @@ namespace PolyVox void setVoxelAt(uint16_t uXPos, uint16_t uYPos, uint16_t uZPos, VoxelType tValue); void setVoxelAt(const Vector3DUint16& v3dPos, VoxelType tValue); - void createUncompressedData(void); - void destroyUncompressedData(void); - uint32_t calculateSizeInBytes(void); public: diff --git a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl index bebcfb35..bdba78f9 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl @@ -66,12 +66,12 @@ namespace PolyVox //Temporarily create the block data. This is just so we can compress it an discard it. // FIXME - this is a temporary solution. - const uint32_t uNoOfVoxels = m_uSideLength * m_uSideLength * m_uSideLength; + /*const uint32_t uNoOfVoxels = m_uSideLength * m_uSideLength * m_uSideLength; m_tUncompressedData = new VoxelType[uNoOfVoxels]; std::fill(m_tUncompressedData, m_tUncompressedData + uNoOfVoxels, VoxelType()); m_bIsUncompressedDataModified = true; - destroyUncompressedData(); + destroyUncompressedData();*/ } template @@ -127,7 +127,7 @@ namespace PolyVox template void Block::setCompressedData(const uint8_t* const data, uint32_t dataLength) { - POLYVOX_ASSERT(m_pCompressedData, "Compressed data is NULL"); + //POLYVOX_ASSERT(m_pCompressedData, "Compressed data is NULL"); POLYVOX_ASSERT(m_pCompressedData != data, "Attempting to copy data onto itself"); delete[] m_pCompressedData; @@ -163,109 +163,6 @@ namespace PolyVox setVoxelAt(v3dPos.getX(), v3dPos.getY(), v3dPos.getZ(), tValue); } - template - void Block::destroyUncompressedData() - { - if(!hasUncompressedData()) - { - POLYVOX_THROW(invalid_operation, "No uncompressed data to compress."); - } - - POLYVOX_ASSERT(m_tUncompressedData != 0, "No uncompressed data is present."); - - //If the uncompressed data hasn't actually been - //modified then we don't need to redo the compression. - if(m_bIsUncompressedDataModified) - { - // Delete the old compressed data as we'll create a new one - delete[] m_pCompressedData; - m_pCompressedData = 0; - - void* pSrcData = reinterpret_cast(m_tUncompressedData); - uint32_t uSrcLength = m_uSideLength * m_uSideLength * m_uSideLength * sizeof(VoxelType); - - uint8_t tempBuffer[10000]; - void* pDstData = reinterpret_cast( tempBuffer ); - uint32_t uDstLength = 10000; - - uint32_t uCompressedLength = 0; - - try - { - uCompressedLength = m_pCompressor->compress(pSrcData, uSrcLength, pDstData, uDstLength); - - // Create new compressed data and copy across - m_pCompressedData = new uint8_t[uCompressedLength]; - memcpy(m_pCompressedData, pDstData, uCompressedLength); - m_uCompressedDataLength = uCompressedLength; - } - catch(std::exception&) - { - // It is possible for the compression to fail. A common cause for this would be if the destination - // buffer is not big enough. So now we try again using a buffer that is definitely big enough. - // Note that ideally we will choose our earlier buffer size so that this almost never happens. - logWarning() << "The compressor failed to compress the block, proabaly due to the buffer being too small."; - logWarning() << "The compression will be tried again with a larger buffer"; - uint32_t uMaxCompressedSize = m_pCompressor->getMaxCompressedSize(uSrcLength); - uint8_t* buffer = new uint8_t[ uMaxCompressedSize ]; - - pDstData = reinterpret_cast( buffer ); - uDstLength = uMaxCompressedSize; - - try - { - uCompressedLength = m_pCompressor->compress(pSrcData, uSrcLength, pDstData, uDstLength); - - // Create new compressed data and copy across - m_pCompressedData = new uint8_t[uCompressedLength]; - memcpy(m_pCompressedData, pDstData, uCompressedLength); - m_uCompressedDataLength = uCompressedLength; - } - catch(std::exception&) - { - // At this point it didn't work even with a bigger buffer. - // Not much more we can do so just rethrow the exception. - delete[] buffer; - POLYVOX_THROW(std::runtime_error, "Failed to compress block data"); - } - - delete[] buffer; - } - } - - //Flag the uncompressed data as no longer being used. - delete[] m_tUncompressedData; - m_tUncompressedData = 0; - } - - template - void Block::createUncompressedData() - { - if(hasUncompressedData()) - { - POLYVOX_THROW(invalid_operation, "Uncompressed data already exists."); - } - - POLYVOX_ASSERT(m_tUncompressedData == 0, "Uncompressed data already exists."); - - m_tUncompressedData = new VoxelType[m_uSideLength * m_uSideLength * m_uSideLength]; - - void* pSrcData = reinterpret_cast(m_pCompressedData); - void* pDstData = reinterpret_cast(m_tUncompressedData); - uint32_t uSrcLength = m_uCompressedDataLength; - uint32_t uDstLength = m_uSideLength * m_uSideLength * m_uSideLength * sizeof(VoxelType); - - //MinizCompressor compressor; - //RLECompressor compressor; - uint32_t uUncompressedLength = m_pCompressor->decompress(pSrcData, uSrcLength, pDstData, uDstLength); - - POLYVOX_ASSERT(uUncompressedLength == m_uSideLength * m_uSideLength * m_uSideLength * sizeof(VoxelType), "Destination length has changed."); - - //m_tUncompressedData = reinterpret_cast(uncompressedResult.ptr); - - m_bIsUncompressedDataModified = false; - } - template uint32_t Block::calculateSizeInBytes(void) { diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h index 03b4ac01..86e36381 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h @@ -337,11 +337,6 @@ namespace PolyVox mutable std::map m_pUncompressedBlockCache; mutable std::map, BlockPositionCompare> m_pBlocks; - // The cache of uncompressed blocks. The uncompressed block data and the timestamps are stored here rather - // than in the Block class. This is so that in the future each VolumeIterator might to maintain its own cache - // of blocks. However, this could mean the same block data is uncompressed and modified in more than one - // location in memory... could be messy with threading. - mutable std::vector< Block* > m_vecBlocksWithUncompressedData; mutable uint32_t m_uTimestamper; mutable Vector3DInt32 m_v3dLastAccessedBlockPos; mutable VoxelType* m_pLastAccessedBlock; diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl index e8194b2f..770c94ee 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl @@ -102,7 +102,7 @@ namespace PolyVox template LargeVolume::~LargeVolume() { - flushAll(); + //flushAll(); // Only delete the compressor if it was created by us (in the constructor), not by the user. if(m_bIsOurCompressor) @@ -509,11 +509,6 @@ namespace PolyVox template void LargeVolume::clearBlockCache(void) { - for(uint32_t ct = 0; ct < m_vecBlocksWithUncompressedData.size(); ct++) - { - m_vecBlocksWithUncompressedData[ct]->destroyUncompressedData(); - } - m_vecBlocksWithUncompressedData.clear(); } //////////////////////////////////////////////////////////////////////////////// @@ -571,7 +566,9 @@ namespace PolyVox template void LargeVolume::eraseBlock(typename std::map, BlockPositionCompare>::iterator itBlock) const { - if(itBlock->second.hasUncompressedData()) + POLYVOX_ASSERT(false, "This function has not been implemented properly"); + + /*if(itBlock->second.hasUncompressedData()) { itBlock->second.destroyUncompressedData(); } @@ -585,9 +582,11 @@ namespace PolyVox Region reg(v3dLower, v3dUpper); m_pPager->pageOut(reg, &(itBlock->second)); - } + }*/ + - for(uint32_t ct = 0; ct < m_vecBlocksWithUncompressedData.size(); ct++) + // FIXME - the code below used to make sure the uncompressed version of the data was removed. Reinstate something similar. + /*for(uint32_t ct = 0; ct < m_vecBlocksWithUncompressedData.size(); ct++) { // find the block in the uncompressed cache if(m_vecBlocksWithUncompressedData[ct] == &(itBlock->second)) @@ -598,7 +597,7 @@ namespace PolyVox m_vecBlocksWithUncompressedData.resize(m_vecBlocksWithUncompressedData.size()-1); break; } - } + }*/ m_pBlocks.erase(itBlock); } @@ -676,47 +675,6 @@ namespace PolyVox m_v3dLastAccessedBlockPos = v3dBlockPos; return m_pLastAccessedBlock; - - if(block->hasUncompressedData()) - { - return block->m_tUncompressedData; - } - - //If we are allowed to compress then check whether we need to - if(m_vecBlocksWithUncompressedData.size() == m_uMaxNumberOfUncompressedBlocks) - { - int32_t leastRecentlyUsedBlockIndex = -1; - uint32_t uLeastRecentTimestamp = (std::numeric_limits::max)(); - - //Currently we find the oldest block by iterating over the whole array. Of course we could store the blocks sorted by - //timestamp (set, priority_queue, etc) but then we'll need to move them around as the timestamp changes. Can come back - //to this if it proves to be a bottleneck (compraed to the cost of actually doing the compression/decompression). - for(uint32_t ct = 0; ct < m_vecBlocksWithUncompressedData.size(); ct++) - { - if(m_vecBlocksWithUncompressedData[ct]->timestamp < uLeastRecentTimestamp) - { - uLeastRecentTimestamp = m_vecBlocksWithUncompressedData[ct]->timestamp; - leastRecentlyUsedBlockIndex = ct; - } - } - - //Compress the least recently used block. - m_vecBlocksWithUncompressedData[leastRecentlyUsedBlockIndex]->destroyUncompressedData(); - - //We don't actually remove any elements from this vector, we - //simply change the pointer to point at the new uncompressed bloack. - m_vecBlocksWithUncompressedData[leastRecentlyUsedBlockIndex] = block; - } - else - { - m_vecBlocksWithUncompressedData.push_back(block); - } - - block->createUncompressedData(); - - m_pLastAccessedBlock = block->m_tUncompressedData; - POLYVOX_ASSERT(m_pLastAccessedBlock, "Block has no uncompressed data"); - return m_pLastAccessedBlock; } //////////////////////////////////////////////////////////////////////////////// @@ -747,8 +705,8 @@ namespace PolyVox } //Memory used by the block cache. - uSizeInBytes += m_vecBlocksWithUncompressedData.capacity() * sizeof(Block); - uSizeInBytes += m_vecBlocksWithUncompressedData.size() * m_uBlockSideLength * m_uBlockSideLength * m_uBlockSideLength * sizeof(VoxelType); + //uSizeInBytes += m_vecBlocksWithUncompressedData.capacity() * sizeof(Block); + //uSizeInBytes += m_vecBlocksWithUncompressedData.size() * m_uBlockSideLength * m_uBlockSideLength * m_uBlockSideLength * sizeof(VoxelType); return uSizeInBytes; } From 00eb281990f4a813b166fe24edb4c854a40a3143 Mon Sep 17 00:00:00 2001 From: David Williams Date: Thu, 11 Jul 2013 16:30:19 +0200 Subject: [PATCH 07/38] Stripping down Block class. --- .../include/PolyVoxCore/FilePager.h | 4 +- .../include/PolyVoxCore/Impl/Block.h | 13 --- .../include/PolyVoxCore/Impl/Block.inl | 81 +------------------ .../include/PolyVoxCore/LargeVolume.inl | 2 +- 4 files changed, 4 insertions(+), 96 deletions(-) diff --git a/library/PolyVoxCore/include/PolyVoxCore/FilePager.h b/library/PolyVoxCore/include/PolyVoxCore/FilePager.h index 1f3115ab..7fb79f1c 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/FilePager.h +++ b/library/PolyVoxCore/include/PolyVoxCore/FilePager.h @@ -55,7 +55,7 @@ namespace PolyVox virtual void pageIn(const Region& region, Block* pBlockData) { POLYVOX_ASSERT(pBlockData, "Attempting to page in NULL block"); - POLYVOX_ASSERT(pBlockData->hasUncompressedData() == false, "Block should not have uncompressed data"); + //POLYVOX_ASSERT(pBlockData->hasUncompressedData() == false, "Block should not have uncompressed data"); std::stringstream ss; ss << region.getLowerX() << "_" << region.getLowerY() << "_" << region.getLowerZ() << "_" @@ -96,7 +96,7 @@ namespace PolyVox virtual void pageOut(const Region& region, Block* pBlockData) { POLYVOX_ASSERT(pBlockData, "Attempting to page out NULL block"); - POLYVOX_ASSERT(pBlockData->hasUncompressedData() == false, "Block should not have uncompressed data"); + //POLYVOX_ASSERT(pBlockData->hasUncompressedData() == false, "Block should not have uncompressed data"); logTrace() << "Paging out data for " << region; diff --git a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h index 509b2dce..86d6d08b 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h +++ b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h @@ -42,27 +42,14 @@ namespace PolyVox const uint8_t* getCompressedData(void) const; uint32_t getCompressedDataLength(void) const; - uint16_t getSideLength(void) const; - VoxelType getVoxel(uint16_t uXPos, uint16_t uYPos, uint16_t uZPos) const; - VoxelType getVoxel(const Vector3DUint16& v3dPos) const; - - bool hasUncompressedData(void) const; void setCompressedData(const uint8_t* const data, uint32_t dataLength); - void setVoxelAt(uint16_t uXPos, uint16_t uYPos, uint16_t uZPos, VoxelType tValue); - void setVoxelAt(const Vector3DUint16& v3dPos, VoxelType tValue); uint32_t calculateSizeInBytes(void); public: - Compressor* m_pCompressor; uint8_t* m_pCompressedData; uint32_t m_uCompressedDataLength; - VoxelType* m_tUncompressedData; - uint16_t m_uSideLength; - uint8_t m_uSideLengthPower; - bool m_bIsUncompressedDataModified; - uint32_t timestamp; }; } diff --git a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl index bdba78f9..3f311e0d 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl @@ -37,13 +37,8 @@ namespace PolyVox { template Block::Block(uint16_t uSideLength, Compressor* pCompressor) - :m_pCompressor(pCompressor) - ,m_pCompressedData(0) + :m_pCompressedData(0) ,m_uCompressedDataLength(0) - ,m_tUncompressedData(0) - ,m_uSideLength(0) - ,m_uSideLengthPower(0) - ,m_bIsUncompressedDataModified(true) { if(uSideLength == 0) { @@ -60,18 +55,6 @@ namespace PolyVox POLYVOX_THROW(std::invalid_argument, "Block must be provided with a valid compressor."); } - //Compute the side length - m_uSideLength = uSideLength; - m_uSideLengthPower = logBase2(uSideLength); - - //Temporarily create the block data. This is just so we can compress it an discard it. - // FIXME - this is a temporary solution. - /*const uint32_t uNoOfVoxels = m_uSideLength * m_uSideLength * m_uSideLength; - m_tUncompressedData = new VoxelType[uNoOfVoxels]; - std::fill(m_tUncompressedData, m_tUncompressedData + uNoOfVoxels, VoxelType()); - m_bIsUncompressedDataModified = true; - - destroyUncompressedData();*/ } template @@ -88,42 +71,6 @@ namespace PolyVox return m_uCompressedDataLength; } - template - uint16_t Block::getSideLength(void) const - { - return m_uSideLength; - } - - template - VoxelType Block::getVoxel(uint16_t uXPos, uint16_t uYPos, uint16_t uZPos) const - { - // This is internal code not directly called by the user. For efficiency we assert rather than throwing. - POLYVOX_ASSERT(uXPos < m_uSideLength, "Supplied position is outside of the block"); - POLYVOX_ASSERT(uYPos < m_uSideLength, "Supplied position is outside of the block"); - POLYVOX_ASSERT(uZPos < m_uSideLength, "Supplied position is outside of the block"); - POLYVOX_ASSERT(hasUncompressedData(), "The block must have uncompressed data to call getVoxel()"); - POLYVOX_ASSERT(m_tUncompressedData, "No uncompressed data - block must be decompressed before accessing voxels."); - - return m_tUncompressedData - [ - uXPos + - uYPos * m_uSideLength + - uZPos * m_uSideLength * m_uSideLength - ]; - } - - template - VoxelType Block::getVoxel(const Vector3DUint16& v3dPos) const - { - return getVoxel(v3dPos.getX(), v3dPos.getY(), v3dPos.getZ()); - } - - template - bool Block::hasUncompressedData(void) const - { - return m_tUncompressedData != 0; - } - template void Block::setCompressedData(const uint8_t* const data, uint32_t dataLength) { @@ -137,32 +84,6 @@ namespace PolyVox memcpy(m_pCompressedData, data, dataLength); } - template - void Block::setVoxelAt(uint16_t uXPos, uint16_t uYPos, uint16_t uZPos, VoxelType tValue) - { - // This is internal code not directly called by the user. For efficiency we assert rather than throwing. - POLYVOX_ASSERT(uXPos < m_uSideLength, "Supplied position is outside of the block"); - POLYVOX_ASSERT(uYPos < m_uSideLength, "Supplied position is outside of the block"); - POLYVOX_ASSERT(uZPos < m_uSideLength, "Supplied position is outside of the block"); - POLYVOX_ASSERT(hasUncompressedData(), "The block must have uncompressed data to call setVoxelAt()"); - POLYVOX_ASSERT(m_tUncompressedData, "No uncompressed data - block must be decompressed before accessing voxels."); - - m_tUncompressedData - [ - uXPos + - uYPos * m_uSideLength + - uZPos * m_uSideLength * m_uSideLength - ] = tValue; - - m_bIsUncompressedDataModified = true; - } - - template - void Block::setVoxelAt(const Vector3DUint16& v3dPos, VoxelType tValue) - { - setVoxelAt(v3dPos.getX(), v3dPos.getY(), v3dPos.getZ(), tValue); - } - template uint32_t Block::calculateSizeInBytes(void) { diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl index 770c94ee..46b89c36 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl @@ -746,7 +746,7 @@ namespace PolyVox } } - POLYVOX_ASSERT(itUnloadBlock->second.hasUncompressedData() == false, "This function should never flush blocks with uncompressed data."); + //POLYVOX_ASSERT(itUnloadBlock->second.hasUncompressedData() == false, "This function should never flush blocks with uncompressed data."); uMemoryToReclaim -= itUnloadBlock->second.calculateSizeInBytes(); From a00574351ffa94225dd7272618e2d7d8a6a4d885 Mon Sep 17 00:00:00 2001 From: David Williams Date: Thu, 11 Jul 2013 16:38:28 +0200 Subject: [PATCH 08/38] Renamed Block to CompressedBlock --- examples/Paging/main.cpp | 4 +-- .../include/PolyVoxCore/FilePager.h | 4 +-- .../include/PolyVoxCore/Impl/Block.h | 4 +-- .../include/PolyVoxCore/Impl/Block.inl | 12 ++++----- .../include/PolyVoxCore/LargeVolume.h | 6 ++--- .../include/PolyVoxCore/LargeVolume.inl | 26 +++++++++---------- .../PolyVoxCore/include/PolyVoxCore/Pager.h | 4 +-- 7 files changed, 30 insertions(+), 30 deletions(-) diff --git a/examples/Paging/main.cpp b/examples/Paging/main.cpp index c497a255..07877405 100644 --- a/examples/Paging/main.cpp +++ b/examples/Paging/main.cpp @@ -91,7 +91,7 @@ public: /// Destructor virtual ~PerlinNoisePager() {}; - virtual void pageIn(const PolyVox::Region& region, Block* pBlockData) + virtual void pageIn(const PolyVox::Region& region, CompressedBlock* pBlockData) { /*pBlockData->createUncompressedData(); @@ -137,7 +137,7 @@ public: }*/ } - virtual void pageOut(const PolyVox::Region& region, Block* /*pBlockData*/) + virtual void pageOut(const PolyVox::Region& region, CompressedBlock* /*pBlockData*/) { std::cout << "warning unloading region: " << region.getLowerCorner() << " -> " << region.getUpperCorner() << std::endl; } diff --git a/library/PolyVoxCore/include/PolyVoxCore/FilePager.h b/library/PolyVoxCore/include/PolyVoxCore/FilePager.h index 7fb79f1c..80c4f05a 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/FilePager.h +++ b/library/PolyVoxCore/include/PolyVoxCore/FilePager.h @@ -52,7 +52,7 @@ namespace PolyVox /// Destructor virtual ~FilePager() {}; - virtual void pageIn(const Region& region, Block* pBlockData) + virtual void pageIn(const Region& region, CompressedBlock* pBlockData) { POLYVOX_ASSERT(pBlockData, "Attempting to page in NULL block"); //POLYVOX_ASSERT(pBlockData->hasUncompressedData() == false, "Block should not have uncompressed data"); @@ -93,7 +93,7 @@ namespace PolyVox } } - virtual void pageOut(const Region& region, Block* pBlockData) + virtual void pageOut(const Region& region, CompressedBlock* pBlockData) { POLYVOX_ASSERT(pBlockData, "Attempting to page out NULL block"); //POLYVOX_ASSERT(pBlockData->hasUncompressedData() == false, "Block should not have uncompressed data"); diff --git a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h index 86d6d08b..f9ce6f3c 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h +++ b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h @@ -35,10 +35,10 @@ freely, subject to the following restrictions: namespace PolyVox { template - class Block + class CompressedBlock { public: - Block(uint16_t uSideLength, Compressor* pCompressor); + CompressedBlock(uint16_t uSideLength, Compressor* pCompressor); const uint8_t* getCompressedData(void) const; uint32_t getCompressedDataLength(void) const; diff --git a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl index 3f311e0d..8f54c9a3 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl @@ -36,7 +36,7 @@ freely, subject to the following restrictions: namespace PolyVox { template - Block::Block(uint16_t uSideLength, Compressor* pCompressor) + CompressedBlock::CompressedBlock(uint16_t uSideLength, Compressor* pCompressor) :m_pCompressedData(0) ,m_uCompressedDataLength(0) { @@ -58,21 +58,21 @@ namespace PolyVox } template - const uint8_t* Block::getCompressedData(void) const + const uint8_t* CompressedBlock::getCompressedData(void) const { POLYVOX_ASSERT(m_pCompressedData, "Compressed data is NULL"); return m_pCompressedData; } template - uint32_t Block::getCompressedDataLength(void) const + uint32_t CompressedBlock::getCompressedDataLength(void) const { POLYVOX_ASSERT(m_pCompressedData, "Compressed data is NULL"); return m_uCompressedDataLength; } template - void Block::setCompressedData(const uint8_t* const data, uint32_t dataLength) + void CompressedBlock::setCompressedData(const uint8_t* const data, uint32_t dataLength) { //POLYVOX_ASSERT(m_pCompressedData, "Compressed data is NULL"); POLYVOX_ASSERT(m_pCompressedData != data, "Attempting to copy data onto itself"); @@ -85,11 +85,11 @@ namespace PolyVox } template - uint32_t Block::calculateSizeInBytes(void) + uint32_t CompressedBlock::calculateSizeInBytes(void) { // Returns the size of this class plus the size of the compressed data. It // doesn't include the uncompressed data cache as that is owned by the volume. - uint32_t uSizeInBytes = sizeof(Block) + m_uCompressedDataLength; + uint32_t uSizeInBytes = sizeof(CompressedBlock) + m_uCompressedDataLength; return uSizeInBytes; } } diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h index 86e36381..c4d3c777 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h @@ -329,13 +329,13 @@ namespace PolyVox VoxelType getVoxelImpl(int32_t uXPos, int32_t uYPos, int32_t uZPos, WrapModeType, VoxelType tBorder) const; VoxelType getVoxelImpl(int32_t uXPos, int32_t uYPos, int32_t uZPos, WrapModeType, VoxelType tBorder) const; - Block* getCompressedBlock(int32_t uBlockX, int32_t uBlockY, int32_t uBlockZ) const; + CompressedBlock* getCompressedBlock(int32_t uBlockX, int32_t uBlockY, int32_t uBlockZ) const; VoxelType* getUncompressedBlock(int32_t uBlockX, int32_t uBlockY, int32_t uBlockZ) const; - void eraseBlock(typename std::map, BlockPositionCompare>::iterator itBlock) const; + void eraseBlock(typename std::map, BlockPositionCompare>::iterator itBlock) const; // The block data mutable std::map m_pUncompressedBlockCache; - mutable std::map, BlockPositionCompare> m_pBlocks; + mutable std::map, BlockPositionCompare> m_pBlocks; mutable uint32_t m_uTimestamper; mutable Vector3DInt32 m_v3dLastAccessedBlockPos; diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl index 46b89c36..ff72cde0 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl @@ -421,7 +421,7 @@ namespace PolyVox for(int32_t z = v3dStart.getZ(); z <= v3dEnd.getZ(); z++) { Vector3DInt32 pos(x,y,z); - typename std::map, BlockPositionCompare>::iterator itBlock = m_pBlocks.find(pos); + typename std::map, BlockPositionCompare>::iterator itBlock = m_pBlocks.find(pos); if(itBlock != m_pBlocks.end()) { @@ -452,7 +452,7 @@ namespace PolyVox template void LargeVolume::flushAll() { - typename std::map, BlockPositionCompare>::iterator i; + typename std::map, BlockPositionCompare>::iterator i; //Replaced the for loop here as the call to //eraseBlock was invalidating the iterator. while(m_pBlocks.size() > 0) @@ -486,7 +486,7 @@ namespace PolyVox for(int32_t z = v3dStart.getZ(); z <= v3dEnd.getZ(); z++) { Vector3DInt32 pos(x,y,z); - typename std::map, BlockPositionCompare>::iterator itBlock = m_pBlocks.find(pos); + typename std::map, BlockPositionCompare>::iterator itBlock = m_pBlocks.find(pos); if(itBlock == m_pBlocks.end()) { // not loaded, not unloading @@ -564,7 +564,7 @@ namespace PolyVox } template - void LargeVolume::eraseBlock(typename std::map, BlockPositionCompare>::iterator itBlock) const + void LargeVolume::eraseBlock(typename std::map, BlockPositionCompare>::iterator itBlock) const { POLYVOX_ASSERT(false, "This function has not been implemented properly"); @@ -603,16 +603,16 @@ namespace PolyVox } template - Block* LargeVolume::getCompressedBlock(int32_t uBlockX, int32_t uBlockY, int32_t uBlockZ) const + CompressedBlock* LargeVolume::getCompressedBlock(int32_t uBlockX, int32_t uBlockY, int32_t uBlockZ) const { Vector3DInt32 v3dBlockPos(uBlockX, uBlockY, uBlockZ); - typename std::map, BlockPositionCompare>::iterator itBlock = m_pBlocks.find(v3dBlockPos); + typename std::map, BlockPositionCompare>::iterator itBlock = m_pBlocks.find(v3dBlockPos); // check whether the block is already loaded if(itBlock == m_pBlocks.end()) { //The block is not in the map, so we will have to create a new block and add it. - Block newBlock(m_uBlockSideLength, m_pCompressor); + CompressedBlock newBlock(m_uBlockSideLength, m_pCompressor); itBlock = m_pBlocks.insert(std::make_pair(v3dBlockPos, newBlock)).first; // Now use the pager to fill the block with it's initial data. @@ -626,7 +626,7 @@ namespace PolyVox } //Get the block and mark that we accessed it - Block& block = itBlock->second; + CompressedBlock& block = itBlock->second; block.timestamp = ++m_uTimestamper; //m_v3dLastAccessedBlockPos = v3dBlockPos; @@ -648,7 +648,7 @@ namespace PolyVox } //Get the block and mark that we accessed it - Block* block = getCompressedBlock(uBlockX, uBlockY, uBlockZ); + CompressedBlock* block = getCompressedBlock(uBlockX, uBlockY, uBlockZ); typename std::map::iterator itUncompressedBlock = m_pUncompressedBlockCache.find(v3dBlockPos); @@ -697,7 +697,7 @@ namespace PolyVox uint32_t uSizeInBytes = sizeof(LargeVolume); //Memory used by the blocks - typename std::map, BlockPositionCompare>::iterator i; + typename std::map, BlockPositionCompare>::iterator i; for(i = m_pBlocks.begin(); i != m_pBlocks.end(); i++) { //Inaccurate - account for rest of loaded block. @@ -716,7 +716,7 @@ namespace PolyVox { uint32_t uMemoryUsage = 0; - typename std::map, BlockPositionCompare>::iterator i; + typename std::map, BlockPositionCompare>::iterator i; for(i = m_pBlocks.begin(); i != m_pBlocks.end(); i++) { //Inaccurate - account for rest of loaded block. @@ -736,8 +736,8 @@ namespace PolyVox while(calculateBlockMemoryUsage() > m_uCompressedBlockMemoryLimitInBytes) //FIXME - This calculation of size is slow and should be outside the loop. { // find the least recently used block - typename std::map, BlockPositionCompare>::iterator i; - typename std::map, BlockPositionCompare>::iterator itUnloadBlock = m_pBlocks.begin(); + typename std::map, BlockPositionCompare>::iterator i; + typename std::map, BlockPositionCompare>::iterator itUnloadBlock = m_pBlocks.begin(); for(i = m_pBlocks.begin(); i != m_pBlocks.end(); i++) { if(i->second.timestamp < itUnloadBlock->second.timestamp) diff --git a/library/PolyVoxCore/include/PolyVoxCore/Pager.h b/library/PolyVoxCore/include/PolyVoxCore/Pager.h index 4cb95adc..af1e3387 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/Pager.h +++ b/library/PolyVoxCore/include/PolyVoxCore/Pager.h @@ -41,8 +41,8 @@ namespace PolyVox /// Destructor virtual ~Pager() {}; - virtual void pageIn(const Region& region, Block* pBlockData) = 0; - virtual void pageOut(const Region& region, Block* pBlockData) = 0; + virtual void pageIn(const Region& region, CompressedBlock* pBlockData) = 0; + virtual void pageOut(const Region& region, CompressedBlock* pBlockData) = 0; }; } From 0cfb9f51961324d533d52deeb04a3e3da2198729 Mon Sep 17 00:00:00 2001 From: David Williams Date: Tue, 16 Jul 2013 14:42:43 +0200 Subject: [PATCH 09/38] Splitting 'Block into CompressedBlock and UncompressedBlock. --- .../include/PolyVoxCore/Impl/Block.h | 20 ++++ .../include/PolyVoxCore/Impl/Block.inl | 99 +++++++++++++++++-- .../include/PolyVoxCore/LargeVolume.h | 6 +- .../include/PolyVoxCore/LargeVolume.inl | 46 +++------ .../PolyVoxCore/LargeVolumeSampler.inl | 4 +- 5 files changed, 127 insertions(+), 48 deletions(-) diff --git a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h index f9ce6f3c..edd3a893 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h +++ b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h @@ -52,6 +52,26 @@ namespace PolyVox uint32_t m_uCompressedDataLength; uint32_t timestamp; }; + + template + class UncompressedBlock + { + public: + UncompressedBlock(uint16_t uSideLength); + ~UncompressedBlock(); + + uint16_t getSideLength(void) const; + VoxelType getVoxel(uint16_t uXPos, uint16_t uYPos, uint16_t uZPos) const; + VoxelType getVoxel(const Vector3DUint16& v3dPos) const; + + void setVoxelAt(uint16_t uXPos, uint16_t uYPos, uint16_t uZPos, VoxelType tValue); + void setVoxelAt(const Vector3DUint16& v3dPos, VoxelType tValue); + + VoxelType* m_tUncompressedData; + uint16_t m_uSideLength; + uint8_t m_uSideLengthPower; + bool m_bIsUncompressedDataModified; + }; } #include "PolyVoxCore/Impl/Block.inl" diff --git a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl index 8f54c9a3..19f50146 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl @@ -9,16 +9,16 @@ Permission is granted to anyone to use this software for any purpose, including commercial applications, and to alter it and redistribute it freely, subject to the following restrictions: - 1. The origin of this software must not be misrepresented; you must not - claim that you wrote the original software. If you use this software - in a product, an acknowledgment in the product documentation would be - appreciated but is not required. +1. The origin of this software must not be misrepresented; you must not +claim that you wrote the original software. If you use this software +in a product, an acknowledgment in the product documentation would be +appreciated but is not required. - 2. Altered source versions must be plainly marked as such, and must not be - misrepresented as being the original software. +2. Altered source versions must be plainly marked as such, and must not be +misrepresented as being the original software. - 3. This notice may not be removed or altered from any source - distribution. +3. This notice may not be removed or altered from any source +distribution. *******************************************************************************/ #include "PolyVoxCore/Impl/ErrorHandling.h" @@ -35,6 +35,8 @@ freely, subject to the following restrictions: namespace PolyVox { + //////////////////////////////////////////////////////////////////////////////// + template CompressedBlock::CompressedBlock(uint16_t uSideLength, Compressor* pCompressor) :m_pCompressedData(0) @@ -63,7 +65,7 @@ namespace PolyVox POLYVOX_ASSERT(m_pCompressedData, "Compressed data is NULL"); return m_pCompressedData; } - + template uint32_t CompressedBlock::getCompressedDataLength(void) const { @@ -92,4 +94,83 @@ namespace PolyVox uint32_t uSizeInBytes = sizeof(CompressedBlock) + m_uCompressedDataLength; return uSizeInBytes; } + + //////////////////////////////////////////////////////////////////////////////// + + template + UncompressedBlock::UncompressedBlock(uint16_t uSideLength) + :m_tUncompressedData(0) + ,m_uSideLength(0) + ,m_uSideLengthPower(0) + ,m_bIsUncompressedDataModified(true) + { + // Compute the side length + m_uSideLength = uSideLength; + m_uSideLengthPower = logBase2(uSideLength); + + // Allocate the data + const uint32_t uNoOfVoxels = m_uSideLength * m_uSideLength * m_uSideLength; + m_tUncompressedData = new VoxelType[uNoOfVoxels]; + } + + template + UncompressedBlock::~UncompressedBlock() + { + delete m_tUncompressedData; + m_tUncompressedData = 0; + } + + template + uint16_t UncompressedBlock::getSideLength(void) const + { + return m_uSideLength; + } + + template + VoxelType UncompressedBlock::getVoxel(uint16_t uXPos, uint16_t uYPos, uint16_t uZPos) const + { + // This is internal code not directly called by the user. For efficiency we assert rather than throwing. + POLYVOX_ASSERT(uXPos < m_uSideLength, "Supplied position is outside of the block"); + POLYVOX_ASSERT(uYPos < m_uSideLength, "Supplied position is outside of the block"); + POLYVOX_ASSERT(uZPos < m_uSideLength, "Supplied position is outside of the block"); + POLYVOX_ASSERT(m_tUncompressedData, "No uncompressed data - block must be decompressed before accessing voxels."); + + return m_tUncompressedData + [ + uXPos + + uYPos * m_uSideLength + + uZPos * m_uSideLength * m_uSideLength + ]; + } + + template + VoxelType UncompressedBlock::getVoxel(const Vector3DUint16& v3dPos) const + { + return getVoxel(v3dPos.getX(), v3dPos.getY(), v3dPos.getZ()); + } + + template + void UncompressedBlock::setVoxelAt(uint16_t uXPos, uint16_t uYPos, uint16_t uZPos, VoxelType tValue) + { + // This is internal code not directly called by the user. For efficiency we assert rather than throwing. + POLYVOX_ASSERT(uXPos < m_uSideLength, "Supplied position is outside of the block"); + POLYVOX_ASSERT(uYPos < m_uSideLength, "Supplied position is outside of the block"); + POLYVOX_ASSERT(uZPos < m_uSideLength, "Supplied position is outside of the block"); + POLYVOX_ASSERT(m_tUncompressedData, "No uncompressed data - block must be decompressed before accessing voxels."); + + m_tUncompressedData + [ + uXPos + + uYPos * m_uSideLength + + uZPos * m_uSideLength * m_uSideLength + ] = tValue; + + m_bIsUncompressedDataModified = true; + } + + template + void UncompressedBlock::setVoxelAt(const Vector3DUint16& v3dPos, VoxelType tValue) + { + setVoxelAt(v3dPos.getX(), v3dPos.getY(), v3dPos.getZ(), tValue); + } } diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h index c4d3c777..65e9dec4 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h @@ -330,16 +330,16 @@ namespace PolyVox VoxelType getVoxelImpl(int32_t uXPos, int32_t uYPos, int32_t uZPos, WrapModeType, VoxelType tBorder) const; CompressedBlock* getCompressedBlock(int32_t uBlockX, int32_t uBlockY, int32_t uBlockZ) const; - VoxelType* getUncompressedBlock(int32_t uBlockX, int32_t uBlockY, int32_t uBlockZ) const; + UncompressedBlock* getUncompressedBlock(int32_t uBlockX, int32_t uBlockY, int32_t uBlockZ) const; void eraseBlock(typename std::map, BlockPositionCompare>::iterator itBlock) const; // The block data - mutable std::map m_pUncompressedBlockCache; + mutable std::map*, BlockPositionCompare> m_pUncompressedBlockCache; mutable std::map, BlockPositionCompare> m_pBlocks; mutable uint32_t m_uTimestamper; mutable Vector3DInt32 m_v3dLastAccessedBlockPos; - mutable VoxelType* m_pLastAccessedBlock; + mutable UncompressedBlock* m_pLastAccessedBlock; uint32_t m_uMaxNumberOfUncompressedBlocks; uint32_t m_uCompressedBlockMemoryLimitInBytes; diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl index ff72cde0..3357f60c 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl @@ -223,14 +223,9 @@ namespace PolyVox const uint16_t yOffset = static_cast(uYPos - (blockY << m_uBlockSideLengthPower)); const uint16_t zOffset = static_cast(uZPos - (blockZ << m_uBlockSideLengthPower)); - VoxelType* pUncompressedBlock = getUncompressedBlock(blockX, blockY, blockZ); + const UncompressedBlock* pUncompressedBlock = getUncompressedBlock(blockX, blockY, blockZ); - return pUncompressedBlock - [ - xOffset + - yOffset * m_uBlockSideLength + - zOffset * m_uBlockSideLength * m_uBlockSideLength - ]; + return pUncompressedBlock->getVoxel(xOffset, yOffset, zOffset); } else { @@ -317,14 +312,8 @@ namespace PolyVox const uint16_t yOffset = static_cast(uYPos - (blockY << m_uBlockSideLengthPower)); const uint16_t zOffset = static_cast(uZPos - (blockZ << m_uBlockSideLengthPower)); - VoxelType* pUncompressedBlock = getUncompressedBlock(blockX, blockY, blockZ); - - pUncompressedBlock - [ - xOffset + - yOffset * m_uBlockSideLength + - zOffset * m_uBlockSideLength * m_uBlockSideLength - ] = tValue; + UncompressedBlock* pUncompressedBlock = getUncompressedBlock(blockX, blockY, blockZ); + pUncompressedBlock->setVoxelAt(xOffset, yOffset, zOffset); } //////////////////////////////////////////////////////////////////////////////// @@ -360,14 +349,9 @@ namespace PolyVox const uint16_t yOffset = static_cast(uYPos - (blockY << m_uBlockSideLengthPower)); const uint16_t zOffset = static_cast(uZPos - (blockZ << m_uBlockSideLengthPower)); - VoxelType* pUncompressedBlock = getUncompressedBlock(blockX, blockY, blockZ); + UncompressedBlock* pUncompressedBlock = getUncompressedBlock(blockX, blockY, blockZ); - pUncompressedBlock - [ - xOffset + - yOffset * m_uBlockSideLength + - zOffset * m_uBlockSideLength * m_uBlockSideLength - ] = tValue; + pUncompressedBlock->setVoxelAt(xOffset, yOffset, zOffset, tValue); //Return true to indicate that we modified a voxel. return true; @@ -634,7 +618,7 @@ namespace PolyVox } template - VoxelType* LargeVolume::getUncompressedBlock(int32_t uBlockX, int32_t uBlockY, int32_t uBlockZ) const + UncompressedBlock* LargeVolume::getUncompressedBlock(int32_t uBlockX, int32_t uBlockY, int32_t uBlockZ) const { Vector3DInt32 v3dBlockPos(uBlockX, uBlockY, uBlockZ); @@ -651,14 +635,14 @@ namespace PolyVox CompressedBlock* block = getCompressedBlock(uBlockX, uBlockY, uBlockZ); - typename std::map::iterator itUncompressedBlock = m_pUncompressedBlockCache.find(v3dBlockPos); + typename std::map*, BlockPositionCompare>::iterator itUncompressedBlock = m_pUncompressedBlockCache.find(v3dBlockPos); // check whether the block is already loaded if(itUncompressedBlock == m_pUncompressedBlockCache.end()) { - VoxelType* pUncompressedBlock = new VoxelType[m_uBlockSideLength * m_uBlockSideLength * m_uBlockSideLength]; + UncompressedBlock* pUncompressedBlock = new UncompressedBlock(m_uBlockSideLength); void* pSrcData = reinterpret_cast(block->m_pCompressedData); - void* pDstData = reinterpret_cast(pUncompressedBlock); + void* pDstData = reinterpret_cast(pUncompressedBlock->m_tUncompressedData); uint32_t uSrcLength = block->m_uCompressedDataLength; uint32_t uDstLength = m_uBlockSideLength * m_uBlockSideLength * m_uBlockSideLength * sizeof(VoxelType); @@ -816,14 +800,8 @@ namespace PolyVox const uint16_t yOffset = static_cast(uYPos - (blockY << m_uBlockSideLengthPower)); const uint16_t zOffset = static_cast(uZPos - (blockZ << m_uBlockSideLengthPower)); - VoxelType* pUncompressedBlock = getUncompressedBlock(blockX, blockY, blockZ); - - return pUncompressedBlock - [ - xOffset + - yOffset * m_uBlockSideLength + - zOffset * m_uBlockSideLength * m_uBlockSideLength - ]; + UncompressedBlock* pUncompressedBlock = getUncompressedBlock(blockX, blockY, blockZ); + return pUncompressedBlock->getVoxel(xOffset, yOffset, zOffset); } } diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolumeSampler.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolumeSampler.inl index d90ea5f4..8bbad0f4 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolumeSampler.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolumeSampler.inl @@ -119,9 +119,9 @@ namespace PolyVox uYPosInBlock * this->mVolume->m_uBlockSideLength + uZPosInBlock * this->mVolume->m_uBlockSideLength * this->mVolume->m_uBlockSideLength; - VoxelType* pUncompressedCurrentBlock = this->mVolume->getUncompressedBlock(uXBlock, uYBlock, uZBlock); + UncompressedBlock* pUncompressedCurrentBlock = this->mVolume->getUncompressedBlock(uXBlock, uYBlock, uZBlock); - mCurrentVoxel = pUncompressedCurrentBlock + uVoxelIndexInBlock; + mCurrentVoxel = pUncompressedCurrentBlock->m_tUncompressedData + uVoxelIndexInBlock; } else { From 3904c9aa8f798795c16e5ce9688a530686b306e9 Mon Sep 17 00:00:00 2001 From: David Williams Date: Tue, 16 Jul 2013 15:59:06 +0200 Subject: [PATCH 10/38] Tidying and renaming... --- .../include/PolyVoxCore/FilePager.h | 4 +- .../include/PolyVoxCore/Impl/Block.h | 35 ++++++++---- .../include/PolyVoxCore/Impl/Block.inl | 54 +++++++------------ .../include/PolyVoxCore/LargeVolume.inl | 10 ++-- 4 files changed, 52 insertions(+), 51 deletions(-) diff --git a/library/PolyVoxCore/include/PolyVoxCore/FilePager.h b/library/PolyVoxCore/include/PolyVoxCore/FilePager.h index 80c4f05a..66091d9c 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/FilePager.h +++ b/library/PolyVoxCore/include/PolyVoxCore/FilePager.h @@ -77,7 +77,7 @@ namespace PolyVox uint8_t* buffer = new uint8_t[fileSizeInBytes]; fread(buffer, sizeof(uint8_t), fileSizeInBytes, pFile); - pBlockData->setCompressedData(buffer, fileSizeInBytes); + pBlockData->setData(buffer, fileSizeInBytes); delete[] buffer; if(ferror(pFile)) @@ -115,7 +115,7 @@ namespace PolyVox POLYVOX_THROW(std::runtime_error, "Unable to open file to write out block data."); } - fwrite(pBlockData->getCompressedData(), sizeof(uint8_t), pBlockData->getCompressedDataLength(), pFile); + fwrite(pBlockData->getData(), sizeof(uint8_t), pBlockData->getDataSizeInBytes(), pFile); if(ferror(pFile)) { diff --git a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h index edd3a893..2b9893b5 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h +++ b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h @@ -35,26 +35,42 @@ freely, subject to the following restrictions: namespace PolyVox { template - class CompressedBlock + class Block + { + friend LargeVolume; + + public: + Block() + :m_uBlockLastAccessed(0) + ,m_bDataModified(false) + { + } + + protected: + uint32_t m_uBlockLastAccessed; + bool m_bDataModified; + }; + + template + class CompressedBlock : public Block { public: - CompressedBlock(uint16_t uSideLength, Compressor* pCompressor); + CompressedBlock(); - const uint8_t* getCompressedData(void) const; - uint32_t getCompressedDataLength(void) const; + const uint8_t* getData(void) const; + uint32_t getDataSizeInBytes(void) const; - void setCompressedData(const uint8_t* const data, uint32_t dataLength); + void setData(const uint8_t* const pData, uint32_t uDataSizeInBytes); uint32_t calculateSizeInBytes(void); public: - uint8_t* m_pCompressedData; - uint32_t m_uCompressedDataLength; - uint32_t timestamp; + uint8_t* m_pData; + uint32_t m_uDataSizeInBytes; }; template - class UncompressedBlock + class UncompressedBlock : public Block { public: UncompressedBlock(uint16_t uSideLength); @@ -70,7 +86,6 @@ namespace PolyVox VoxelType* m_tUncompressedData; uint16_t m_uSideLength; uint8_t m_uSideLengthPower; - bool m_bIsUncompressedDataModified; }; } diff --git a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl index 19f50146..492d829f 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl @@ -38,52 +38,39 @@ namespace PolyVox //////////////////////////////////////////////////////////////////////////////// template - CompressedBlock::CompressedBlock(uint16_t uSideLength, Compressor* pCompressor) - :m_pCompressedData(0) - ,m_uCompressedDataLength(0) + CompressedBlock::CompressedBlock() + :m_pData(0) + ,m_uDataSizeInBytes(0) { - if(uSideLength == 0) - { - POLYVOX_THROW(std::invalid_argument, "Block side length cannot be zero."); - } - - if(!isPowerOf2(uSideLength)) - { - POLYVOX_THROW(std::invalid_argument, "Block side length must be a power of two."); - } - - if(pCompressor == 0) - { - POLYVOX_THROW(std::invalid_argument, "Block must be provided with a valid compressor."); - } - } template - const uint8_t* CompressedBlock::getCompressedData(void) const + const uint8_t* CompressedBlock::getData(void) const { - POLYVOX_ASSERT(m_pCompressedData, "Compressed data is NULL"); - return m_pCompressedData; + return m_pData; } template - uint32_t CompressedBlock::getCompressedDataLength(void) const + uint32_t CompressedBlock::getDataSizeInBytes(void) const { - POLYVOX_ASSERT(m_pCompressedData, "Compressed data is NULL"); - return m_uCompressedDataLength; + return m_uDataSizeInBytes; } template - void CompressedBlock::setCompressedData(const uint8_t* const data, uint32_t dataLength) + void CompressedBlock::setData(const uint8_t* const pData, uint32_t uDataSizeInBytes) { - //POLYVOX_ASSERT(m_pCompressedData, "Compressed data is NULL"); - POLYVOX_ASSERT(m_pCompressedData != data, "Attempting to copy data onto itself"); + POLYVOX_THROW_IF(pData == 0, std::invalid_argument, "Pointer to data cannot be null"); + POLYVOX_THROW_IF(m_pData == pData, std::invalid_argument, "Attempting to copy data onto itself"); - delete[] m_pCompressedData; + // Delete any existing data + delete[] m_pData; - m_uCompressedDataLength = dataLength; - m_pCompressedData = new uint8_t[dataLength]; - memcpy(m_pCompressedData, data, dataLength); + // Allocate new data + m_uDataSizeInBytes = uDataSizeInBytes; + m_pData = new uint8_t[uDataSizeInBytes]; + + // Copy the data across + memcpy(m_pData, pData, uDataSizeInBytes); } template @@ -91,7 +78,7 @@ namespace PolyVox { // Returns the size of this class plus the size of the compressed data. It // doesn't include the uncompressed data cache as that is owned by the volume. - uint32_t uSizeInBytes = sizeof(CompressedBlock) + m_uCompressedDataLength; + uint32_t uSizeInBytes = sizeof(CompressedBlock) + m_uDataSizeInBytes; return uSizeInBytes; } @@ -102,7 +89,6 @@ namespace PolyVox :m_tUncompressedData(0) ,m_uSideLength(0) ,m_uSideLengthPower(0) - ,m_bIsUncompressedDataModified(true) { // Compute the side length m_uSideLength = uSideLength; @@ -165,7 +151,7 @@ namespace PolyVox uZPos * m_uSideLength * m_uSideLength ] = tValue; - m_bIsUncompressedDataModified = true; + m_bDataModified = true; } template diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl index 3357f60c..ce482006 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl @@ -596,7 +596,7 @@ namespace PolyVox if(itBlock == m_pBlocks.end()) { //The block is not in the map, so we will have to create a new block and add it. - CompressedBlock newBlock(m_uBlockSideLength, m_pCompressor); + CompressedBlock newBlock; itBlock = m_pBlocks.insert(std::make_pair(v3dBlockPos, newBlock)).first; // Now use the pager to fill the block with it's initial data. @@ -611,7 +611,7 @@ namespace PolyVox //Get the block and mark that we accessed it CompressedBlock& block = itBlock->second; - block.timestamp = ++m_uTimestamper; + block.m_uBlockLastAccessed = ++m_uTimestamper; //m_v3dLastAccessedBlockPos = v3dBlockPos; return █ @@ -641,9 +641,9 @@ namespace PolyVox { UncompressedBlock* pUncompressedBlock = new UncompressedBlock(m_uBlockSideLength); - void* pSrcData = reinterpret_cast(block->m_pCompressedData); + void* pSrcData = reinterpret_cast(block->m_pData); void* pDstData = reinterpret_cast(pUncompressedBlock->m_tUncompressedData); - uint32_t uSrcLength = block->m_uCompressedDataLength; + uint32_t uSrcLength = block->m_uDataSizeInBytes; uint32_t uDstLength = m_uBlockSideLength * m_uBlockSideLength * m_uBlockSideLength * sizeof(VoxelType); //MinizCompressor compressor; @@ -724,7 +724,7 @@ namespace PolyVox typename std::map, BlockPositionCompare>::iterator itUnloadBlock = m_pBlocks.begin(); for(i = m_pBlocks.begin(); i != m_pBlocks.end(); i++) { - if(i->second.timestamp < itUnloadBlock->second.timestamp) + if(i->second.m_uBlockLastAccessed < itUnloadBlock->second.m_uBlockLastAccessed) { itUnloadBlock = i; } From b5d930062bae0ca20ef5df64a7f71cb8360bb406 Mon Sep 17 00:00:00 2001 From: David Williams Date: Tue, 16 Jul 2013 16:09:57 +0200 Subject: [PATCH 11/38] Const fixes. --- library/PolyVoxCore/include/PolyVoxCore/Compressor.h | 4 ++-- library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h | 3 ++- .../PolyVoxCore/include/PolyVoxCore/Impl/Block.inl | 10 ++++++++++ .../PolyVoxCore/include/PolyVoxCore/LargeVolume.inl | 4 ++-- .../include/PolyVoxCore/MinizCompressor.h | 4 ++-- .../PolyVoxCore/include/PolyVoxCore/RLECompressor.h | 4 ++-- .../include/PolyVoxCore/RLECompressor.inl | 12 ++++++------ library/PolyVoxCore/source/MinizCompressor.cpp | 4 ++-- 8 files changed, 28 insertions(+), 17 deletions(-) diff --git a/library/PolyVoxCore/include/PolyVoxCore/Compressor.h b/library/PolyVoxCore/include/PolyVoxCore/Compressor.h index 8c00be5e..e85610de 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/Compressor.h +++ b/library/PolyVoxCore/include/PolyVoxCore/Compressor.h @@ -76,7 +76,7 @@ namespace PolyVox * \param uDstLength The length of the destination buffer (compression will fail if this isn't big enough). * \return The size of the resulting compressed data. */ - virtual uint32_t compress(void* pSrcData, uint32_t uSrcLength, void* pDstData, uint32_t uDstLength) = 0; + virtual uint32_t compress(const void* pSrcData, uint32_t uSrcLength, void* pDstData, uint32_t uDstLength) = 0; /** * Decompresses the data. @@ -93,7 +93,7 @@ namespace PolyVox * \param uDstLength The length of the destination buffer (decompression will fail if this isn't big enough). * \return The size of the resulting uncompressed data. */ - virtual uint32_t decompress(void* pSrcData, uint32_t uSrcLength, void* pDstData, uint32_t uDstLength) = 0; + virtual uint32_t decompress(const void* pSrcData, uint32_t uSrcLength, void* pDstData, uint32_t uDstLength) = 0; }; } diff --git a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h index 2b9893b5..32cb9857 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h +++ b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h @@ -56,6 +56,7 @@ namespace PolyVox { public: CompressedBlock(); + ~CompressedBlock(); const uint8_t* getData(void) const; uint32_t getDataSizeInBytes(void) const; @@ -64,7 +65,7 @@ namespace PolyVox uint32_t calculateSizeInBytes(void); - public: + private: uint8_t* m_pData; uint32_t m_uDataSizeInBytes; }; diff --git a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl index 492d829f..b59d6f0a 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl @@ -44,6 +44,13 @@ namespace PolyVox { } + template + CompressedBlock::~CompressedBlock() + { + delete[] m_pData; + m_pData = 0; + } + template const uint8_t* CompressedBlock::getData(void) const { @@ -71,6 +78,9 @@ namespace PolyVox // Copy the data across memcpy(m_pData, pData, uDataSizeInBytes); + + // Flag as modified + m_bDataModified = true; } template diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl index ce482006..fdf4bcac 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl @@ -641,9 +641,9 @@ namespace PolyVox { UncompressedBlock* pUncompressedBlock = new UncompressedBlock(m_uBlockSideLength); - void* pSrcData = reinterpret_cast(block->m_pData); + const void* pSrcData = reinterpret_cast(block->getData()); void* pDstData = reinterpret_cast(pUncompressedBlock->m_tUncompressedData); - uint32_t uSrcLength = block->m_uDataSizeInBytes; + uint32_t uSrcLength = block->getDataSizeInBytes(); uint32_t uDstLength = m_uBlockSideLength * m_uBlockSideLength * m_uBlockSideLength * sizeof(VoxelType); //MinizCompressor compressor; diff --git a/library/PolyVoxCore/include/PolyVoxCore/MinizCompressor.h b/library/PolyVoxCore/include/PolyVoxCore/MinizCompressor.h index fb2b2473..7ea8f967 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/MinizCompressor.h +++ b/library/PolyVoxCore/include/PolyVoxCore/MinizCompressor.h @@ -25,8 +25,8 @@ namespace PolyVox // API documentation is in base class and gets inherited by Doxygen. uint32_t getMaxCompressedSize(uint32_t uUncompressedInputSize); - uint32_t compress(void* pSrcData, uint32_t uSrcLength, void* pDstData, uint32_t uDstLength); - uint32_t decompress(void* pSrcData, uint32_t uSrcLength, void* pDstData, uint32_t uDstLength); + uint32_t compress(const void* pSrcData, uint32_t uSrcLength, void* pDstData, uint32_t uDstLength); + uint32_t decompress(const void* pSrcData, uint32_t uSrcLength, void* pDstData, uint32_t uDstLength); private: unsigned int m_uCompressionFlags; diff --git a/library/PolyVoxCore/include/PolyVoxCore/RLECompressor.h b/library/PolyVoxCore/include/PolyVoxCore/RLECompressor.h index 10b52b90..03fd3318 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/RLECompressor.h +++ b/library/PolyVoxCore/include/PolyVoxCore/RLECompressor.h @@ -30,8 +30,8 @@ namespace PolyVox // API documentation is in base class and gets inherited by Doxygen. uint32_t getMaxCompressedSize(uint32_t uUncompressedInputSize); - uint32_t compress(void* pSrcData, uint32_t uSrcLength, void* pDstData, uint32_t uDstLength); - uint32_t decompress(void* pSrcData, uint32_t uSrcLength, void* pDstData, uint32_t uDstLength); + uint32_t compress(const void* pSrcData, uint32_t uSrcLength, void* pDstData, uint32_t uDstLength); + uint32_t decompress(const void* pSrcData, uint32_t uSrcLength, void* pDstData, uint32_t uDstLength); }; } diff --git a/library/PolyVoxCore/include/PolyVoxCore/RLECompressor.inl b/library/PolyVoxCore/include/PolyVoxCore/RLECompressor.inl index 0b43a301..cf65bc03 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/RLECompressor.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/RLECompressor.inl @@ -22,7 +22,7 @@ namespace PolyVox } template - uint32_t RLECompressor::compress(void* pSrcData, uint32_t uSrcLength, void* pDstData, uint32_t uDstLength) + uint32_t RLECompressor::compress(const void* pSrcData, uint32_t uSrcLength, void* pDstData, uint32_t uDstLength) { if(uSrcLength % sizeof(ValueType) != 0) { @@ -34,11 +34,11 @@ namespace PolyVox uDstLength /= sizeof(Run); // Get data pointers in the appropriate type - ValueType* pSrcDataAsType = reinterpret_cast(pSrcData); + const ValueType* pSrcDataAsType = reinterpret_cast(pSrcData); Run* pDstDataAsRun = reinterpret_cast(pDstData); // Pointers to just past the end of the data - ValueType* pSrcDataEnd = pSrcDataAsType + uSrcLength; + const ValueType* pSrcDataEnd = pSrcDataAsType + uSrcLength; Run* pDstDataEnd = pDstDataAsRun + uDstLength; //Counter for the output length @@ -83,7 +83,7 @@ namespace PolyVox } template - uint32_t RLECompressor::decompress(void* pSrcData, uint32_t uSrcLength, void* pDstData, uint32_t uDstLength) + uint32_t RLECompressor::decompress(const void* pSrcData, uint32_t uSrcLength, void* pDstData, uint32_t uDstLength) { if(uSrcLength % sizeof(Run) != 0) { @@ -95,11 +95,11 @@ namespace PolyVox uDstLength /= sizeof(ValueType); // Get data pointers in the appropriate type - Run* pSrcDataAsRun = reinterpret_cast(pSrcData); + const Run* pSrcDataAsRun = reinterpret_cast(pSrcData); ValueType* pDstDataAsType = reinterpret_cast(pDstData); // Pointers to just past the end of the data - Run* pSrcDataEnd = pSrcDataAsRun + uSrcLength; + const Run* pSrcDataEnd = pSrcDataAsRun + uSrcLength; ValueType* pDstDataEnd = pDstDataAsType + uDstLength; //Counter for the output length diff --git a/library/PolyVoxCore/source/MinizCompressor.cpp b/library/PolyVoxCore/source/MinizCompressor.cpp index a341ff81..aa35c644 100644 --- a/library/PolyVoxCore/source/MinizCompressor.cpp +++ b/library/PolyVoxCore/source/MinizCompressor.cpp @@ -70,7 +70,7 @@ namespace PolyVox return MZ_MAX(128 + (source_len * 110) / 100, 128 + source_len + ((source_len / (31 * 1024)) + 1) * 5); } - uint32_t MinizCompressor::compress(void* pSrcData, uint32_t uSrcLength, void* pDstData, uint32_t uDstLength) + uint32_t MinizCompressor::compress(const void* pSrcData, uint32_t uSrcLength, void* pDstData, uint32_t uDstLength) { //Get the deflator tdefl_compressor* pDeflator = reinterpret_cast(m_pDeflator); @@ -103,7 +103,7 @@ namespace PolyVox return ulDstLength; } - uint32_t MinizCompressor::decompress(void* pSrcData, uint32_t uSrcLength, void* pDstData, uint32_t uDstLength) + uint32_t MinizCompressor::decompress(const void* pSrcData, uint32_t uSrcLength, void* pDstData, uint32_t uDstLength) { // I don't know exactly why this limitation exists but it's an implementation detail of miniz. It shouldn't matter for our purposes // as our detination is a Block and those are always a power of two. If you need to use this class for other purposes then you'll From dea7e6a4e9055904a99fb7613b5a882ad1daab9c Mon Sep 17 00:00:00 2001 From: David Williams Date: Tue, 16 Jul 2013 16:50:04 +0200 Subject: [PATCH 12/38] Tidying up block classes. --- .../include/PolyVoxCore/Impl/Block.h | 20 ++++++++-- .../include/PolyVoxCore/Impl/Block.inl | 39 ++++++++++--------- .../include/PolyVoxCore/LargeVolume.inl | 2 +- .../PolyVoxCore/LargeVolumeSampler.inl | 2 +- 4 files changed, 40 insertions(+), 23 deletions(-) diff --git a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h index 32cb9857..e78ecb48 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h +++ b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h @@ -47,13 +47,19 @@ namespace PolyVox } protected: + // This is updated by the LargeVolume and used to discard the least recently used blocks. uint32_t m_uBlockLastAccessed; + + // This is so we can tell whether a uncompressed block has to be recompressed and whether + // a compressed block has to be paged back to disk, or whether they can just be discarded. bool m_bDataModified; }; template class CompressedBlock : public Block { + friend LargeVolume; + public: CompressedBlock(); ~CompressedBlock(); @@ -63,9 +69,11 @@ namespace PolyVox void setData(const uint8_t* const pData, uint32_t uDataSizeInBytes); + private: + // Made this private to avoid any confusion with getDataSizeInBytes(). + // Users shouldn't really need this for CompressedBlock anyway. uint32_t calculateSizeInBytes(void); - private: uint8_t* m_pData; uint32_t m_uDataSizeInBytes; }; @@ -73,18 +81,24 @@ namespace PolyVox template class UncompressedBlock : public Block { + friend LargeVolume; + public: UncompressedBlock(uint16_t uSideLength); ~UncompressedBlock(); - uint16_t getSideLength(void) const; VoxelType getVoxel(uint16_t uXPos, uint16_t uYPos, uint16_t uZPos) const; VoxelType getVoxel(const Vector3DUint16& v3dPos) const; void setVoxelAt(uint16_t uXPos, uint16_t uYPos, uint16_t uZPos, VoxelType tValue); void setVoxelAt(const Vector3DUint16& v3dPos, VoxelType tValue); - VoxelType* m_tUncompressedData; + private: + // Made this private for consistancy with CompressedBlock. + // Users shouldn't really need this for UncompressedBlock anyway. + uint32_t calculateSizeInBytes(void); + + VoxelType* m_tData; uint16_t m_uSideLength; uint8_t m_uSideLengthPower; }; diff --git a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl index b59d6f0a..dab2f35d 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl @@ -86,8 +86,7 @@ namespace PolyVox template uint32_t CompressedBlock::calculateSizeInBytes(void) { - // Returns the size of this class plus the size of the compressed data. It - // doesn't include the uncompressed data cache as that is owned by the volume. + // Returns the size of this class plus the size of the compressed data. uint32_t uSizeInBytes = sizeof(CompressedBlock) + m_uDataSizeInBytes; return uSizeInBytes; } @@ -96,7 +95,7 @@ namespace PolyVox template UncompressedBlock::UncompressedBlock(uint16_t uSideLength) - :m_tUncompressedData(0) + :m_tData(0) ,m_uSideLength(0) ,m_uSideLengthPower(0) { @@ -106,32 +105,27 @@ namespace PolyVox // Allocate the data const uint32_t uNoOfVoxels = m_uSideLength * m_uSideLength * m_uSideLength; - m_tUncompressedData = new VoxelType[uNoOfVoxels]; + m_tData = new VoxelType[uNoOfVoxels]; } template UncompressedBlock::~UncompressedBlock() { - delete m_tUncompressedData; - m_tUncompressedData = 0; - } - - template - uint16_t UncompressedBlock::getSideLength(void) const - { - return m_uSideLength; + delete m_tData; + m_tData = 0; } template VoxelType UncompressedBlock::getVoxel(uint16_t uXPos, uint16_t uYPos, uint16_t uZPos) const { - // This is internal code not directly called by the user. For efficiency we assert rather than throwing. + // This code is not usually expected to be called by the user, with the exception of when implementing paging + // of uncompressed data. It's a performance critical code path so we use asserts rather than exceptions. POLYVOX_ASSERT(uXPos < m_uSideLength, "Supplied position is outside of the block"); POLYVOX_ASSERT(uYPos < m_uSideLength, "Supplied position is outside of the block"); POLYVOX_ASSERT(uZPos < m_uSideLength, "Supplied position is outside of the block"); - POLYVOX_ASSERT(m_tUncompressedData, "No uncompressed data - block must be decompressed before accessing voxels."); + POLYVOX_ASSERT(m_tData, "No uncompressed data - block must be decompressed before accessing voxels."); - return m_tUncompressedData + return m_tData [ uXPos + uYPos * m_uSideLength + @@ -148,13 +142,14 @@ namespace PolyVox template void UncompressedBlock::setVoxelAt(uint16_t uXPos, uint16_t uYPos, uint16_t uZPos, VoxelType tValue) { - // This is internal code not directly called by the user. For efficiency we assert rather than throwing. + // This code is not usually expected to be called by the user, with the exception of when implementing paging + // of uncompressed data. It's a performance critical code path so we use asserts rather than exceptions. POLYVOX_ASSERT(uXPos < m_uSideLength, "Supplied position is outside of the block"); POLYVOX_ASSERT(uYPos < m_uSideLength, "Supplied position is outside of the block"); POLYVOX_ASSERT(uZPos < m_uSideLength, "Supplied position is outside of the block"); - POLYVOX_ASSERT(m_tUncompressedData, "No uncompressed data - block must be decompressed before accessing voxels."); + POLYVOX_ASSERT(m_tData, "No uncompressed data - block must be decompressed before accessing voxels."); - m_tUncompressedData + m_tData [ uXPos + uYPos * m_uSideLength + @@ -169,4 +164,12 @@ namespace PolyVox { setVoxelAt(v3dPos.getX(), v3dPos.getY(), v3dPos.getZ(), tValue); } + + template + uint32_t UncompressedBlock::calculateSizeInBytes(void) + { + // Returns the size of this class plus the size of the uncompressed data. + uint32_t uSizeInBytes = sizeof(UncompressedBlock) + (m_uSideLength * m_uSideLength * m_uSideLength * sizeof(VoxelType)); + return uSizeInBytes; + } } diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl index fdf4bcac..731a6c01 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl @@ -642,7 +642,7 @@ namespace PolyVox UncompressedBlock* pUncompressedBlock = new UncompressedBlock(m_uBlockSideLength); const void* pSrcData = reinterpret_cast(block->getData()); - void* pDstData = reinterpret_cast(pUncompressedBlock->m_tUncompressedData); + void* pDstData = reinterpret_cast(pUncompressedBlock->m_tData); uint32_t uSrcLength = block->getDataSizeInBytes(); uint32_t uDstLength = m_uBlockSideLength * m_uBlockSideLength * m_uBlockSideLength * sizeof(VoxelType); diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolumeSampler.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolumeSampler.inl index 8bbad0f4..c5e4e12c 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolumeSampler.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolumeSampler.inl @@ -121,7 +121,7 @@ namespace PolyVox UncompressedBlock* pUncompressedCurrentBlock = this->mVolume->getUncompressedBlock(uXBlock, uYBlock, uZBlock); - mCurrentVoxel = pUncompressedCurrentBlock->m_tUncompressedData + uVoxelIndexInBlock; + mCurrentVoxel = pUncompressedCurrentBlock->m_tData + uVoxelIndexInBlock; } else { From 5b99854c02a2aa46a74895ef15ecea839158c24d Mon Sep 17 00:00:00 2001 From: David Williams Date: Wed, 17 Jul 2013 16:23:46 +0200 Subject: [PATCH 13/38] Making block copy constructors and assignment operators private to prevent accidental copying. --- .../include/PolyVoxCore/Impl/Block.h | 19 ++++++++++ .../include/PolyVoxCore/LargeVolume.h | 4 +-- .../include/PolyVoxCore/LargeVolume.inl | 36 +++++++++---------- 3 files changed, 39 insertions(+), 20 deletions(-) diff --git a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h index e78ecb48..3d795e26 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h +++ b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h @@ -53,6 +53,13 @@ namespace PolyVox // This is so we can tell whether a uncompressed block has to be recompressed and whether // a compressed block has to be paged back to disk, or whether they can just be discarded. bool m_bDataModified; + + private: + /// Private copy constructor to prevent accisdental copying + Block(const Block& /*rhs*/) {}; + + /// Private assignment operator to prevent accisdental copying + Block& operator=(const Block& /*rhs*/) {}; }; template @@ -70,6 +77,12 @@ namespace PolyVox void setData(const uint8_t* const pData, uint32_t uDataSizeInBytes); private: + /// Private copy constructor to prevent accisdental copying + CompressedBlock(const CompressedBlock& /*rhs*/) {}; + + /// Private assignment operator to prevent accisdental copying + CompressedBlock& operator=(const CompressedBlock& /*rhs*/) {}; + // Made this private to avoid any confusion with getDataSizeInBytes(). // Users shouldn't really need this for CompressedBlock anyway. uint32_t calculateSizeInBytes(void); @@ -94,6 +107,12 @@ namespace PolyVox void setVoxelAt(const Vector3DUint16& v3dPos, VoxelType tValue); private: + /// Private copy constructor to prevent accisdental copying + UncompressedBlock(const UncompressedBlock& /*rhs*/) {}; + + /// Private assignment operator to prevent accisdental copying + UncompressedBlock& operator=(const UncompressedBlock& /*rhs*/) {}; + // Made this private for consistancy with CompressedBlock. // Users shouldn't really need this for UncompressedBlock anyway. uint32_t calculateSizeInBytes(void); diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h index 65e9dec4..01781cc6 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h @@ -331,11 +331,11 @@ namespace PolyVox CompressedBlock* getCompressedBlock(int32_t uBlockX, int32_t uBlockY, int32_t uBlockZ) const; UncompressedBlock* getUncompressedBlock(int32_t uBlockX, int32_t uBlockY, int32_t uBlockZ) const; - void eraseBlock(typename std::map, BlockPositionCompare>::iterator itBlock) const; + void eraseBlock(typename std::map*, BlockPositionCompare>::iterator itBlock) const; // The block data mutable std::map*, BlockPositionCompare> m_pUncompressedBlockCache; - mutable std::map, BlockPositionCompare> m_pBlocks; + mutable std::map*, BlockPositionCompare> m_pBlocks; mutable uint32_t m_uTimestamper; mutable Vector3DInt32 m_v3dLastAccessedBlockPos; diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl index 731a6c01..42692706 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl @@ -405,7 +405,7 @@ namespace PolyVox for(int32_t z = v3dStart.getZ(); z <= v3dEnd.getZ(); z++) { Vector3DInt32 pos(x,y,z); - typename std::map, BlockPositionCompare>::iterator itBlock = m_pBlocks.find(pos); + typename std::map*, BlockPositionCompare>::iterator itBlock = m_pBlocks.find(pos); if(itBlock != m_pBlocks.end()) { @@ -436,7 +436,7 @@ namespace PolyVox template void LargeVolume::flushAll() { - typename std::map, BlockPositionCompare>::iterator i; + typename std::map*, BlockPositionCompare>::iterator i; //Replaced the for loop here as the call to //eraseBlock was invalidating the iterator. while(m_pBlocks.size() > 0) @@ -470,7 +470,7 @@ namespace PolyVox for(int32_t z = v3dStart.getZ(); z <= v3dEnd.getZ(); z++) { Vector3DInt32 pos(x,y,z); - typename std::map, BlockPositionCompare>::iterator itBlock = m_pBlocks.find(pos); + typename std::map*, BlockPositionCompare>::iterator itBlock = m_pBlocks.find(pos); if(itBlock == m_pBlocks.end()) { // not loaded, not unloading @@ -548,7 +548,7 @@ namespace PolyVox } template - void LargeVolume::eraseBlock(typename std::map, BlockPositionCompare>::iterator itBlock) const + void LargeVolume::eraseBlock(typename std::map*, BlockPositionCompare>::iterator itBlock) const { POLYVOX_ASSERT(false, "This function has not been implemented properly"); @@ -591,30 +591,30 @@ namespace PolyVox { Vector3DInt32 v3dBlockPos(uBlockX, uBlockY, uBlockZ); - typename std::map, BlockPositionCompare>::iterator itBlock = m_pBlocks.find(v3dBlockPos); + typename std::map*, BlockPositionCompare>::iterator itBlock = m_pBlocks.find(v3dBlockPos); // check whether the block is already loaded if(itBlock == m_pBlocks.end()) { //The block is not in the map, so we will have to create a new block and add it. - CompressedBlock newBlock; + CompressedBlock* newBlock = new CompressedBlock; itBlock = m_pBlocks.insert(std::make_pair(v3dBlockPos, newBlock)).first; // Now use the pager to fill the block with it's initial data. Vector3DInt32 v3dLower(v3dBlockPos.getX() << m_uBlockSideLengthPower, v3dBlockPos.getY() << m_uBlockSideLengthPower, v3dBlockPos.getZ() << m_uBlockSideLengthPower); Vector3DInt32 v3dUpper = v3dLower + Vector3DInt32(m_uBlockSideLength-1, m_uBlockSideLength-1, m_uBlockSideLength-1); Region reg(v3dLower, v3dUpper); - m_pPager->pageIn(reg, &(itBlock->second)); + m_pPager->pageIn(reg, itBlock->second); // Paging in this new block may mean we are now using too much memory. If necessary, flush some old blocks. flushOldestExcessiveBlocks(); } //Get the block and mark that we accessed it - CompressedBlock& block = itBlock->second; - block.m_uBlockLastAccessed = ++m_uTimestamper; + CompressedBlock* block = itBlock->second; + block->m_uBlockLastAccessed = ++m_uTimestamper; //m_v3dLastAccessedBlockPos = v3dBlockPos; - return █ + return block; } template @@ -681,11 +681,11 @@ namespace PolyVox uint32_t uSizeInBytes = sizeof(LargeVolume); //Memory used by the blocks - typename std::map, BlockPositionCompare>::iterator i; + typename std::map*, BlockPositionCompare>::iterator i; for(i = m_pBlocks.begin(); i != m_pBlocks.end(); i++) { //Inaccurate - account for rest of loaded block. - uSizeInBytes += i->second.calculateSizeInBytes(); + uSizeInBytes += i->second->calculateSizeInBytes(); } //Memory used by the block cache. @@ -700,11 +700,11 @@ namespace PolyVox { uint32_t uMemoryUsage = 0; - typename std::map, BlockPositionCompare>::iterator i; + typename std::map*, BlockPositionCompare>::iterator i; for(i = m_pBlocks.begin(); i != m_pBlocks.end(); i++) { //Inaccurate - account for rest of loaded block. - uMemoryUsage += i->second.calculateSizeInBytes(); + uMemoryUsage += i->second->calculateSizeInBytes(); } return uMemoryUsage; @@ -720,11 +720,11 @@ namespace PolyVox while(calculateBlockMemoryUsage() > m_uCompressedBlockMemoryLimitInBytes) //FIXME - This calculation of size is slow and should be outside the loop. { // find the least recently used block - typename std::map, BlockPositionCompare>::iterator i; - typename std::map, BlockPositionCompare>::iterator itUnloadBlock = m_pBlocks.begin(); + typename std::map*, BlockPositionCompare>::iterator i; + typename std::map*, BlockPositionCompare>::iterator itUnloadBlock = m_pBlocks.begin(); for(i = m_pBlocks.begin(); i != m_pBlocks.end(); i++) { - if(i->second.m_uBlockLastAccessed < itUnloadBlock->second.m_uBlockLastAccessed) + if(i->second->m_uBlockLastAccessed < itUnloadBlock->second->m_uBlockLastAccessed) { itUnloadBlock = i; } @@ -732,7 +732,7 @@ namespace PolyVox //POLYVOX_ASSERT(itUnloadBlock->second.hasUncompressedData() == false, "This function should never flush blocks with uncompressed data."); - uMemoryToReclaim -= itUnloadBlock->second.calculateSizeInBytes(); + uMemoryToReclaim -= itUnloadBlock->second->calculateSizeInBytes(); eraseBlock(itUnloadBlock); } From 404f12a43e57ea4222bd1a515389cc54ce101a86 Mon Sep 17 00:00:00 2001 From: David Williams Date: Wed, 17 Jul 2013 16:33:46 +0200 Subject: [PATCH 14/38] Added typedefs to simplify code. --- .../include/PolyVoxCore/LargeVolume.h | 11 ++++++---- .../include/PolyVoxCore/LargeVolume.inl | 20 +++++++++---------- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h index 01781cc6..02e226a5 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h @@ -312,7 +312,10 @@ namespace PolyVox } return false; } - }; + }; + + typedef std::map*, BlockPositionCompare> CompressedBlockMap; + typedef std::map*, BlockPositionCompare> UncompressedBlockMap; uint32_t calculateBlockMemoryUsage(void) const; @@ -331,11 +334,11 @@ namespace PolyVox CompressedBlock* getCompressedBlock(int32_t uBlockX, int32_t uBlockY, int32_t uBlockZ) const; UncompressedBlock* getUncompressedBlock(int32_t uBlockX, int32_t uBlockY, int32_t uBlockZ) const; - void eraseBlock(typename std::map*, BlockPositionCompare>::iterator itBlock) const; + void eraseBlock(typename CompressedBlockMap::iterator itBlock) const; // The block data - mutable std::map*, BlockPositionCompare> m_pUncompressedBlockCache; - mutable std::map*, BlockPositionCompare> m_pBlocks; + mutable CompressedBlockMap m_pBlocks; + mutable UncompressedBlockMap m_pUncompressedBlockCache; mutable uint32_t m_uTimestamper; mutable Vector3DInt32 m_v3dLastAccessedBlockPos; diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl index 42692706..5ba67966 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl @@ -405,7 +405,7 @@ namespace PolyVox for(int32_t z = v3dStart.getZ(); z <= v3dEnd.getZ(); z++) { Vector3DInt32 pos(x,y,z); - typename std::map*, BlockPositionCompare>::iterator itBlock = m_pBlocks.find(pos); + typename CompressedBlockMap::iterator itBlock = m_pBlocks.find(pos); if(itBlock != m_pBlocks.end()) { @@ -436,7 +436,7 @@ namespace PolyVox template void LargeVolume::flushAll() { - typename std::map*, BlockPositionCompare>::iterator i; + typename CompressedBlockMap::iterator i; //Replaced the for loop here as the call to //eraseBlock was invalidating the iterator. while(m_pBlocks.size() > 0) @@ -470,7 +470,7 @@ namespace PolyVox for(int32_t z = v3dStart.getZ(); z <= v3dEnd.getZ(); z++) { Vector3DInt32 pos(x,y,z); - typename std::map*, BlockPositionCompare>::iterator itBlock = m_pBlocks.find(pos); + typename CompressedBlockMap::iterator itBlock = m_pBlocks.find(pos); if(itBlock == m_pBlocks.end()) { // not loaded, not unloading @@ -548,7 +548,7 @@ namespace PolyVox } template - void LargeVolume::eraseBlock(typename std::map*, BlockPositionCompare>::iterator itBlock) const + void LargeVolume::eraseBlock(typename CompressedBlockMap::iterator itBlock) const { POLYVOX_ASSERT(false, "This function has not been implemented properly"); @@ -591,7 +591,7 @@ namespace PolyVox { Vector3DInt32 v3dBlockPos(uBlockX, uBlockY, uBlockZ); - typename std::map*, BlockPositionCompare>::iterator itBlock = m_pBlocks.find(v3dBlockPos); + typename CompressedBlockMap::iterator itBlock = m_pBlocks.find(v3dBlockPos); // check whether the block is already loaded if(itBlock == m_pBlocks.end()) { @@ -635,7 +635,7 @@ namespace PolyVox CompressedBlock* block = getCompressedBlock(uBlockX, uBlockY, uBlockZ); - typename std::map*, BlockPositionCompare>::iterator itUncompressedBlock = m_pUncompressedBlockCache.find(v3dBlockPos); + typename UncompressedBlockMap::iterator itUncompressedBlock = m_pUncompressedBlockCache.find(v3dBlockPos); // check whether the block is already loaded if(itUncompressedBlock == m_pUncompressedBlockCache.end()) { @@ -681,7 +681,7 @@ namespace PolyVox uint32_t uSizeInBytes = sizeof(LargeVolume); //Memory used by the blocks - typename std::map*, BlockPositionCompare>::iterator i; + typename CompressedBlockMap::iterator i; for(i = m_pBlocks.begin(); i != m_pBlocks.end(); i++) { //Inaccurate - account for rest of loaded block. @@ -700,7 +700,7 @@ namespace PolyVox { uint32_t uMemoryUsage = 0; - typename std::map*, BlockPositionCompare>::iterator i; + typename CompressedBlockMap::iterator i; for(i = m_pBlocks.begin(); i != m_pBlocks.end(); i++) { //Inaccurate - account for rest of loaded block. @@ -720,8 +720,8 @@ namespace PolyVox while(calculateBlockMemoryUsage() > m_uCompressedBlockMemoryLimitInBytes) //FIXME - This calculation of size is slow and should be outside the loop. { // find the least recently used block - typename std::map*, BlockPositionCompare>::iterator i; - typename std::map*, BlockPositionCompare>::iterator itUnloadBlock = m_pBlocks.begin(); + typename CompressedBlockMap::iterator i; + typename CompressedBlockMap::iterator itUnloadBlock = m_pBlocks.begin(); for(i = m_pBlocks.begin(); i != m_pBlocks.end(); i++) { if(i->second->m_uBlockLastAccessed < itUnloadBlock->second->m_uBlockLastAccessed) From 1e986f9aa6f3e654d19f02e9f682e4d91e040e99 Mon Sep 17 00:00:00 2001 From: David Williams Date: Wed, 17 Jul 2013 17:00:12 +0200 Subject: [PATCH 15/38] Reimplemented eraseBlock for compressed blocks. --- .../include/PolyVoxCore/LargeVolume.h | 4 +- .../include/PolyVoxCore/LargeVolume.inl | 47 +++++++------------ 2 files changed, 21 insertions(+), 30 deletions(-) diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h index 02e226a5..809b9020 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h @@ -334,7 +334,9 @@ namespace PolyVox CompressedBlock* getCompressedBlock(int32_t uBlockX, int32_t uBlockY, int32_t uBlockZ) const; UncompressedBlock* getUncompressedBlock(int32_t uBlockX, int32_t uBlockY, int32_t uBlockZ) const; - void eraseBlock(typename CompressedBlockMap::iterator itBlock) const; + + void eraseBlock(typename CompressedBlockMap::iterator itCompressedBlock) const; + void eraseBlock(typename UncompressedBlockMap::iterator itUncompressedBlock) const; // The block data mutable CompressedBlockMap m_pBlocks; diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl index 5ba67966..c8dec3e3 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl @@ -548,42 +548,31 @@ namespace PolyVox } template - void LargeVolume::eraseBlock(typename CompressedBlockMap::iterator itBlock) const + void LargeVolume::eraseBlock(typename CompressedBlockMap::iterator itCompressedBlock) const { - POLYVOX_ASSERT(false, "This function has not been implemented properly"); - - /*if(itBlock->second.hasUncompressedData()) + // Before deleting the block we may need to page out it's data. We + // only do this if the data has been modified since it was paged in. + CompressedBlock* pCompressedBlock = itCompressedBlock->second; + if(pCompressedBlock->m_bDataModified) { - itBlock->second.destroyUncompressedData(); - } + // The position of the block within the volume. + Vector3DInt32 v3dBlockPos = itCompressedBlock->first; - if(m_pPager) - { - Vector3DInt32 v3dPos = itBlock->first; - Vector3DInt32 v3dLower(v3dPos.getX() << m_uBlockSideLengthPower, v3dPos.getY() << m_uBlockSideLengthPower, v3dPos.getZ() << m_uBlockSideLengthPower); + // From the coordinates of the block we deduce the coordinates of the contained voxels. + Vector3DInt32 v3dLower(v3dBlockPos.getX() << m_uBlockSideLengthPower, v3dBlockPos.getY() << m_uBlockSideLengthPower, v3dBlockPos.getZ() << m_uBlockSideLengthPower); Vector3DInt32 v3dUpper = v3dLower + Vector3DInt32(m_uBlockSideLength-1, m_uBlockSideLength-1, m_uBlockSideLength-1); - Region reg(v3dLower, v3dUpper); + // Page the data out + m_pPager->pageOut(Region(v3dLower, v3dUpper), pCompressedBlock); + } - m_pPager->pageOut(reg, &(itBlock->second)); - }*/ + // We can now remove the block data from memory. + m_pBlocks.erase(itCompressedBlock); + } - - // FIXME - the code below used to make sure the uncompressed version of the data was removed. Reinstate something similar. - /*for(uint32_t ct = 0; ct < m_vecBlocksWithUncompressedData.size(); ct++) - { - // find the block in the uncompressed cache - if(m_vecBlocksWithUncompressedData[ct] == &(itBlock->second)) - { - // put last object in cache here - m_vecBlocksWithUncompressedData[ct] = m_vecBlocksWithUncompressedData.back(); - // decrease cache size by one since last element is now in here twice - m_vecBlocksWithUncompressedData.resize(m_vecBlocksWithUncompressedData.size()-1); - break; - } - }*/ - - m_pBlocks.erase(itBlock); + template + void LargeVolume::eraseBlock(typename UncompressedBlockMap::iterator itUncompressedBlock) const + { } template From 88cbeb309c8297bf3489b0c8bc463e8307e61490 Mon Sep 17 00:00:00 2001 From: David Williams Date: Thu, 18 Jul 2013 16:08:18 +0200 Subject: [PATCH 16/38] Reimplemented eraseBlock for uncompressed blocks. --- .../include/PolyVoxCore/LargeVolume.inl | 76 ++++++++++++++++++- 1 file changed, 74 insertions(+), 2 deletions(-) diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl index c8dec3e3..99e75787 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl @@ -550,9 +550,10 @@ namespace PolyVox template void LargeVolume::eraseBlock(typename CompressedBlockMap::iterator itCompressedBlock) const { - // Before deleting the block we may need to page out it's data. We - // only do this if the data has been modified since it was paged in. CompressedBlock* pCompressedBlock = itCompressedBlock->second; + + // Before deleting the block we may need to page out its data. We + // only do this if the data has been modified since it was paged in. if(pCompressedBlock->m_bDataModified) { // The position of the block within the volume. @@ -573,6 +574,77 @@ namespace PolyVox template void LargeVolume::eraseBlock(typename UncompressedBlockMap::iterator itUncompressedBlock) const { + UncompressedBlock* pUncompressedBlock = itUncompressedBlock->second; + + // This should not often happen as blocks are normally deleted based on being least recently used. + // However, I can imagine that flushing a large number of blocks could cause this to occur. Just + // to be safe we handle it by invalidating the last accessed block pointer. + if(pUncompressedBlock == m_pLastAccessedBlock) + { + logWarning() << "The last accessed block is being erased from the uncompressed cache."; + m_pLastAccessedBlock = 0; + } + + // Before deleting the block we may need to recompress its data. We + // only do this if the data has been modified since it was decompressed. + if(pUncompressedBlock->m_bDataModified) + { + // Get the compressed block which we will copy the data back in to. + Vector3DInt32 v3dBlockPos = itCompressedBlock->first; + CompressedBlock* pCompressedBlock = getCompressedBlock(v3dBlockPos); + + void* pSrcData = reinterpret_cast(pUncompressedBlock->m_tData); + uint32_t uSrcLength = m_uSideLength * m_uSideLength * m_uSideLength * sizeof(VoxelType); + + uint8_t tempBuffer[10000]; + void* pDstData = reinterpret_cast( tempBuffer ); + uint32_t uDstLength = 10000; + + uint32_t uCompressedLength = 0; + + try + { + // Perform the compression + uCompressedLength = m_pCompressor->compress(pSrcData, uSrcLength, pDstData, uDstLength); + + // Copy the resulting compressed data into the compressed block + pCompressedBlock->setData(pDstData, uDstLength); + } + catch(std::exception&) + { + // It is possible for the compression to fail. A common cause for this would be if the destination + // buffer is not big enough. So now we try again using a buffer that is definitely big enough. + // Note that ideally we will choose our earlier buffer size so that this almost never happens. + logWarning() << "The compressor failed to compress the block, proabaly due to the buffer being too small."; + logWarning() << "The compression will be tried again with a larger buffer"; + uint32_t uMaxCompressedSize = m_pCompressor->getMaxCompressedSize(uSrcLength); + uint8_t* buffer = new uint8_t[ uMaxCompressedSize ]; + + pDstData = reinterpret_cast( buffer ); + uDstLength = uMaxCompressedSize; + + try + { + // Perform the compression + uCompressedLength = m_pCompressor->compress(pSrcData, uSrcLength, pDstData, uDstLength); + + // Copy the resulting compressed data into the compressed block + pCompressedBlock->setData(pDstData, uDstLength); + } + catch(std::exception&) + { + // At this point it didn't work even with a bigger buffer. + // Not much more we can do so just rethrow the exception. + delete[] buffer; + POLYVOX_THROW(std::runtime_error, "Failed to compress block data"); + } + + delete[] buffer; + } + } + + // We can now remove the block data from memory. + m_pUncompressedBlockCache.erase(itUncompressedBlock); } template From ef6bd31651f19af278e7ab21f1fcdbf84e8aed21 Mon Sep 17 00:00:00 2001 From: David Williams Date: Thu, 18 Jul 2013 17:03:14 +0200 Subject: [PATCH 17/38] Small fixes. --- .../PolyVoxCore/include/PolyVoxCore/LargeVolume.inl | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl index 99e75787..dae9d2d4 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl @@ -567,6 +567,8 @@ namespace PolyVox m_pPager->pageOut(Region(v3dLower, v3dUpper), pCompressedBlock); } + delete itCompressedBlock->second; + // We can now remove the block data from memory. m_pBlocks.erase(itCompressedBlock); } @@ -615,7 +617,7 @@ namespace PolyVox // It is possible for the compression to fail. A common cause for this would be if the destination // buffer is not big enough. So now we try again using a buffer that is definitely big enough. // Note that ideally we will choose our earlier buffer size so that this almost never happens. - logWarning() << "The compressor failed to compress the block, proabaly due to the buffer being too small."; + logWarning() << "The compressor failed to compress the block, probabaly due to the buffer being too small."; logWarning() << "The compression will be tried again with a larger buffer"; uint32_t uMaxCompressedSize = m_pCompressor->getMaxCompressedSize(uSrcLength); uint8_t* buffer = new uint8_t[ uMaxCompressedSize ]; @@ -658,6 +660,7 @@ namespace PolyVox { //The block is not in the map, so we will have to create a new block and add it. CompressedBlock* newBlock = new CompressedBlock; + newBlock->m_uBlockLastAccessed = ++m_uTimestamper; itBlock = m_pBlocks.insert(std::make_pair(v3dBlockPos, newBlock)).first; // Now use the pager to fill the block with it's initial data. @@ -692,15 +695,15 @@ namespace PolyVox return m_pLastAccessedBlock; } - //Get the block and mark that we accessed it + //Gets the block and marks that we accessed it CompressedBlock* block = getCompressedBlock(uBlockX, uBlockY, uBlockZ); - typename UncompressedBlockMap::iterator itUncompressedBlock = m_pUncompressedBlockCache.find(v3dBlockPos); // check whether the block is already loaded if(itUncompressedBlock == m_pUncompressedBlockCache.end()) { UncompressedBlock* pUncompressedBlock = new UncompressedBlock(m_uBlockSideLength); + pUncompressedBlock->m_uBlockLastAccessed = ++m_uTimestamper; const void* pSrcData = reinterpret_cast(block->getData()); void* pDstData = reinterpret_cast(pUncompressedBlock->m_tData); @@ -775,7 +778,7 @@ namespace PolyVox void LargeVolume::flushOldestExcessiveBlocks(void) const { const uint32_t uMemoryUsedForCompressedBlocks = calculateBlockMemoryUsage(); - uint32_t uMemoryToReclaim = uMemoryUsedForCompressedBlocks - m_uCompressedBlockMemoryLimitInBytes; + //uint32_t uMemoryToReclaim = uMemoryUsedForCompressedBlocks - m_uCompressedBlockMemoryLimitInBytes; //while(uMemoryToReclaim > 0) while(calculateBlockMemoryUsage() > m_uCompressedBlockMemoryLimitInBytes) //FIXME - This calculation of size is slow and should be outside the loop. @@ -793,7 +796,7 @@ namespace PolyVox //POLYVOX_ASSERT(itUnloadBlock->second.hasUncompressedData() == false, "This function should never flush blocks with uncompressed data."); - uMemoryToReclaim -= itUnloadBlock->second->calculateSizeInBytes(); + //uMemoryToReclaim -= itUnloadBlock->second->calculateSizeInBytes(); eraseBlock(itUnloadBlock); } From 7cd115b33d5b7f0d3ae23254441833fbc16e48bf Mon Sep 17 00:00:00 2001 From: David Williams Date: Fri, 19 Jul 2013 16:43:59 +0200 Subject: [PATCH 18/38] Work on code which computes how the memory should be split between compressed and uncompressed data. --- .../include/PolyVoxCore/LargeVolume.h | 2 +- .../include/PolyVoxCore/LargeVolume.inl | 60 +++++++++++++++---- 2 files changed, 50 insertions(+), 12 deletions(-) diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h index 809b9020..499a5f91 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h @@ -347,7 +347,7 @@ namespace PolyVox mutable UncompressedBlock* m_pLastAccessedBlock; uint32_t m_uMaxNumberOfUncompressedBlocks; - uint32_t m_uCompressedBlockMemoryLimitInBytes; + uint32_t m_uTargetMemoryLimitInBytes; // The size of the volume Region m_regValidRegionInBlocks; diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl index dae9d2d4..e174a578 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl @@ -246,23 +246,57 @@ namespace PolyVox template void LargeVolume::setTargetMemoryLimitInBytes(uint32_t uTargetMemoryLimitInBytes) { + // Set the limit, and then work out how big the uncompressed cache should be. + m_uTargetMemoryLimitInBytes = uTargetMemoryLimitInBytes; + // The size of a single uncompressed block of data. uint32_t uUncompressedBlockSizeInBytes = m_uBlockSideLength * m_uBlockSideLength * m_uBlockSideLength * sizeof(VoxelType); - // The idea number of uncompressed blocks is chosen by gut feeling as much as anything. Part of the - // rationale is that it should let us iterate along any edge without data being pushed out of the cache. - uint32_t uIdealNoOfUncompressedBlocks = m_regValidRegionInBlocks.getWidthInVoxels() + m_regValidRegionInBlocks.getHeightInVoxels() * m_regValidRegionInBlocks.getDepthInVoxels(); + // When deciding how to split the memory between compressed and uncompressed blocks it is useful to know the dimensions of the + // volume in blocks. After sorting, the array below will hold the smallest dimension in [0] and the largest dimension in [2]. + const uint32_t uNoOfDimensions = 3; + uint32_t uSortedDimensionsInBlocks[uNoOfDimensions]; + uSortedDimensionsInBlocks[0] = (getEnclosingRegion().getWidthInVoxels() >> m_uBlockSideLengthPower) + 1; + uSortedDimensionsInBlocks[1] = (getEnclosingRegion().getHeightInVoxels() >> m_uBlockSideLengthPower) + 1; + uSortedDimensionsInBlocks[2] = (getEnclosingRegion().getDepthInVoxels() >> m_uBlockSideLengthPower) + 1; + std::sort(uSortedDimensionsInBlocks, uSortedDimensionsInBlocks + uNoOfDimensions); - // Let's (arbitrarily?) say that we should never use more than half the available memory for the uncompressed block cache. - uint32_t uMaxMemoryForUncompressedBlocks = uTargetMemoryLimitInBytes / 2; + const uint32_t uLongestEdgeInBlocks = uSortedDimensionsInBlocks[2]; + const uint32_t uLargestFaceSizeInBlocks = uSortedDimensionsInBlocks[1] * uSortedDimensionsInBlocks[2]; - uint32_t uMaxFittableNoOfUncompressedBlocks = uMaxMemoryForUncompressedBlocks / uUncompressedBlockSizeInBytes; + // In the ideal situation we will be able to keep an entire face of blocks decompressed in the cache, and still have significant memory + // left for storing a large number of compressed block. Failing this, we would at least like to be able to keep enough blocks decompressed + // to iterate over the longest edge. Otherwise we will find that excessive compression and decompresion occurs. In both cases we multiply + // by an extra factor make sure there is space for storing some compressed data as well. + const uint32_t uFactor = 2; // Just to make sure we leave some space for compressed data as well + const uint32_t uIdealMemoryTarget = uLargestFaceSizeInBlocks * uUncompressedBlockSizeInBytes * uFactor; + const uint32_t uAcceptableMemoryTarget = uLongestEdgeInBlocks * uUncompressedBlockSizeInBytes * uFactor; - setMaxNumberOfUncompressedBlocks((std::min)(uIdealNoOfUncompressedBlocks, uMaxFittableNoOfUncompressedBlocks)); + if(uTargetMemoryLimitInBytes >= uIdealMemoryTarget) + { + logDebug() << "The target memory limit (" << uTargetMemoryLimitInBytes << "bytes) is plenty for a volume of this size)"; - uint32_t uUncompressedBlockCacheSizeInBytes = m_uMaxNumberOfUncompressedBlocks * uUncompressedBlockSizeInBytes; + // The added value below is just some extra 'grace' space. + setMaxNumberOfUncompressedBlocks(uLargestFaceSizeInBlocks + uSortedDimensionsInBlocks[1]); + } + else if(uTargetMemoryLimitInBytes >= uAcceptableMemoryTarget) + { + logDebug() << "The target memory limit (" << uTargetMemoryLimitInBytes << "bytes) should be acceptable for a volume of this size)"; - m_uCompressedBlockMemoryLimitInBytes = uTargetMemoryLimitInBytes - uUncompressedBlockCacheSizeInBytes; + // Let's (arbitrarily) use 1/4 of the available memory for the cache, while making sure that it is as long as the longest side. + uint32_t uMaxFittableUncompressedBlocks = uTargetMemoryLimitInBytes / uUncompressedBlockSizeInBytes; + setMaxNumberOfUncompressedBlocks((std::max)(uMaxFittableUncompressedBlocks / 4, uLongestEdgeInBlocks)); + } + else + { + logWarning() << "The target memory limit (" << uTargetMemoryLimitInBytes << "bytes) is too low for a volume of this size)"; + logWarning() << "Excessive block thrashing may occur and the target memory limit may not be honored"; + + // We still have to do something. Let's still use 1/4 of the memory for the cache, but also set a minimum size. + const uint32_t uMinNoOfUncompressedBlocks = 8; // Chosen because it just feels about right. + uint32_t uMaxFittableUncompressedBlocks = uTargetMemoryLimitInBytes / uUncompressedBlockSizeInBytes; + setMaxNumberOfUncompressedBlocks((std::max)(uMaxFittableUncompressedBlocks / 4, uMinNoOfUncompressedBlocks)); + } } //////////////////////////////////////////////////////////////////////////////// @@ -277,6 +311,10 @@ namespace PolyVox clearBlockCache(); m_uMaxNumberOfUncompressedBlocks = uMaxNumberOfUncompressedBlocks; + + uint32_t uUncompressedBlockSizeInBytes = m_uBlockSideLength * m_uBlockSideLength * m_uBlockSideLength * sizeof(VoxelType); + logDebug() << "The maximum number of uncompresed blocks has been set to " << m_uMaxNumberOfUncompressedBlocks + << ", which is " << m_uMaxNumberOfUncompressedBlocks * uUncompressedBlockSizeInBytes << " bytes"; } //////////////////////////////////////////////////////////////////////////////// @@ -781,7 +819,7 @@ namespace PolyVox //uint32_t uMemoryToReclaim = uMemoryUsedForCompressedBlocks - m_uCompressedBlockMemoryLimitInBytes; //while(uMemoryToReclaim > 0) - while(calculateBlockMemoryUsage() > m_uCompressedBlockMemoryLimitInBytes) //FIXME - This calculation of size is slow and should be outside the loop. + /*while(calculateBlockMemoryUsage() > m_uCompressedBlockMemoryLimitInBytes) //FIXME - This calculation of size is slow and should be outside the loop. { // find the least recently used block typename CompressedBlockMap::iterator i; @@ -799,7 +837,7 @@ namespace PolyVox //uMemoryToReclaim -= itUnloadBlock->second->calculateSizeInBytes(); eraseBlock(itUnloadBlock); - } + }*/ } template From 7146b5ecdb400dd44068b1ea3720b7c1aa068ac8 Mon Sep 17 00:00:00 2001 From: David Williams Date: Tue, 23 Jul 2013 13:15:01 +0200 Subject: [PATCH 19/38] The idea of a 'setTargetMemoryLimitInBytes' function was not really working out. It was too comp-lex and not really clear what the ideal size would be. I think user code needs to call setMaxNumberOfBlocksInMemory() and setMaxNumberOfUncompressedBlocks() directly, if they see from profiling toat too much camressing or paging is taking place. --- examples/Paging/main.cpp | 5 +- .../include/PolyVoxCore/LargeVolume.h | 7 +- .../include/PolyVoxCore/LargeVolume.inl | 71 ++++--------------- tests/testvolume.cpp | 5 +- 4 files changed, 22 insertions(+), 66 deletions(-) diff --git a/examples/Paging/main.cpp b/examples/Paging/main.cpp index 07877405..117145b7 100644 --- a/examples/Paging/main.cpp +++ b/examples/Paging/main.cpp @@ -153,9 +153,8 @@ int main(int argc, char *argv[]) RLECompressor* compressor = new RLECompressor(); PerlinNoisePager* pager = new PerlinNoisePager(); LargeVolume volData(PolyVox::Region::MaxRegion, compressor, pager, 256); - //volData.setMaxNumberOfBlocksInMemory(4096); - //volData.setMaxNumberOfUncompressedBlocks(64); - volData.setTargetMemoryLimitInBytes(128 * 1024 * 1024); + volData.setMaxNumberOfBlocksInMemory(4096); + volData.setMaxNumberOfUncompressedBlocks(64); //volData.setMaxNumberOfUncompressedBlocks(4096); //createSphereInVolume(volData, 30); diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h index 499a5f91..e1786609 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h @@ -263,10 +263,10 @@ namespace PolyVox /// Gets a voxel at the position given by a 3D vector POLYVOX_DEPRECATED VoxelType getVoxelAt(const Vector3DInt32& v3dPos) const; - void setTargetMemoryLimitInBytes(uint32_t uTargetMemoryLimitInBytes); - /// Sets the number of blocks for which uncompressed data is stored void setMaxNumberOfUncompressedBlocks(uint32_t uMaxNumberOfUncompressedBlocks); + /// Sets the number of blocks which can be in memory before the paging system starts unloading them + void setMaxNumberOfBlocksInMemory(uint32_t uMaxNumberOfBlocksInMemory); /// Sets the voxel at the position given by x,y,z coordinates void setVoxel(int32_t uXPos, int32_t uYPos, int32_t uZPos, VoxelType tValue, WrapMode eWrapMode = WrapModes::Validate); /// Sets the voxel at the position given by a 3D vector @@ -346,8 +346,7 @@ namespace PolyVox mutable Vector3DInt32 m_v3dLastAccessedBlockPos; mutable UncompressedBlock* m_pLastAccessedBlock; uint32_t m_uMaxNumberOfUncompressedBlocks; - - uint32_t m_uTargetMemoryLimitInBytes; + uint32_t m_uMaxNumberOfBlocksInMemory; // The size of the volume Region m_regValidRegionInBlocks; diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl index e174a578..04b68a6d 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl @@ -243,62 +243,6 @@ namespace PolyVox return getVoxelAt(v3dPos.getX(), v3dPos.getY(), v3dPos.getZ()); } - template - void LargeVolume::setTargetMemoryLimitInBytes(uint32_t uTargetMemoryLimitInBytes) - { - // Set the limit, and then work out how big the uncompressed cache should be. - m_uTargetMemoryLimitInBytes = uTargetMemoryLimitInBytes; - - // The size of a single uncompressed block of data. - uint32_t uUncompressedBlockSizeInBytes = m_uBlockSideLength * m_uBlockSideLength * m_uBlockSideLength * sizeof(VoxelType); - - // When deciding how to split the memory between compressed and uncompressed blocks it is useful to know the dimensions of the - // volume in blocks. After sorting, the array below will hold the smallest dimension in [0] and the largest dimension in [2]. - const uint32_t uNoOfDimensions = 3; - uint32_t uSortedDimensionsInBlocks[uNoOfDimensions]; - uSortedDimensionsInBlocks[0] = (getEnclosingRegion().getWidthInVoxels() >> m_uBlockSideLengthPower) + 1; - uSortedDimensionsInBlocks[1] = (getEnclosingRegion().getHeightInVoxels() >> m_uBlockSideLengthPower) + 1; - uSortedDimensionsInBlocks[2] = (getEnclosingRegion().getDepthInVoxels() >> m_uBlockSideLengthPower) + 1; - std::sort(uSortedDimensionsInBlocks, uSortedDimensionsInBlocks + uNoOfDimensions); - - const uint32_t uLongestEdgeInBlocks = uSortedDimensionsInBlocks[2]; - const uint32_t uLargestFaceSizeInBlocks = uSortedDimensionsInBlocks[1] * uSortedDimensionsInBlocks[2]; - - // In the ideal situation we will be able to keep an entire face of blocks decompressed in the cache, and still have significant memory - // left for storing a large number of compressed block. Failing this, we would at least like to be able to keep enough blocks decompressed - // to iterate over the longest edge. Otherwise we will find that excessive compression and decompresion occurs. In both cases we multiply - // by an extra factor make sure there is space for storing some compressed data as well. - const uint32_t uFactor = 2; // Just to make sure we leave some space for compressed data as well - const uint32_t uIdealMemoryTarget = uLargestFaceSizeInBlocks * uUncompressedBlockSizeInBytes * uFactor; - const uint32_t uAcceptableMemoryTarget = uLongestEdgeInBlocks * uUncompressedBlockSizeInBytes * uFactor; - - if(uTargetMemoryLimitInBytes >= uIdealMemoryTarget) - { - logDebug() << "The target memory limit (" << uTargetMemoryLimitInBytes << "bytes) is plenty for a volume of this size)"; - - // The added value below is just some extra 'grace' space. - setMaxNumberOfUncompressedBlocks(uLargestFaceSizeInBlocks + uSortedDimensionsInBlocks[1]); - } - else if(uTargetMemoryLimitInBytes >= uAcceptableMemoryTarget) - { - logDebug() << "The target memory limit (" << uTargetMemoryLimitInBytes << "bytes) should be acceptable for a volume of this size)"; - - // Let's (arbitrarily) use 1/4 of the available memory for the cache, while making sure that it is as long as the longest side. - uint32_t uMaxFittableUncompressedBlocks = uTargetMemoryLimitInBytes / uUncompressedBlockSizeInBytes; - setMaxNumberOfUncompressedBlocks((std::max)(uMaxFittableUncompressedBlocks / 4, uLongestEdgeInBlocks)); - } - else - { - logWarning() << "The target memory limit (" << uTargetMemoryLimitInBytes << "bytes) is too low for a volume of this size)"; - logWarning() << "Excessive block thrashing may occur and the target memory limit may not be honored"; - - // We still have to do something. Let's still use 1/4 of the memory for the cache, but also set a minimum size. - const uint32_t uMinNoOfUncompressedBlocks = 8; // Chosen because it just feels about right. - uint32_t uMaxFittableUncompressedBlocks = uTargetMemoryLimitInBytes / uUncompressedBlockSizeInBytes; - setMaxNumberOfUncompressedBlocks((std::max)(uMaxFittableUncompressedBlocks / 4, uMinNoOfUncompressedBlocks)); - } - } - //////////////////////////////////////////////////////////////////////////////// /// Increasing the size of the block cache will increase memory but may improve performance. /// You may want to set this to a large value (e.g. 1024) when you are first loading your @@ -317,6 +261,21 @@ namespace PolyVox << ", which is " << m_uMaxNumberOfUncompressedBlocks * uUncompressedBlockSizeInBytes << " bytes"; } + //////////////////////////////////////////////////////////////////////////////// + /// Increasing the number of blocks in memory causes less paging to occur + /// \param uMaxNumberOfBlocksInMemory The number of blocks + //////////////////////////////////////////////////////////////////////////////// + template + void LargeVolume::setMaxNumberOfBlocksInMemory(uint32_t uMaxNumberOfBlocksInMemory) + { + if(m_pBlocks.size() > uMaxNumberOfBlocksInMemory) + { + flushAll(); + } + + m_uMaxNumberOfBlocksInMemory = uMaxNumberOfBlocksInMemory; + } + //////////////////////////////////////////////////////////////////////////////// /// \param uXPos the \c x position of the voxel /// \param uYPos the \c y position of the voxel diff --git a/tests/testvolume.cpp b/tests/testvolume.cpp index 2063e8e3..77ad1063 100644 --- a/tests/testvolume.cpp +++ b/tests/testvolume.cpp @@ -281,9 +281,8 @@ TestVolume::TestVolume() m_pSimpleVolume = new SimpleVolume(region); m_pLargeVolume = new LargeVolume(region, m_pCompressor, m_pFilePager, 32); - //m_pLargeVolume->setMaxNumberOfBlocksInMemory(32); - //m_pLargeVolume->setMaxNumberOfUncompressedBlocks(16); - m_pLargeVolume->setTargetMemoryLimitInBytes(4 * 1024 * 1024); + m_pLargeVolume->setMaxNumberOfBlocksInMemory(32); + m_pLargeVolume->setMaxNumberOfUncompressedBlocks(16); //Fill the volume with some data for(int z = region.getLowerZ(); z <= region.getUpperZ(); z++) From 7c11a53484e37bcc34d92313aa9cb0965683052d Mon Sep 17 00:00:00 2001 From: David Williams Date: Tue, 23 Jul 2013 13:31:24 +0200 Subject: [PATCH 20/38] Fixed flushOldestExcessiveBlocks(). --- .../include/PolyVoxCore/LargeVolume.inl | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl index 04b68a6d..c77b2883 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl @@ -775,12 +775,11 @@ namespace PolyVox void LargeVolume::flushOldestExcessiveBlocks(void) const { const uint32_t uMemoryUsedForCompressedBlocks = calculateBlockMemoryUsage(); - //uint32_t uMemoryToReclaim = uMemoryUsedForCompressedBlocks - m_uCompressedBlockMemoryLimitInBytes; - //while(uMemoryToReclaim > 0) - /*while(calculateBlockMemoryUsage() > m_uCompressedBlockMemoryLimitInBytes) //FIXME - This calculation of size is slow and should be outside the loop. + while(m_pBlocks.size() > m_uMaxNumberOfBlocksInMemory) { - // find the least recently used block + // Find the least recently used block. This is somewhat inefficient as it searches through + // the map, so if it proves to be a bottleneck we may want some kind of sorted structure. typename CompressedBlockMap::iterator i; typename CompressedBlockMap::iterator itUnloadBlock = m_pBlocks.begin(); for(i = m_pBlocks.begin(); i != m_pBlocks.end(); i++) @@ -791,12 +790,9 @@ namespace PolyVox } } - //POLYVOX_ASSERT(itUnloadBlock->second.hasUncompressedData() == false, "This function should never flush blocks with uncompressed data."); - - //uMemoryToReclaim -= itUnloadBlock->second->calculateSizeInBytes(); - + // Erase the least recently used block eraseBlock(itUnloadBlock); - }*/ + } } template From 69066a068e6d6618b941ea4474f1dc8a879c442c Mon Sep 17 00:00:00 2001 From: David Williams Date: Tue, 23 Jul 2013 13:48:04 +0200 Subject: [PATCH 21/38] Implemented flushExcessiveCacheEntries() --- .../include/PolyVoxCore/LargeVolume.inl | 32 +++++++++++++++---- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl index c77b2883..e82bfc0f 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl @@ -589,14 +589,14 @@ namespace PolyVox if(pUncompressedBlock->m_bDataModified) { // Get the compressed block which we will copy the data back in to. - Vector3DInt32 v3dBlockPos = itCompressedBlock->first; - CompressedBlock* pCompressedBlock = getCompressedBlock(v3dBlockPos); + Vector3DInt32 v3dBlockPos = itUncompressedBlock->first; + CompressedBlock* pCompressedBlock = getCompressedBlock(v3dBlockPos.getX(), v3dBlockPos.getY(), v3dBlockPos.getZ()); void* pSrcData = reinterpret_cast(pUncompressedBlock->m_tData); - uint32_t uSrcLength = m_uSideLength * m_uSideLength * m_uSideLength * sizeof(VoxelType); + uint32_t uSrcLength = m_uBlockSideLength * m_uBlockSideLength * m_uBlockSideLength * sizeof(VoxelType); uint8_t tempBuffer[10000]; - void* pDstData = reinterpret_cast( tempBuffer ); + uint8_t* pDstData = reinterpret_cast( tempBuffer ); uint32_t uDstLength = 10000; uint32_t uCompressedLength = 0; @@ -619,7 +619,7 @@ namespace PolyVox uint32_t uMaxCompressedSize = m_pCompressor->getMaxCompressedSize(uSrcLength); uint8_t* buffer = new uint8_t[ uMaxCompressedSize ]; - pDstData = reinterpret_cast( buffer ); + pDstData = reinterpret_cast( buffer ); uDstLength = uMaxCompressedSize; try @@ -714,6 +714,9 @@ namespace PolyVox POLYVOX_ASSERT(uUncompressedLength == m_uBlockSideLength * m_uBlockSideLength * m_uBlockSideLength * sizeof(VoxelType), "Destination length has changed."); itUncompressedBlock = m_pUncompressedBlockCache.insert(std::make_pair(v3dBlockPos, pUncompressedBlock)).first; + + // Our block cache may now have grown too large. Fluch some entries is necessary. + flushExcessiveCacheEntries(); } m_pLastAccessedBlock = (*itUncompressedBlock).second; @@ -774,8 +777,6 @@ namespace PolyVox template void LargeVolume::flushOldestExcessiveBlocks(void) const { - const uint32_t uMemoryUsedForCompressedBlocks = calculateBlockMemoryUsage(); - while(m_pBlocks.size() > m_uMaxNumberOfBlocksInMemory) { // Find the least recently used block. This is somewhat inefficient as it searches through @@ -798,6 +799,23 @@ namespace PolyVox template void LargeVolume::flushExcessiveCacheEntries(void) const { + while(m_pUncompressedBlockCache.size() > m_uMaxNumberOfUncompressedBlocks) + { + // Find the least recently used block. The uncompressed block cache should be + // much smaller than the total number of blocks, so hopefully this isn't too slow. + typename UncompressedBlockMap::iterator i; + typename UncompressedBlockMap::iterator itUnloadBlock = m_pUncompressedBlockCache.begin(); + for(i = m_pUncompressedBlockCache.begin(); i != m_pUncompressedBlockCache.end(); i++) + { + if(i->second->m_uBlockLastAccessed < itUnloadBlock->second->m_uBlockLastAccessed) + { + itUnloadBlock = i; + } + } + + // Erase the least recently used block + eraseBlock(itUnloadBlock); + } } template From b07dafc9fa3fc4d58b527b8fb69deea644a06dd2 Mon Sep 17 00:00:00 2001 From: David Williams Date: Wed, 24 Jul 2013 16:39:31 +0200 Subject: [PATCH 22/38] Added code to initilise empty compressed blocks. --- .../include/PolyVoxCore/LargeVolume.h | 3 +++ .../include/PolyVoxCore/LargeVolume.inl | 26 ++++++++++++++++++- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h index e1786609..9f409571 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h @@ -359,6 +359,9 @@ namespace PolyVox Compressor* m_pCompressor; Pager* m_pPager; + // Compressed data for an empty block (sometimes needed for initialisation). + CompressedBlock* m_pCompressedEmptyBlock; + // Whether we created the compressor or whether it was provided // by the user. This controls whether we delete it on destruction. bool m_bIsOurCompressor; diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl index e82bfc0f..10069d49 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl @@ -109,6 +109,8 @@ namespace PolyVox { delete m_pCompressor; } + + delete m_pCompressedEmptyBlock; } //////////////////////////////////////////////////////////////////////////////// @@ -542,6 +544,21 @@ namespace PolyVox this->m_uLongestSideLength = (std::max)((std::max)(this->getWidth(),this->getHeight()),this->getDepth()); this->m_uShortestSideLength = (std::min)((std::min)(this->getWidth(),this->getHeight()),this->getDepth()); this->m_fDiagonalLength = sqrtf(static_cast(this->getWidth() * this->getWidth() + this->getHeight() * this->getHeight() + this->getDepth() * this->getDepth())); + + // This is used for initialising empty blocks. + // FIXME - Should probably build an UncompressedBlock and then call some 'fill()' method. Ideally we should set voxels to an 'empty' (default?) value rather than zeros. + VoxelType* pZeros = new VoxelType[m_uBlockSideLength * m_uBlockSideLength * m_uBlockSideLength]; + uint32_t uSrcLength = m_uBlockSideLength * m_uBlockSideLength * m_uBlockSideLength * sizeof(VoxelType); + memset(pZeros, 0, uSrcLength); + uint32_t uDstLength = 100000; + uint8_t* pCompressedZeros = new uint8_t[uDstLength]; //Should be plenty as zeros compress down very small. + uint32_t uCompressedSize = m_pCompressor->compress(reinterpret_cast(pZeros), uSrcLength, reinterpret_cast(pCompressedZeros), uDstLength); + + m_pCompressedEmptyBlock = new CompressedBlock; + m_pCompressedEmptyBlock->setData(pCompressedZeros, uCompressedSize); + + delete[] pZeros; + delete[] pCompressedZeros; } template @@ -664,7 +681,14 @@ namespace PolyVox Vector3DInt32 v3dLower(v3dBlockPos.getX() << m_uBlockSideLengthPower, v3dBlockPos.getY() << m_uBlockSideLengthPower, v3dBlockPos.getZ() << m_uBlockSideLengthPower); Vector3DInt32 v3dUpper = v3dLower + Vector3DInt32(m_uBlockSideLength-1, m_uBlockSideLength-1, m_uBlockSideLength-1); Region reg(v3dLower, v3dUpper); - m_pPager->pageIn(reg, itBlock->second); + m_pPager->pageIn(reg, newBlock); + + // The pager may not actually have given us any data (perhaps there wasn't any) so in that case we need to create some ourselves. + if(newBlock->m_bDataModified == false) + { + // Internally this performs a memcpy() of the data. + newBlock->setData(m_pCompressedEmptyBlock->getData(), m_pCompressedEmptyBlock->getDataSizeInBytes()); + } // Paging in this new block may mean we are now using too much memory. If necessary, flush some old blocks. flushOldestExcessiveBlocks(); From cf9b54e5ab231d99d31a66e0b564b157f038c38a Mon Sep 17 00:00:00 2001 From: David Williams Date: Wed, 24 Jul 2013 16:53:40 +0200 Subject: [PATCH 23/38] Properly deleting data (delete vs. delete[]). --- library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl | 2 +- library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl index dab2f35d..16927b4e 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl @@ -111,7 +111,7 @@ namespace PolyVox template UncompressedBlock::~UncompressedBlock() { - delete m_tData; + delete[] m_tData; m_tData = 0; } diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl index 10069d49..ad67d849 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl @@ -659,6 +659,8 @@ namespace PolyVox } } + delete itUncompressedBlock->second; + // We can now remove the block data from memory. m_pUncompressedBlockCache.erase(itUncompressedBlock); } From f4f85551c6179a1b8bb50feaaee4a6bf9b270629 Mon Sep 17 00:00:00 2001 From: David Williams Date: Thu, 25 Jul 2013 15:51:30 +0200 Subject: [PATCH 24/38] Starting some refactoring of the LargeVolume such that the uncompressed blocks are the 'main' representation, rather than the compressed block being the main version and the uncompressed blocks simply being a cached version. I hope this simplifies and improves the code. --- .../include/PolyVoxCore/LargeVolume.inl | 34 ++++++++++++++++--- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl index ad67d849..65bbbed3 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl @@ -713,13 +713,39 @@ namespace PolyVox //the time stamp. If we updated it everytime then that would be every time we touched //a voxel, which would overflow a uint32_t and require us to use a uint64_t instead. //This check should also provide a significant speed boost as usually it is true. - if((v3dBlockPos == m_v3dLastAccessedBlockPos) && (m_pLastAccessedBlock != 0)) + /*if((v3dBlockPos == m_v3dLastAccessedBlockPos) && (m_pLastAccessedBlock != 0)) { return m_pLastAccessedBlock; + }*/ + + UncompressedBlock* pUncompressedBlock = 0; + + typename UncompressedBlockMap::iterator itUncompressedBlock = m_pUncompressedBlockCache.find(v3dBlockPos); + // check whether the block is already loaded + if(itUncompressedBlock != m_pUncompressedBlockCache.end()) + { + pUncompressedBlock = itUncompressedBlock->second; + + } + //else if(check compressed list...) + //{ + //} + else + { + // At this point we just create a new block. + pUncompressedBlock = new UncompressedBlock(m_uBlockSideLength); + + m_pUncompressedBlockCache.insert(std::make_pair(v3dBlockPos, pUncompressedBlock)); + + // Our block cache may now have grown too large. Flush some entries is necessary. + // FIXME - Watch out for flushing the block we just created! + //flushExcessiveCacheEntries(); } + pUncompressedBlock->m_uBlockLastAccessed = ++m_uTimestamper; + //Gets the block and marks that we accessed it - CompressedBlock* block = getCompressedBlock(uBlockX, uBlockY, uBlockZ); + /*CompressedBlock* block = getCompressedBlock(uBlockX, uBlockY, uBlockZ); typename UncompressedBlockMap::iterator itUncompressedBlock = m_pUncompressedBlockCache.find(v3dBlockPos); // check whether the block is already loaded @@ -743,9 +769,9 @@ namespace PolyVox // Our block cache may now have grown too large. Fluch some entries is necessary. flushExcessiveCacheEntries(); - } + }*/ - m_pLastAccessedBlock = (*itUncompressedBlock).second; + m_pLastAccessedBlock = pUncompressedBlock; m_v3dLastAccessedBlockPos = v3dBlockPos; return m_pLastAccessedBlock; From b767d9b89661e84fa4462599a3fb8f9d9bd5b194 Mon Sep 17 00:00:00 2001 From: David Williams Date: Thu, 25 Jul 2013 17:07:50 +0200 Subject: [PATCH 25/38] Tidying up. --- .../include/PolyVoxCore/Impl/Block.h | 2 +- .../include/PolyVoxCore/LargeVolume.inl | 105 +++++++++++------- 2 files changed, 65 insertions(+), 42 deletions(-) diff --git a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h index 3d795e26..aeb309cc 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h +++ b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h @@ -42,7 +42,7 @@ namespace PolyVox public: Block() :m_uBlockLastAccessed(0) - ,m_bDataModified(false) + ,m_bDataModified(true) { } diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl index 65bbbed3..f0d15bdc 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl @@ -625,6 +625,9 @@ namespace PolyVox // Copy the resulting compressed data into the compressed block pCompressedBlock->setData(pDstData, uDstLength); + + // The compressed data has been updated, so the uncompressed data is no longer modified with respect to it. + pUncompressedBlock->m_bDataModified = false; } catch(std::exception&) { @@ -646,6 +649,9 @@ namespace PolyVox // Copy the resulting compressed data into the compressed block pCompressedBlock->setData(pDstData, uDstLength); + + // The compressed data has been updated, so the uncompressed data is no longer modified with respect to it. + pUncompressedBlock->m_bDataModified = false; } catch(std::exception&) { @@ -670,7 +676,28 @@ namespace PolyVox { Vector3DInt32 v3dBlockPos(uBlockX, uBlockY, uBlockZ); + CompressedBlock* pBlock = 0; + typename CompressedBlockMap::iterator itBlock = m_pBlocks.find(v3dBlockPos); + if(itBlock != m_pBlocks.end()) + { + pBlock = itBlock->second; + } + /*else if(check on disk) + { + }*/ + else + { + pBlock = new CompressedBlock; + pBlock->m_uBlockLastAccessed = ++m_uTimestamper; + itBlock = m_pBlocks.insert(std::make_pair(v3dBlockPos, pBlock)).first; + } + + pBlock->m_uBlockLastAccessed = ++m_uTimestamper; + + return pBlock; + + /*typename CompressedBlockMap::iterator itBlock = m_pBlocks.find(v3dBlockPos); // check whether the block is already loaded if(itBlock == m_pBlocks.end()) { @@ -701,7 +728,7 @@ namespace PolyVox block->m_uBlockLastAccessed = ++m_uTimestamper; //m_v3dLastAccessedBlockPos = v3dBlockPos; - return block; + return block;*/ } template @@ -718,63 +745,59 @@ namespace PolyVox return m_pLastAccessedBlock; }*/ - UncompressedBlock* pUncompressedBlock = 0; - typename UncompressedBlockMap::iterator itUncompressedBlock = m_pUncompressedBlockCache.find(v3dBlockPos); // check whether the block is already loaded if(itUncompressedBlock != m_pUncompressedBlockCache.end()) { - pUncompressedBlock = itUncompressedBlock->second; - + UncompressedBlock* pUncompressedBlock = itUncompressedBlock->second; + + pUncompressedBlock->m_uBlockLastAccessed = ++m_uTimestamper; + m_pLastAccessedBlock = pUncompressedBlock; + m_v3dLastAccessedBlockPos = v3dBlockPos; + + return pUncompressedBlock; } - //else if(check compressed list...) - //{ - //} else { // At this point we just create a new block. - pUncompressedBlock = new UncompressedBlock(m_uBlockSideLength); - - m_pUncompressedBlockCache.insert(std::make_pair(v3dBlockPos, pUncompressedBlock)); - - // Our block cache may now have grown too large. Flush some entries is necessary. - // FIXME - Watch out for flushing the block we just created! - //flushExcessiveCacheEntries(); - } - - pUncompressedBlock->m_uBlockLastAccessed = ++m_uTimestamper; - - //Gets the block and marks that we accessed it - /*CompressedBlock* block = getCompressedBlock(uBlockX, uBlockY, uBlockZ); - - typename UncompressedBlockMap::iterator itUncompressedBlock = m_pUncompressedBlockCache.find(v3dBlockPos); - // check whether the block is already loaded - if(itUncompressedBlock == m_pUncompressedBlockCache.end()) - { UncompressedBlock* pUncompressedBlock = new UncompressedBlock(m_uBlockSideLength); + + // It's important to set the timestamp before we flush later. pUncompressedBlock->m_uBlockLastAccessed = ++m_uTimestamper; - const void* pSrcData = reinterpret_cast(block->getData()); - void* pDstData = reinterpret_cast(pUncompressedBlock->m_tData); - uint32_t uSrcLength = block->getDataSizeInBytes(); - uint32_t uDstLength = m_uBlockSideLength * m_uBlockSideLength * m_uBlockSideLength * sizeof(VoxelType); + // We set these before flushing because the flush can cause block to be erased, and there + // is a test to make sure the block which is being erase is not the last accessed block. + m_pLastAccessedBlock = pUncompressedBlock; + m_v3dLastAccessedBlockPos = v3dBlockPos; - //MinizCompressor compressor; - //RLECompressor compressor; - uint32_t uUncompressedLength = m_pCompressor->decompress(pSrcData, uSrcLength, pDstData, uDstLength); + // An uncompressed bock is always backed by a compressed one, and this is created by getCompressedBlock() if it doesn't + // already exist. If it does already exist and has data then we bring this across into the ucompressed version. + if(getCompressedBlock(uBlockX, uBlockY, uBlockZ)->getData() != 0) + { + // FIXME - multiple getCompressedBlock() calls (including the one above) + CompressedBlock* pBlock = getCompressedBlock(uBlockX, uBlockY, uBlockZ); - POLYVOX_ASSERT(uUncompressedLength == m_uBlockSideLength * m_uBlockSideLength * m_uBlockSideLength * sizeof(VoxelType), "Destination length has changed."); + const void* pSrcData = reinterpret_cast(pBlock->getData()); + void* pDstData = reinterpret_cast(pUncompressedBlock->m_tData); + uint32_t uSrcLength = pBlock->getDataSizeInBytes(); + uint32_t uDstLength = m_uBlockSideLength * m_uBlockSideLength * m_uBlockSideLength * sizeof(VoxelType); - itUncompressedBlock = m_pUncompressedBlockCache.insert(std::make_pair(v3dBlockPos, pUncompressedBlock)).first; + //MinizCompressor compressor; + //RLECompressor compressor; + uint32_t uUncompressedLength = m_pCompressor->decompress(pSrcData, uSrcLength, pDstData, uDstLength); - // Our block cache may now have grown too large. Fluch some entries is necessary. + POLYVOX_ASSERT(uUncompressedLength == m_uBlockSideLength * m_uBlockSideLength * m_uBlockSideLength * sizeof(VoxelType), "Destination length has changed."); + } + + // Addd our new block to the map. + m_pUncompressedBlockCache.insert(std::make_pair(v3dBlockPos, pUncompressedBlock)); + + // Our block cache may now have grown too large. Flush some entries if necessary. + // FIXME - Watch out for flushing the block we just created! flushExcessiveCacheEntries(); - }*/ - m_pLastAccessedBlock = pUncompressedBlock; - m_v3dLastAccessedBlockPos = v3dBlockPos; - - return m_pLastAccessedBlock; + return pUncompressedBlock; + } } //////////////////////////////////////////////////////////////////////////////// From c3c4ead1f339427607a2526dd2ec0fb4fcc52b0b Mon Sep 17 00:00:00 2001 From: David Williams Date: Fri, 26 Jul 2013 15:38:03 +0200 Subject: [PATCH 26/38] Work on paging to files. --- .../include/PolyVoxCore/LargeVolume.inl | 36 ++++++++++++++----- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl index f0d15bdc..e40d3c8a 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl @@ -102,7 +102,7 @@ namespace PolyVox template LargeVolume::~LargeVolume() { - //flushAll(); + flushAll(); // Only delete the compressor if it was created by us (in the constructor), not by the user. if(m_bIsOurCompressor) @@ -579,6 +579,9 @@ namespace PolyVox // Page the data out m_pPager->pageOut(Region(v3dLower, v3dUpper), pCompressedBlock); + + // The compressed data is no longer modified with respect to the data on disk + pCompressedBlock->m_bDataModified = false; } delete itCompressedBlock->second; @@ -682,20 +685,35 @@ namespace PolyVox if(itBlock != m_pBlocks.end()) { pBlock = itBlock->second; + pBlock->m_uBlockLastAccessed = ++m_uTimestamper; + return pBlock; } - /*else if(check on disk) - { - }*/ else { + // The block wasn't found so we create a new one pBlock = new CompressedBlock; + + // It's important to set the timestamp before we flush later. pBlock->m_uBlockLastAccessed = ++m_uTimestamper; + + // Pass the block to the Pager to give it a chance to initialise it with any data + Vector3DInt32 v3dLower(v3dBlockPos.getX() << m_uBlockSideLengthPower, v3dBlockPos.getY() << m_uBlockSideLengthPower, v3dBlockPos.getZ() << m_uBlockSideLengthPower); + Vector3DInt32 v3dUpper = v3dLower + Vector3DInt32(m_uBlockSideLength-1, m_uBlockSideLength-1, m_uBlockSideLength-1); + Region reg(v3dLower, v3dUpper); + m_pPager->pageIn(reg, pBlock); + + // Add the block to the map itBlock = m_pBlocks.insert(std::make_pair(v3dBlockPos, pBlock)).first; + + // Paging in this new block may mean we are now using too much memory. If necessary, flush some old blocks. + flushOldestExcessiveBlocks(); + + return pBlock; } - pBlock->m_uBlockLastAccessed = ++m_uTimestamper; + - return pBlock; + /*typename CompressedBlockMap::iterator itBlock = m_pBlocks.find(v3dBlockPos); // check whether the block is already loaded @@ -740,10 +758,10 @@ namespace PolyVox //the time stamp. If we updated it everytime then that would be every time we touched //a voxel, which would overflow a uint32_t and require us to use a uint64_t instead. //This check should also provide a significant speed boost as usually it is true. - /*if((v3dBlockPos == m_v3dLastAccessedBlockPos) && (m_pLastAccessedBlock != 0)) + if((v3dBlockPos == m_v3dLastAccessedBlockPos) && (m_pLastAccessedBlock != 0)) { return m_pLastAccessedBlock; - }*/ + } typename UncompressedBlockMap::iterator itUncompressedBlock = m_pUncompressedBlockCache.find(v3dBlockPos); // check whether the block is already loaded @@ -789,7 +807,7 @@ namespace PolyVox POLYVOX_ASSERT(uUncompressedLength == m_uBlockSideLength * m_uBlockSideLength * m_uBlockSideLength * sizeof(VoxelType), "Destination length has changed."); } - // Addd our new block to the map. + // Add our new block to the map. m_pUncompressedBlockCache.insert(std::make_pair(v3dBlockPos, pUncompressedBlock)); // Our block cache may now have grown too large. Flush some entries if necessary. From f54532a9052870b09e7277581d91e3be02c49411 Mon Sep 17 00:00:00 2001 From: David Williams Date: Fri, 26 Jul 2013 16:00:29 +0200 Subject: [PATCH 27/38] Started moving compression code into separate class. --- library/PolyVoxCore/CMakeLists.txt | 2 + .../include/PolyVoxCore/Impl/Block.h | 3 + .../include/PolyVoxCore/Impl/Block.inl | 12 ++++ .../include/PolyVoxCore/LargeVolume.h | 2 + .../include/PolyVoxCore/LargeVolume.inl | 8 ++- .../PolyVoxCore/MinizBlockCompressor.h | 52 +++++++++++++++++ .../PolyVoxCore/MinizBlockCompressor.inl | 57 +++++++++++++++++++ 7 files changed, 134 insertions(+), 2 deletions(-) create mode 100644 library/PolyVoxCore/include/PolyVoxCore/MinizBlockCompressor.h create mode 100644 library/PolyVoxCore/include/PolyVoxCore/MinizBlockCompressor.inl diff --git a/library/PolyVoxCore/CMakeLists.txt b/library/PolyVoxCore/CMakeLists.txt index a8ce8a0c..f79df326 100644 --- a/library/PolyVoxCore/CMakeLists.txt +++ b/library/PolyVoxCore/CMakeLists.txt @@ -71,6 +71,8 @@ SET(CORE_INC_FILES include/PolyVoxCore/MarchingCubesSurfaceExtractor.inl include/PolyVoxCore/Material.h include/PolyVoxCore/MaterialDensityPair.h + include/PolyVoxCore/MinizBlockCompressor.h + include/PolyVoxCore/MinizBlockCompressor.inl include/PolyVoxCore/MinizCompressor.h include/PolyVoxCore/Pager.h include/PolyVoxCore/PolyVoxForwardDeclarations.h diff --git a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h index aeb309cc..554bc3c2 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h +++ b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h @@ -100,6 +100,9 @@ namespace PolyVox UncompressedBlock(uint16_t uSideLength); ~UncompressedBlock(); + VoxelType* getData(void) const; + uint32_t getDataSizeInBytes(void) const; + VoxelType getVoxel(uint16_t uXPos, uint16_t uYPos, uint16_t uZPos) const; VoxelType getVoxel(const Vector3DUint16& v3dPos) const; diff --git a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl index 16927b4e..ba156ef4 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl @@ -115,6 +115,18 @@ namespace PolyVox m_tData = 0; } + template + VoxelType* UncompressedBlock::getData(void) const + { + return m_tData; + } + + template + uint32_t UncompressedBlock::getDataSizeInBytes(void) const + { + return m_uSideLength * m_uSideLength * m_uSideLength * sizeof(VoxelType); + } + template VoxelType UncompressedBlock::getVoxel(uint16_t uXPos, uint16_t uYPos, uint16_t uZPos) const { diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h index 9f409571..e5095c3f 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h @@ -27,6 +27,7 @@ freely, subject to the following restrictions: #include "PolyVoxCore/BaseVolume.h" #include "Impl/Block.h" #include "PolyVoxCore/Compressor.h" +#include "PolyVoxCore/MinizBlockCompressor.h" //Shouldn't include the implementation here. #include "PolyVoxCore/Pager.h" #include "PolyVoxCore/Region.h" #include "PolyVoxCore/Vector.h" @@ -357,6 +358,7 @@ namespace PolyVox // The compressor used by the Blocks to compress their data if required. Compressor* m_pCompressor; + MinizBlockCompressor* m_pBlockCompressor; Pager* m_pPager; // Compressed data for an empty block (sometimes needed for initialisation). diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl index e40d3c8a..c6cb68c3 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl @@ -559,6 +559,8 @@ namespace PolyVox delete[] pZeros; delete[] pCompressedZeros; + + m_pBlockCompressor = new MinizBlockCompressor; } template @@ -795,7 +797,7 @@ namespace PolyVox // FIXME - multiple getCompressedBlock() calls (including the one above) CompressedBlock* pBlock = getCompressedBlock(uBlockX, uBlockY, uBlockZ); - const void* pSrcData = reinterpret_cast(pBlock->getData()); + /*const void* pSrcData = reinterpret_cast(pBlock->getData()); void* pDstData = reinterpret_cast(pUncompressedBlock->m_tData); uint32_t uSrcLength = pBlock->getDataSizeInBytes(); uint32_t uDstLength = m_uBlockSideLength * m_uBlockSideLength * m_uBlockSideLength * sizeof(VoxelType); @@ -804,7 +806,9 @@ namespace PolyVox //RLECompressor compressor; uint32_t uUncompressedLength = m_pCompressor->decompress(pSrcData, uSrcLength, pDstData, uDstLength); - POLYVOX_ASSERT(uUncompressedLength == m_uBlockSideLength * m_uBlockSideLength * m_uBlockSideLength * sizeof(VoxelType), "Destination length has changed."); + POLYVOX_ASSERT(uUncompressedLength == m_uBlockSideLength * m_uBlockSideLength * m_uBlockSideLength * sizeof(VoxelType), "Destination length has changed.");*/ + + m_pBlockCompressor->decompress(pBlock, pUncompressedBlock); } // Add our new block to the map. diff --git a/library/PolyVoxCore/include/PolyVoxCore/MinizBlockCompressor.h b/library/PolyVoxCore/include/PolyVoxCore/MinizBlockCompressor.h new file mode 100644 index 00000000..b6053fbc --- /dev/null +++ b/library/PolyVoxCore/include/PolyVoxCore/MinizBlockCompressor.h @@ -0,0 +1,52 @@ +/******************************************************************************* +Copyright (c) 2005-2013 David Williams and Matthew Williams + +This software is provided 'as-is', without any express or implied +warranty. In no event will the authors be held liable for any damages +arising from the use of this software. + +Permission is granted to anyone to use this software for any purpose, +including commercial applications, and to alter it and redistribute it +freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + + 3. This notice may not be removed or altered from any source + distribution. +*******************************************************************************/ + +#ifndef __PolyVox_MinizBlockCompressor_H__ +#define __PolyVox_MinizBlockCompressor_H__ + +#include "PolyVoxCore/Impl/Block.h" + +#include "PolyVoxCore/MinizCompressor.h" + +namespace PolyVox +{ + /** + * Provides an interface for performing paging of data. + */ + template + class MinizBlockCompressor + { + public: + MinizBlockCompressor(); + ~MinizBlockCompressor(); + + void compress(UncompressedBlock* pSrcBlock, CompressedBlock* pDstBlock); + void decompress(CompressedBlock* pSrcBlock, UncompressedBlock* pDstBlock); + + MinizCompressor* m_pCompressor; + }; +} + +#include "PolyVoxCore/MinizBlockCompressor.inl" + +#endif //__PolyVox_MinizBlockCompressor_H__ diff --git a/library/PolyVoxCore/include/PolyVoxCore/MinizBlockCompressor.inl b/library/PolyVoxCore/include/PolyVoxCore/MinizBlockCompressor.inl new file mode 100644 index 00000000..69f6b2e9 --- /dev/null +++ b/library/PolyVoxCore/include/PolyVoxCore/MinizBlockCompressor.inl @@ -0,0 +1,57 @@ +/******************************************************************************* +Copyright (c) 2005-2013 David Williams and Matthew Williams + +This software is provided 'as-is', without any express or implied +warranty. In no event will the authors be held liable for any damages +arising from the use of this software. + +Permission is granted to anyone to use this software for any purpose, +including commercial applications, and to alter it and redistribute it +freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + + 3. This notice may not be removed or altered from any source + distribution. +*******************************************************************************/ + +namespace PolyVox +{ + template + MinizBlockCompressor::MinizBlockCompressor() + { + m_pCompressor = new MinizCompressor; + } + + template + MinizBlockCompressor::~MinizBlockCompressor() + { + delete m_pCompressor; + } + + template + void MinizBlockCompressor::compress(UncompressedBlock* pSrcBlock, CompressedBlock* pDstBlock) + { + } + + template + void MinizBlockCompressor::decompress(CompressedBlock* pSrcBlock, UncompressedBlock* pDstBlock) + { + const void* pSrcData = reinterpret_cast(pSrcBlock->getData()); + void* pDstData = reinterpret_cast(pDstBlock->getData()); + uint32_t uSrcLength = pSrcBlock->getDataSizeInBytes(); + uint32_t uDstLength = pDstBlock->getDataSizeInBytes(); + + + //RLECompressor compressor; + uint32_t uUncompressedLength = m_pCompressor->decompress(pSrcData, uSrcLength, pDstData, uDstLength); + + POLYVOX_ASSERT(uUncompressedLength == pDstBlock->getDataSizeInBytes(), "Destination length has changed."); + } +} \ No newline at end of file From ef59f329b61de55dccdadcb8a61cf22cb0c9d6c3 Mon Sep 17 00:00:00 2001 From: David Williams Date: Fri, 26 Jul 2013 16:25:50 +0200 Subject: [PATCH 28/38] Reorganising compression code. --- .../include/PolyVoxCore/LargeVolume.inl | 57 +++---------------- .../PolyVoxCore/MinizBlockCompressor.inl | 48 ++++++++++++++++ 2 files changed, 55 insertions(+), 50 deletions(-) diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl index c6cb68c3..7534e8bb 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl @@ -614,7 +614,12 @@ namespace PolyVox Vector3DInt32 v3dBlockPos = itUncompressedBlock->first; CompressedBlock* pCompressedBlock = getCompressedBlock(v3dBlockPos.getX(), v3dBlockPos.getY(), v3dBlockPos.getZ()); - void* pSrcData = reinterpret_cast(pUncompressedBlock->m_tData); + m_pBlockCompressor->compress(pUncompressedBlock, pCompressedBlock); + + // The compressed data has been updated, so the uncompressed data is no longer modified with respect to it. + pUncompressedBlock->m_bDataModified = false; + + /*void* pSrcData = reinterpret_cast(pUncompressedBlock->m_tData); uint32_t uSrcLength = m_uBlockSideLength * m_uBlockSideLength * m_uBlockSideLength * sizeof(VoxelType); uint8_t tempBuffer[10000]; @@ -667,7 +672,7 @@ namespace PolyVox } delete[] buffer; - } + }*/ } delete itUncompressedBlock->second; @@ -712,43 +717,6 @@ namespace PolyVox return pBlock; } - - - - - - /*typename CompressedBlockMap::iterator itBlock = m_pBlocks.find(v3dBlockPos); - // check whether the block is already loaded - if(itBlock == m_pBlocks.end()) - { - //The block is not in the map, so we will have to create a new block and add it. - CompressedBlock* newBlock = new CompressedBlock; - newBlock->m_uBlockLastAccessed = ++m_uTimestamper; - itBlock = m_pBlocks.insert(std::make_pair(v3dBlockPos, newBlock)).first; - - // Now use the pager to fill the block with it's initial data. - Vector3DInt32 v3dLower(v3dBlockPos.getX() << m_uBlockSideLengthPower, v3dBlockPos.getY() << m_uBlockSideLengthPower, v3dBlockPos.getZ() << m_uBlockSideLengthPower); - Vector3DInt32 v3dUpper = v3dLower + Vector3DInt32(m_uBlockSideLength-1, m_uBlockSideLength-1, m_uBlockSideLength-1); - Region reg(v3dLower, v3dUpper); - m_pPager->pageIn(reg, newBlock); - - // The pager may not actually have given us any data (perhaps there wasn't any) so in that case we need to create some ourselves. - if(newBlock->m_bDataModified == false) - { - // Internally this performs a memcpy() of the data. - newBlock->setData(m_pCompressedEmptyBlock->getData(), m_pCompressedEmptyBlock->getDataSizeInBytes()); - } - - // Paging in this new block may mean we are now using too much memory. If necessary, flush some old blocks. - flushOldestExcessiveBlocks(); - } - - //Get the block and mark that we accessed it - CompressedBlock* block = itBlock->second; - block->m_uBlockLastAccessed = ++m_uTimestamper; - //m_v3dLastAccessedBlockPos = v3dBlockPos; - - return block;*/ } template @@ -797,17 +765,6 @@ namespace PolyVox // FIXME - multiple getCompressedBlock() calls (including the one above) CompressedBlock* pBlock = getCompressedBlock(uBlockX, uBlockY, uBlockZ); - /*const void* pSrcData = reinterpret_cast(pBlock->getData()); - void* pDstData = reinterpret_cast(pUncompressedBlock->m_tData); - uint32_t uSrcLength = pBlock->getDataSizeInBytes(); - uint32_t uDstLength = m_uBlockSideLength * m_uBlockSideLength * m_uBlockSideLength * sizeof(VoxelType); - - //MinizCompressor compressor; - //RLECompressor compressor; - uint32_t uUncompressedLength = m_pCompressor->decompress(pSrcData, uSrcLength, pDstData, uDstLength); - - POLYVOX_ASSERT(uUncompressedLength == m_uBlockSideLength * m_uBlockSideLength * m_uBlockSideLength * sizeof(VoxelType), "Destination length has changed.");*/ - m_pBlockCompressor->decompress(pBlock, pUncompressedBlock); } diff --git a/library/PolyVoxCore/include/PolyVoxCore/MinizBlockCompressor.inl b/library/PolyVoxCore/include/PolyVoxCore/MinizBlockCompressor.inl index 69f6b2e9..67501ea9 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/MinizBlockCompressor.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/MinizBlockCompressor.inl @@ -38,6 +38,54 @@ namespace PolyVox template void MinizBlockCompressor::compress(UncompressedBlock* pSrcBlock, CompressedBlock* pDstBlock) { + void* pSrcData = reinterpret_cast(pSrcBlock->getData()); + uint32_t uSrcLength = pSrcBlock->getDataSizeInBytes(); + + uint8_t tempBuffer[10000]; + uint8_t* pDstData = reinterpret_cast( tempBuffer ); + uint32_t uDstLength = 10000; + + uint32_t uCompressedLength = 0; + + try + { + // Perform the compression + uCompressedLength = m_pCompressor->compress(pSrcData, uSrcLength, pDstData, uDstLength); + + // Copy the resulting compressed data into the compressed block + pDstBlock->setData(pDstData, uDstLength); + } + catch(std::exception&) + { + // It is possible for the compression to fail. A common cause for this would be if the destination + // buffer is not big enough. So now we try again using a buffer that is definitely big enough. + // Note that ideally we will choose our earlier buffer size so that this almost never happens. + logWarning() << "The compressor failed to compress the block, probabaly due to the buffer being too small."; + logWarning() << "The compression will be tried again with a larger buffer"; + uint32_t uMaxCompressedSize = m_pCompressor->getMaxCompressedSize(uSrcLength); + uint8_t* buffer = new uint8_t[ uMaxCompressedSize ]; + + pDstData = reinterpret_cast( buffer ); + uDstLength = uMaxCompressedSize; + + try + { + // Perform the compression + uCompressedLength = m_pCompressor->compress(pSrcData, uSrcLength, pDstData, uDstLength); + + // Copy the resulting compressed data into the compressed block + pDstBlock->setData(pDstData, uDstLength); + } + catch(std::exception&) + { + // At this point it didn't work even with a bigger buffer. + // Not much more we can do so just rethrow the exception. + delete[] buffer; + POLYVOX_THROW(std::runtime_error, "Failed to compress block data"); + } + + delete[] buffer; + } } template From a4fc814a511409ac3de7f21a2bd0d9120a6a5bd5 Mon Sep 17 00:00:00 2001 From: David Williams Date: Mon, 29 Jul 2013 16:04:01 +0200 Subject: [PATCH 29/38] Rearranging and renaming some code. --- library/PolyVoxCore/CMakeLists.txt | 1 + .../include/PolyVoxCore/BlockCompressor.h | 46 ++++++++++ .../include/PolyVoxCore/LargeVolume.h | 12 ++- .../include/PolyVoxCore/LargeVolume.inl | 89 ++----------------- .../PolyVoxCore/MinizBlockCompressor.h | 4 +- .../PolyVoxCore/PolyVoxForwardDeclarations.h | 5 ++ tests/testvolume.cpp | 8 +- tests/testvolume.h | 2 +- 8 files changed, 72 insertions(+), 95 deletions(-) create mode 100644 library/PolyVoxCore/include/PolyVoxCore/BlockCompressor.h diff --git a/library/PolyVoxCore/CMakeLists.txt b/library/PolyVoxCore/CMakeLists.txt index f79df326..11951b0d 100644 --- a/library/PolyVoxCore/CMakeLists.txt +++ b/library/PolyVoxCore/CMakeLists.txt @@ -48,6 +48,7 @@ SET(CORE_INC_FILES include/PolyVoxCore/BaseVolume.h include/PolyVoxCore/BaseVolume.inl include/PolyVoxCore/BaseVolumeSampler.inl + include/PolyVoxCore/BlockCompressor.h include/PolyVoxCore/Compressor.h include/PolyVoxCore/CubicSurfaceExtractor.h include/PolyVoxCore/CubicSurfaceExtractor.inl diff --git a/library/PolyVoxCore/include/PolyVoxCore/BlockCompressor.h b/library/PolyVoxCore/include/PolyVoxCore/BlockCompressor.h new file mode 100644 index 00000000..21338de7 --- /dev/null +++ b/library/PolyVoxCore/include/PolyVoxCore/BlockCompressor.h @@ -0,0 +1,46 @@ +/******************************************************************************* +Copyright (c) 2005-2013 David Williams and Matthew Williams + +This software is provided 'as-is', without any express or implied +warranty. In no event will the authors be held liable for any damages +arising from the use of this software. + +Permission is granted to anyone to use this software for any purpose, +including commercial applications, and to alter it and redistribute it +freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + + 3. This notice may not be removed or altered from any source + distribution. +*******************************************************************************/ + +#ifndef __PolyVox_BlockCompressor_H__ +#define __PolyVox_BlockCompressor_H__ + +#include "PolyVoxCore/Impl/Block.h" + +namespace PolyVox +{ + /** + * Provides an interface for performing compression of blocks. + */ + template + class BlockCompressor + { + public: + BlockCompressor() {}; + virtual ~BlockCompressor() {}; + + virtual void compress(UncompressedBlock* pSrcBlock, CompressedBlock* pDstBlock) = 0; + virtual void decompress(CompressedBlock* pSrcBlock, UncompressedBlock* pDstBlock) = 0; + }; +} + +#endif //__PolyVox_BlockCompressor_H__ diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h index e5095c3f..986bcf2b 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h @@ -25,9 +25,8 @@ freely, subject to the following restrictions: #define __PolyVox_LargeVolume_H__ #include "PolyVoxCore/BaseVolume.h" -#include "Impl/Block.h" -#include "PolyVoxCore/Compressor.h" -#include "PolyVoxCore/MinizBlockCompressor.h" //Shouldn't include the implementation here. +#include "PolyVoxCore/Impl/Block.h" +#include "PolyVoxCore/BlockCompressor.h" #include "PolyVoxCore/Pager.h" #include "PolyVoxCore/Region.h" #include "PolyVoxCore/Vector.h" @@ -240,7 +239,7 @@ namespace PolyVox LargeVolume ( const Region& regValid, - Compressor* pCompressor, + BlockCompressor* pBlockCompressor, Pager* pPager , uint16_t uBlockSideLength = 32 ); @@ -357,12 +356,11 @@ namespace PolyVox uint8_t m_uBlockSideLengthPower; // The compressor used by the Blocks to compress their data if required. - Compressor* m_pCompressor; - MinizBlockCompressor* m_pBlockCompressor; + BlockCompressor* m_pBlockCompressor; Pager* m_pPager; // Compressed data for an empty block (sometimes needed for initialisation). - CompressedBlock* m_pCompressedEmptyBlock; + //CompressedBlock* m_pCompressedEmptyBlock; // Whether we created the compressor or whether it was provided // by the user. This controls whether we delete it on destruction. diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl index 7534e8bb..7be743fd 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl @@ -23,7 +23,7 @@ freely, subject to the following restrictions: #include "PolyVoxCore/Impl/ErrorHandling.h" -#include "PolyVoxCore/MinizCompressor.h" +#include "PolyVoxCore/MinizBlockCompressor.h" // For creating a default compressor when none is provided. #include @@ -45,7 +45,7 @@ namespace PolyVox { m_uBlockSideLength = uBlockSideLength; - m_pCompressor = new MinizCompressor(); + m_pBlockCompressor = new MinizBlockCompressor(); m_bIsOurCompressor = true; m_pPager = 0; @@ -56,7 +56,7 @@ namespace PolyVox //////////////////////////////////////////////////////////////////////////////// /// This constructor creates a volume with a fixed size which is specified as a parameter. By default this constructor will not enable paging but you can override this if desired. If you do wish to enable paging then you are required to provide the call back function (see the other LargeVolume constructor). /// \param regValid Specifies the minimum and maximum valid voxel positions. - /// \param pCompressor An implementation of the Compressor interface which is used to compress blocks in memory. + /// \param pBlockCompressor An implementation of the Compressor interface which is used to compress blocks in memory. /// \param dataRequiredHandler The callback function which will be called when PolyVox tries to use data which is not currently in momory. /// \param dataOverflowHandler The callback function which will be called when PolyVox has too much data and needs to remove some from memory. /// \param bPagingEnabled Controls whether or not paging is enabled for this LargeVolume. @@ -66,16 +66,15 @@ namespace PolyVox LargeVolume::LargeVolume ( const Region& regValid, - Compressor* pCompressor, + BlockCompressor* pBlockCompressor, Pager* pPager, uint16_t uBlockSideLength ) :BaseVolume(regValid) { - m_uBlockSideLength = uBlockSideLength; - m_pCompressor = pCompressor; + m_pBlockCompressor = pBlockCompressor; m_bIsOurCompressor = false; m_pPager = pPager; @@ -107,10 +106,10 @@ namespace PolyVox // Only delete the compressor if it was created by us (in the constructor), not by the user. if(m_bIsOurCompressor) { - delete m_pCompressor; + delete m_pBlockCompressor; } - delete m_pCompressedEmptyBlock; + //delete m_pCompressedEmptyBlock; } //////////////////////////////////////////////////////////////////////////////// @@ -511,7 +510,7 @@ namespace PolyVox POLYVOX_THROW(std::invalid_argument, "Block side length must be a power of two."); } - if(!m_pCompressor) + if(!m_pBlockCompressor) { POLYVOX_THROW(std::invalid_argument, "You must provide a valid compressor for the LargeVolume to use."); } @@ -544,23 +543,6 @@ namespace PolyVox this->m_uLongestSideLength = (std::max)((std::max)(this->getWidth(),this->getHeight()),this->getDepth()); this->m_uShortestSideLength = (std::min)((std::min)(this->getWidth(),this->getHeight()),this->getDepth()); this->m_fDiagonalLength = sqrtf(static_cast(this->getWidth() * this->getWidth() + this->getHeight() * this->getHeight() + this->getDepth() * this->getDepth())); - - // This is used for initialising empty blocks. - // FIXME - Should probably build an UncompressedBlock and then call some 'fill()' method. Ideally we should set voxels to an 'empty' (default?) value rather than zeros. - VoxelType* pZeros = new VoxelType[m_uBlockSideLength * m_uBlockSideLength * m_uBlockSideLength]; - uint32_t uSrcLength = m_uBlockSideLength * m_uBlockSideLength * m_uBlockSideLength * sizeof(VoxelType); - memset(pZeros, 0, uSrcLength); - uint32_t uDstLength = 100000; - uint8_t* pCompressedZeros = new uint8_t[uDstLength]; //Should be plenty as zeros compress down very small. - uint32_t uCompressedSize = m_pCompressor->compress(reinterpret_cast(pZeros), uSrcLength, reinterpret_cast(pCompressedZeros), uDstLength); - - m_pCompressedEmptyBlock = new CompressedBlock; - m_pCompressedEmptyBlock->setData(pCompressedZeros, uCompressedSize); - - delete[] pZeros; - delete[] pCompressedZeros; - - m_pBlockCompressor = new MinizBlockCompressor; } template @@ -618,61 +600,6 @@ namespace PolyVox // The compressed data has been updated, so the uncompressed data is no longer modified with respect to it. pUncompressedBlock->m_bDataModified = false; - - /*void* pSrcData = reinterpret_cast(pUncompressedBlock->m_tData); - uint32_t uSrcLength = m_uBlockSideLength * m_uBlockSideLength * m_uBlockSideLength * sizeof(VoxelType); - - uint8_t tempBuffer[10000]; - uint8_t* pDstData = reinterpret_cast( tempBuffer ); - uint32_t uDstLength = 10000; - - uint32_t uCompressedLength = 0; - - try - { - // Perform the compression - uCompressedLength = m_pCompressor->compress(pSrcData, uSrcLength, pDstData, uDstLength); - - // Copy the resulting compressed data into the compressed block - pCompressedBlock->setData(pDstData, uDstLength); - - // The compressed data has been updated, so the uncompressed data is no longer modified with respect to it. - pUncompressedBlock->m_bDataModified = false; - } - catch(std::exception&) - { - // It is possible for the compression to fail. A common cause for this would be if the destination - // buffer is not big enough. So now we try again using a buffer that is definitely big enough. - // Note that ideally we will choose our earlier buffer size so that this almost never happens. - logWarning() << "The compressor failed to compress the block, probabaly due to the buffer being too small."; - logWarning() << "The compression will be tried again with a larger buffer"; - uint32_t uMaxCompressedSize = m_pCompressor->getMaxCompressedSize(uSrcLength); - uint8_t* buffer = new uint8_t[ uMaxCompressedSize ]; - - pDstData = reinterpret_cast( buffer ); - uDstLength = uMaxCompressedSize; - - try - { - // Perform the compression - uCompressedLength = m_pCompressor->compress(pSrcData, uSrcLength, pDstData, uDstLength); - - // Copy the resulting compressed data into the compressed block - pCompressedBlock->setData(pDstData, uDstLength); - - // The compressed data has been updated, so the uncompressed data is no longer modified with respect to it. - pUncompressedBlock->m_bDataModified = false; - } - catch(std::exception&) - { - // At this point it didn't work even with a bigger buffer. - // Not much more we can do so just rethrow the exception. - delete[] buffer; - POLYVOX_THROW(std::runtime_error, "Failed to compress block data"); - } - - delete[] buffer; - }*/ } delete itUncompressedBlock->second; diff --git a/library/PolyVoxCore/include/PolyVoxCore/MinizBlockCompressor.h b/library/PolyVoxCore/include/PolyVoxCore/MinizBlockCompressor.h index b6053fbc..e4b57cf6 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/MinizBlockCompressor.h +++ b/library/PolyVoxCore/include/PolyVoxCore/MinizBlockCompressor.h @@ -24,7 +24,7 @@ freely, subject to the following restrictions: #ifndef __PolyVox_MinizBlockCompressor_H__ #define __PolyVox_MinizBlockCompressor_H__ -#include "PolyVoxCore/Impl/Block.h" +#include "PolyVoxCore/BlockCompressor.h" #include "PolyVoxCore/MinizCompressor.h" @@ -34,7 +34,7 @@ namespace PolyVox * Provides an interface for performing paging of data. */ template - class MinizBlockCompressor + class MinizBlockCompressor : public BlockCompressor { public: MinizBlockCompressor(); diff --git a/library/PolyVoxCore/include/PolyVoxCore/PolyVoxForwardDeclarations.h b/library/PolyVoxCore/include/PolyVoxCore/PolyVoxForwardDeclarations.h index 05092ab1..2eef3077 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/PolyVoxForwardDeclarations.h +++ b/library/PolyVoxCore/include/PolyVoxCore/PolyVoxForwardDeclarations.h @@ -60,6 +60,11 @@ namespace PolyVox typedef Array<3,int32_t> Array3DInt32; typedef Array<3,uint32_t> Array3DUint32; + //////////////////////////////////////////////////////////////////////////////// + // BlockCompressor + //////////////////////////////////////////////////////////////////////////////// + template class BlockCompressor; + //////////////////////////////////////////////////////////////////////////////// // Compressor //////////////////////////////////////////////////////////////////////////////// diff --git a/tests/testvolume.cpp b/tests/testvolume.cpp index 77ad1063..d15e1fcc 100644 --- a/tests/testvolume.cpp +++ b/tests/testvolume.cpp @@ -25,7 +25,7 @@ freely, subject to the following restrictions: #include "PolyVoxCore/FilePager.h" #include "PolyVoxCore/LargeVolume.h" -#include "PolyVoxCore/MinizCompressor.h" +#include "PolyVoxCore/MinizBlockCompressor.h" #include "PolyVoxCore/RawVolume.h" #include "PolyVoxCore/RLECompressor.h" #include "PolyVoxCore/SimpleVolume.h" @@ -273,13 +273,13 @@ TestVolume::TestVolume() Region region(-57, -31, 12, 64, 96, 131); // Deliberatly awkward size //m_pCompressor = new RLECompressor; - m_pCompressor = new MinizCompressor; + m_pBlockCompressor = new MinizBlockCompressor; m_pFilePager = new FilePager("./"); //Create the volumes m_pRawVolume = new RawVolume(region); m_pSimpleVolume = new SimpleVolume(region); - m_pLargeVolume = new LargeVolume(region, m_pCompressor, m_pFilePager, 32); + m_pLargeVolume = new LargeVolume(region, m_pBlockCompressor, m_pFilePager, 32); m_pLargeVolume->setMaxNumberOfBlocksInMemory(32); m_pLargeVolume->setMaxNumberOfUncompressedBlocks(16); @@ -307,7 +307,7 @@ TestVolume::~TestVolume() delete m_pLargeVolume; delete m_pFilePager; - delete m_pCompressor; + delete m_pBlockCompressor; } /* diff --git a/tests/testvolume.h b/tests/testvolume.h index 8a3e46d7..9db5f7a0 100644 --- a/tests/testvolume.h +++ b/tests/testvolume.h @@ -65,7 +65,7 @@ private slots: void testLargeVolumeSamplersWithExternalBackwards(); private: - PolyVox::Compressor* m_pCompressor; + PolyVox::BlockCompressor* m_pBlockCompressor; PolyVox::FilePager* m_pFilePager; PolyVox::RawVolume* m_pRawVolume; From 020ac9655a008de379b624076d57d54265218295 Mon Sep 17 00:00:00 2001 From: David Williams Date: Tue, 30 Jul 2013 15:42:51 +0200 Subject: [PATCH 30/38] Added random prefix to filename to prevent name conflicts. --- .../include/PolyVoxCore/FilePager.h | 23 ++++++++++++++----- 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/library/PolyVoxCore/include/PolyVoxCore/FilePager.h b/library/PolyVoxCore/include/PolyVoxCore/FilePager.h index 66091d9c..4f8f15c8 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/FilePager.h +++ b/library/PolyVoxCore/include/PolyVoxCore/FilePager.h @@ -29,6 +29,8 @@ freely, subject to the following restrictions: #include "PolyVoxCore/Pager.h" #include "PolyVoxCore/Region.h" +#include +#include #include #include #include @@ -47,6 +49,12 @@ namespace PolyVox :Pager() ,m_strFolderName(strFolderName) { + srand(time(0)); + int iRandomValue = rand(); + + std::stringstream ss; + ss << std::hex << iRandomValue; + m_strRandomPrefix = ss.str(); } /// Destructor @@ -57,11 +65,12 @@ namespace PolyVox POLYVOX_ASSERT(pBlockData, "Attempting to page in NULL block"); //POLYVOX_ASSERT(pBlockData->hasUncompressedData() == false, "Block should not have uncompressed data"); - std::stringstream ss; - ss << region.getLowerX() << "_" << region.getLowerY() << "_" << region.getLowerZ() << "_" + std::stringstream ssFilename; + ssFilename << m_strFolderName << "/" << m_strRandomPrefix << "-" + << region.getLowerX() << "_" << region.getLowerY() << "_" << region.getLowerZ() << "_" << region.getUpperX() << "_" << region.getUpperY() << "_" << region.getUpperZ(); - std::string filename = m_strFolderName + ss.str(); + std::string filename = ssFilename.str(); // FIXME - This should be replaced by C++ style IO, but currently this causes problems with // the gameplay-cubiquity integration. See: https://github.com/blackberry/GamePlay/issues/919 @@ -100,11 +109,12 @@ namespace PolyVox logTrace() << "Paging out data for " << region; - std::stringstream ss; - ss << region.getLowerX() << "_" << region.getLowerY() << "_" << region.getLowerZ() << "_" + std::stringstream ssFilename; + ssFilename << m_strFolderName << "/" << m_strRandomPrefix << "-" + << region.getLowerX() << "_" << region.getLowerY() << "_" << region.getLowerZ() << "_" << region.getUpperX() << "_" << region.getUpperY() << "_" << region.getUpperZ(); - std::string filename = m_strFolderName + ss.str(); + std::string filename = ssFilename.str(); // FIXME - This should be replaced by C++ style IO, but currently this causes problems with // the gameplay-cubiquity integration. See: https://github.com/blackberry/GamePlay/issues/919 @@ -127,6 +137,7 @@ namespace PolyVox protected: std::string m_strFolderName; + std::string m_strRandomPrefix; }; } From d1138dcdb12d35824b49b05e61cc39ee1ff5c396 Mon Sep 17 00:00:00 2001 From: David Williams Date: Tue, 30 Jul 2013 16:01:03 +0200 Subject: [PATCH 31/38] Some new block compression code as I rework the previous code... --- library/PolyVoxCore/CMakeLists.txt | 2 + .../include/PolyVoxCore/RLEBlockCompressor.h | 61 +++++++ .../PolyVoxCore/RLEBlockCompressor.inl | 169 ++++++++++++++++++ 3 files changed, 232 insertions(+) create mode 100644 library/PolyVoxCore/include/PolyVoxCore/RLEBlockCompressor.h create mode 100644 library/PolyVoxCore/include/PolyVoxCore/RLEBlockCompressor.inl diff --git a/library/PolyVoxCore/CMakeLists.txt b/library/PolyVoxCore/CMakeLists.txt index 11951b0d..018efe2d 100644 --- a/library/PolyVoxCore/CMakeLists.txt +++ b/library/PolyVoxCore/CMakeLists.txt @@ -85,6 +85,8 @@ SET(CORE_INC_FILES include/PolyVoxCore/Raycast.h include/PolyVoxCore/Raycast.inl include/PolyVoxCore/Region.h + include/PolyVoxCore/RLEBlockCompressor.h + include/PolyVoxCore/RLEBlockCompressor.inl include/PolyVoxCore/RLECompressor.h include/PolyVoxCore/RLECompressor.inl include/PolyVoxCore/SimpleVolume.h diff --git a/library/PolyVoxCore/include/PolyVoxCore/RLEBlockCompressor.h b/library/PolyVoxCore/include/PolyVoxCore/RLEBlockCompressor.h new file mode 100644 index 00000000..fd77522f --- /dev/null +++ b/library/PolyVoxCore/include/PolyVoxCore/RLEBlockCompressor.h @@ -0,0 +1,61 @@ +/******************************************************************************* +Copyright (c) 2005-2013 David Williams and Matthew Williams + +This software is provided 'as-is', without any express or implied +warranty. In no event will the authors be held liable for any damages +arising from the use of this software. + +Permission is granted to anyone to use this software for any purpose, +including commercial applications, and to alter it and redistribute it +freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + + 3. This notice may not be removed or altered from any source + distribution. +*******************************************************************************/ + +#ifndef __PolyVox_RLEBlockCompressor_H__ +#define __PolyVox_RLEBlockCompressor_H__ + +#include "PolyVoxCore/BlockCompressor.h" + +#include "PolyVoxCore/MinizCompressor.h" + +namespace PolyVox +{ + template + class Run + { + public: + typedef uint16_t LengthType; + VoxelType value; + LengthType length; + }; + + /** + * Provides an interface for performing paging of data. + */ + template + class RLEBlockCompressor : public BlockCompressor + { + + + public: + RLEBlockCompressor(); + ~RLEBlockCompressor(); + + void compress(UncompressedBlock* pSrcBlock, CompressedBlock* pDstBlock); + void decompress(CompressedBlock* pSrcBlock, UncompressedBlock* pDstBlock); + }; +} + +#include "PolyVoxCore/RLEBlockCompressor.inl" + +#endif //__PolyVox_RLEBlockCompressor_H__ diff --git a/library/PolyVoxCore/include/PolyVoxCore/RLEBlockCompressor.inl b/library/PolyVoxCore/include/PolyVoxCore/RLEBlockCompressor.inl new file mode 100644 index 00000000..4fc90b2e --- /dev/null +++ b/library/PolyVoxCore/include/PolyVoxCore/RLEBlockCompressor.inl @@ -0,0 +1,169 @@ +/******************************************************************************* +Copyright (c) 2005-2013 David Williams and Matthew Williams + +This software is provided 'as-is', without any express or implied +warranty. In no event will the authors be held liable for any damages +arising from the use of this software. + +Permission is granted to anyone to use this software for any purpose, +including commercial applications, and to alter it and redistribute it +freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + + 3. This notice may not be removed or altered from any source + distribution. +*******************************************************************************/ + +#include + +#include "RLECompressor.h" + +namespace PolyVox +{ + template + RLEBlockCompressor::RLEBlockCompressor() + { + } + + template + RLEBlockCompressor::~RLEBlockCompressor() + { + } + + template + void RLEBlockCompressor::compress(UncompressedBlock* pSrcBlock, CompressedBlock* pDstBlock) + { + void* pSrcData = reinterpret_cast(pSrcBlock->getData()); + uint32_t uSrcLength = pSrcBlock->getDataSizeInBytes(); + + if(uSrcLength % sizeof(VoxelType) != 0) + { + POLYVOX_THROW(std::length_error, "Source length must be a integer multiple of the ValueType size"); + } + + // Lengths provided are in bytes, so convert them to be in terms of our types. + uSrcLength /= sizeof(VoxelType); + //uDstLength /= sizeof(Run); + + // Get data pointers in the appropriate type + const VoxelType* pSrcDataAsType = reinterpret_cast(pSrcData); + //Run* pDstDataAsRun = reinterpret_cast(pDstData); + std::vector< Run > vecDstDataAsRuns; + + // Pointers to just past the end of the data + const VoxelType* pSrcDataEnd = pSrcDataAsType + uSrcLength; + //Run* pDstDataEnd = pDstDataAsRun + uDstLength; + + //Counter for the output length + //uint32_t uDstLengthInBytes = 0; + + // Read the first element of the source and set up the first run based on it. + /*pDstDataAsRun->value = *pSrcDataAsType; + pSrcDataAsType++; + pDstDataAsRun->length = 1; + uDstLengthInBytes += sizeof(Run);*/ + Run firstRun; + firstRun.value = *pSrcDataAsType; + firstRun.length = 1; + vecDstDataAsRuns.push_back(firstRun); + + pSrcDataAsType++; + + //Now process all remaining elements of the source. + while(pSrcDataAsType < pSrcDataEnd) + { + // If the value is the same as the current run (and we have not + // reached the maximum run length) then extend the current run. + std::vector< Run >::iterator currentRun = (vecDstDataAsRuns.end() - 1); + if((*pSrcDataAsType == currentRun->value) && (currentRun->length < (std::numeric_limits::LengthType>::max)())) + { + currentRun->length++; + } + // Otherwise we need to start a new Run. + else + { + /*pDstDataAsRun++; + + // Check if we have enough space in the destination buffer. + if(pDstDataAsRun >= pDstDataEnd) + { + POLYVOX_THROW(std::runtime_error, "Insufficient space in destination buffer."); + }*/ + + // Create the new run. + /*pDstDataAsRun->value = *pSrcDataAsType; + pDstDataAsRun->length = 1; + uDstLengthInBytes += sizeof(Run);*/ + + Run newRun; + newRun.value = *pSrcDataAsType; + newRun.length = 1; + vecDstDataAsRuns.push_back(newRun); + } + + pSrcDataAsType++; + } + + //Now copy the data into the compressed block. + + pDstBlock->setData(reinterpret_cast(&(vecDstDataAsRuns[0])), vecDstDataAsRuns.size() * sizeof(Run)); + } + + template + void RLEBlockCompressor::decompress(CompressedBlock* pSrcBlock, UncompressedBlock* pDstBlock) + { + const void* pSrcData = reinterpret_cast(pSrcBlock->getData()); + uint32_t uSrcLength = pSrcBlock->getDataSizeInBytes(); + + void* pDstData = pDstBlock->getData(); + uint32_t uDstLength = pDstBlock->getDataSizeInBytes(); + + if(uSrcLength % sizeof(Run) != 0) + { + POLYVOX_THROW(std::length_error, "Source length must be a integer multiple of the Run size"); + } + + // Lengths provided are in bytes, so convert them to be in terms of our types. + uSrcLength /= sizeof(Run); + uDstLength /= sizeof(VoxelType); + + // Get data pointers in the appropriate type + const Run* pSrcDataAsRun = reinterpret_cast*>(pSrcData); + VoxelType* pDstDataAsType = reinterpret_cast(pDstData); + + // Pointers to just past the end of the data + const Run* pSrcDataEnd = pSrcDataAsRun + uSrcLength; + VoxelType* pDstDataEnd = pDstDataAsType + uDstLength; + + //Counter for the output length + uint32_t uDstLengthInBytes = 0; + + while(pSrcDataAsRun < pSrcDataEnd) + { + // Check if we have enough space in the destination buffer. + if(pDstDataAsType + pSrcDataAsRun->length > pDstDataEnd) + { + POLYVOX_THROW(std::runtime_error, "Insufficient space in destination buffer."); + } + else + { + + // Write the run into the destination + std::fill(pDstDataAsType, pDstDataAsType + pSrcDataAsRun->length, pSrcDataAsRun->value); + pDstDataAsType += pSrcDataAsRun->length; + + uDstLengthInBytes += pSrcDataAsRun->length * sizeof(VoxelType); + } + pSrcDataAsRun++; + } + + POLYVOX_ASSERT(uDstLengthInBytes == pDstBlock->getDataSizeInBytes(), "Uncompressed data does not have the correct length."); + } +} \ No newline at end of file From fa8813ba86cfdb3752daf8c69c8dc8eb48fee657 Mon Sep 17 00:00:00 2001 From: David Williams Date: Tue, 30 Jul 2013 16:34:40 +0200 Subject: [PATCH 32/38] FilePager now cleans up after itself. --- .../include/PolyVoxCore/FilePager.h | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/library/PolyVoxCore/include/PolyVoxCore/FilePager.h b/library/PolyVoxCore/include/PolyVoxCore/FilePager.h index 4f8f15c8..3e087b5f 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/FilePager.h +++ b/library/PolyVoxCore/include/PolyVoxCore/FilePager.h @@ -58,7 +58,18 @@ namespace PolyVox } /// Destructor - virtual ~FilePager() {}; + virtual ~FilePager() + { + for(std::vector::iterator iter = m_vecCreatedFiles.begin(); iter < m_vecCreatedFiles.end(); iter++) + { + if(!std::remove(iter->c_str())) + { + logWarning() << "Failed to delete '" << *iter << "' when destroying FilePager"; + } + } + + m_vecCreatedFiles.clear(); + } virtual void pageIn(const Region& region, CompressedBlock* pBlockData) { @@ -125,6 +136,9 @@ namespace PolyVox POLYVOX_THROW(std::runtime_error, "Unable to open file to write out block data."); } + //The file has been created, so add it to the list to delete on shutdown. + m_vecCreatedFiles.push_back(filename); + fwrite(pBlockData->getData(), sizeof(uint8_t), pBlockData->getDataSizeInBytes(), pFile); if(ferror(pFile)) @@ -138,6 +152,8 @@ namespace PolyVox protected: std::string m_strFolderName; std::string m_strRandomPrefix; + + std::vector m_vecCreatedFiles; }; } From 4478e365c9c6d3d4533f0198842c8cc61df6ddf2 Mon Sep 17 00:00:00 2001 From: David Williams Date: Tue, 30 Jul 2013 17:01:27 +0200 Subject: [PATCH 33/38] Removed RLECompressor as the code is now in RLEBlockCompressor. Things are starting to get back under control. All tests pass, and all examples except PagingExample work. --- examples/OpenGL/main.cpp | 8 +- examples/Paging/main.cpp | 4 +- library/PolyVoxCore/CMakeLists.txt | 2 - .../PolyVoxCore/RLEBlockCompressor.inl | 2 - .../include/PolyVoxCore/RLECompressor.h | 40 ------ .../include/PolyVoxCore/RLECompressor.inl | 126 ------------------ tests/testvolume.cpp | 1 - 7 files changed, 7 insertions(+), 176 deletions(-) delete mode 100644 library/PolyVoxCore/include/PolyVoxCore/RLECompressor.h delete mode 100644 library/PolyVoxCore/include/PolyVoxCore/RLECompressor.inl diff --git a/examples/OpenGL/main.cpp b/examples/OpenGL/main.cpp index ccae9941..e6ae33d7 100644 --- a/examples/OpenGL/main.cpp +++ b/examples/OpenGL/main.cpp @@ -21,11 +21,12 @@ freely, subject to the following restrictions: distribution. *******************************************************************************/ +#include "PolyVoxCore/FilePager.h" #include "PolyVoxCore/MaterialDensityPair.h" #include "PolyVoxCore/LargeVolume.h" #include "PolyVoxCore/LowPassFilter.h" #include "PolyVoxCore/RawVolume.h" -#include "PolyVoxCore/RLECompressor.h" +#include "PolyVoxCore/RLEBlockCompressor.h" #include "PolyVoxCore/SurfaceMesh.h" #include "PolyVoxCore/Impl/Utility.h" @@ -49,8 +50,9 @@ using namespace std; int main(int argc, char *argv[]) { - RLECompressor* compressor = new RLECompressor(); - LargeVolume volData(PolyVox::Region(Vector3DInt32(0,0,0), Vector3DInt32(g_uVolumeSideLength-1, g_uVolumeSideLength-1, g_uVolumeSideLength-1)), compressor, 0); + RLEBlockCompressor* compressor = new RLEBlockCompressor(); + FilePager* pager = new FilePager("./"); + LargeVolume volData(PolyVox::Region(Vector3DInt32(0,0,0), Vector3DInt32(g_uVolumeSideLength-1, g_uVolumeSideLength-1, g_uVolumeSideLength-1)), compressor, pager); //Make our volume contain a sphere in the center. int32_t minPos = 0; diff --git a/examples/Paging/main.cpp b/examples/Paging/main.cpp index 117145b7..f28c54e6 100644 --- a/examples/Paging/main.cpp +++ b/examples/Paging/main.cpp @@ -28,7 +28,7 @@ freely, subject to the following restrictions: #include "PolyVoxCore/CubicSurfaceExtractorWithNormals.h" #include "PolyVoxCore/MarchingCubesSurfaceExtractor.h" #include "PolyVoxCore/Pager.h" -#include "PolyVoxCore/RLECompressor.h" +#include "PolyVoxCore/RLEBlockCompressor.h" #include "PolyVoxCore/SurfaceMesh.h" #include "PolyVoxCore/LargeVolume.h" @@ -150,7 +150,7 @@ int main(int argc, char *argv[]) OpenGLWidget openGLWidget(0); openGLWidget.show(); - RLECompressor* compressor = new RLECompressor(); + RLEBlockCompressor* compressor = new RLEBlockCompressor(); PerlinNoisePager* pager = new PerlinNoisePager(); LargeVolume volData(PolyVox::Region::MaxRegion, compressor, pager, 256); volData.setMaxNumberOfBlocksInMemory(4096); diff --git a/library/PolyVoxCore/CMakeLists.txt b/library/PolyVoxCore/CMakeLists.txt index 018efe2d..f618c9ae 100644 --- a/library/PolyVoxCore/CMakeLists.txt +++ b/library/PolyVoxCore/CMakeLists.txt @@ -87,8 +87,6 @@ SET(CORE_INC_FILES include/PolyVoxCore/Region.h include/PolyVoxCore/RLEBlockCompressor.h include/PolyVoxCore/RLEBlockCompressor.inl - include/PolyVoxCore/RLECompressor.h - include/PolyVoxCore/RLECompressor.inl include/PolyVoxCore/SimpleVolume.h include/PolyVoxCore/SimpleVolume.inl include/PolyVoxCore/SimpleVolumeBlock.inl diff --git a/library/PolyVoxCore/include/PolyVoxCore/RLEBlockCompressor.inl b/library/PolyVoxCore/include/PolyVoxCore/RLEBlockCompressor.inl index 4fc90b2e..aba87809 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/RLEBlockCompressor.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/RLEBlockCompressor.inl @@ -23,8 +23,6 @@ freely, subject to the following restrictions: #include -#include "RLECompressor.h" - namespace PolyVox { template diff --git a/library/PolyVoxCore/include/PolyVoxCore/RLECompressor.h b/library/PolyVoxCore/include/PolyVoxCore/RLECompressor.h deleted file mode 100644 index 03fd3318..00000000 --- a/library/PolyVoxCore/include/PolyVoxCore/RLECompressor.h +++ /dev/null @@ -1,40 +0,0 @@ -#ifndef __PolyVox_RLECompressor_H__ -#define __PolyVox_RLECompressor_H__ - -#include "PolyVoxCore/Compressor.h" - -namespace PolyVox -{ - /** - * Performs compression of data using Run Length Encoding (RLE). - * - * This compressor is designed for voxel data which contains long runs of the same value. Minecraft-style terrain and other - * cubic-style terrains are likely to fall under this category, whereas density fields for Marching Cubes terrain will not. Please - * see the following article if you want more details of how RLE compression works: http://en.wikipedia.org/wiki/Run-length_encoding - * - * \sa MinizCompressor - */ - template - class RLECompressor : public Compressor - { - struct Run - { - ValueType value; - LengthType length; - }; - public: - /// Constructor - RLECompressor(); - /// Destructor - ~RLECompressor(); - - // API documentation is in base class and gets inherited by Doxygen. - uint32_t getMaxCompressedSize(uint32_t uUncompressedInputSize); - uint32_t compress(const void* pSrcData, uint32_t uSrcLength, void* pDstData, uint32_t uDstLength); - uint32_t decompress(const void* pSrcData, uint32_t uSrcLength, void* pDstData, uint32_t uDstLength); - }; -} - -#include "RLECompressor.inl" - -#endif //__PolyVox_RLECompressor_H__ diff --git a/library/PolyVoxCore/include/PolyVoxCore/RLECompressor.inl b/library/PolyVoxCore/include/PolyVoxCore/RLECompressor.inl deleted file mode 100644 index cf65bc03..00000000 --- a/library/PolyVoxCore/include/PolyVoxCore/RLECompressor.inl +++ /dev/null @@ -1,126 +0,0 @@ -#include -#include -#include - -namespace PolyVox -{ - template - RLECompressor::RLECompressor() - { - } - - template - RLECompressor::~RLECompressor() - { - } - - template - uint32_t RLECompressor::getMaxCompressedSize(uint32_t uUncompressedInputSize) - { - // In the worst case we will have a seperate Run (of length one) for each element of the input data. - return uUncompressedInputSize * sizeof(Run); - } - - template - uint32_t RLECompressor::compress(const void* pSrcData, uint32_t uSrcLength, void* pDstData, uint32_t uDstLength) - { - if(uSrcLength % sizeof(ValueType) != 0) - { - POLYVOX_THROW(std::length_error, "Source length must be a integer multiple of the ValueType size"); - } - - // Lengths provided are in bytes, so convert them to be in terms of our types. - uSrcLength /= sizeof(ValueType); - uDstLength /= sizeof(Run); - - // Get data pointers in the appropriate type - const ValueType* pSrcDataAsType = reinterpret_cast(pSrcData); - Run* pDstDataAsRun = reinterpret_cast(pDstData); - - // Pointers to just past the end of the data - const ValueType* pSrcDataEnd = pSrcDataAsType + uSrcLength; - Run* pDstDataEnd = pDstDataAsRun + uDstLength; - - //Counter for the output length - uint32_t uDstLengthInBytes = 0; - - // Read the first element of the source and set up the first run based on it. - pDstDataAsRun->value = *pSrcDataAsType; - pSrcDataAsType++; - pDstDataAsRun->length = 1; - uDstLengthInBytes += sizeof(Run); - - //Now process all remaining elements of the source. - while(pSrcDataAsType < pSrcDataEnd) - { - // If the value is the same as the current run (and we have not - // reached the maximum run length) then extend the current run. - if((*pSrcDataAsType == pDstDataAsRun->value) && (pDstDataAsRun->length < (std::numeric_limits::max)())) - { - pDstDataAsRun->length++; - } - // Otherwise we need to start a new Run. - else - { - pDstDataAsRun++; - - // Check if we have enough space in the destination buffer. - if(pDstDataAsRun >= pDstDataEnd) - { - POLYVOX_THROW(std::runtime_error, "Insufficient space in destination buffer."); - } - - // Create the new run. - pDstDataAsRun->value = *pSrcDataAsType; - pDstDataAsRun->length = 1; - uDstLengthInBytes += sizeof(Run); - } - - pSrcDataAsType++; - } - - return uDstLengthInBytes; - } - - template - uint32_t RLECompressor::decompress(const void* pSrcData, uint32_t uSrcLength, void* pDstData, uint32_t uDstLength) - { - if(uSrcLength % sizeof(Run) != 0) - { - POLYVOX_THROW(std::length_error, "Source length must be a integer multiple of the Run size"); - } - - // Lengths provided are in bytes, so convert them to be in terms of our types. - uSrcLength /= sizeof(Run); - uDstLength /= sizeof(ValueType); - - // Get data pointers in the appropriate type - const Run* pSrcDataAsRun = reinterpret_cast(pSrcData); - ValueType* pDstDataAsType = reinterpret_cast(pDstData); - - // Pointers to just past the end of the data - const Run* pSrcDataEnd = pSrcDataAsRun + uSrcLength; - ValueType* pDstDataEnd = pDstDataAsType + uDstLength; - - //Counter for the output length - uint32_t uDstLengthInBytes = 0; - - while(pSrcDataAsRun < pSrcDataEnd) - { - // Check if we have enough space in the destination buffer. - if(pDstDataAsType + pSrcDataAsRun->length > pDstDataEnd) - { - POLYVOX_THROW(std::runtime_error, "Insufficient space in destination buffer."); - } - - // Write the run into the destination - std::fill(pDstDataAsType, pDstDataAsType + pSrcDataAsRun->length, pSrcDataAsRun->value); - pDstDataAsType += pSrcDataAsRun->length; - - uDstLengthInBytes += pSrcDataAsRun->length * sizeof(ValueType); - pSrcDataAsRun++; - } - - return uDstLengthInBytes; - } -} diff --git a/tests/testvolume.cpp b/tests/testvolume.cpp index d15e1fcc..f879f1bf 100644 --- a/tests/testvolume.cpp +++ b/tests/testvolume.cpp @@ -27,7 +27,6 @@ freely, subject to the following restrictions: #include "PolyVoxCore/LargeVolume.h" #include "PolyVoxCore/MinizBlockCompressor.h" #include "PolyVoxCore/RawVolume.h" -#include "PolyVoxCore/RLECompressor.h" #include "PolyVoxCore/SimpleVolume.h" #include From e35b58ba188335613481f623bb6bd8bfecff7765 Mon Sep 17 00:00:00 2001 From: David Williams Date: Wed, 31 Jul 2013 15:49:00 +0200 Subject: [PATCH 34/38] Fixed paging example. --- examples/Paging/main.cpp | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/examples/Paging/main.cpp b/examples/Paging/main.cpp index f28c54e6..27236afd 100644 --- a/examples/Paging/main.cpp +++ b/examples/Paging/main.cpp @@ -93,7 +93,9 @@ public: virtual void pageIn(const PolyVox::Region& region, CompressedBlock* pBlockData) { - /*pBlockData->createUncompressedData(); + // FIXME - this isn't a great example... it's a shame we have to hard clode the block size and also create/destroy + // a compressor each time. These could at least be moved outside somewhere if we can't fix it in a better way... + UncompressedBlock block(256); Perlin perlin(2,2,1,234); @@ -131,10 +133,15 @@ public: // Voxel position within a block always start from zero. So if a block represents region (4, 8, 12) to (11, 19, 15) // then the valid block voxels are from (0, 0, 0) to (7, 11, 3). Hence we subtract the lower corner position of the // region from the volume space position in order to get the block space position. - pBlockData->setVoxelAt(x - region.getLowerX(), y - region.getLowerY(), z - region.getLowerZ(), voxel); + block.setVoxelAt(x - region.getLowerX(), y - region.getLowerY(), z - region.getLowerZ(), voxel); } } - }*/ + } + + // Now compress the computed data into the provided block. + RLEBlockCompressor* compressor = new RLEBlockCompressor(); + compressor->compress(&block, pBlockData); + delete compressor; } virtual void pageOut(const PolyVox::Region& region, CompressedBlock* /*pBlockData*/) From 9503d975c1b5b4f797e321194229bd463ec9058e Mon Sep 17 00:00:00 2001 From: David Williams Date: Wed, 31 Jul 2013 16:37:39 +0200 Subject: [PATCH 35/38] GCC compile fixes. --- library/PolyVoxCore/include/PolyVoxCore/BlockCompressor.h | 1 + library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h | 6 +++--- library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl | 4 ++-- library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl | 2 +- .../PolyVoxCore/include/PolyVoxCore/RLEBlockCompressor.inl | 4 ++-- 5 files changed, 9 insertions(+), 8 deletions(-) diff --git a/library/PolyVoxCore/include/PolyVoxCore/BlockCompressor.h b/library/PolyVoxCore/include/PolyVoxCore/BlockCompressor.h index 21338de7..dbef7a02 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/BlockCompressor.h +++ b/library/PolyVoxCore/include/PolyVoxCore/BlockCompressor.h @@ -24,6 +24,7 @@ freely, subject to the following restrictions: #ifndef __PolyVox_BlockCompressor_H__ #define __PolyVox_BlockCompressor_H__ +#include "PolyVoxCore/PolyVoxForwardDeclarations.h" #include "PolyVoxCore/Impl/Block.h" namespace PolyVox diff --git a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h index 554bc3c2..718ee9db 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h +++ b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h @@ -37,7 +37,7 @@ namespace PolyVox template class Block { - friend LargeVolume; + friend class LargeVolume; public: Block() @@ -65,7 +65,7 @@ namespace PolyVox template class CompressedBlock : public Block { - friend LargeVolume; + friend class LargeVolume; public: CompressedBlock(); @@ -94,7 +94,7 @@ namespace PolyVox template class UncompressedBlock : public Block { - friend LargeVolume; + friend class LargeVolume; public: UncompressedBlock(uint16_t uSideLength); diff --git a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl index ba156ef4..279dbf0e 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl @@ -80,7 +80,7 @@ namespace PolyVox memcpy(m_pData, pData, uDataSizeInBytes); // Flag as modified - m_bDataModified = true; + this->m_bDataModified = true; } template @@ -168,7 +168,7 @@ namespace PolyVox uZPos * m_uSideLength * m_uSideLength ] = tValue; - m_bDataModified = true; + this->m_bDataModified = true; } template diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl index 7be743fd..e18568bd 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl @@ -45,7 +45,7 @@ namespace PolyVox { m_uBlockSideLength = uBlockSideLength; - m_pBlockCompressor = new MinizBlockCompressor(); + m_pBlockCompressor = new MinizBlockCompressor(); m_bIsOurCompressor = true; m_pPager = 0; diff --git a/library/PolyVoxCore/include/PolyVoxCore/RLEBlockCompressor.inl b/library/PolyVoxCore/include/PolyVoxCore/RLEBlockCompressor.inl index aba87809..369fb912 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/RLEBlockCompressor.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/RLEBlockCompressor.inl @@ -79,8 +79,8 @@ namespace PolyVox { // If the value is the same as the current run (and we have not // reached the maximum run length) then extend the current run. - std::vector< Run >::iterator currentRun = (vecDstDataAsRuns.end() - 1); - if((*pSrcDataAsType == currentRun->value) && (currentRun->length < (std::numeric_limits::LengthType>::max)())) + typename std::vector< Run< VoxelType> >::iterator currentRun = (vecDstDataAsRuns.end() - 1); + if((*pSrcDataAsType == currentRun->value) && (currentRun->length < (std::numeric_limits::LengthType>::max)())) { currentRun->length++; } From 6f17e0faa67f53d6f4ac8b0093474dbe3d50fb1f Mon Sep 17 00:00:00 2001 From: David Williams Date: Thu, 1 Aug 2013 15:19:10 +0200 Subject: [PATCH 36/38] The 'Block' class is no longer considered just to be an implementation details of LargeVolume. Users may need to interact with them directly of they implement their own paging or compression systems. --- library/PolyVoxCore/CMakeLists.txt | 4 ++-- library/PolyVoxCore/include/PolyVoxCore/{Impl => }/Block.h | 2 +- library/PolyVoxCore/include/PolyVoxCore/{Impl => }/Block.inl | 0 library/PolyVoxCore/include/PolyVoxCore/BlockCompressor.h | 2 +- library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h | 2 +- library/PolyVoxCore/include/PolyVoxCore/Pager.h | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) rename library/PolyVoxCore/include/PolyVoxCore/{Impl => }/Block.h (99%) rename library/PolyVoxCore/include/PolyVoxCore/{Impl => }/Block.inl (100%) diff --git a/library/PolyVoxCore/CMakeLists.txt b/library/PolyVoxCore/CMakeLists.txt index f618c9ae..5f2d49b3 100644 --- a/library/PolyVoxCore/CMakeLists.txt +++ b/library/PolyVoxCore/CMakeLists.txt @@ -48,6 +48,8 @@ SET(CORE_INC_FILES include/PolyVoxCore/BaseVolume.h include/PolyVoxCore/BaseVolume.inl include/PolyVoxCore/BaseVolumeSampler.inl + include/PolyVoxCore/Block.h + include/PolyVoxCore/Block.inl include/PolyVoxCore/BlockCompressor.h include/PolyVoxCore/Compressor.h include/PolyVoxCore/CubicSurfaceExtractor.h @@ -114,8 +116,6 @@ SET(IMPL_INC_FILES include/PolyVoxCore/Impl/ArraySizesImpl.h include/PolyVoxCore/Impl/ArraySizesImpl.inl include/PolyVoxCore/Impl/AStarPathfinderImpl.h - include/PolyVoxCore/Impl/Block.h - include/PolyVoxCore/Impl/Block.inl include/PolyVoxCore/Impl/CompilerCapabilities.h include/PolyVoxCore/Impl/Config.h include/PolyVoxCore/Impl/ErrorHandling.h diff --git a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h b/library/PolyVoxCore/include/PolyVoxCore/Block.h similarity index 99% rename from library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h rename to library/PolyVoxCore/include/PolyVoxCore/Block.h index 718ee9db..a1567552 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h +++ b/library/PolyVoxCore/include/PolyVoxCore/Block.h @@ -126,6 +126,6 @@ namespace PolyVox }; } -#include "PolyVoxCore/Impl/Block.inl" +#include "PolyVoxCore/Block.inl" #endif diff --git a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl b/library/PolyVoxCore/include/PolyVoxCore/Block.inl similarity index 100% rename from library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl rename to library/PolyVoxCore/include/PolyVoxCore/Block.inl diff --git a/library/PolyVoxCore/include/PolyVoxCore/BlockCompressor.h b/library/PolyVoxCore/include/PolyVoxCore/BlockCompressor.h index dbef7a02..2646e94c 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/BlockCompressor.h +++ b/library/PolyVoxCore/include/PolyVoxCore/BlockCompressor.h @@ -25,7 +25,7 @@ freely, subject to the following restrictions: #define __PolyVox_BlockCompressor_H__ #include "PolyVoxCore/PolyVoxForwardDeclarations.h" -#include "PolyVoxCore/Impl/Block.h" +#include "PolyVoxCore/Block.h" namespace PolyVox { diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h index 986bcf2b..a26bc29b 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h @@ -25,7 +25,7 @@ freely, subject to the following restrictions: #define __PolyVox_LargeVolume_H__ #include "PolyVoxCore/BaseVolume.h" -#include "PolyVoxCore/Impl/Block.h" +#include "PolyVoxCore/Block.h" #include "PolyVoxCore/BlockCompressor.h" #include "PolyVoxCore/Pager.h" #include "PolyVoxCore/Region.h" diff --git a/library/PolyVoxCore/include/PolyVoxCore/Pager.h b/library/PolyVoxCore/include/PolyVoxCore/Pager.h index af1e3387..6189566a 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/Pager.h +++ b/library/PolyVoxCore/include/PolyVoxCore/Pager.h @@ -24,7 +24,7 @@ freely, subject to the following restrictions: #ifndef __PolyVox_Pager_H__ #define __PolyVox_Pager_H__ -#include "PolyVoxCore/Impl/Block.h" +#include "PolyVoxCore/Block.h" #include "PolyVoxCore/Impl/TypeDef.h" namespace PolyVox From dcf86a905c64cc755790054a6565062c51479b4c Mon Sep 17 00:00:00 2001 From: David Williams Date: Thu, 1 Aug 2013 15:32:12 +0200 Subject: [PATCH 37/38] Split Block, CompressedBlock and UncompressedBlock into separate files. It's tidier, and maybe better for SWIG? --- library/PolyVoxCore/CMakeLists.txt | 5 +- .../PolyVoxCore/include/PolyVoxCore/Block.h | 67 +------------- .../include/PolyVoxCore/BlockCompressor.h | 3 +- .../include/PolyVoxCore/CompressedBlock.h | 63 +++++++++++++ .../include/PolyVoxCore/CompressedBlock.inl | 79 +++++++++++++++++ .../PolyVoxCore/include/PolyVoxCore/Pager.h | 2 +- .../include/PolyVoxCore/UncompressedBlock.h | 68 ++++++++++++++ .../{Block.inl => UncompressedBlock.inl} | 88 ++----------------- 8 files changed, 227 insertions(+), 148 deletions(-) create mode 100644 library/PolyVoxCore/include/PolyVoxCore/CompressedBlock.h create mode 100644 library/PolyVoxCore/include/PolyVoxCore/CompressedBlock.inl create mode 100644 library/PolyVoxCore/include/PolyVoxCore/UncompressedBlock.h rename library/PolyVoxCore/include/PolyVoxCore/{Block.inl => UncompressedBlock.inl} (62%) diff --git a/library/PolyVoxCore/CMakeLists.txt b/library/PolyVoxCore/CMakeLists.txt index 5f2d49b3..bb28c01e 100644 --- a/library/PolyVoxCore/CMakeLists.txt +++ b/library/PolyVoxCore/CMakeLists.txt @@ -49,8 +49,9 @@ SET(CORE_INC_FILES include/PolyVoxCore/BaseVolume.inl include/PolyVoxCore/BaseVolumeSampler.inl include/PolyVoxCore/Block.h - include/PolyVoxCore/Block.inl include/PolyVoxCore/BlockCompressor.h + include/PolyVoxCore/CompressedBlock.h + include/PolyVoxCore/CompressedBlock.inl include/PolyVoxCore/Compressor.h include/PolyVoxCore/CubicSurfaceExtractor.h include/PolyVoxCore/CubicSurfaceExtractor.inl @@ -95,6 +96,8 @@ SET(CORE_INC_FILES include/PolyVoxCore/SimpleVolumeSampler.inl include/PolyVoxCore/SurfaceMesh.h include/PolyVoxCore/SurfaceMesh.inl + include/PolyVoxCore/UncompressedBlock.h + include/PolyVoxCore/UncompressedBlock.inl include/PolyVoxCore/Vector.h include/PolyVoxCore/Vector.inl include/PolyVoxCore/VertexTypes.h diff --git a/library/PolyVoxCore/include/PolyVoxCore/Block.h b/library/PolyVoxCore/include/PolyVoxCore/Block.h index a1567552..e2bb16cb 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/Block.h +++ b/library/PolyVoxCore/include/PolyVoxCore/Block.h @@ -1,5 +1,5 @@ /******************************************************************************* -Copyright (c) 2005-2009 David Williams +Copyright (c) 2005-2013 David Williams and Matthew Williams This software is provided 'as-is', without any express or implied warranty. In no event will the authors be held liable for any damages @@ -61,71 +61,6 @@ namespace PolyVox /// Private assignment operator to prevent accisdental copying Block& operator=(const Block& /*rhs*/) {}; }; - - template - class CompressedBlock : public Block - { - friend class LargeVolume; - - public: - CompressedBlock(); - ~CompressedBlock(); - - const uint8_t* getData(void) const; - uint32_t getDataSizeInBytes(void) const; - - void setData(const uint8_t* const pData, uint32_t uDataSizeInBytes); - - private: - /// Private copy constructor to prevent accisdental copying - CompressedBlock(const CompressedBlock& /*rhs*/) {}; - - /// Private assignment operator to prevent accisdental copying - CompressedBlock& operator=(const CompressedBlock& /*rhs*/) {}; - - // Made this private to avoid any confusion with getDataSizeInBytes(). - // Users shouldn't really need this for CompressedBlock anyway. - uint32_t calculateSizeInBytes(void); - - uint8_t* m_pData; - uint32_t m_uDataSizeInBytes; - }; - - template - class UncompressedBlock : public Block - { - friend class LargeVolume; - - public: - UncompressedBlock(uint16_t uSideLength); - ~UncompressedBlock(); - - VoxelType* getData(void) const; - uint32_t getDataSizeInBytes(void) const; - - VoxelType getVoxel(uint16_t uXPos, uint16_t uYPos, uint16_t uZPos) const; - VoxelType getVoxel(const Vector3DUint16& v3dPos) const; - - void setVoxelAt(uint16_t uXPos, uint16_t uYPos, uint16_t uZPos, VoxelType tValue); - void setVoxelAt(const Vector3DUint16& v3dPos, VoxelType tValue); - - private: - /// Private copy constructor to prevent accisdental copying - UncompressedBlock(const UncompressedBlock& /*rhs*/) {}; - - /// Private assignment operator to prevent accisdental copying - UncompressedBlock& operator=(const UncompressedBlock& /*rhs*/) {}; - - // Made this private for consistancy with CompressedBlock. - // Users shouldn't really need this for UncompressedBlock anyway. - uint32_t calculateSizeInBytes(void); - - VoxelType* m_tData; - uint16_t m_uSideLength; - uint8_t m_uSideLengthPower; - }; } -#include "PolyVoxCore/Block.inl" - #endif diff --git a/library/PolyVoxCore/include/PolyVoxCore/BlockCompressor.h b/library/PolyVoxCore/include/PolyVoxCore/BlockCompressor.h index 2646e94c..5af6dbfb 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/BlockCompressor.h +++ b/library/PolyVoxCore/include/PolyVoxCore/BlockCompressor.h @@ -25,7 +25,8 @@ freely, subject to the following restrictions: #define __PolyVox_BlockCompressor_H__ #include "PolyVoxCore/PolyVoxForwardDeclarations.h" -#include "PolyVoxCore/Block.h" +#include "PolyVoxCore/CompressedBlock.h" +#include "PolyVoxCore/UncompressedBlock.h" namespace PolyVox { diff --git a/library/PolyVoxCore/include/PolyVoxCore/CompressedBlock.h b/library/PolyVoxCore/include/PolyVoxCore/CompressedBlock.h new file mode 100644 index 00000000..5a8ba89a --- /dev/null +++ b/library/PolyVoxCore/include/PolyVoxCore/CompressedBlock.h @@ -0,0 +1,63 @@ +/******************************************************************************* +Copyright (c) 2005-2013 David Williams and Matthew Williams + +This software is provided 'as-is', without any express or implied +warranty. In no event will the authors be held liable for any damages +arising from the use of this software. + +Permission is granted to anyone to use this software for any purpose, +including commercial applications, and to alter it and redistribute it +freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + + 3. This notice may not be removed or altered from any source + distribution. +*******************************************************************************/ + +#ifndef __PolyVox_CompressedBlock_H__ +#define __PolyVox_CompressedBlock_H__ + +#include "PolyVoxCore/Block.h" + +namespace PolyVox +{ + template + class CompressedBlock : public Block + { + friend class LargeVolume; + + public: + CompressedBlock(); + ~CompressedBlock(); + + const uint8_t* getData(void) const; + uint32_t getDataSizeInBytes(void) const; + + void setData(const uint8_t* const pData, uint32_t uDataSizeInBytes); + + private: + /// Private copy constructor to prevent accisdental copying + CompressedBlock(const CompressedBlock& /*rhs*/) {}; + + /// Private assignment operator to prevent accisdental copying + CompressedBlock& operator=(const CompressedBlock& /*rhs*/) {}; + + // Made this private to avoid any confusion with getDataSizeInBytes(). + // Users shouldn't really need this for CompressedBlock anyway. + uint32_t calculateSizeInBytes(void); + + uint8_t* m_pData; + uint32_t m_uDataSizeInBytes; + }; +} + +#include "PolyVoxCore/CompressedBlock.inl" + +#endif diff --git a/library/PolyVoxCore/include/PolyVoxCore/CompressedBlock.inl b/library/PolyVoxCore/include/PolyVoxCore/CompressedBlock.inl new file mode 100644 index 00000000..e98df8e3 --- /dev/null +++ b/library/PolyVoxCore/include/PolyVoxCore/CompressedBlock.inl @@ -0,0 +1,79 @@ +/******************************************************************************* +Copyright (c) 2005-2013 David Williams and Matthew Williams + +This software is provided 'as-is', without any express or implied +warranty. In no event will the authors be held liable for any damages +arising from the use of this software. + +Permission is granted to anyone to use this software for any purpose, +including commercial applications, and to alter it and redistribute it +freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + + 3. This notice may not be removed or altered from any source + distribution. +*******************************************************************************/ + +namespace PolyVox +{ + template + CompressedBlock::CompressedBlock() + :m_pData(0) + ,m_uDataSizeInBytes(0) + { + } + + template + CompressedBlock::~CompressedBlock() + { + delete[] m_pData; + m_pData = 0; + } + + template + const uint8_t* CompressedBlock::getData(void) const + { + return m_pData; + } + + template + uint32_t CompressedBlock::getDataSizeInBytes(void) const + { + return m_uDataSizeInBytes; + } + + template + void CompressedBlock::setData(const uint8_t* const pData, uint32_t uDataSizeInBytes) + { + POLYVOX_THROW_IF(pData == 0, std::invalid_argument, "Pointer to data cannot be null"); + POLYVOX_THROW_IF(m_pData == pData, std::invalid_argument, "Attempting to copy data onto itself"); + + // Delete any existing data + delete[] m_pData; + + // Allocate new data + m_uDataSizeInBytes = uDataSizeInBytes; + m_pData = new uint8_t[uDataSizeInBytes]; + + // Copy the data across + memcpy(m_pData, pData, uDataSizeInBytes); + + // Flag as modified + this->m_bDataModified = true; + } + + template + uint32_t CompressedBlock::calculateSizeInBytes(void) + { + // Returns the size of this class plus the size of the compressed data. + uint32_t uSizeInBytes = sizeof(CompressedBlock) + m_uDataSizeInBytes; + return uSizeInBytes; + } +} diff --git a/library/PolyVoxCore/include/PolyVoxCore/Pager.h b/library/PolyVoxCore/include/PolyVoxCore/Pager.h index 6189566a..eb37e4f7 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/Pager.h +++ b/library/PolyVoxCore/include/PolyVoxCore/Pager.h @@ -24,7 +24,7 @@ freely, subject to the following restrictions: #ifndef __PolyVox_Pager_H__ #define __PolyVox_Pager_H__ -#include "PolyVoxCore/Block.h" +#include "PolyVoxCore/CompressedBlock.h" #include "PolyVoxCore/Impl/TypeDef.h" namespace PolyVox diff --git a/library/PolyVoxCore/include/PolyVoxCore/UncompressedBlock.h b/library/PolyVoxCore/include/PolyVoxCore/UncompressedBlock.h new file mode 100644 index 00000000..54a1aacb --- /dev/null +++ b/library/PolyVoxCore/include/PolyVoxCore/UncompressedBlock.h @@ -0,0 +1,68 @@ +/******************************************************************************* +Copyright (c) 2005-2013 David Williams and Matthew Williams + +This software is provided 'as-is', without any express or implied +warranty. In no event will the authors be held liable for any damages +arising from the use of this software. + +Permission is granted to anyone to use this software for any purpose, +including commercial applications, and to alter it and redistribute it +freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + + 3. This notice may not be removed or altered from any source + distribution. +*******************************************************************************/ + +#ifndef __PolyVox_UncompressedBlock_H__ +#define __PolyVox_UncompressedBlock_H__ + +#include "PolyVoxCore/Block.h" + +namespace PolyVox +{ + template + class UncompressedBlock : public Block + { + friend class LargeVolume; + + public: + UncompressedBlock(uint16_t uSideLength); + ~UncompressedBlock(); + + VoxelType* getData(void) const; + uint32_t getDataSizeInBytes(void) const; + + VoxelType getVoxel(uint16_t uXPos, uint16_t uYPos, uint16_t uZPos) const; + VoxelType getVoxel(const Vector3DUint16& v3dPos) const; + + void setVoxelAt(uint16_t uXPos, uint16_t uYPos, uint16_t uZPos, VoxelType tValue); + void setVoxelAt(const Vector3DUint16& v3dPos, VoxelType tValue); + + private: + /// Private copy constructor to prevent accisdental copying + UncompressedBlock(const UncompressedBlock& /*rhs*/) {}; + + /// Private assignment operator to prevent accisdental copying + UncompressedBlock& operator=(const UncompressedBlock& /*rhs*/) {}; + + // Made this private for consistancy with CompressedBlock. + // Users shouldn't really need this for UncompressedBlock anyway. + uint32_t calculateSizeInBytes(void); + + VoxelType* m_tData; + uint16_t m_uSideLength; + uint8_t m_uSideLengthPower; + }; +} + +#include "PolyVoxCore/UncompressedBlock.inl" + +#endif diff --git a/library/PolyVoxCore/include/PolyVoxCore/Block.inl b/library/PolyVoxCore/include/PolyVoxCore/UncompressedBlock.inl similarity index 62% rename from library/PolyVoxCore/include/PolyVoxCore/Block.inl rename to library/PolyVoxCore/include/PolyVoxCore/UncompressedBlock.inl index 279dbf0e..4d506d74 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/Block.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/UncompressedBlock.inl @@ -1,5 +1,5 @@ /******************************************************************************* -Copyright (c) 2005-2009 David Williams +Copyright (c) 2005-2013 David Williams and Matthew Williams This software is provided 'as-is', without any express or implied warranty. In no event will the authors be held liable for any damages @@ -9,90 +9,20 @@ Permission is granted to anyone to use this software for any purpose, including commercial applications, and to alter it and redistribute it freely, subject to the following restrictions: -1. The origin of this software must not be misrepresented; you must not -claim that you wrote the original software. If you use this software -in a product, an acknowledgment in the product documentation would be -appreciated but is not required. + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. -2. Altered source versions must be plainly marked as such, and must not be -misrepresented as being the original software. + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. -3. This notice may not be removed or altered from any source -distribution. + 3. This notice may not be removed or altered from any source + distribution. *******************************************************************************/ -#include "PolyVoxCore/Impl/ErrorHandling.h" -#include "PolyVoxCore/Impl/Utility.h" - -#include "PolyVoxCore/Compressor.h" -#include "PolyVoxCore/Vector.h" - -#include "PolyVoxCore/Impl/ErrorHandling.h" - -#include //For memcpy -#include -#include //for std::invalid_argument - namespace PolyVox { - //////////////////////////////////////////////////////////////////////////////// - - template - CompressedBlock::CompressedBlock() - :m_pData(0) - ,m_uDataSizeInBytes(0) - { - } - - template - CompressedBlock::~CompressedBlock() - { - delete[] m_pData; - m_pData = 0; - } - - template - const uint8_t* CompressedBlock::getData(void) const - { - return m_pData; - } - - template - uint32_t CompressedBlock::getDataSizeInBytes(void) const - { - return m_uDataSizeInBytes; - } - - template - void CompressedBlock::setData(const uint8_t* const pData, uint32_t uDataSizeInBytes) - { - POLYVOX_THROW_IF(pData == 0, std::invalid_argument, "Pointer to data cannot be null"); - POLYVOX_THROW_IF(m_pData == pData, std::invalid_argument, "Attempting to copy data onto itself"); - - // Delete any existing data - delete[] m_pData; - - // Allocate new data - m_uDataSizeInBytes = uDataSizeInBytes; - m_pData = new uint8_t[uDataSizeInBytes]; - - // Copy the data across - memcpy(m_pData, pData, uDataSizeInBytes); - - // Flag as modified - this->m_bDataModified = true; - } - - template - uint32_t CompressedBlock::calculateSizeInBytes(void) - { - // Returns the size of this class plus the size of the compressed data. - uint32_t uSizeInBytes = sizeof(CompressedBlock) + m_uDataSizeInBytes; - return uSizeInBytes; - } - - //////////////////////////////////////////////////////////////////////////////// - template UncompressedBlock::UncompressedBlock(uint16_t uSideLength) :m_tData(0) From e485265dd85050df6dc7afd2f7fbdf2e2a5bf9da Mon Sep 17 00:00:00 2001 From: David Williams Date: Thu, 1 Aug 2013 16:08:44 +0200 Subject: [PATCH 38/38] Fixes to allow the SWIG/Python bindings to build again. --- library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl | 2 +- library/bindings/Block.i | 9 +++++++++ library/bindings/BlockCompressor.i | 8 ++++++++ library/bindings/CompressedBlock.i | 8 ++++++++ library/bindings/MinizBlockCompressor.i | 6 ++++++ library/bindings/MinizCompressor.i | 6 ------ library/bindings/PolyVoxCore.i | 8 ++++++-- library/bindings/RLEBlockCompressor.i | 6 ++++++ library/bindings/RLECompressor.i | 6 ------ library/bindings/UncompressedBlock.i | 8 ++++++++ 10 files changed, 52 insertions(+), 15 deletions(-) create mode 100644 library/bindings/Block.i create mode 100644 library/bindings/BlockCompressor.i create mode 100644 library/bindings/CompressedBlock.i create mode 100644 library/bindings/MinizBlockCompressor.i delete mode 100644 library/bindings/MinizCompressor.i create mode 100644 library/bindings/RLEBlockCompressor.i delete mode 100644 library/bindings/RLECompressor.i create mode 100644 library/bindings/UncompressedBlock.i diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl index e18568bd..c9fe8fa6 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl @@ -311,7 +311,7 @@ namespace PolyVox const uint16_t zOffset = static_cast(uZPos - (blockZ << m_uBlockSideLengthPower)); UncompressedBlock* pUncompressedBlock = getUncompressedBlock(blockX, blockY, blockZ); - pUncompressedBlock->setVoxelAt(xOffset, yOffset, zOffset); + pUncompressedBlock->setVoxelAt(xOffset, yOffset, zOffset, tValue); } //////////////////////////////////////////////////////////////////////////////// diff --git a/library/bindings/Block.i b/library/bindings/Block.i new file mode 100644 index 00000000..d5aa8da7 --- /dev/null +++ b/library/bindings/Block.i @@ -0,0 +1,9 @@ +%module Block +%{ +#include "Block.h" +%} + +%include "Block.h" + +VOLUMETYPES(Block) + \ No newline at end of file diff --git a/library/bindings/BlockCompressor.i b/library/bindings/BlockCompressor.i new file mode 100644 index 00000000..ddb6eb41 --- /dev/null +++ b/library/bindings/BlockCompressor.i @@ -0,0 +1,8 @@ +%module BlockCompressor +%{ +#include "BlockCompressor.h" +%} + +%include "BlockCompressor.h" + +VOLUMETYPES(BlockCompressor) \ No newline at end of file diff --git a/library/bindings/CompressedBlock.i b/library/bindings/CompressedBlock.i new file mode 100644 index 00000000..c27ef617 --- /dev/null +++ b/library/bindings/CompressedBlock.i @@ -0,0 +1,8 @@ +%module CompressedBlock +%{ +#include "CompressedBlock.h" +%} + +%include "CompressedBlock.h" + +VOLUMETYPES(CompressedBlock) diff --git a/library/bindings/MinizBlockCompressor.i b/library/bindings/MinizBlockCompressor.i new file mode 100644 index 00000000..7f2f0c9a --- /dev/null +++ b/library/bindings/MinizBlockCompressor.i @@ -0,0 +1,6 @@ +%module MinizBlockCompressor +%{ +#include "MinizBlockCompressor.h" +%} + +%include "MinizBlockCompressor.h" diff --git a/library/bindings/MinizCompressor.i b/library/bindings/MinizCompressor.i deleted file mode 100644 index f846265f..00000000 --- a/library/bindings/MinizCompressor.i +++ /dev/null @@ -1,6 +0,0 @@ -%module MinizCompressor -%{ -#include "MinizCompressor.h" -%} - -%include "MinizCompressor.h" diff --git a/library/bindings/PolyVoxCore.i b/library/bindings/PolyVoxCore.i index 30a71407..7e6bf245 100644 --- a/library/bindings/PolyVoxCore.i +++ b/library/bindings/PolyVoxCore.i @@ -75,11 +75,15 @@ EXTRACTOR(shortname, LargeVolume) %include "Vector.i" %include "DefaultMarchingCubesController.i" %include "Region.i" +%include "Block.i" +%include "CompressedBlock.i" +%include "UncompressedBlock.i" %include "Compressor.i" +%include "BlockCompressor.i" %include "Pager.i" %include "FilePager.i" -%include "MinizCompressor.i" -%include "RLECompressor.i" +%include "MinizBlockCompressor.i" +%include "RLEBlockCompressor.i" %include "BaseVolume.i" %include "SimpleVolume.i" %include "RawVolume.i" diff --git a/library/bindings/RLEBlockCompressor.i b/library/bindings/RLEBlockCompressor.i new file mode 100644 index 00000000..b1ea1810 --- /dev/null +++ b/library/bindings/RLEBlockCompressor.i @@ -0,0 +1,6 @@ +%module RLEBlockCompressor +%{ +#include "RLEBlockCompressor.h" +%} + +%include "RLEBlockCompressor.h" diff --git a/library/bindings/RLECompressor.i b/library/bindings/RLECompressor.i deleted file mode 100644 index 01023c27..00000000 --- a/library/bindings/RLECompressor.i +++ /dev/null @@ -1,6 +0,0 @@ -%module RLECompressor -%{ -#include "RLECompressor.h" -%} - -%include "RLECompressor.h" diff --git a/library/bindings/UncompressedBlock.i b/library/bindings/UncompressedBlock.i new file mode 100644 index 00000000..26598f31 --- /dev/null +++ b/library/bindings/UncompressedBlock.i @@ -0,0 +1,8 @@ +%module UncompressedBlock +%{ +#include "UncompressedBlock.h" +%} + +%include "UncompressedBlock.h" + +VOLUMETYPES(UncompressedBlock)