From c843e7e705646539c796169e421bcdd2605e4583 Mon Sep 17 00:00:00 2001 From: Matt Williams Date: Mon, 1 Jul 2013 12:23:21 +0100 Subject: [PATCH 01/55] Remove unnecessary consts. Since a copy is being made in the return, the const qualifiers are ignored anyway. --- library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h | 4 ++-- library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h index 8a9932c0..06963f7c 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h +++ b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h @@ -40,8 +40,8 @@ namespace PolyVox public: Block(uint16_t uSideLength, Compressor* pCompressor); - const uint8_t* const getCompressedData(void) const; - const uint32_t getCompressedDataLength(void) const; + const uint8_t* getCompressedData(void) const; + uint32_t getCompressedDataLength(void) const; uint16_t getSideLength(void) const; VoxelType getVoxel(uint16_t uXPos, uint16_t uYPos, uint16_t uZPos) const; VoxelType getVoxel(const Vector3DUint16& v3dPos) const; diff --git a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl index 3076d2bb..cfeb00c1 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl @@ -75,14 +75,14 @@ namespace PolyVox } template - const uint8_t* const Block::getCompressedData(void) const + const uint8_t* Block::getCompressedData(void) const { POLYVOX_ASSERT(m_pCompressedData, "Compressed data is NULL"); return m_pCompressedData; } template - const uint32_t Block::getCompressedDataLength(void) const + uint32_t Block::getCompressedDataLength(void) const { POLYVOX_ASSERT(m_pCompressedData, "Compressed data is NULL"); return m_uCompressedDataLength; From 26f512eba7ea9bf27582db55b8bcab13d450b095 Mon Sep 17 00:00:00 2001 From: Daviw Williams Date: Tue, 2 Jul 2013 16:08:52 +0200 Subject: [PATCH 02/55] Fixed compile warning. --- examples/Paging/main.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/Paging/main.cpp b/examples/Paging/main.cpp index 6d183033..cac31ff3 100644 --- a/examples/Paging/main.cpp +++ b/examples/Paging/main.cpp @@ -137,7 +137,7 @@ public: } } - virtual void pageOut(const PolyVox::Region& region, Block* pBlockData) + virtual void pageOut(const PolyVox::Region& region, Block* /*pBlockData*/) { std::cout << "warning unloading region: " << region.getLowerCorner() << " -> " << region.getUpperCorner() << std::endl; } From bd60f34bd71bb724ae04b3921b46530c0e19f88b Mon Sep 17 00:00:00 2001 From: Daviw Williams Date: Thu, 4 Jul 2013 16:23:58 +0200 Subject: [PATCH 03/55] Removing explicit functions to control the number of compressed and uncompressed blocks in memory, and letting the user set a memory limit instead. --- examples/Paging/main.cpp | 5 +- .../include/PolyVoxCore/Impl/Block.inl | 5 +- .../include/PolyVoxCore/LargeVolume.h | 9 ++- .../include/PolyVoxCore/LargeVolume.inl | 67 +++++++++++++------ tests/testvolume.cpp | 5 +- 5 files changed, 61 insertions(+), 30 deletions(-) diff --git a/examples/Paging/main.cpp b/examples/Paging/main.cpp index cac31ff3..f172a214 100644 --- a/examples/Paging/main.cpp +++ b/examples/Paging/main.cpp @@ -153,8 +153,9 @@ int main(int argc, char *argv[]) RLECompressor* compressor = new RLECompressor(); PerlinNoisePager* pager = new PerlinNoisePager(); LargeVolume volData(PolyVox::Region::MaxRegion, compressor, pager, 256); - volData.setMaxNumberOfBlocksInMemory(4096); - volData.setMaxNumberOfUncompressedBlocks(64); + //volData.setMaxNumberOfBlocksInMemory(4096); + //volData.setMaxNumberOfUncompressedBlocks(64); + volData.setTargetMemoryLimitInBytes(128 * 1024 * 1024); //volData.setMaxNumberOfUncompressedBlocks(4096); //createSphereInVolume(volData, 30); diff --git a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl index cfeb00c1..bebcfb35 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl @@ -269,8 +269,9 @@ namespace PolyVox template uint32_t Block::calculateSizeInBytes(void) { - //FIXME - This function is incomplete. - uint32_t uSizeInBytes = sizeof(Block); + // Returns the size of this class plus the size of the compressed data. It + // doesn't include the uncompressed data cache as that is owned by the volume. + uint32_t uSizeInBytes = sizeof(Block) + m_uCompressedDataLength; return uSizeInBytes; } } diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h index 103d7ca8..f46562d7 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h @@ -263,10 +263,10 @@ namespace PolyVox /// Gets a voxel at the position given by a 3D vector POLYVOX_DEPRECATED VoxelType getVoxelAt(const Vector3DInt32& v3dPos) const; + void setTargetMemoryLimitInBytes(uint32_t uTargetMemoryLimitInBytes); + /// Sets the number of blocks for which uncompressed data is stored void setMaxNumberOfUncompressedBlocks(uint32_t uMaxNumberOfUncompressedBlocks); - /// Sets the number of blocks which can be in memory before the paging system starts unloading them - void setMaxNumberOfBlocksInMemory(uint32_t uMaxNumberOfBlocksInMemory); /// Sets the voxel at the position given by x,y,z coordinates void setVoxel(int32_t uXPos, int32_t uYPos, int32_t uZPos, VoxelType tValue, WrapMode eWrapMode = WrapModes::Validate); /// Sets the voxel at the position given by a 3D vector @@ -315,6 +315,8 @@ namespace PolyVox }; void initialise(); + uint32_t calculateBlockMemoryUsage(void) const; + // A trick to implement specialization of template member functions in template classes. See http://stackoverflow.com/a/4951057 template VoxelType getVoxelImpl(int32_t uXPos, int32_t uYPos, int32_t uZPos, WrapModeType, VoxelType tBorder) const; @@ -338,7 +340,8 @@ namespace PolyVox mutable Vector3DInt32 m_v3dLastAccessedBlockPos; mutable Block* m_pLastAccessedBlock; uint32_t m_uMaxNumberOfUncompressedBlocks; - uint32_t m_uMaxNumberOfBlocksInMemory; + + uint32_t m_uCompressedBlockMemoryLimitInBytes; // The size of the volume Region m_regValidRegionInBlocks; diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl index 19b7d114..b50ef2bf 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl @@ -25,6 +25,8 @@ freely, subject to the following restrictions: #include "PolyVoxCore/MinizCompressor.h" +#include + namespace PolyVox { //////////////////////////////////////////////////////////////////////////////// @@ -241,6 +243,25 @@ namespace PolyVox return getVoxelAt(v3dPos.getX(), v3dPos.getY(), v3dPos.getZ()); } + template + void LargeVolume::setTargetMemoryLimitInBytes(uint32_t uTargetMemoryLimitInBytes) + { + uint32_t uUncompressedBlockSizeInBytes = m_uBlockSideLength * m_uBlockSideLength * m_uBlockSideLength * sizeof(VoxelType); + + uint32_t uIdealNoOfUncompressedBlocks = m_regValidRegionInBlocks.getWidthInVoxels() + m_regValidRegionInBlocks.getHeightInVoxels() * m_regValidRegionInBlocks.getDepthInVoxels(); + + // Let's say that we should never use more than half the available memory for the uncompressed block cache. + uint32_t uMaxMemoryForUncompressedBlocks = uTargetMemoryLimitInBytes / 2; + + uint32_t uMaxFittableNoOfUncompressedBlocks = uMaxMemoryForUncompressedBlocks / uUncompressedBlockSizeInBytes; + + setMaxNumberOfUncompressedBlocks((std::min)(uIdealNoOfUncompressedBlocks, uMaxFittableNoOfUncompressedBlocks)); + + uint32_t uUncompressedBlockCacheSizeInBytes = m_uMaxNumberOfUncompressedBlocks * uUncompressedBlockSizeInBytes; + + m_uCompressedBlockMemoryLimitInBytes = uTargetMemoryLimitInBytes - uUncompressedBlockCacheSizeInBytes; + } + //////////////////////////////////////////////////////////////////////////////// /// Increasing the size of the block cache will increase memory but may improve performance. /// You may want to set this to a large value (e.g. 1024) when you are first loading your @@ -255,20 +276,6 @@ namespace PolyVox m_uMaxNumberOfUncompressedBlocks = uMaxNumberOfUncompressedBlocks; } - //////////////////////////////////////////////////////////////////////////////// - /// Increasing the number of blocks in memory causes fewer calls to dataRequiredHandler()/dataOverflowHandler() - /// \param uMaxNumberOfBlocksInMemory The number of blocks - //////////////////////////////////////////////////////////////////////////////// - template - void LargeVolume::setMaxNumberOfBlocksInMemory(uint32_t uMaxNumberOfBlocksInMemory) - { - if(m_pBlocks.size() > uMaxNumberOfBlocksInMemory) - { - flushAll(); - } - m_uMaxNumberOfBlocksInMemory = uMaxNumberOfBlocksInMemory; - } - //////////////////////////////////////////////////////////////////////////////// /// \param uXPos the \c x position of the voxel /// \param uYPos the \c y position of the voxel @@ -361,7 +368,7 @@ namespace PolyVox //////////////////////////////////////////////////////////////////////////////// - /// Note that if MaxNumberOfBlocksInMemory is not large enough to support the region this function will only load part of the region. In this case it is undefined which parts will actually be loaded. If all the voxels in the given region are already loaded, this function will not do anything. Other voxels might be unloaded to make space for the new voxels. + /// Note that if *NOTE - update docs - MaxNumberOfBlocksInMemory no longer exists* MaxNumberOfBlocksInMemory is not large enough to support the region this function will only load part of the region. In this case it is undefined which parts will actually be loaded. If all the voxels in the given region are already loaded, this function will not do anything. Other voxels might be unloaded to make space for the new voxels. /// \param regPrefetch The Region of voxels to prefetch into memory. //////////////////////////////////////////////////////////////////////////////// template @@ -381,11 +388,14 @@ namespace PolyVox Vector3DInt32 v3dSize = v3dEnd - v3dStart + Vector3DInt32(1,1,1); uint32_t numblocks = static_cast(v3dSize.getX() * v3dSize.getY() * v3dSize.getZ()); - if(numblocks > m_uMaxNumberOfBlocksInMemory) + + // FIXME - reinstate some logic to handle when the prefetched region is too large. + + /*if(numblocks > m_uMaxNumberOfBlocksInMemory) { // cannot support the amount of blocks... so only load the maximum possible numblocks = m_uMaxNumberOfBlocksInMemory; - } + }*/ for(int32_t x = v3dStart.getX(); x <= v3dEnd.getX(); x++) { for(int32_t y = v3dStart.getY(); y <= v3dEnd.getY(); y++) @@ -511,8 +521,8 @@ namespace PolyVox } m_uTimestamper = 0; - m_uMaxNumberOfUncompressedBlocks = 16; - m_uMaxNumberOfBlocksInMemory = 1024; + //m_uMaxNumberOfUncompressedBlocks = 16; + //m_uMaxNumberOfBlocksInMemory = 1024; m_v3dLastAccessedBlockPos = Vector3DInt32(0,0,0); //There are no invalid positions, but initially the m_pLastAccessedBlock pointer will be null; m_pLastAccessedBlock = 0; @@ -526,7 +536,7 @@ namespace PolyVox m_regValidRegionInBlocks.setUpperY(this->m_regValidRegion.getUpperY() >> m_uBlockSideLengthPower); m_regValidRegionInBlocks.setUpperZ(this->m_regValidRegion.getUpperZ() >> m_uBlockSideLengthPower); - setMaxNumberOfUncompressedBlocks(m_uMaxNumberOfUncompressedBlocks); + //setMaxNumberOfUncompressedBlocks(m_uMaxNumberOfUncompressedBlocks); //Clear the previous data m_pBlocks.clear(); @@ -600,7 +610,7 @@ namespace PolyVox if(m_pPager) { // check wether another block needs to be unloaded before this one can be loaded - if(m_pBlocks.size() == m_uMaxNumberOfBlocksInMemory) + while(calculateBlockMemoryUsage() > m_uCompressedBlockMemoryLimitInBytes) //FIXME - This calculation of size is slow and should be outside the loop. { // find the least recently used block typename std::map, BlockPositionCompare>::iterator i; @@ -721,6 +731,21 @@ namespace PolyVox return uSizeInBytes; } + template + uint32_t LargeVolume::calculateBlockMemoryUsage(void) const + { + uint32_t uMemoryUsage = 0; + + typename std::map, BlockPositionCompare>::iterator i; + for(i = m_pBlocks.begin(); i != m_pBlocks.end(); i++) + { + //Inaccurate - account for rest of loaded block. + uMemoryUsage += i->second.calculateSizeInBytes(); + } + + return uMemoryUsage; + } + template template VoxelType LargeVolume::getVoxelImpl(int32_t uXPos, int32_t uYPos, int32_t uZPos, WrapModeType, VoxelType tBorder) const diff --git a/tests/testvolume.cpp b/tests/testvolume.cpp index 77ad1063..2063e8e3 100644 --- a/tests/testvolume.cpp +++ b/tests/testvolume.cpp @@ -281,8 +281,9 @@ TestVolume::TestVolume() m_pSimpleVolume = new SimpleVolume(region); m_pLargeVolume = new LargeVolume(region, m_pCompressor, m_pFilePager, 32); - m_pLargeVolume->setMaxNumberOfBlocksInMemory(32); - m_pLargeVolume->setMaxNumberOfUncompressedBlocks(16); + //m_pLargeVolume->setMaxNumberOfBlocksInMemory(32); + //m_pLargeVolume->setMaxNumberOfUncompressedBlocks(16); + m_pLargeVolume->setTargetMemoryLimitInBytes(4 * 1024 * 1024); //Fill the volume with some data for(int z = region.getLowerZ(); z <= region.getUpperZ(); z++) From 9f7f893b68e3ba069657f98960951f992590197c Mon Sep 17 00:00:00 2001 From: Matt Williams Date: Wed, 3 Jul 2013 19:14:00 +0100 Subject: [PATCH 04/55] Change the types of the edgeTable and triTable The range on values in these tables is much less than needs an int so making them specific width types packs them smaller. I measure a 5% decrease in the size of the .so file created at -O3. --- .../include/PolyVoxCore/Impl/MarchingCubesTables.h | 4 ++-- library/PolyVoxCore/source/Impl/MarchingCubesTables.cpp | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/library/PolyVoxCore/include/PolyVoxCore/Impl/MarchingCubesTables.h b/library/PolyVoxCore/include/PolyVoxCore/Impl/MarchingCubesTables.h index 3dab0ffa..19aece87 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/Impl/MarchingCubesTables.h +++ b/library/PolyVoxCore/include/PolyVoxCore/Impl/MarchingCubesTables.h @@ -28,8 +28,8 @@ freely, subject to the following restrictions: namespace PolyVox { - extern const POLYVOX_API int edgeTable[256]; - extern const POLYVOX_API int triTable[256][16]; + extern const POLYVOX_API uint16_t edgeTable[256]; + extern const POLYVOX_API int8_t triTable[256][16]; } #endif diff --git a/library/PolyVoxCore/source/Impl/MarchingCubesTables.cpp b/library/PolyVoxCore/source/Impl/MarchingCubesTables.cpp index 38f1a56c..15769007 100644 --- a/library/PolyVoxCore/source/Impl/MarchingCubesTables.cpp +++ b/library/PolyVoxCore/source/Impl/MarchingCubesTables.cpp @@ -30,7 +30,7 @@ freely, subject to the following restrictions: namespace PolyVox { - const int edgeTable[256]= + const uint16_t edgeTable[256]= { 0x000, 0x109, 0x203, 0x30a, 0x80c, 0x905, 0xa0f, 0xb06, 0x406, 0x50f, 0x605, 0x70c, 0xc0a, 0xd03, 0xe09, 0xf00, @@ -66,7 +66,7 @@ namespace PolyVox 0xb06, 0xa0f, 0x905, 0x80c, 0x30a, 0x203, 0x109, 0x000 }; - const int triTable[256][16] = + const int8_t triTable[256][16] = { {-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, }, { 0, 8, 3, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, }, From 8027f9904de5263813cce2b6ad4e64d9c1cfd809 Mon Sep 17 00:00:00 2001 From: Matt Williams Date: Thu, 4 Jul 2013 19:34:49 +0100 Subject: [PATCH 05/55] On systems that support it, raise SIGTRAP to drop into the debugger Clang was being clever and giving a warning for this line so it prompted me to fix it. I believe that SIGTRAP is the correct way to get the debugger to work here. It does a compile-time check for the platform when not using MSVC. Discussed at http://www.volumesoffun.com/phpBB3/viewtopic.php?p=3766#p3766 --- .../PolyVoxCore/include/PolyVoxCore/Impl/ErrorHandling.h | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/library/PolyVoxCore/include/PolyVoxCore/Impl/ErrorHandling.h b/library/PolyVoxCore/include/PolyVoxCore/Impl/ErrorHandling.h index afc452c0..5c6f6b0f 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/Impl/ErrorHandling.h +++ b/library/PolyVoxCore/include/PolyVoxCore/Impl/ErrorHandling.h @@ -31,6 +31,7 @@ freely, subject to the following restrictions: #include #include #include // Exception constuctors take strings. +#include #if defined(_MSC_VER) // In Visual Studio we can use this function to go into the debugger. @@ -38,7 +39,11 @@ freely, subject to the following restrictions: #else // On other platforms we just halt by forcing a crash. // Hopefully this puts us in the debugger if one is running - #define POLYVOX_HALT() *((unsigned int*)0) = 0xDEAD + #if defined(__linux__) || defined(__APPLE__) + #define POLYVOX_HALT() raise(SIGTRAP) + #else + #define POLYVOX_HALT() *((unsigned int*)0) = 0xDEAD + #endif #endif // Macros cannot contain #ifdefs, but some of our macros need to disable warnings and such warning supression is From 2f2475ad0ccb0398ed945d944915e0c2704a8c47 Mon Sep 17 00:00:00 2001 From: Matt Williams Date: Thu, 4 Jul 2013 22:05:06 +0100 Subject: [PATCH 06/55] Avoid ambiguous function resolution of getVoxelImpl Clang was complaining that the function couldn't access the *Volumes*'s ``getVoxelImpl``. Since we actually want the ``BaseVolume::Sampler``'s version this solves the problem. --- library/PolyVoxCore/include/PolyVoxCore/LargeVolumeSampler.inl | 2 +- library/PolyVoxCore/include/PolyVoxCore/RawVolumeSampler.inl | 2 +- library/PolyVoxCore/include/PolyVoxCore/SimpleVolumeSampler.inl | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolumeSampler.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolumeSampler.inl index 33acc503..0d71a68f 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolumeSampler.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolumeSampler.inl @@ -88,7 +88,7 @@ namespace PolyVox } else { - return getVoxelImpl(this->mXPosInVolume, this->mYPosInVolume, this->mZPosInVolume); + return this->getVoxelImpl(this->mXPosInVolume, this->mYPosInVolume, this->mZPosInVolume); } } diff --git a/library/PolyVoxCore/include/PolyVoxCore/RawVolumeSampler.inl b/library/PolyVoxCore/include/PolyVoxCore/RawVolumeSampler.inl index 6de3f3df..7460c19c 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/RawVolumeSampler.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/RawVolumeSampler.inl @@ -51,7 +51,7 @@ namespace PolyVox } else { - return getVoxelImpl(this->mXPosInVolume, this->mYPosInVolume, this->mZPosInVolume); + return this->getVoxelImpl(this->mXPosInVolume, this->mYPosInVolume, this->mZPosInVolume); } } diff --git a/library/PolyVoxCore/include/PolyVoxCore/SimpleVolumeSampler.inl b/library/PolyVoxCore/include/PolyVoxCore/SimpleVolumeSampler.inl index a4892f26..6cf16bef 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/SimpleVolumeSampler.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/SimpleVolumeSampler.inl @@ -94,7 +94,7 @@ namespace PolyVox } else { - return getVoxelImpl(this->mXPosInVolume, this->mYPosInVolume, this->mZPosInVolume); + return this->getVoxelImpl(this->mXPosInVolume, this->mYPosInVolume, this->mZPosInVolume); } } From 0c5593824282d7e192d2ce428e1c4311ff59f23c Mon Sep 17 00:00:00 2001 From: Daviw Williams Date: Fri, 5 Jul 2013 15:41:16 +0200 Subject: [PATCH 07/55] Work on the code that frees up memory in LargeVolume. --- .../include/PolyVoxCore/LargeVolume.h | 8 +++- .../include/PolyVoxCore/LargeVolume.inl | 46 +++++++++++++++++-- 2 files changed, 48 insertions(+), 6 deletions(-) diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h index f46562d7..cd773156 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h @@ -312,11 +312,15 @@ namespace PolyVox } return false; } - }; - void initialise(); + }; uint32_t calculateBlockMemoryUsage(void) const; + void flushOldestExcessiveBlocks(void) const; + void flushExcessiveCacheEntries(void) const; + + void initialise(); + // A trick to implement specialization of template member functions in template classes. See http://stackoverflow.com/a/4951057 template VoxelType getVoxelImpl(int32_t uXPos, int32_t uYPos, int32_t uZPos, WrapModeType, VoxelType tBorder) const; diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl index b50ef2bf..c46549af 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl @@ -246,11 +246,14 @@ namespace PolyVox template void LargeVolume::setTargetMemoryLimitInBytes(uint32_t uTargetMemoryLimitInBytes) { + // The size of a single uncompressed block of data. uint32_t uUncompressedBlockSizeInBytes = m_uBlockSideLength * m_uBlockSideLength * m_uBlockSideLength * sizeof(VoxelType); + // The idea number of uncompressed blocks is chosen by gut feeling as much as anything. Part of the + // rationale is that it should let us iterate along any edge without data being pushed out of the cache. uint32_t uIdealNoOfUncompressedBlocks = m_regValidRegionInBlocks.getWidthInVoxels() + m_regValidRegionInBlocks.getHeightInVoxels() * m_regValidRegionInBlocks.getDepthInVoxels(); - // Let's say that we should never use more than half the available memory for the uncompressed block cache. + // Let's (arbitrarily?) say that we should never use more than half the available memory for the uncompressed block cache. uint32_t uMaxMemoryForUncompressedBlocks = uTargetMemoryLimitInBytes / 2; uint32_t uMaxFittableNoOfUncompressedBlocks = uMaxMemoryForUncompressedBlocks / uUncompressedBlockSizeInBytes; @@ -607,9 +610,9 @@ namespace PolyVox //The block is not in the map, so we will have to create a new block and add it. //Before we do so, we might want to dump some existing data to make space. We //Only do this if paging is enabled. - if(m_pPager) + /*if(m_pPager) { - // check wether another block needs to be unloaded before this one can be loaded + // check whether another block needs to be unloaded before this one can be loaded while(calculateBlockMemoryUsage() > m_uCompressedBlockMemoryLimitInBytes) //FIXME - This calculation of size is slow and should be outside the loop. { // find the least recently used block @@ -624,7 +627,9 @@ namespace PolyVox } eraseBlock(itUnloadBlock); } - } + }*/ + + flushOldestExcessiveBlocks(); // create the new block Block newBlock(m_uBlockSideLength, m_pCompressor); @@ -746,6 +751,39 @@ namespace PolyVox return uMemoryUsage; } + template + void LargeVolume::flushOldestExcessiveBlocks(void) const + { + const uint32_t uMemoryUsedForCompressedBlocks = calculateBlockMemoryUsage(); + uint32_t uMemoryToReclaim = uMemoryUsedForCompressedBlocks - m_uCompressedBlockMemoryLimitInBytes; + + //while(uMemoryToReclaim > 0) + while(calculateBlockMemoryUsage() > m_uCompressedBlockMemoryLimitInBytes) //FIXME - This calculation of size is slow and should be outside the loop. + { + // find the least recently used block + typename std::map, BlockPositionCompare>::iterator i; + typename std::map, BlockPositionCompare>::iterator itUnloadBlock = m_pBlocks.begin(); + for(i = m_pBlocks.begin(); i != m_pBlocks.end(); i++) + { + if(i->second.timestamp < itUnloadBlock->second.timestamp) + { + itUnloadBlock = i; + } + } + + //POLYVOX_ASSERT(itUnloadBlock->second.hasUncompressedData() == false, "This function should never flush blocks with uncompressed data."); + + uMemoryToReclaim -= itUnloadBlock->second.calculateSizeInBytes(); + + eraseBlock(itUnloadBlock); + } + } + + template + void LargeVolume::flushExcessiveCacheEntries(void) const + { + } + template template VoxelType LargeVolume::getVoxelImpl(int32_t uXPos, int32_t uYPos, int32_t uZPos, WrapModeType, VoxelType tBorder) const From 056064409dd1cfcf01202e2646450371bd9c6d43 Mon Sep 17 00:00:00 2001 From: Daviw Williams Date: Fri, 5 Jul 2013 16:07:38 +0200 Subject: [PATCH 08/55] Work on LargeVolume. --- .../include/PolyVoxCore/LargeVolume.inl | 49 +++---------------- 1 file changed, 8 insertions(+), 41 deletions(-) diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl index c46549af..662d10db 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl @@ -608,50 +608,17 @@ namespace PolyVox if(itBlock == m_pBlocks.end()) { //The block is not in the map, so we will have to create a new block and add it. - //Before we do so, we might want to dump some existing data to make space. We - //Only do this if paging is enabled. - /*if(m_pPager) - { - // check whether another block needs to be unloaded before this one can be loaded - while(calculateBlockMemoryUsage() > m_uCompressedBlockMemoryLimitInBytes) //FIXME - This calculation of size is slow and should be outside the loop. - { - // find the least recently used block - typename std::map, BlockPositionCompare>::iterator i; - typename std::map, BlockPositionCompare>::iterator itUnloadBlock = m_pBlocks.begin(); - for(i = m_pBlocks.begin(); i != m_pBlocks.end(); i++) - { - if(i->second.timestamp < itUnloadBlock->second.timestamp) - { - itUnloadBlock = i; - } - } - eraseBlock(itUnloadBlock); - } - }*/ - - flushOldestExcessiveBlocks(); - - // create the new block Block newBlock(m_uBlockSideLength, m_pCompressor); - itBlock = m_pBlocks.insert(std::make_pair(v3dBlockPos, newBlock)).first; - //We have created the new block. If paging is enabled it should be used to - //fill in the required data. Otherwise it is just left in the default state. - if(m_pPager) - { - //if(m_funcDataRequiredHandler) - { - // "load" will actually call setVoxel, which will in turn call this function again but the block will be found - // so this if(itBlock == m_pBlocks.end()) never is entered - //FIXME - can we pass the block around so that we don't have to find it again when we recursively call this function? - Vector3DInt32 v3dLower(v3dBlockPos.getX() << m_uBlockSideLengthPower, v3dBlockPos.getY() << m_uBlockSideLengthPower, v3dBlockPos.getZ() << m_uBlockSideLengthPower); - Vector3DInt32 v3dUpper = v3dLower + Vector3DInt32(m_uBlockSideLength-1, m_uBlockSideLength-1, m_uBlockSideLength-1); - Region reg(v3dLower, v3dUpper); + // Now use the pager to fill the block with it's initial data. + Vector3DInt32 v3dLower(v3dBlockPos.getX() << m_uBlockSideLengthPower, v3dBlockPos.getY() << m_uBlockSideLengthPower, v3dBlockPos.getZ() << m_uBlockSideLengthPower); + Vector3DInt32 v3dUpper = v3dLower + Vector3DInt32(m_uBlockSideLength-1, m_uBlockSideLength-1, m_uBlockSideLength-1); + Region reg(v3dLower, v3dUpper); + m_pPager->pageIn(reg, &(itBlock->second)); - m_pPager->pageIn(reg, &(itBlock->second)); - } - } + // Paging in this new block may mean we are now using too much memory. If necessary, flush some old blocks. + flushOldestExcessiveBlocks(); } //Get the block and mark that we accessed it @@ -771,7 +738,7 @@ namespace PolyVox } } - //POLYVOX_ASSERT(itUnloadBlock->second.hasUncompressedData() == false, "This function should never flush blocks with uncompressed data."); + POLYVOX_ASSERT(itUnloadBlock->second.hasUncompressedData() == false, "This function should never flush blocks with uncompressed data."); uMemoryToReclaim -= itUnloadBlock->second.calculateSizeInBytes(); From a589c6e4acd4a215e37f36cca99c9524758e6751 Mon Sep 17 00:00:00 2001 From: David Williams Date: Thu, 11 Jul 2013 14:26:38 +0200 Subject: [PATCH 09/55] Split come code into getCompressedBlock() function. --- .../include/PolyVoxCore/LargeVolume.h | 1 + .../include/PolyVoxCore/LargeVolume.inl | 45 ++++++++++++------- 2 files changed, 29 insertions(+), 17 deletions(-) diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h index cd773156..934336bc 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h @@ -329,6 +329,7 @@ namespace PolyVox VoxelType getVoxelImpl(int32_t uXPos, int32_t uYPos, int32_t uZPos, WrapModeType, VoxelType tBorder) const; VoxelType getVoxelImpl(int32_t uXPos, int32_t uYPos, int32_t uZPos, WrapModeType, VoxelType tBorder) const; + Block* getCompressedBlock(int32_t uBlockX, int32_t uBlockY, int32_t uBlockZ) const; Block* getUncompressedBlock(int32_t uBlockX, int32_t uBlockY, int32_t uBlockZ) const; void eraseBlock(typename std::map, BlockPositionCompare>::iterator itBlock) const; diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl index 662d10db..fac37e2a 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl @@ -589,20 +589,10 @@ namespace PolyVox } template - Block* LargeVolume::getUncompressedBlock(int32_t uBlockX, int32_t uBlockY, int32_t uBlockZ) const + Block* LargeVolume::getCompressedBlock(int32_t uBlockX, int32_t uBlockY, int32_t uBlockZ) const { Vector3DInt32 v3dBlockPos(uBlockX, uBlockY, uBlockZ); - //Check if we have the same block as last time, if so there's no need to even update - //the time stamp. If we updated it everytime then that would be every time we touched - //a voxel, which would overflow a uint32_t and require us to use a uint64_t instead. - //This check should also provide a significant speed boost as usually it is true. - if((v3dBlockPos == m_v3dLastAccessedBlockPos) && (m_pLastAccessedBlock != 0)) - { - POLYVOX_ASSERT(m_pLastAccessedBlock->hasUncompressedData(), "Last accessed block has no uncompressed data."); - return m_pLastAccessedBlock; - } - typename std::map, BlockPositionCompare>::iterator itBlock = m_pBlocks.find(v3dBlockPos); // check whether the block is already loaded if(itBlock == m_pBlocks.end()) @@ -627,9 +617,30 @@ namespace PolyVox m_v3dLastAccessedBlockPos = v3dBlockPos; m_pLastAccessedBlock = █ - if(block.hasUncompressedData()) - { + return m_pLastAccessedBlock; + } + + template + Block* LargeVolume::getUncompressedBlock(int32_t uBlockX, int32_t uBlockY, int32_t uBlockZ) const + { + Vector3DInt32 v3dBlockPos(uBlockX, uBlockY, uBlockZ); + + //Check if we have the same block as last time, if so there's no need to even update + //the time stamp. If we updated it everytime then that would be every time we touched + //a voxel, which would overflow a uint32_t and require us to use a uint64_t instead. + //This check should also provide a significant speed boost as usually it is true. + if((v3dBlockPos == m_v3dLastAccessedBlockPos) && (m_pLastAccessedBlock != 0)) + { + POLYVOX_ASSERT(m_pLastAccessedBlock->hasUncompressedData(), "Last accessed block has no uncompressed data."); return m_pLastAccessedBlock; + } + + //Get the block and mark that we accessed it + Block* block = getCompressedBlock(uBlockX, uBlockY, uBlockZ); + + if(block->hasUncompressedData()) + { + return block; } //If we are allowed to compress then check whether we need to @@ -655,16 +666,16 @@ namespace PolyVox //We don't actually remove any elements from this vector, we //simply change the pointer to point at the new uncompressed bloack. - m_vecBlocksWithUncompressedData[leastRecentlyUsedBlockIndex] = █ + m_vecBlocksWithUncompressedData[leastRecentlyUsedBlockIndex] = block; } else { - m_vecBlocksWithUncompressedData.push_back(&block); + m_vecBlocksWithUncompressedData.push_back(block); } - block.createUncompressedData(); + block->createUncompressedData(); - m_pLastAccessedBlock = &(block); + m_pLastAccessedBlock = block; POLYVOX_ASSERT(m_pLastAccessedBlock->m_tUncompressedData, "Block has no uncompressed data"); return m_pLastAccessedBlock; } From abd1920d80c32e25795a1973de841a492d834840 Mon Sep 17 00:00:00 2001 From: David Williams Date: Thu, 11 Jul 2013 14:50:06 +0200 Subject: [PATCH 10/55] Adding new caching mechanism. --- .../include/PolyVoxCore/LargeVolume.h | 1 + .../include/PolyVoxCore/LargeVolume.inl | 21 +++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h index 934336bc..a87f2796 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h @@ -334,6 +334,7 @@ namespace PolyVox void eraseBlock(typename std::map, BlockPositionCompare>::iterator itBlock) const; // The block data + mutable std::map m_pUncompressedBlockCache; mutable std::map, BlockPositionCompare> m_pBlocks; // The cache of uncompressed blocks. The uncompressed block data and the timestamps are stored here rather diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl index fac37e2a..db604532 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl @@ -638,6 +638,27 @@ namespace PolyVox //Get the block and mark that we accessed it Block* block = getCompressedBlock(uBlockX, uBlockY, uBlockZ); + + typename std::map::iterator itUncompressedBlock = m_pUncompressedBlockCache.find(v3dBlockPos); + // check whether the block is already loaded + if(itUncompressedBlock == m_pUncompressedBlockCache.end()) + { + VoxelType* pUncompressedBlock = new VoxelType[m_uBlockSideLength * m_uBlockSideLength * m_uBlockSideLength]; + + void* pSrcData = reinterpret_cast(block->m_pCompressedData); + void* pDstData = reinterpret_cast(pUncompressedBlock); + uint32_t uSrcLength = block->m_uCompressedDataLength; + uint32_t uDstLength = m_uBlockSideLength * m_uBlockSideLength * m_uBlockSideLength * sizeof(VoxelType); + + //MinizCompressor compressor; + //RLECompressor compressor; + uint32_t uUncompressedLength = m_pCompressor->decompress(pSrcData, uSrcLength, pDstData, uDstLength); + + POLYVOX_ASSERT(uUncompressedLength == m_uBlockSideLength * m_uBlockSideLength * m_uBlockSideLength * sizeof(VoxelType), "Destination length has changed."); + + itUncompressedBlock = m_pUncompressedBlockCache.insert(std::make_pair(v3dBlockPos, pUncompressedBlock)).first; + } + if(block->hasUncompressedData()) { return block; From 884fe04c1224b513f9f6aa66748252ab90d5fd4c Mon Sep 17 00:00:00 2001 From: David Williams Date: Thu, 11 Jul 2013 15:01:18 +0200 Subject: [PATCH 11/55] Tidying up... --- .../include/PolyVoxCore/LargeVolume.inl | 36 ++++++++++++++----- .../PolyVoxCore/LargeVolumeSampler.inl | 4 +-- 2 files changed, 30 insertions(+), 10 deletions(-) diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl index db604532..502ac706 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl @@ -223,9 +223,14 @@ namespace PolyVox const uint16_t yOffset = static_cast(uYPos - (blockY << m_uBlockSideLengthPower)); const uint16_t zOffset = static_cast(uZPos - (blockZ << m_uBlockSideLengthPower)); - Block* pUncompressedBlock = getUncompressedBlock(blockX, blockY, blockZ); + VoxelType* pUncompressedBlock = getUncompressedBlock(blockX, blockY, blockZ)->m_tUncompressedData; - return pUncompressedBlock->getVoxel(xOffset,yOffset,zOffset); + return pUncompressedBlock + [ + xOffset + + yOffset * m_uBlockSideLength + + zOffset * m_uBlockSideLength * m_uBlockSideLength + ]; } else { @@ -312,9 +317,14 @@ namespace PolyVox const uint16_t yOffset = static_cast(uYPos - (blockY << m_uBlockSideLengthPower)); const uint16_t zOffset = static_cast(uZPos - (blockZ << m_uBlockSideLengthPower)); - Block* pUncompressedBlock = getUncompressedBlock(blockX, blockY, blockZ); + VoxelType* pUncompressedBlock = getUncompressedBlock(blockX, blockY, blockZ)->m_tUncompressedData; - pUncompressedBlock->setVoxelAt(xOffset,yOffset,zOffset, tValue); + pUncompressedBlock + [ + xOffset + + yOffset * m_uBlockSideLength + + zOffset * m_uBlockSideLength * m_uBlockSideLength + ] = tValue; } //////////////////////////////////////////////////////////////////////////////// @@ -350,9 +360,14 @@ namespace PolyVox const uint16_t yOffset = static_cast(uYPos - (blockY << m_uBlockSideLengthPower)); const uint16_t zOffset = static_cast(uZPos - (blockZ << m_uBlockSideLengthPower)); - Block* pUncompressedBlock = getUncompressedBlock(blockX, blockY, blockZ); + VoxelType* pUncompressedBlock = getUncompressedBlock(blockX, blockY, blockZ)->m_tUncompressedData; - pUncompressedBlock->setVoxelAt(xOffset,yOffset,zOffset, tValue); + pUncompressedBlock + [ + xOffset + + yOffset * m_uBlockSideLength + + zOffset * m_uBlockSideLength * m_uBlockSideLength + ] = tValue; //Return true to indicate that we modified a voxel. return true; @@ -840,9 +855,14 @@ namespace PolyVox const uint16_t yOffset = static_cast(uYPos - (blockY << m_uBlockSideLengthPower)); const uint16_t zOffset = static_cast(uZPos - (blockZ << m_uBlockSideLengthPower)); - Block* pUncompressedBlock = getUncompressedBlock(blockX, blockY, blockZ); + VoxelType* pUncompressedBlock = getUncompressedBlock(blockX, blockY, blockZ)->m_tUncompressedData; - return pUncompressedBlock->getVoxel(xOffset,yOffset,zOffset); + return pUncompressedBlock + [ + xOffset + + yOffset * m_uBlockSideLength + + zOffset * m_uBlockSideLength * m_uBlockSideLength + ]; } } diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolumeSampler.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolumeSampler.inl index 0d71a68f..dd61fcf0 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolumeSampler.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolumeSampler.inl @@ -119,9 +119,9 @@ namespace PolyVox uYPosInBlock * this->mVolume->m_uBlockSideLength + uZPosInBlock * this->mVolume->m_uBlockSideLength * this->mVolume->m_uBlockSideLength; - Block* pUncompressedCurrentBlock = this->mVolume->getUncompressedBlock(uXBlock, uYBlock, uZBlock); + VoxelType* pUncompressedCurrentBlock = this->mVolume->getUncompressedBlock(uXBlock, uYBlock, uZBlock)->m_tUncompressedData; - mCurrentVoxel = pUncompressedCurrentBlock->m_tUncompressedData + uVoxelIndexInBlock; + mCurrentVoxel = pUncompressedCurrentBlock + uVoxelIndexInBlock; } else { From e6cbc09e83ed36390e08b56334b4d6dbf77f3ca7 Mon Sep 17 00:00:00 2001 From: David Williams Date: Thu, 11 Jul 2013 15:30:15 +0200 Subject: [PATCH 12/55] getUncompressedBlock now return raw pointer instead of block pointer. --- .../include/PolyVoxCore/LargeVolume.h | 4 +-- .../include/PolyVoxCore/LargeVolume.inl | 25 ++++++++++--------- .../PolyVoxCore/LargeVolumeSampler.inl | 2 +- 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h index a87f2796..03b4ac01 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h @@ -330,7 +330,7 @@ namespace PolyVox VoxelType getVoxelImpl(int32_t uXPos, int32_t uYPos, int32_t uZPos, WrapModeType, VoxelType tBorder) const; Block* getCompressedBlock(int32_t uBlockX, int32_t uBlockY, int32_t uBlockZ) const; - Block* getUncompressedBlock(int32_t uBlockX, int32_t uBlockY, int32_t uBlockZ) const; + VoxelType* getUncompressedBlock(int32_t uBlockX, int32_t uBlockY, int32_t uBlockZ) const; void eraseBlock(typename std::map, BlockPositionCompare>::iterator itBlock) const; // The block data @@ -344,7 +344,7 @@ namespace PolyVox mutable std::vector< Block* > m_vecBlocksWithUncompressedData; mutable uint32_t m_uTimestamper; mutable Vector3DInt32 m_v3dLastAccessedBlockPos; - mutable Block* m_pLastAccessedBlock; + mutable VoxelType* m_pLastAccessedBlock; uint32_t m_uMaxNumberOfUncompressedBlocks; uint32_t m_uCompressedBlockMemoryLimitInBytes; diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl index 502ac706..794b283f 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl @@ -223,7 +223,7 @@ namespace PolyVox const uint16_t yOffset = static_cast(uYPos - (blockY << m_uBlockSideLengthPower)); const uint16_t zOffset = static_cast(uZPos - (blockZ << m_uBlockSideLengthPower)); - VoxelType* pUncompressedBlock = getUncompressedBlock(blockX, blockY, blockZ)->m_tUncompressedData; + VoxelType* pUncompressedBlock = getUncompressedBlock(blockX, blockY, blockZ); return pUncompressedBlock [ @@ -317,7 +317,7 @@ namespace PolyVox const uint16_t yOffset = static_cast(uYPos - (blockY << m_uBlockSideLengthPower)); const uint16_t zOffset = static_cast(uZPos - (blockZ << m_uBlockSideLengthPower)); - VoxelType* pUncompressedBlock = getUncompressedBlock(blockX, blockY, blockZ)->m_tUncompressedData; + VoxelType* pUncompressedBlock = getUncompressedBlock(blockX, blockY, blockZ); pUncompressedBlock [ @@ -360,7 +360,7 @@ namespace PolyVox const uint16_t yOffset = static_cast(uYPos - (blockY << m_uBlockSideLengthPower)); const uint16_t zOffset = static_cast(uZPos - (blockZ << m_uBlockSideLengthPower)); - VoxelType* pUncompressedBlock = getUncompressedBlock(blockX, blockY, blockZ)->m_tUncompressedData; + VoxelType* pUncompressedBlock = getUncompressedBlock(blockX, blockY, blockZ); pUncompressedBlock [ @@ -629,14 +629,13 @@ namespace PolyVox //Get the block and mark that we accessed it Block& block = itBlock->second; block.timestamp = ++m_uTimestamper; - m_v3dLastAccessedBlockPos = v3dBlockPos; - m_pLastAccessedBlock = █ + //m_v3dLastAccessedBlockPos = v3dBlockPos; - return m_pLastAccessedBlock; + return █ } template - Block* LargeVolume::getUncompressedBlock(int32_t uBlockX, int32_t uBlockY, int32_t uBlockZ) const + VoxelType* LargeVolume::getUncompressedBlock(int32_t uBlockX, int32_t uBlockY, int32_t uBlockZ) const { Vector3DInt32 v3dBlockPos(uBlockX, uBlockY, uBlockZ); @@ -646,7 +645,6 @@ namespace PolyVox //This check should also provide a significant speed boost as usually it is true. if((v3dBlockPos == m_v3dLastAccessedBlockPos) && (m_pLastAccessedBlock != 0)) { - POLYVOX_ASSERT(m_pLastAccessedBlock->hasUncompressedData(), "Last accessed block has no uncompressed data."); return m_pLastAccessedBlock; } @@ -674,9 +672,12 @@ namespace PolyVox itUncompressedBlock = m_pUncompressedBlockCache.insert(std::make_pair(v3dBlockPos, pUncompressedBlock)).first; } + m_pLastAccessedBlock = block->m_tUncompressedData; + m_v3dLastAccessedBlockPos = v3dBlockPos; + if(block->hasUncompressedData()) { - return block; + return block->m_tUncompressedData; } //If we are allowed to compress then check whether we need to @@ -711,8 +712,8 @@ namespace PolyVox block->createUncompressedData(); - m_pLastAccessedBlock = block; - POLYVOX_ASSERT(m_pLastAccessedBlock->m_tUncompressedData, "Block has no uncompressed data"); + m_pLastAccessedBlock = block->m_tUncompressedData; + POLYVOX_ASSERT(m_pLastAccessedBlock, "Block has no uncompressed data"); return m_pLastAccessedBlock; } @@ -855,7 +856,7 @@ namespace PolyVox const uint16_t yOffset = static_cast(uYPos - (blockY << m_uBlockSideLengthPower)); const uint16_t zOffset = static_cast(uZPos - (blockZ << m_uBlockSideLengthPower)); - VoxelType* pUncompressedBlock = getUncompressedBlock(blockX, blockY, blockZ)->m_tUncompressedData; + VoxelType* pUncompressedBlock = getUncompressedBlock(blockX, blockY, blockZ); return pUncompressedBlock [ diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolumeSampler.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolumeSampler.inl index dd61fcf0..d90ea5f4 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolumeSampler.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolumeSampler.inl @@ -119,7 +119,7 @@ namespace PolyVox uYPosInBlock * this->mVolume->m_uBlockSideLength + uZPosInBlock * this->mVolume->m_uBlockSideLength * this->mVolume->m_uBlockSideLength; - VoxelType* pUncompressedCurrentBlock = this->mVolume->getUncompressedBlock(uXBlock, uYBlock, uZBlock)->m_tUncompressedData; + VoxelType* pUncompressedCurrentBlock = this->mVolume->getUncompressedBlock(uXBlock, uYBlock, uZBlock); mCurrentVoxel = pUncompressedCurrentBlock + uVoxelIndexInBlock; } From 9ff95f7061eebe00e29db6607d5f00804d3eb1dd Mon Sep 17 00:00:00 2001 From: David Williams Date: Thu, 11 Jul 2013 15:55:34 +0200 Subject: [PATCH 13/55] Minor fixes. --- library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl index 794b283f..e8194b2f 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl @@ -646,7 +646,7 @@ namespace PolyVox if((v3dBlockPos == m_v3dLastAccessedBlockPos) && (m_pLastAccessedBlock != 0)) { return m_pLastAccessedBlock; - } + } //Get the block and mark that we accessed it Block* block = getCompressedBlock(uBlockX, uBlockY, uBlockZ); @@ -672,9 +672,11 @@ namespace PolyVox itUncompressedBlock = m_pUncompressedBlockCache.insert(std::make_pair(v3dBlockPos, pUncompressedBlock)).first; } - m_pLastAccessedBlock = block->m_tUncompressedData; + m_pLastAccessedBlock = (*itUncompressedBlock).second; m_v3dLastAccessedBlockPos = v3dBlockPos; + return m_pLastAccessedBlock; + if(block->hasUncompressedData()) { return block->m_tUncompressedData; From 0d92bc6c8cfb1c0db967d31581510455fbf423e1 Mon Sep 17 00:00:00 2001 From: David Williams Date: Thu, 11 Jul 2013 16:17:48 +0200 Subject: [PATCH 14/55] Starting to split Block class into CompressedBlock and UncompressedBlock. --- examples/Paging/main.cpp | 4 +- .../include/PolyVoxCore/Impl/Block.h | 3 - .../include/PolyVoxCore/Impl/Block.inl | 109 +----------------- .../include/PolyVoxCore/LargeVolume.h | 5 - .../include/PolyVoxCore/LargeVolume.inl | 64 ++-------- 5 files changed, 16 insertions(+), 169 deletions(-) diff --git a/examples/Paging/main.cpp b/examples/Paging/main.cpp index f172a214..c497a255 100644 --- a/examples/Paging/main.cpp +++ b/examples/Paging/main.cpp @@ -93,7 +93,7 @@ public: virtual void pageIn(const PolyVox::Region& region, Block* pBlockData) { - pBlockData->createUncompressedData(); + /*pBlockData->createUncompressedData(); Perlin perlin(2,2,1,234); @@ -134,7 +134,7 @@ public: pBlockData->setVoxelAt(x - region.getLowerX(), y - region.getLowerY(), z - region.getLowerZ(), voxel); } } - } + }*/ } virtual void pageOut(const PolyVox::Region& region, Block* /*pBlockData*/) diff --git a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h index 06963f7c..509b2dce 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h +++ b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h @@ -52,9 +52,6 @@ namespace PolyVox void setVoxelAt(uint16_t uXPos, uint16_t uYPos, uint16_t uZPos, VoxelType tValue); void setVoxelAt(const Vector3DUint16& v3dPos, VoxelType tValue); - void createUncompressedData(void); - void destroyUncompressedData(void); - uint32_t calculateSizeInBytes(void); public: diff --git a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl index bebcfb35..bdba78f9 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl @@ -66,12 +66,12 @@ namespace PolyVox //Temporarily create the block data. This is just so we can compress it an discard it. // FIXME - this is a temporary solution. - const uint32_t uNoOfVoxels = m_uSideLength * m_uSideLength * m_uSideLength; + /*const uint32_t uNoOfVoxels = m_uSideLength * m_uSideLength * m_uSideLength; m_tUncompressedData = new VoxelType[uNoOfVoxels]; std::fill(m_tUncompressedData, m_tUncompressedData + uNoOfVoxels, VoxelType()); m_bIsUncompressedDataModified = true; - destroyUncompressedData(); + destroyUncompressedData();*/ } template @@ -127,7 +127,7 @@ namespace PolyVox template void Block::setCompressedData(const uint8_t* const data, uint32_t dataLength) { - POLYVOX_ASSERT(m_pCompressedData, "Compressed data is NULL"); + //POLYVOX_ASSERT(m_pCompressedData, "Compressed data is NULL"); POLYVOX_ASSERT(m_pCompressedData != data, "Attempting to copy data onto itself"); delete[] m_pCompressedData; @@ -163,109 +163,6 @@ namespace PolyVox setVoxelAt(v3dPos.getX(), v3dPos.getY(), v3dPos.getZ(), tValue); } - template - void Block::destroyUncompressedData() - { - if(!hasUncompressedData()) - { - POLYVOX_THROW(invalid_operation, "No uncompressed data to compress."); - } - - POLYVOX_ASSERT(m_tUncompressedData != 0, "No uncompressed data is present."); - - //If the uncompressed data hasn't actually been - //modified then we don't need to redo the compression. - if(m_bIsUncompressedDataModified) - { - // Delete the old compressed data as we'll create a new one - delete[] m_pCompressedData; - m_pCompressedData = 0; - - void* pSrcData = reinterpret_cast(m_tUncompressedData); - uint32_t uSrcLength = m_uSideLength * m_uSideLength * m_uSideLength * sizeof(VoxelType); - - uint8_t tempBuffer[10000]; - void* pDstData = reinterpret_cast( tempBuffer ); - uint32_t uDstLength = 10000; - - uint32_t uCompressedLength = 0; - - try - { - uCompressedLength = m_pCompressor->compress(pSrcData, uSrcLength, pDstData, uDstLength); - - // Create new compressed data and copy across - m_pCompressedData = new uint8_t[uCompressedLength]; - memcpy(m_pCompressedData, pDstData, uCompressedLength); - m_uCompressedDataLength = uCompressedLength; - } - catch(std::exception&) - { - // It is possible for the compression to fail. A common cause for this would be if the destination - // buffer is not big enough. So now we try again using a buffer that is definitely big enough. - // Note that ideally we will choose our earlier buffer size so that this almost never happens. - logWarning() << "The compressor failed to compress the block, proabaly due to the buffer being too small."; - logWarning() << "The compression will be tried again with a larger buffer"; - uint32_t uMaxCompressedSize = m_pCompressor->getMaxCompressedSize(uSrcLength); - uint8_t* buffer = new uint8_t[ uMaxCompressedSize ]; - - pDstData = reinterpret_cast( buffer ); - uDstLength = uMaxCompressedSize; - - try - { - uCompressedLength = m_pCompressor->compress(pSrcData, uSrcLength, pDstData, uDstLength); - - // Create new compressed data and copy across - m_pCompressedData = new uint8_t[uCompressedLength]; - memcpy(m_pCompressedData, pDstData, uCompressedLength); - m_uCompressedDataLength = uCompressedLength; - } - catch(std::exception&) - { - // At this point it didn't work even with a bigger buffer. - // Not much more we can do so just rethrow the exception. - delete[] buffer; - POLYVOX_THROW(std::runtime_error, "Failed to compress block data"); - } - - delete[] buffer; - } - } - - //Flag the uncompressed data as no longer being used. - delete[] m_tUncompressedData; - m_tUncompressedData = 0; - } - - template - void Block::createUncompressedData() - { - if(hasUncompressedData()) - { - POLYVOX_THROW(invalid_operation, "Uncompressed data already exists."); - } - - POLYVOX_ASSERT(m_tUncompressedData == 0, "Uncompressed data already exists."); - - m_tUncompressedData = new VoxelType[m_uSideLength * m_uSideLength * m_uSideLength]; - - void* pSrcData = reinterpret_cast(m_pCompressedData); - void* pDstData = reinterpret_cast(m_tUncompressedData); - uint32_t uSrcLength = m_uCompressedDataLength; - uint32_t uDstLength = m_uSideLength * m_uSideLength * m_uSideLength * sizeof(VoxelType); - - //MinizCompressor compressor; - //RLECompressor compressor; - uint32_t uUncompressedLength = m_pCompressor->decompress(pSrcData, uSrcLength, pDstData, uDstLength); - - POLYVOX_ASSERT(uUncompressedLength == m_uSideLength * m_uSideLength * m_uSideLength * sizeof(VoxelType), "Destination length has changed."); - - //m_tUncompressedData = reinterpret_cast(uncompressedResult.ptr); - - m_bIsUncompressedDataModified = false; - } - template uint32_t Block::calculateSizeInBytes(void) { diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h index 03b4ac01..86e36381 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h @@ -337,11 +337,6 @@ namespace PolyVox mutable std::map m_pUncompressedBlockCache; mutable std::map, BlockPositionCompare> m_pBlocks; - // The cache of uncompressed blocks. The uncompressed block data and the timestamps are stored here rather - // than in the Block class. This is so that in the future each VolumeIterator might to maintain its own cache - // of blocks. However, this could mean the same block data is uncompressed and modified in more than one - // location in memory... could be messy with threading. - mutable std::vector< Block* > m_vecBlocksWithUncompressedData; mutable uint32_t m_uTimestamper; mutable Vector3DInt32 m_v3dLastAccessedBlockPos; mutable VoxelType* m_pLastAccessedBlock; diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl index e8194b2f..770c94ee 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl @@ -102,7 +102,7 @@ namespace PolyVox template LargeVolume::~LargeVolume() { - flushAll(); + //flushAll(); // Only delete the compressor if it was created by us (in the constructor), not by the user. if(m_bIsOurCompressor) @@ -509,11 +509,6 @@ namespace PolyVox template void LargeVolume::clearBlockCache(void) { - for(uint32_t ct = 0; ct < m_vecBlocksWithUncompressedData.size(); ct++) - { - m_vecBlocksWithUncompressedData[ct]->destroyUncompressedData(); - } - m_vecBlocksWithUncompressedData.clear(); } //////////////////////////////////////////////////////////////////////////////// @@ -571,7 +566,9 @@ namespace PolyVox template void LargeVolume::eraseBlock(typename std::map, BlockPositionCompare>::iterator itBlock) const { - if(itBlock->second.hasUncompressedData()) + POLYVOX_ASSERT(false, "This function has not been implemented properly"); + + /*if(itBlock->second.hasUncompressedData()) { itBlock->second.destroyUncompressedData(); } @@ -585,9 +582,11 @@ namespace PolyVox Region reg(v3dLower, v3dUpper); m_pPager->pageOut(reg, &(itBlock->second)); - } + }*/ + - for(uint32_t ct = 0; ct < m_vecBlocksWithUncompressedData.size(); ct++) + // FIXME - the code below used to make sure the uncompressed version of the data was removed. Reinstate something similar. + /*for(uint32_t ct = 0; ct < m_vecBlocksWithUncompressedData.size(); ct++) { // find the block in the uncompressed cache if(m_vecBlocksWithUncompressedData[ct] == &(itBlock->second)) @@ -598,7 +597,7 @@ namespace PolyVox m_vecBlocksWithUncompressedData.resize(m_vecBlocksWithUncompressedData.size()-1); break; } - } + }*/ m_pBlocks.erase(itBlock); } @@ -676,47 +675,6 @@ namespace PolyVox m_v3dLastAccessedBlockPos = v3dBlockPos; return m_pLastAccessedBlock; - - if(block->hasUncompressedData()) - { - return block->m_tUncompressedData; - } - - //If we are allowed to compress then check whether we need to - if(m_vecBlocksWithUncompressedData.size() == m_uMaxNumberOfUncompressedBlocks) - { - int32_t leastRecentlyUsedBlockIndex = -1; - uint32_t uLeastRecentTimestamp = (std::numeric_limits::max)(); - - //Currently we find the oldest block by iterating over the whole array. Of course we could store the blocks sorted by - //timestamp (set, priority_queue, etc) but then we'll need to move them around as the timestamp changes. Can come back - //to this if it proves to be a bottleneck (compraed to the cost of actually doing the compression/decompression). - for(uint32_t ct = 0; ct < m_vecBlocksWithUncompressedData.size(); ct++) - { - if(m_vecBlocksWithUncompressedData[ct]->timestamp < uLeastRecentTimestamp) - { - uLeastRecentTimestamp = m_vecBlocksWithUncompressedData[ct]->timestamp; - leastRecentlyUsedBlockIndex = ct; - } - } - - //Compress the least recently used block. - m_vecBlocksWithUncompressedData[leastRecentlyUsedBlockIndex]->destroyUncompressedData(); - - //We don't actually remove any elements from this vector, we - //simply change the pointer to point at the new uncompressed bloack. - m_vecBlocksWithUncompressedData[leastRecentlyUsedBlockIndex] = block; - } - else - { - m_vecBlocksWithUncompressedData.push_back(block); - } - - block->createUncompressedData(); - - m_pLastAccessedBlock = block->m_tUncompressedData; - POLYVOX_ASSERT(m_pLastAccessedBlock, "Block has no uncompressed data"); - return m_pLastAccessedBlock; } //////////////////////////////////////////////////////////////////////////////// @@ -747,8 +705,8 @@ namespace PolyVox } //Memory used by the block cache. - uSizeInBytes += m_vecBlocksWithUncompressedData.capacity() * sizeof(Block); - uSizeInBytes += m_vecBlocksWithUncompressedData.size() * m_uBlockSideLength * m_uBlockSideLength * m_uBlockSideLength * sizeof(VoxelType); + //uSizeInBytes += m_vecBlocksWithUncompressedData.capacity() * sizeof(Block); + //uSizeInBytes += m_vecBlocksWithUncompressedData.size() * m_uBlockSideLength * m_uBlockSideLength * m_uBlockSideLength * sizeof(VoxelType); return uSizeInBytes; } From 00eb281990f4a813b166fe24edb4c854a40a3143 Mon Sep 17 00:00:00 2001 From: David Williams Date: Thu, 11 Jul 2013 16:30:19 +0200 Subject: [PATCH 15/55] Stripping down Block class. --- .../include/PolyVoxCore/FilePager.h | 4 +- .../include/PolyVoxCore/Impl/Block.h | 13 --- .../include/PolyVoxCore/Impl/Block.inl | 81 +------------------ .../include/PolyVoxCore/LargeVolume.inl | 2 +- 4 files changed, 4 insertions(+), 96 deletions(-) diff --git a/library/PolyVoxCore/include/PolyVoxCore/FilePager.h b/library/PolyVoxCore/include/PolyVoxCore/FilePager.h index 1f3115ab..7fb79f1c 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/FilePager.h +++ b/library/PolyVoxCore/include/PolyVoxCore/FilePager.h @@ -55,7 +55,7 @@ namespace PolyVox virtual void pageIn(const Region& region, Block* pBlockData) { POLYVOX_ASSERT(pBlockData, "Attempting to page in NULL block"); - POLYVOX_ASSERT(pBlockData->hasUncompressedData() == false, "Block should not have uncompressed data"); + //POLYVOX_ASSERT(pBlockData->hasUncompressedData() == false, "Block should not have uncompressed data"); std::stringstream ss; ss << region.getLowerX() << "_" << region.getLowerY() << "_" << region.getLowerZ() << "_" @@ -96,7 +96,7 @@ namespace PolyVox virtual void pageOut(const Region& region, Block* pBlockData) { POLYVOX_ASSERT(pBlockData, "Attempting to page out NULL block"); - POLYVOX_ASSERT(pBlockData->hasUncompressedData() == false, "Block should not have uncompressed data"); + //POLYVOX_ASSERT(pBlockData->hasUncompressedData() == false, "Block should not have uncompressed data"); logTrace() << "Paging out data for " << region; diff --git a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h index 509b2dce..86d6d08b 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h +++ b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h @@ -42,27 +42,14 @@ namespace PolyVox const uint8_t* getCompressedData(void) const; uint32_t getCompressedDataLength(void) const; - uint16_t getSideLength(void) const; - VoxelType getVoxel(uint16_t uXPos, uint16_t uYPos, uint16_t uZPos) const; - VoxelType getVoxel(const Vector3DUint16& v3dPos) const; - - bool hasUncompressedData(void) const; void setCompressedData(const uint8_t* const data, uint32_t dataLength); - void setVoxelAt(uint16_t uXPos, uint16_t uYPos, uint16_t uZPos, VoxelType tValue); - void setVoxelAt(const Vector3DUint16& v3dPos, VoxelType tValue); uint32_t calculateSizeInBytes(void); public: - Compressor* m_pCompressor; uint8_t* m_pCompressedData; uint32_t m_uCompressedDataLength; - VoxelType* m_tUncompressedData; - uint16_t m_uSideLength; - uint8_t m_uSideLengthPower; - bool m_bIsUncompressedDataModified; - uint32_t timestamp; }; } diff --git a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl index bdba78f9..3f311e0d 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl @@ -37,13 +37,8 @@ namespace PolyVox { template Block::Block(uint16_t uSideLength, Compressor* pCompressor) - :m_pCompressor(pCompressor) - ,m_pCompressedData(0) + :m_pCompressedData(0) ,m_uCompressedDataLength(0) - ,m_tUncompressedData(0) - ,m_uSideLength(0) - ,m_uSideLengthPower(0) - ,m_bIsUncompressedDataModified(true) { if(uSideLength == 0) { @@ -60,18 +55,6 @@ namespace PolyVox POLYVOX_THROW(std::invalid_argument, "Block must be provided with a valid compressor."); } - //Compute the side length - m_uSideLength = uSideLength; - m_uSideLengthPower = logBase2(uSideLength); - - //Temporarily create the block data. This is just so we can compress it an discard it. - // FIXME - this is a temporary solution. - /*const uint32_t uNoOfVoxels = m_uSideLength * m_uSideLength * m_uSideLength; - m_tUncompressedData = new VoxelType[uNoOfVoxels]; - std::fill(m_tUncompressedData, m_tUncompressedData + uNoOfVoxels, VoxelType()); - m_bIsUncompressedDataModified = true; - - destroyUncompressedData();*/ } template @@ -88,42 +71,6 @@ namespace PolyVox return m_uCompressedDataLength; } - template - uint16_t Block::getSideLength(void) const - { - return m_uSideLength; - } - - template - VoxelType Block::getVoxel(uint16_t uXPos, uint16_t uYPos, uint16_t uZPos) const - { - // This is internal code not directly called by the user. For efficiency we assert rather than throwing. - POLYVOX_ASSERT(uXPos < m_uSideLength, "Supplied position is outside of the block"); - POLYVOX_ASSERT(uYPos < m_uSideLength, "Supplied position is outside of the block"); - POLYVOX_ASSERT(uZPos < m_uSideLength, "Supplied position is outside of the block"); - POLYVOX_ASSERT(hasUncompressedData(), "The block must have uncompressed data to call getVoxel()"); - POLYVOX_ASSERT(m_tUncompressedData, "No uncompressed data - block must be decompressed before accessing voxels."); - - return m_tUncompressedData - [ - uXPos + - uYPos * m_uSideLength + - uZPos * m_uSideLength * m_uSideLength - ]; - } - - template - VoxelType Block::getVoxel(const Vector3DUint16& v3dPos) const - { - return getVoxel(v3dPos.getX(), v3dPos.getY(), v3dPos.getZ()); - } - - template - bool Block::hasUncompressedData(void) const - { - return m_tUncompressedData != 0; - } - template void Block::setCompressedData(const uint8_t* const data, uint32_t dataLength) { @@ -137,32 +84,6 @@ namespace PolyVox memcpy(m_pCompressedData, data, dataLength); } - template - void Block::setVoxelAt(uint16_t uXPos, uint16_t uYPos, uint16_t uZPos, VoxelType tValue) - { - // This is internal code not directly called by the user. For efficiency we assert rather than throwing. - POLYVOX_ASSERT(uXPos < m_uSideLength, "Supplied position is outside of the block"); - POLYVOX_ASSERT(uYPos < m_uSideLength, "Supplied position is outside of the block"); - POLYVOX_ASSERT(uZPos < m_uSideLength, "Supplied position is outside of the block"); - POLYVOX_ASSERT(hasUncompressedData(), "The block must have uncompressed data to call setVoxelAt()"); - POLYVOX_ASSERT(m_tUncompressedData, "No uncompressed data - block must be decompressed before accessing voxels."); - - m_tUncompressedData - [ - uXPos + - uYPos * m_uSideLength + - uZPos * m_uSideLength * m_uSideLength - ] = tValue; - - m_bIsUncompressedDataModified = true; - } - - template - void Block::setVoxelAt(const Vector3DUint16& v3dPos, VoxelType tValue) - { - setVoxelAt(v3dPos.getX(), v3dPos.getY(), v3dPos.getZ(), tValue); - } - template uint32_t Block::calculateSizeInBytes(void) { diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl index 770c94ee..46b89c36 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl @@ -746,7 +746,7 @@ namespace PolyVox } } - POLYVOX_ASSERT(itUnloadBlock->second.hasUncompressedData() == false, "This function should never flush blocks with uncompressed data."); + //POLYVOX_ASSERT(itUnloadBlock->second.hasUncompressedData() == false, "This function should never flush blocks with uncompressed data."); uMemoryToReclaim -= itUnloadBlock->second.calculateSizeInBytes(); From a00574351ffa94225dd7272618e2d7d8a6a4d885 Mon Sep 17 00:00:00 2001 From: David Williams Date: Thu, 11 Jul 2013 16:38:28 +0200 Subject: [PATCH 16/55] Renamed Block to CompressedBlock --- examples/Paging/main.cpp | 4 +-- .../include/PolyVoxCore/FilePager.h | 4 +-- .../include/PolyVoxCore/Impl/Block.h | 4 +-- .../include/PolyVoxCore/Impl/Block.inl | 12 ++++----- .../include/PolyVoxCore/LargeVolume.h | 6 ++--- .../include/PolyVoxCore/LargeVolume.inl | 26 +++++++++---------- .../PolyVoxCore/include/PolyVoxCore/Pager.h | 4 +-- 7 files changed, 30 insertions(+), 30 deletions(-) diff --git a/examples/Paging/main.cpp b/examples/Paging/main.cpp index c497a255..07877405 100644 --- a/examples/Paging/main.cpp +++ b/examples/Paging/main.cpp @@ -91,7 +91,7 @@ public: /// Destructor virtual ~PerlinNoisePager() {}; - virtual void pageIn(const PolyVox::Region& region, Block* pBlockData) + virtual void pageIn(const PolyVox::Region& region, CompressedBlock* pBlockData) { /*pBlockData->createUncompressedData(); @@ -137,7 +137,7 @@ public: }*/ } - virtual void pageOut(const PolyVox::Region& region, Block* /*pBlockData*/) + virtual void pageOut(const PolyVox::Region& region, CompressedBlock* /*pBlockData*/) { std::cout << "warning unloading region: " << region.getLowerCorner() << " -> " << region.getUpperCorner() << std::endl; } diff --git a/library/PolyVoxCore/include/PolyVoxCore/FilePager.h b/library/PolyVoxCore/include/PolyVoxCore/FilePager.h index 7fb79f1c..80c4f05a 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/FilePager.h +++ b/library/PolyVoxCore/include/PolyVoxCore/FilePager.h @@ -52,7 +52,7 @@ namespace PolyVox /// Destructor virtual ~FilePager() {}; - virtual void pageIn(const Region& region, Block* pBlockData) + virtual void pageIn(const Region& region, CompressedBlock* pBlockData) { POLYVOX_ASSERT(pBlockData, "Attempting to page in NULL block"); //POLYVOX_ASSERT(pBlockData->hasUncompressedData() == false, "Block should not have uncompressed data"); @@ -93,7 +93,7 @@ namespace PolyVox } } - virtual void pageOut(const Region& region, Block* pBlockData) + virtual void pageOut(const Region& region, CompressedBlock* pBlockData) { POLYVOX_ASSERT(pBlockData, "Attempting to page out NULL block"); //POLYVOX_ASSERT(pBlockData->hasUncompressedData() == false, "Block should not have uncompressed data"); diff --git a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h index 86d6d08b..f9ce6f3c 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h +++ b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h @@ -35,10 +35,10 @@ freely, subject to the following restrictions: namespace PolyVox { template - class Block + class CompressedBlock { public: - Block(uint16_t uSideLength, Compressor* pCompressor); + CompressedBlock(uint16_t uSideLength, Compressor* pCompressor); const uint8_t* getCompressedData(void) const; uint32_t getCompressedDataLength(void) const; diff --git a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl index 3f311e0d..8f54c9a3 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl @@ -36,7 +36,7 @@ freely, subject to the following restrictions: namespace PolyVox { template - Block::Block(uint16_t uSideLength, Compressor* pCompressor) + CompressedBlock::CompressedBlock(uint16_t uSideLength, Compressor* pCompressor) :m_pCompressedData(0) ,m_uCompressedDataLength(0) { @@ -58,21 +58,21 @@ namespace PolyVox } template - const uint8_t* Block::getCompressedData(void) const + const uint8_t* CompressedBlock::getCompressedData(void) const { POLYVOX_ASSERT(m_pCompressedData, "Compressed data is NULL"); return m_pCompressedData; } template - uint32_t Block::getCompressedDataLength(void) const + uint32_t CompressedBlock::getCompressedDataLength(void) const { POLYVOX_ASSERT(m_pCompressedData, "Compressed data is NULL"); return m_uCompressedDataLength; } template - void Block::setCompressedData(const uint8_t* const data, uint32_t dataLength) + void CompressedBlock::setCompressedData(const uint8_t* const data, uint32_t dataLength) { //POLYVOX_ASSERT(m_pCompressedData, "Compressed data is NULL"); POLYVOX_ASSERT(m_pCompressedData != data, "Attempting to copy data onto itself"); @@ -85,11 +85,11 @@ namespace PolyVox } template - uint32_t Block::calculateSizeInBytes(void) + uint32_t CompressedBlock::calculateSizeInBytes(void) { // Returns the size of this class plus the size of the compressed data. It // doesn't include the uncompressed data cache as that is owned by the volume. - uint32_t uSizeInBytes = sizeof(Block) + m_uCompressedDataLength; + uint32_t uSizeInBytes = sizeof(CompressedBlock) + m_uCompressedDataLength; return uSizeInBytes; } } diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h index 86e36381..c4d3c777 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h @@ -329,13 +329,13 @@ namespace PolyVox VoxelType getVoxelImpl(int32_t uXPos, int32_t uYPos, int32_t uZPos, WrapModeType, VoxelType tBorder) const; VoxelType getVoxelImpl(int32_t uXPos, int32_t uYPos, int32_t uZPos, WrapModeType, VoxelType tBorder) const; - Block* getCompressedBlock(int32_t uBlockX, int32_t uBlockY, int32_t uBlockZ) const; + CompressedBlock* getCompressedBlock(int32_t uBlockX, int32_t uBlockY, int32_t uBlockZ) const; VoxelType* getUncompressedBlock(int32_t uBlockX, int32_t uBlockY, int32_t uBlockZ) const; - void eraseBlock(typename std::map, BlockPositionCompare>::iterator itBlock) const; + void eraseBlock(typename std::map, BlockPositionCompare>::iterator itBlock) const; // The block data mutable std::map m_pUncompressedBlockCache; - mutable std::map, BlockPositionCompare> m_pBlocks; + mutable std::map, BlockPositionCompare> m_pBlocks; mutable uint32_t m_uTimestamper; mutable Vector3DInt32 m_v3dLastAccessedBlockPos; diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl index 46b89c36..ff72cde0 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl @@ -421,7 +421,7 @@ namespace PolyVox for(int32_t z = v3dStart.getZ(); z <= v3dEnd.getZ(); z++) { Vector3DInt32 pos(x,y,z); - typename std::map, BlockPositionCompare>::iterator itBlock = m_pBlocks.find(pos); + typename std::map, BlockPositionCompare>::iterator itBlock = m_pBlocks.find(pos); if(itBlock != m_pBlocks.end()) { @@ -452,7 +452,7 @@ namespace PolyVox template void LargeVolume::flushAll() { - typename std::map, BlockPositionCompare>::iterator i; + typename std::map, BlockPositionCompare>::iterator i; //Replaced the for loop here as the call to //eraseBlock was invalidating the iterator. while(m_pBlocks.size() > 0) @@ -486,7 +486,7 @@ namespace PolyVox for(int32_t z = v3dStart.getZ(); z <= v3dEnd.getZ(); z++) { Vector3DInt32 pos(x,y,z); - typename std::map, BlockPositionCompare>::iterator itBlock = m_pBlocks.find(pos); + typename std::map, BlockPositionCompare>::iterator itBlock = m_pBlocks.find(pos); if(itBlock == m_pBlocks.end()) { // not loaded, not unloading @@ -564,7 +564,7 @@ namespace PolyVox } template - void LargeVolume::eraseBlock(typename std::map, BlockPositionCompare>::iterator itBlock) const + void LargeVolume::eraseBlock(typename std::map, BlockPositionCompare>::iterator itBlock) const { POLYVOX_ASSERT(false, "This function has not been implemented properly"); @@ -603,16 +603,16 @@ namespace PolyVox } template - Block* LargeVolume::getCompressedBlock(int32_t uBlockX, int32_t uBlockY, int32_t uBlockZ) const + CompressedBlock* LargeVolume::getCompressedBlock(int32_t uBlockX, int32_t uBlockY, int32_t uBlockZ) const { Vector3DInt32 v3dBlockPos(uBlockX, uBlockY, uBlockZ); - typename std::map, BlockPositionCompare>::iterator itBlock = m_pBlocks.find(v3dBlockPos); + typename std::map, BlockPositionCompare>::iterator itBlock = m_pBlocks.find(v3dBlockPos); // check whether the block is already loaded if(itBlock == m_pBlocks.end()) { //The block is not in the map, so we will have to create a new block and add it. - Block newBlock(m_uBlockSideLength, m_pCompressor); + CompressedBlock newBlock(m_uBlockSideLength, m_pCompressor); itBlock = m_pBlocks.insert(std::make_pair(v3dBlockPos, newBlock)).first; // Now use the pager to fill the block with it's initial data. @@ -626,7 +626,7 @@ namespace PolyVox } //Get the block and mark that we accessed it - Block& block = itBlock->second; + CompressedBlock& block = itBlock->second; block.timestamp = ++m_uTimestamper; //m_v3dLastAccessedBlockPos = v3dBlockPos; @@ -648,7 +648,7 @@ namespace PolyVox } //Get the block and mark that we accessed it - Block* block = getCompressedBlock(uBlockX, uBlockY, uBlockZ); + CompressedBlock* block = getCompressedBlock(uBlockX, uBlockY, uBlockZ); typename std::map::iterator itUncompressedBlock = m_pUncompressedBlockCache.find(v3dBlockPos); @@ -697,7 +697,7 @@ namespace PolyVox uint32_t uSizeInBytes = sizeof(LargeVolume); //Memory used by the blocks - typename std::map, BlockPositionCompare>::iterator i; + typename std::map, BlockPositionCompare>::iterator i; for(i = m_pBlocks.begin(); i != m_pBlocks.end(); i++) { //Inaccurate - account for rest of loaded block. @@ -716,7 +716,7 @@ namespace PolyVox { uint32_t uMemoryUsage = 0; - typename std::map, BlockPositionCompare>::iterator i; + typename std::map, BlockPositionCompare>::iterator i; for(i = m_pBlocks.begin(); i != m_pBlocks.end(); i++) { //Inaccurate - account for rest of loaded block. @@ -736,8 +736,8 @@ namespace PolyVox while(calculateBlockMemoryUsage() > m_uCompressedBlockMemoryLimitInBytes) //FIXME - This calculation of size is slow and should be outside the loop. { // find the least recently used block - typename std::map, BlockPositionCompare>::iterator i; - typename std::map, BlockPositionCompare>::iterator itUnloadBlock = m_pBlocks.begin(); + typename std::map, BlockPositionCompare>::iterator i; + typename std::map, BlockPositionCompare>::iterator itUnloadBlock = m_pBlocks.begin(); for(i = m_pBlocks.begin(); i != m_pBlocks.end(); i++) { if(i->second.timestamp < itUnloadBlock->second.timestamp) diff --git a/library/PolyVoxCore/include/PolyVoxCore/Pager.h b/library/PolyVoxCore/include/PolyVoxCore/Pager.h index 4cb95adc..af1e3387 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/Pager.h +++ b/library/PolyVoxCore/include/PolyVoxCore/Pager.h @@ -41,8 +41,8 @@ namespace PolyVox /// Destructor virtual ~Pager() {}; - virtual void pageIn(const Region& region, Block* pBlockData) = 0; - virtual void pageOut(const Region& region, Block* pBlockData) = 0; + virtual void pageIn(const Region& region, CompressedBlock* pBlockData) = 0; + virtual void pageOut(const Region& region, CompressedBlock* pBlockData) = 0; }; } From 2acb98bdcbf5d0ffa3d098846ba70f45577a48f1 Mon Sep 17 00:00:00 2001 From: David Williams Date: Tue, 16 Jul 2013 11:50:59 +0200 Subject: [PATCH 17/55] Compile warning fixes. --- library/PolyVoxCore/include/PolyVoxCore/BaseVolume.inl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/library/PolyVoxCore/include/PolyVoxCore/BaseVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/BaseVolume.inl index a78804ec..dc280ee2 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/BaseVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/BaseVolume.inl @@ -165,7 +165,7 @@ namespace PolyVox //////////////////////////////////////////////////////////////////////////////// template template - VoxelType BaseVolume::getVoxel(int32_t uXPos, int32_t uYPos, int32_t uZPos, VoxelType tBorder) const + VoxelType BaseVolume::getVoxel(int32_t /*uXPos*/, int32_t /*uYPos*/, int32_t /*uZPos*/, VoxelType /*tBorder*/) const { POLYVOX_ASSERT(false, "You should never call the base class version of this function."); return VoxelType(); @@ -183,7 +183,7 @@ namespace PolyVox //////////////////////////////////////////////////////////////////////////////// template template - VoxelType BaseVolume::getVoxel(const Vector3DInt32& v3dPos, VoxelType tBorder) const + VoxelType BaseVolume::getVoxel(const Vector3DInt32& /*v3dPos*/, VoxelType /*tBorder*/) const { POLYVOX_ASSERT(false, "You should never call the base class version of this function."); return VoxelType(); @@ -200,7 +200,7 @@ namespace PolyVox /// \return The voxel value //////////////////////////////////////////////////////////////////////////////// template - VoxelType BaseVolume::getVoxel(int32_t uXPos, int32_t uYPos, int32_t uZPos, WrapMode eWrapMode, VoxelType tBorder) const + VoxelType BaseVolume::getVoxel(int32_t /*uXPos*/, int32_t /*uYPos*/, int32_t /*uZPos*/, WrapMode /*eWrapMode*/, VoxelType /*tBorder*/) const { POLYVOX_ASSERT(false, "You should never call the base class version of this function."); return VoxelType(); @@ -215,7 +215,7 @@ namespace PolyVox /// \return The voxel value //////////////////////////////////////////////////////////////////////////////// template - VoxelType BaseVolume::getVoxel(const Vector3DInt32& v3dPos, WrapMode eWrapMode, VoxelType tBorder) const + VoxelType BaseVolume::getVoxel(const Vector3DInt32& /*v3dPos*/, WrapMode /*eWrapMode*/, VoxelType /*tBorder*/) const { POLYVOX_ASSERT(false, "You should never call the base class version of this function."); return VoxelType(); From 0cfb9f51961324d533d52deeb04a3e3da2198729 Mon Sep 17 00:00:00 2001 From: David Williams Date: Tue, 16 Jul 2013 14:42:43 +0200 Subject: [PATCH 18/55] Splitting 'Block into CompressedBlock and UncompressedBlock. --- .../include/PolyVoxCore/Impl/Block.h | 20 ++++ .../include/PolyVoxCore/Impl/Block.inl | 99 +++++++++++++++++-- .../include/PolyVoxCore/LargeVolume.h | 6 +- .../include/PolyVoxCore/LargeVolume.inl | 46 +++------ .../PolyVoxCore/LargeVolumeSampler.inl | 4 +- 5 files changed, 127 insertions(+), 48 deletions(-) diff --git a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h index f9ce6f3c..edd3a893 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h +++ b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h @@ -52,6 +52,26 @@ namespace PolyVox uint32_t m_uCompressedDataLength; uint32_t timestamp; }; + + template + class UncompressedBlock + { + public: + UncompressedBlock(uint16_t uSideLength); + ~UncompressedBlock(); + + uint16_t getSideLength(void) const; + VoxelType getVoxel(uint16_t uXPos, uint16_t uYPos, uint16_t uZPos) const; + VoxelType getVoxel(const Vector3DUint16& v3dPos) const; + + void setVoxelAt(uint16_t uXPos, uint16_t uYPos, uint16_t uZPos, VoxelType tValue); + void setVoxelAt(const Vector3DUint16& v3dPos, VoxelType tValue); + + VoxelType* m_tUncompressedData; + uint16_t m_uSideLength; + uint8_t m_uSideLengthPower; + bool m_bIsUncompressedDataModified; + }; } #include "PolyVoxCore/Impl/Block.inl" diff --git a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl index 8f54c9a3..19f50146 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl @@ -9,16 +9,16 @@ Permission is granted to anyone to use this software for any purpose, including commercial applications, and to alter it and redistribute it freely, subject to the following restrictions: - 1. The origin of this software must not be misrepresented; you must not - claim that you wrote the original software. If you use this software - in a product, an acknowledgment in the product documentation would be - appreciated but is not required. +1. The origin of this software must not be misrepresented; you must not +claim that you wrote the original software. If you use this software +in a product, an acknowledgment in the product documentation would be +appreciated but is not required. - 2. Altered source versions must be plainly marked as such, and must not be - misrepresented as being the original software. +2. Altered source versions must be plainly marked as such, and must not be +misrepresented as being the original software. - 3. This notice may not be removed or altered from any source - distribution. +3. This notice may not be removed or altered from any source +distribution. *******************************************************************************/ #include "PolyVoxCore/Impl/ErrorHandling.h" @@ -35,6 +35,8 @@ freely, subject to the following restrictions: namespace PolyVox { + //////////////////////////////////////////////////////////////////////////////// + template CompressedBlock::CompressedBlock(uint16_t uSideLength, Compressor* pCompressor) :m_pCompressedData(0) @@ -63,7 +65,7 @@ namespace PolyVox POLYVOX_ASSERT(m_pCompressedData, "Compressed data is NULL"); return m_pCompressedData; } - + template uint32_t CompressedBlock::getCompressedDataLength(void) const { @@ -92,4 +94,83 @@ namespace PolyVox uint32_t uSizeInBytes = sizeof(CompressedBlock) + m_uCompressedDataLength; return uSizeInBytes; } + + //////////////////////////////////////////////////////////////////////////////// + + template + UncompressedBlock::UncompressedBlock(uint16_t uSideLength) + :m_tUncompressedData(0) + ,m_uSideLength(0) + ,m_uSideLengthPower(0) + ,m_bIsUncompressedDataModified(true) + { + // Compute the side length + m_uSideLength = uSideLength; + m_uSideLengthPower = logBase2(uSideLength); + + // Allocate the data + const uint32_t uNoOfVoxels = m_uSideLength * m_uSideLength * m_uSideLength; + m_tUncompressedData = new VoxelType[uNoOfVoxels]; + } + + template + UncompressedBlock::~UncompressedBlock() + { + delete m_tUncompressedData; + m_tUncompressedData = 0; + } + + template + uint16_t UncompressedBlock::getSideLength(void) const + { + return m_uSideLength; + } + + template + VoxelType UncompressedBlock::getVoxel(uint16_t uXPos, uint16_t uYPos, uint16_t uZPos) const + { + // This is internal code not directly called by the user. For efficiency we assert rather than throwing. + POLYVOX_ASSERT(uXPos < m_uSideLength, "Supplied position is outside of the block"); + POLYVOX_ASSERT(uYPos < m_uSideLength, "Supplied position is outside of the block"); + POLYVOX_ASSERT(uZPos < m_uSideLength, "Supplied position is outside of the block"); + POLYVOX_ASSERT(m_tUncompressedData, "No uncompressed data - block must be decompressed before accessing voxels."); + + return m_tUncompressedData + [ + uXPos + + uYPos * m_uSideLength + + uZPos * m_uSideLength * m_uSideLength + ]; + } + + template + VoxelType UncompressedBlock::getVoxel(const Vector3DUint16& v3dPos) const + { + return getVoxel(v3dPos.getX(), v3dPos.getY(), v3dPos.getZ()); + } + + template + void UncompressedBlock::setVoxelAt(uint16_t uXPos, uint16_t uYPos, uint16_t uZPos, VoxelType tValue) + { + // This is internal code not directly called by the user. For efficiency we assert rather than throwing. + POLYVOX_ASSERT(uXPos < m_uSideLength, "Supplied position is outside of the block"); + POLYVOX_ASSERT(uYPos < m_uSideLength, "Supplied position is outside of the block"); + POLYVOX_ASSERT(uZPos < m_uSideLength, "Supplied position is outside of the block"); + POLYVOX_ASSERT(m_tUncompressedData, "No uncompressed data - block must be decompressed before accessing voxels."); + + m_tUncompressedData + [ + uXPos + + uYPos * m_uSideLength + + uZPos * m_uSideLength * m_uSideLength + ] = tValue; + + m_bIsUncompressedDataModified = true; + } + + template + void UncompressedBlock::setVoxelAt(const Vector3DUint16& v3dPos, VoxelType tValue) + { + setVoxelAt(v3dPos.getX(), v3dPos.getY(), v3dPos.getZ(), tValue); + } } diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h index c4d3c777..65e9dec4 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h @@ -330,16 +330,16 @@ namespace PolyVox VoxelType getVoxelImpl(int32_t uXPos, int32_t uYPos, int32_t uZPos, WrapModeType, VoxelType tBorder) const; CompressedBlock* getCompressedBlock(int32_t uBlockX, int32_t uBlockY, int32_t uBlockZ) const; - VoxelType* getUncompressedBlock(int32_t uBlockX, int32_t uBlockY, int32_t uBlockZ) const; + UncompressedBlock* getUncompressedBlock(int32_t uBlockX, int32_t uBlockY, int32_t uBlockZ) const; void eraseBlock(typename std::map, BlockPositionCompare>::iterator itBlock) const; // The block data - mutable std::map m_pUncompressedBlockCache; + mutable std::map*, BlockPositionCompare> m_pUncompressedBlockCache; mutable std::map, BlockPositionCompare> m_pBlocks; mutable uint32_t m_uTimestamper; mutable Vector3DInt32 m_v3dLastAccessedBlockPos; - mutable VoxelType* m_pLastAccessedBlock; + mutable UncompressedBlock* m_pLastAccessedBlock; uint32_t m_uMaxNumberOfUncompressedBlocks; uint32_t m_uCompressedBlockMemoryLimitInBytes; diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl index ff72cde0..3357f60c 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl @@ -223,14 +223,9 @@ namespace PolyVox const uint16_t yOffset = static_cast(uYPos - (blockY << m_uBlockSideLengthPower)); const uint16_t zOffset = static_cast(uZPos - (blockZ << m_uBlockSideLengthPower)); - VoxelType* pUncompressedBlock = getUncompressedBlock(blockX, blockY, blockZ); + const UncompressedBlock* pUncompressedBlock = getUncompressedBlock(blockX, blockY, blockZ); - return pUncompressedBlock - [ - xOffset + - yOffset * m_uBlockSideLength + - zOffset * m_uBlockSideLength * m_uBlockSideLength - ]; + return pUncompressedBlock->getVoxel(xOffset, yOffset, zOffset); } else { @@ -317,14 +312,8 @@ namespace PolyVox const uint16_t yOffset = static_cast(uYPos - (blockY << m_uBlockSideLengthPower)); const uint16_t zOffset = static_cast(uZPos - (blockZ << m_uBlockSideLengthPower)); - VoxelType* pUncompressedBlock = getUncompressedBlock(blockX, blockY, blockZ); - - pUncompressedBlock - [ - xOffset + - yOffset * m_uBlockSideLength + - zOffset * m_uBlockSideLength * m_uBlockSideLength - ] = tValue; + UncompressedBlock* pUncompressedBlock = getUncompressedBlock(blockX, blockY, blockZ); + pUncompressedBlock->setVoxelAt(xOffset, yOffset, zOffset); } //////////////////////////////////////////////////////////////////////////////// @@ -360,14 +349,9 @@ namespace PolyVox const uint16_t yOffset = static_cast(uYPos - (blockY << m_uBlockSideLengthPower)); const uint16_t zOffset = static_cast(uZPos - (blockZ << m_uBlockSideLengthPower)); - VoxelType* pUncompressedBlock = getUncompressedBlock(blockX, blockY, blockZ); + UncompressedBlock* pUncompressedBlock = getUncompressedBlock(blockX, blockY, blockZ); - pUncompressedBlock - [ - xOffset + - yOffset * m_uBlockSideLength + - zOffset * m_uBlockSideLength * m_uBlockSideLength - ] = tValue; + pUncompressedBlock->setVoxelAt(xOffset, yOffset, zOffset, tValue); //Return true to indicate that we modified a voxel. return true; @@ -634,7 +618,7 @@ namespace PolyVox } template - VoxelType* LargeVolume::getUncompressedBlock(int32_t uBlockX, int32_t uBlockY, int32_t uBlockZ) const + UncompressedBlock* LargeVolume::getUncompressedBlock(int32_t uBlockX, int32_t uBlockY, int32_t uBlockZ) const { Vector3DInt32 v3dBlockPos(uBlockX, uBlockY, uBlockZ); @@ -651,14 +635,14 @@ namespace PolyVox CompressedBlock* block = getCompressedBlock(uBlockX, uBlockY, uBlockZ); - typename std::map::iterator itUncompressedBlock = m_pUncompressedBlockCache.find(v3dBlockPos); + typename std::map*, BlockPositionCompare>::iterator itUncompressedBlock = m_pUncompressedBlockCache.find(v3dBlockPos); // check whether the block is already loaded if(itUncompressedBlock == m_pUncompressedBlockCache.end()) { - VoxelType* pUncompressedBlock = new VoxelType[m_uBlockSideLength * m_uBlockSideLength * m_uBlockSideLength]; + UncompressedBlock* pUncompressedBlock = new UncompressedBlock(m_uBlockSideLength); void* pSrcData = reinterpret_cast(block->m_pCompressedData); - void* pDstData = reinterpret_cast(pUncompressedBlock); + void* pDstData = reinterpret_cast(pUncompressedBlock->m_tUncompressedData); uint32_t uSrcLength = block->m_uCompressedDataLength; uint32_t uDstLength = m_uBlockSideLength * m_uBlockSideLength * m_uBlockSideLength * sizeof(VoxelType); @@ -816,14 +800,8 @@ namespace PolyVox const uint16_t yOffset = static_cast(uYPos - (blockY << m_uBlockSideLengthPower)); const uint16_t zOffset = static_cast(uZPos - (blockZ << m_uBlockSideLengthPower)); - VoxelType* pUncompressedBlock = getUncompressedBlock(blockX, blockY, blockZ); - - return pUncompressedBlock - [ - xOffset + - yOffset * m_uBlockSideLength + - zOffset * m_uBlockSideLength * m_uBlockSideLength - ]; + UncompressedBlock* pUncompressedBlock = getUncompressedBlock(blockX, blockY, blockZ); + return pUncompressedBlock->getVoxel(xOffset, yOffset, zOffset); } } diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolumeSampler.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolumeSampler.inl index d90ea5f4..8bbad0f4 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolumeSampler.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolumeSampler.inl @@ -119,9 +119,9 @@ namespace PolyVox uYPosInBlock * this->mVolume->m_uBlockSideLength + uZPosInBlock * this->mVolume->m_uBlockSideLength * this->mVolume->m_uBlockSideLength; - VoxelType* pUncompressedCurrentBlock = this->mVolume->getUncompressedBlock(uXBlock, uYBlock, uZBlock); + UncompressedBlock* pUncompressedCurrentBlock = this->mVolume->getUncompressedBlock(uXBlock, uYBlock, uZBlock); - mCurrentVoxel = pUncompressedCurrentBlock + uVoxelIndexInBlock; + mCurrentVoxel = pUncompressedCurrentBlock->m_tUncompressedData + uVoxelIndexInBlock; } else { From 3904c9aa8f798795c16e5ce9688a530686b306e9 Mon Sep 17 00:00:00 2001 From: David Williams Date: Tue, 16 Jul 2013 15:59:06 +0200 Subject: [PATCH 19/55] Tidying and renaming... --- .../include/PolyVoxCore/FilePager.h | 4 +- .../include/PolyVoxCore/Impl/Block.h | 35 ++++++++---- .../include/PolyVoxCore/Impl/Block.inl | 54 +++++++------------ .../include/PolyVoxCore/LargeVolume.inl | 10 ++-- 4 files changed, 52 insertions(+), 51 deletions(-) diff --git a/library/PolyVoxCore/include/PolyVoxCore/FilePager.h b/library/PolyVoxCore/include/PolyVoxCore/FilePager.h index 80c4f05a..66091d9c 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/FilePager.h +++ b/library/PolyVoxCore/include/PolyVoxCore/FilePager.h @@ -77,7 +77,7 @@ namespace PolyVox uint8_t* buffer = new uint8_t[fileSizeInBytes]; fread(buffer, sizeof(uint8_t), fileSizeInBytes, pFile); - pBlockData->setCompressedData(buffer, fileSizeInBytes); + pBlockData->setData(buffer, fileSizeInBytes); delete[] buffer; if(ferror(pFile)) @@ -115,7 +115,7 @@ namespace PolyVox POLYVOX_THROW(std::runtime_error, "Unable to open file to write out block data."); } - fwrite(pBlockData->getCompressedData(), sizeof(uint8_t), pBlockData->getCompressedDataLength(), pFile); + fwrite(pBlockData->getData(), sizeof(uint8_t), pBlockData->getDataSizeInBytes(), pFile); if(ferror(pFile)) { diff --git a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h index edd3a893..2b9893b5 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h +++ b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h @@ -35,26 +35,42 @@ freely, subject to the following restrictions: namespace PolyVox { template - class CompressedBlock + class Block + { + friend LargeVolume; + + public: + Block() + :m_uBlockLastAccessed(0) + ,m_bDataModified(false) + { + } + + protected: + uint32_t m_uBlockLastAccessed; + bool m_bDataModified; + }; + + template + class CompressedBlock : public Block { public: - CompressedBlock(uint16_t uSideLength, Compressor* pCompressor); + CompressedBlock(); - const uint8_t* getCompressedData(void) const; - uint32_t getCompressedDataLength(void) const; + const uint8_t* getData(void) const; + uint32_t getDataSizeInBytes(void) const; - void setCompressedData(const uint8_t* const data, uint32_t dataLength); + void setData(const uint8_t* const pData, uint32_t uDataSizeInBytes); uint32_t calculateSizeInBytes(void); public: - uint8_t* m_pCompressedData; - uint32_t m_uCompressedDataLength; - uint32_t timestamp; + uint8_t* m_pData; + uint32_t m_uDataSizeInBytes; }; template - class UncompressedBlock + class UncompressedBlock : public Block { public: UncompressedBlock(uint16_t uSideLength); @@ -70,7 +86,6 @@ namespace PolyVox VoxelType* m_tUncompressedData; uint16_t m_uSideLength; uint8_t m_uSideLengthPower; - bool m_bIsUncompressedDataModified; }; } diff --git a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl index 19f50146..492d829f 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl @@ -38,52 +38,39 @@ namespace PolyVox //////////////////////////////////////////////////////////////////////////////// template - CompressedBlock::CompressedBlock(uint16_t uSideLength, Compressor* pCompressor) - :m_pCompressedData(0) - ,m_uCompressedDataLength(0) + CompressedBlock::CompressedBlock() + :m_pData(0) + ,m_uDataSizeInBytes(0) { - if(uSideLength == 0) - { - POLYVOX_THROW(std::invalid_argument, "Block side length cannot be zero."); - } - - if(!isPowerOf2(uSideLength)) - { - POLYVOX_THROW(std::invalid_argument, "Block side length must be a power of two."); - } - - if(pCompressor == 0) - { - POLYVOX_THROW(std::invalid_argument, "Block must be provided with a valid compressor."); - } - } template - const uint8_t* CompressedBlock::getCompressedData(void) const + const uint8_t* CompressedBlock::getData(void) const { - POLYVOX_ASSERT(m_pCompressedData, "Compressed data is NULL"); - return m_pCompressedData; + return m_pData; } template - uint32_t CompressedBlock::getCompressedDataLength(void) const + uint32_t CompressedBlock::getDataSizeInBytes(void) const { - POLYVOX_ASSERT(m_pCompressedData, "Compressed data is NULL"); - return m_uCompressedDataLength; + return m_uDataSizeInBytes; } template - void CompressedBlock::setCompressedData(const uint8_t* const data, uint32_t dataLength) + void CompressedBlock::setData(const uint8_t* const pData, uint32_t uDataSizeInBytes) { - //POLYVOX_ASSERT(m_pCompressedData, "Compressed data is NULL"); - POLYVOX_ASSERT(m_pCompressedData != data, "Attempting to copy data onto itself"); + POLYVOX_THROW_IF(pData == 0, std::invalid_argument, "Pointer to data cannot be null"); + POLYVOX_THROW_IF(m_pData == pData, std::invalid_argument, "Attempting to copy data onto itself"); - delete[] m_pCompressedData; + // Delete any existing data + delete[] m_pData; - m_uCompressedDataLength = dataLength; - m_pCompressedData = new uint8_t[dataLength]; - memcpy(m_pCompressedData, data, dataLength); + // Allocate new data + m_uDataSizeInBytes = uDataSizeInBytes; + m_pData = new uint8_t[uDataSizeInBytes]; + + // Copy the data across + memcpy(m_pData, pData, uDataSizeInBytes); } template @@ -91,7 +78,7 @@ namespace PolyVox { // Returns the size of this class plus the size of the compressed data. It // doesn't include the uncompressed data cache as that is owned by the volume. - uint32_t uSizeInBytes = sizeof(CompressedBlock) + m_uCompressedDataLength; + uint32_t uSizeInBytes = sizeof(CompressedBlock) + m_uDataSizeInBytes; return uSizeInBytes; } @@ -102,7 +89,6 @@ namespace PolyVox :m_tUncompressedData(0) ,m_uSideLength(0) ,m_uSideLengthPower(0) - ,m_bIsUncompressedDataModified(true) { // Compute the side length m_uSideLength = uSideLength; @@ -165,7 +151,7 @@ namespace PolyVox uZPos * m_uSideLength * m_uSideLength ] = tValue; - m_bIsUncompressedDataModified = true; + m_bDataModified = true; } template diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl index 3357f60c..ce482006 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl @@ -596,7 +596,7 @@ namespace PolyVox if(itBlock == m_pBlocks.end()) { //The block is not in the map, so we will have to create a new block and add it. - CompressedBlock newBlock(m_uBlockSideLength, m_pCompressor); + CompressedBlock newBlock; itBlock = m_pBlocks.insert(std::make_pair(v3dBlockPos, newBlock)).first; // Now use the pager to fill the block with it's initial data. @@ -611,7 +611,7 @@ namespace PolyVox //Get the block and mark that we accessed it CompressedBlock& block = itBlock->second; - block.timestamp = ++m_uTimestamper; + block.m_uBlockLastAccessed = ++m_uTimestamper; //m_v3dLastAccessedBlockPos = v3dBlockPos; return █ @@ -641,9 +641,9 @@ namespace PolyVox { UncompressedBlock* pUncompressedBlock = new UncompressedBlock(m_uBlockSideLength); - void* pSrcData = reinterpret_cast(block->m_pCompressedData); + void* pSrcData = reinterpret_cast(block->m_pData); void* pDstData = reinterpret_cast(pUncompressedBlock->m_tUncompressedData); - uint32_t uSrcLength = block->m_uCompressedDataLength; + uint32_t uSrcLength = block->m_uDataSizeInBytes; uint32_t uDstLength = m_uBlockSideLength * m_uBlockSideLength * m_uBlockSideLength * sizeof(VoxelType); //MinizCompressor compressor; @@ -724,7 +724,7 @@ namespace PolyVox typename std::map, BlockPositionCompare>::iterator itUnloadBlock = m_pBlocks.begin(); for(i = m_pBlocks.begin(); i != m_pBlocks.end(); i++) { - if(i->second.timestamp < itUnloadBlock->second.timestamp) + if(i->second.m_uBlockLastAccessed < itUnloadBlock->second.m_uBlockLastAccessed) { itUnloadBlock = i; } From b5d930062bae0ca20ef5df64a7f71cb8360bb406 Mon Sep 17 00:00:00 2001 From: David Williams Date: Tue, 16 Jul 2013 16:09:57 +0200 Subject: [PATCH 20/55] Const fixes. --- library/PolyVoxCore/include/PolyVoxCore/Compressor.h | 4 ++-- library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h | 3 ++- .../PolyVoxCore/include/PolyVoxCore/Impl/Block.inl | 10 ++++++++++ .../PolyVoxCore/include/PolyVoxCore/LargeVolume.inl | 4 ++-- .../include/PolyVoxCore/MinizCompressor.h | 4 ++-- .../PolyVoxCore/include/PolyVoxCore/RLECompressor.h | 4 ++-- .../include/PolyVoxCore/RLECompressor.inl | 12 ++++++------ library/PolyVoxCore/source/MinizCompressor.cpp | 4 ++-- 8 files changed, 28 insertions(+), 17 deletions(-) diff --git a/library/PolyVoxCore/include/PolyVoxCore/Compressor.h b/library/PolyVoxCore/include/PolyVoxCore/Compressor.h index 8c00be5e..e85610de 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/Compressor.h +++ b/library/PolyVoxCore/include/PolyVoxCore/Compressor.h @@ -76,7 +76,7 @@ namespace PolyVox * \param uDstLength The length of the destination buffer (compression will fail if this isn't big enough). * \return The size of the resulting compressed data. */ - virtual uint32_t compress(void* pSrcData, uint32_t uSrcLength, void* pDstData, uint32_t uDstLength) = 0; + virtual uint32_t compress(const void* pSrcData, uint32_t uSrcLength, void* pDstData, uint32_t uDstLength) = 0; /** * Decompresses the data. @@ -93,7 +93,7 @@ namespace PolyVox * \param uDstLength The length of the destination buffer (decompression will fail if this isn't big enough). * \return The size of the resulting uncompressed data. */ - virtual uint32_t decompress(void* pSrcData, uint32_t uSrcLength, void* pDstData, uint32_t uDstLength) = 0; + virtual uint32_t decompress(const void* pSrcData, uint32_t uSrcLength, void* pDstData, uint32_t uDstLength) = 0; }; } diff --git a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h index 2b9893b5..32cb9857 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h +++ b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h @@ -56,6 +56,7 @@ namespace PolyVox { public: CompressedBlock(); + ~CompressedBlock(); const uint8_t* getData(void) const; uint32_t getDataSizeInBytes(void) const; @@ -64,7 +65,7 @@ namespace PolyVox uint32_t calculateSizeInBytes(void); - public: + private: uint8_t* m_pData; uint32_t m_uDataSizeInBytes; }; diff --git a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl index 492d829f..b59d6f0a 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl @@ -44,6 +44,13 @@ namespace PolyVox { } + template + CompressedBlock::~CompressedBlock() + { + delete[] m_pData; + m_pData = 0; + } + template const uint8_t* CompressedBlock::getData(void) const { @@ -71,6 +78,9 @@ namespace PolyVox // Copy the data across memcpy(m_pData, pData, uDataSizeInBytes); + + // Flag as modified + m_bDataModified = true; } template diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl index ce482006..fdf4bcac 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl @@ -641,9 +641,9 @@ namespace PolyVox { UncompressedBlock* pUncompressedBlock = new UncompressedBlock(m_uBlockSideLength); - void* pSrcData = reinterpret_cast(block->m_pData); + const void* pSrcData = reinterpret_cast(block->getData()); void* pDstData = reinterpret_cast(pUncompressedBlock->m_tUncompressedData); - uint32_t uSrcLength = block->m_uDataSizeInBytes; + uint32_t uSrcLength = block->getDataSizeInBytes(); uint32_t uDstLength = m_uBlockSideLength * m_uBlockSideLength * m_uBlockSideLength * sizeof(VoxelType); //MinizCompressor compressor; diff --git a/library/PolyVoxCore/include/PolyVoxCore/MinizCompressor.h b/library/PolyVoxCore/include/PolyVoxCore/MinizCompressor.h index fb2b2473..7ea8f967 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/MinizCompressor.h +++ b/library/PolyVoxCore/include/PolyVoxCore/MinizCompressor.h @@ -25,8 +25,8 @@ namespace PolyVox // API documentation is in base class and gets inherited by Doxygen. uint32_t getMaxCompressedSize(uint32_t uUncompressedInputSize); - uint32_t compress(void* pSrcData, uint32_t uSrcLength, void* pDstData, uint32_t uDstLength); - uint32_t decompress(void* pSrcData, uint32_t uSrcLength, void* pDstData, uint32_t uDstLength); + uint32_t compress(const void* pSrcData, uint32_t uSrcLength, void* pDstData, uint32_t uDstLength); + uint32_t decompress(const void* pSrcData, uint32_t uSrcLength, void* pDstData, uint32_t uDstLength); private: unsigned int m_uCompressionFlags; diff --git a/library/PolyVoxCore/include/PolyVoxCore/RLECompressor.h b/library/PolyVoxCore/include/PolyVoxCore/RLECompressor.h index 10b52b90..03fd3318 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/RLECompressor.h +++ b/library/PolyVoxCore/include/PolyVoxCore/RLECompressor.h @@ -30,8 +30,8 @@ namespace PolyVox // API documentation is in base class and gets inherited by Doxygen. uint32_t getMaxCompressedSize(uint32_t uUncompressedInputSize); - uint32_t compress(void* pSrcData, uint32_t uSrcLength, void* pDstData, uint32_t uDstLength); - uint32_t decompress(void* pSrcData, uint32_t uSrcLength, void* pDstData, uint32_t uDstLength); + uint32_t compress(const void* pSrcData, uint32_t uSrcLength, void* pDstData, uint32_t uDstLength); + uint32_t decompress(const void* pSrcData, uint32_t uSrcLength, void* pDstData, uint32_t uDstLength); }; } diff --git a/library/PolyVoxCore/include/PolyVoxCore/RLECompressor.inl b/library/PolyVoxCore/include/PolyVoxCore/RLECompressor.inl index 0b43a301..cf65bc03 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/RLECompressor.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/RLECompressor.inl @@ -22,7 +22,7 @@ namespace PolyVox } template - uint32_t RLECompressor::compress(void* pSrcData, uint32_t uSrcLength, void* pDstData, uint32_t uDstLength) + uint32_t RLECompressor::compress(const void* pSrcData, uint32_t uSrcLength, void* pDstData, uint32_t uDstLength) { if(uSrcLength % sizeof(ValueType) != 0) { @@ -34,11 +34,11 @@ namespace PolyVox uDstLength /= sizeof(Run); // Get data pointers in the appropriate type - ValueType* pSrcDataAsType = reinterpret_cast(pSrcData); + const ValueType* pSrcDataAsType = reinterpret_cast(pSrcData); Run* pDstDataAsRun = reinterpret_cast(pDstData); // Pointers to just past the end of the data - ValueType* pSrcDataEnd = pSrcDataAsType + uSrcLength; + const ValueType* pSrcDataEnd = pSrcDataAsType + uSrcLength; Run* pDstDataEnd = pDstDataAsRun + uDstLength; //Counter for the output length @@ -83,7 +83,7 @@ namespace PolyVox } template - uint32_t RLECompressor::decompress(void* pSrcData, uint32_t uSrcLength, void* pDstData, uint32_t uDstLength) + uint32_t RLECompressor::decompress(const void* pSrcData, uint32_t uSrcLength, void* pDstData, uint32_t uDstLength) { if(uSrcLength % sizeof(Run) != 0) { @@ -95,11 +95,11 @@ namespace PolyVox uDstLength /= sizeof(ValueType); // Get data pointers in the appropriate type - Run* pSrcDataAsRun = reinterpret_cast(pSrcData); + const Run* pSrcDataAsRun = reinterpret_cast(pSrcData); ValueType* pDstDataAsType = reinterpret_cast(pDstData); // Pointers to just past the end of the data - Run* pSrcDataEnd = pSrcDataAsRun + uSrcLength; + const Run* pSrcDataEnd = pSrcDataAsRun + uSrcLength; ValueType* pDstDataEnd = pDstDataAsType + uDstLength; //Counter for the output length diff --git a/library/PolyVoxCore/source/MinizCompressor.cpp b/library/PolyVoxCore/source/MinizCompressor.cpp index a341ff81..aa35c644 100644 --- a/library/PolyVoxCore/source/MinizCompressor.cpp +++ b/library/PolyVoxCore/source/MinizCompressor.cpp @@ -70,7 +70,7 @@ namespace PolyVox return MZ_MAX(128 + (source_len * 110) / 100, 128 + source_len + ((source_len / (31 * 1024)) + 1) * 5); } - uint32_t MinizCompressor::compress(void* pSrcData, uint32_t uSrcLength, void* pDstData, uint32_t uDstLength) + uint32_t MinizCompressor::compress(const void* pSrcData, uint32_t uSrcLength, void* pDstData, uint32_t uDstLength) { //Get the deflator tdefl_compressor* pDeflator = reinterpret_cast(m_pDeflator); @@ -103,7 +103,7 @@ namespace PolyVox return ulDstLength; } - uint32_t MinizCompressor::decompress(void* pSrcData, uint32_t uSrcLength, void* pDstData, uint32_t uDstLength) + uint32_t MinizCompressor::decompress(const void* pSrcData, uint32_t uSrcLength, void* pDstData, uint32_t uDstLength) { // I don't know exactly why this limitation exists but it's an implementation detail of miniz. It shouldn't matter for our purposes // as our detination is a Block and those are always a power of two. If you need to use this class for other purposes then you'll From dea7e6a4e9055904a99fb7613b5a882ad1daab9c Mon Sep 17 00:00:00 2001 From: David Williams Date: Tue, 16 Jul 2013 16:50:04 +0200 Subject: [PATCH 21/55] Tidying up block classes. --- .../include/PolyVoxCore/Impl/Block.h | 20 ++++++++-- .../include/PolyVoxCore/Impl/Block.inl | 39 ++++++++++--------- .../include/PolyVoxCore/LargeVolume.inl | 2 +- .../PolyVoxCore/LargeVolumeSampler.inl | 2 +- 4 files changed, 40 insertions(+), 23 deletions(-) diff --git a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h index 32cb9857..e78ecb48 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h +++ b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h @@ -47,13 +47,19 @@ namespace PolyVox } protected: + // This is updated by the LargeVolume and used to discard the least recently used blocks. uint32_t m_uBlockLastAccessed; + + // This is so we can tell whether a uncompressed block has to be recompressed and whether + // a compressed block has to be paged back to disk, or whether they can just be discarded. bool m_bDataModified; }; template class CompressedBlock : public Block { + friend LargeVolume; + public: CompressedBlock(); ~CompressedBlock(); @@ -63,9 +69,11 @@ namespace PolyVox void setData(const uint8_t* const pData, uint32_t uDataSizeInBytes); + private: + // Made this private to avoid any confusion with getDataSizeInBytes(). + // Users shouldn't really need this for CompressedBlock anyway. uint32_t calculateSizeInBytes(void); - private: uint8_t* m_pData; uint32_t m_uDataSizeInBytes; }; @@ -73,18 +81,24 @@ namespace PolyVox template class UncompressedBlock : public Block { + friend LargeVolume; + public: UncompressedBlock(uint16_t uSideLength); ~UncompressedBlock(); - uint16_t getSideLength(void) const; VoxelType getVoxel(uint16_t uXPos, uint16_t uYPos, uint16_t uZPos) const; VoxelType getVoxel(const Vector3DUint16& v3dPos) const; void setVoxelAt(uint16_t uXPos, uint16_t uYPos, uint16_t uZPos, VoxelType tValue); void setVoxelAt(const Vector3DUint16& v3dPos, VoxelType tValue); - VoxelType* m_tUncompressedData; + private: + // Made this private for consistancy with CompressedBlock. + // Users shouldn't really need this for UncompressedBlock anyway. + uint32_t calculateSizeInBytes(void); + + VoxelType* m_tData; uint16_t m_uSideLength; uint8_t m_uSideLengthPower; }; diff --git a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl index b59d6f0a..dab2f35d 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl @@ -86,8 +86,7 @@ namespace PolyVox template uint32_t CompressedBlock::calculateSizeInBytes(void) { - // Returns the size of this class plus the size of the compressed data. It - // doesn't include the uncompressed data cache as that is owned by the volume. + // Returns the size of this class plus the size of the compressed data. uint32_t uSizeInBytes = sizeof(CompressedBlock) + m_uDataSizeInBytes; return uSizeInBytes; } @@ -96,7 +95,7 @@ namespace PolyVox template UncompressedBlock::UncompressedBlock(uint16_t uSideLength) - :m_tUncompressedData(0) + :m_tData(0) ,m_uSideLength(0) ,m_uSideLengthPower(0) { @@ -106,32 +105,27 @@ namespace PolyVox // Allocate the data const uint32_t uNoOfVoxels = m_uSideLength * m_uSideLength * m_uSideLength; - m_tUncompressedData = new VoxelType[uNoOfVoxels]; + m_tData = new VoxelType[uNoOfVoxels]; } template UncompressedBlock::~UncompressedBlock() { - delete m_tUncompressedData; - m_tUncompressedData = 0; - } - - template - uint16_t UncompressedBlock::getSideLength(void) const - { - return m_uSideLength; + delete m_tData; + m_tData = 0; } template VoxelType UncompressedBlock::getVoxel(uint16_t uXPos, uint16_t uYPos, uint16_t uZPos) const { - // This is internal code not directly called by the user. For efficiency we assert rather than throwing. + // This code is not usually expected to be called by the user, with the exception of when implementing paging + // of uncompressed data. It's a performance critical code path so we use asserts rather than exceptions. POLYVOX_ASSERT(uXPos < m_uSideLength, "Supplied position is outside of the block"); POLYVOX_ASSERT(uYPos < m_uSideLength, "Supplied position is outside of the block"); POLYVOX_ASSERT(uZPos < m_uSideLength, "Supplied position is outside of the block"); - POLYVOX_ASSERT(m_tUncompressedData, "No uncompressed data - block must be decompressed before accessing voxels."); + POLYVOX_ASSERT(m_tData, "No uncompressed data - block must be decompressed before accessing voxels."); - return m_tUncompressedData + return m_tData [ uXPos + uYPos * m_uSideLength + @@ -148,13 +142,14 @@ namespace PolyVox template void UncompressedBlock::setVoxelAt(uint16_t uXPos, uint16_t uYPos, uint16_t uZPos, VoxelType tValue) { - // This is internal code not directly called by the user. For efficiency we assert rather than throwing. + // This code is not usually expected to be called by the user, with the exception of when implementing paging + // of uncompressed data. It's a performance critical code path so we use asserts rather than exceptions. POLYVOX_ASSERT(uXPos < m_uSideLength, "Supplied position is outside of the block"); POLYVOX_ASSERT(uYPos < m_uSideLength, "Supplied position is outside of the block"); POLYVOX_ASSERT(uZPos < m_uSideLength, "Supplied position is outside of the block"); - POLYVOX_ASSERT(m_tUncompressedData, "No uncompressed data - block must be decompressed before accessing voxels."); + POLYVOX_ASSERT(m_tData, "No uncompressed data - block must be decompressed before accessing voxels."); - m_tUncompressedData + m_tData [ uXPos + uYPos * m_uSideLength + @@ -169,4 +164,12 @@ namespace PolyVox { setVoxelAt(v3dPos.getX(), v3dPos.getY(), v3dPos.getZ(), tValue); } + + template + uint32_t UncompressedBlock::calculateSizeInBytes(void) + { + // Returns the size of this class plus the size of the uncompressed data. + uint32_t uSizeInBytes = sizeof(UncompressedBlock) + (m_uSideLength * m_uSideLength * m_uSideLength * sizeof(VoxelType)); + return uSizeInBytes; + } } diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl index fdf4bcac..731a6c01 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl @@ -642,7 +642,7 @@ namespace PolyVox UncompressedBlock* pUncompressedBlock = new UncompressedBlock(m_uBlockSideLength); const void* pSrcData = reinterpret_cast(block->getData()); - void* pDstData = reinterpret_cast(pUncompressedBlock->m_tUncompressedData); + void* pDstData = reinterpret_cast(pUncompressedBlock->m_tData); uint32_t uSrcLength = block->getDataSizeInBytes(); uint32_t uDstLength = m_uBlockSideLength * m_uBlockSideLength * m_uBlockSideLength * sizeof(VoxelType); diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolumeSampler.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolumeSampler.inl index 8bbad0f4..c5e4e12c 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolumeSampler.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolumeSampler.inl @@ -121,7 +121,7 @@ namespace PolyVox UncompressedBlock* pUncompressedCurrentBlock = this->mVolume->getUncompressedBlock(uXBlock, uYBlock, uZBlock); - mCurrentVoxel = pUncompressedCurrentBlock->m_tUncompressedData + uVoxelIndexInBlock; + mCurrentVoxel = pUncompressedCurrentBlock->m_tData + uVoxelIndexInBlock; } else { From 5b99854c02a2aa46a74895ef15ecea839158c24d Mon Sep 17 00:00:00 2001 From: David Williams Date: Wed, 17 Jul 2013 16:23:46 +0200 Subject: [PATCH 22/55] Making block copy constructors and assignment operators private to prevent accidental copying. --- .../include/PolyVoxCore/Impl/Block.h | 19 ++++++++++ .../include/PolyVoxCore/LargeVolume.h | 4 +-- .../include/PolyVoxCore/LargeVolume.inl | 36 +++++++++---------- 3 files changed, 39 insertions(+), 20 deletions(-) diff --git a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h index e78ecb48..3d795e26 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h +++ b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h @@ -53,6 +53,13 @@ namespace PolyVox // This is so we can tell whether a uncompressed block has to be recompressed and whether // a compressed block has to be paged back to disk, or whether they can just be discarded. bool m_bDataModified; + + private: + /// Private copy constructor to prevent accisdental copying + Block(const Block& /*rhs*/) {}; + + /// Private assignment operator to prevent accisdental copying + Block& operator=(const Block& /*rhs*/) {}; }; template @@ -70,6 +77,12 @@ namespace PolyVox void setData(const uint8_t* const pData, uint32_t uDataSizeInBytes); private: + /// Private copy constructor to prevent accisdental copying + CompressedBlock(const CompressedBlock& /*rhs*/) {}; + + /// Private assignment operator to prevent accisdental copying + CompressedBlock& operator=(const CompressedBlock& /*rhs*/) {}; + // Made this private to avoid any confusion with getDataSizeInBytes(). // Users shouldn't really need this for CompressedBlock anyway. uint32_t calculateSizeInBytes(void); @@ -94,6 +107,12 @@ namespace PolyVox void setVoxelAt(const Vector3DUint16& v3dPos, VoxelType tValue); private: + /// Private copy constructor to prevent accisdental copying + UncompressedBlock(const UncompressedBlock& /*rhs*/) {}; + + /// Private assignment operator to prevent accisdental copying + UncompressedBlock& operator=(const UncompressedBlock& /*rhs*/) {}; + // Made this private for consistancy with CompressedBlock. // Users shouldn't really need this for UncompressedBlock anyway. uint32_t calculateSizeInBytes(void); diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h index 65e9dec4..01781cc6 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h @@ -331,11 +331,11 @@ namespace PolyVox CompressedBlock* getCompressedBlock(int32_t uBlockX, int32_t uBlockY, int32_t uBlockZ) const; UncompressedBlock* getUncompressedBlock(int32_t uBlockX, int32_t uBlockY, int32_t uBlockZ) const; - void eraseBlock(typename std::map, BlockPositionCompare>::iterator itBlock) const; + void eraseBlock(typename std::map*, BlockPositionCompare>::iterator itBlock) const; // The block data mutable std::map*, BlockPositionCompare> m_pUncompressedBlockCache; - mutable std::map, BlockPositionCompare> m_pBlocks; + mutable std::map*, BlockPositionCompare> m_pBlocks; mutable uint32_t m_uTimestamper; mutable Vector3DInt32 m_v3dLastAccessedBlockPos; diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl index 731a6c01..42692706 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl @@ -405,7 +405,7 @@ namespace PolyVox for(int32_t z = v3dStart.getZ(); z <= v3dEnd.getZ(); z++) { Vector3DInt32 pos(x,y,z); - typename std::map, BlockPositionCompare>::iterator itBlock = m_pBlocks.find(pos); + typename std::map*, BlockPositionCompare>::iterator itBlock = m_pBlocks.find(pos); if(itBlock != m_pBlocks.end()) { @@ -436,7 +436,7 @@ namespace PolyVox template void LargeVolume::flushAll() { - typename std::map, BlockPositionCompare>::iterator i; + typename std::map*, BlockPositionCompare>::iterator i; //Replaced the for loop here as the call to //eraseBlock was invalidating the iterator. while(m_pBlocks.size() > 0) @@ -470,7 +470,7 @@ namespace PolyVox for(int32_t z = v3dStart.getZ(); z <= v3dEnd.getZ(); z++) { Vector3DInt32 pos(x,y,z); - typename std::map, BlockPositionCompare>::iterator itBlock = m_pBlocks.find(pos); + typename std::map*, BlockPositionCompare>::iterator itBlock = m_pBlocks.find(pos); if(itBlock == m_pBlocks.end()) { // not loaded, not unloading @@ -548,7 +548,7 @@ namespace PolyVox } template - void LargeVolume::eraseBlock(typename std::map, BlockPositionCompare>::iterator itBlock) const + void LargeVolume::eraseBlock(typename std::map*, BlockPositionCompare>::iterator itBlock) const { POLYVOX_ASSERT(false, "This function has not been implemented properly"); @@ -591,30 +591,30 @@ namespace PolyVox { Vector3DInt32 v3dBlockPos(uBlockX, uBlockY, uBlockZ); - typename std::map, BlockPositionCompare>::iterator itBlock = m_pBlocks.find(v3dBlockPos); + typename std::map*, BlockPositionCompare>::iterator itBlock = m_pBlocks.find(v3dBlockPos); // check whether the block is already loaded if(itBlock == m_pBlocks.end()) { //The block is not in the map, so we will have to create a new block and add it. - CompressedBlock newBlock; + CompressedBlock* newBlock = new CompressedBlock; itBlock = m_pBlocks.insert(std::make_pair(v3dBlockPos, newBlock)).first; // Now use the pager to fill the block with it's initial data. Vector3DInt32 v3dLower(v3dBlockPos.getX() << m_uBlockSideLengthPower, v3dBlockPos.getY() << m_uBlockSideLengthPower, v3dBlockPos.getZ() << m_uBlockSideLengthPower); Vector3DInt32 v3dUpper = v3dLower + Vector3DInt32(m_uBlockSideLength-1, m_uBlockSideLength-1, m_uBlockSideLength-1); Region reg(v3dLower, v3dUpper); - m_pPager->pageIn(reg, &(itBlock->second)); + m_pPager->pageIn(reg, itBlock->second); // Paging in this new block may mean we are now using too much memory. If necessary, flush some old blocks. flushOldestExcessiveBlocks(); } //Get the block and mark that we accessed it - CompressedBlock& block = itBlock->second; - block.m_uBlockLastAccessed = ++m_uTimestamper; + CompressedBlock* block = itBlock->second; + block->m_uBlockLastAccessed = ++m_uTimestamper; //m_v3dLastAccessedBlockPos = v3dBlockPos; - return █ + return block; } template @@ -681,11 +681,11 @@ namespace PolyVox uint32_t uSizeInBytes = sizeof(LargeVolume); //Memory used by the blocks - typename std::map, BlockPositionCompare>::iterator i; + typename std::map*, BlockPositionCompare>::iterator i; for(i = m_pBlocks.begin(); i != m_pBlocks.end(); i++) { //Inaccurate - account for rest of loaded block. - uSizeInBytes += i->second.calculateSizeInBytes(); + uSizeInBytes += i->second->calculateSizeInBytes(); } //Memory used by the block cache. @@ -700,11 +700,11 @@ namespace PolyVox { uint32_t uMemoryUsage = 0; - typename std::map, BlockPositionCompare>::iterator i; + typename std::map*, BlockPositionCompare>::iterator i; for(i = m_pBlocks.begin(); i != m_pBlocks.end(); i++) { //Inaccurate - account for rest of loaded block. - uMemoryUsage += i->second.calculateSizeInBytes(); + uMemoryUsage += i->second->calculateSizeInBytes(); } return uMemoryUsage; @@ -720,11 +720,11 @@ namespace PolyVox while(calculateBlockMemoryUsage() > m_uCompressedBlockMemoryLimitInBytes) //FIXME - This calculation of size is slow and should be outside the loop. { // find the least recently used block - typename std::map, BlockPositionCompare>::iterator i; - typename std::map, BlockPositionCompare>::iterator itUnloadBlock = m_pBlocks.begin(); + typename std::map*, BlockPositionCompare>::iterator i; + typename std::map*, BlockPositionCompare>::iterator itUnloadBlock = m_pBlocks.begin(); for(i = m_pBlocks.begin(); i != m_pBlocks.end(); i++) { - if(i->second.m_uBlockLastAccessed < itUnloadBlock->second.m_uBlockLastAccessed) + if(i->second->m_uBlockLastAccessed < itUnloadBlock->second->m_uBlockLastAccessed) { itUnloadBlock = i; } @@ -732,7 +732,7 @@ namespace PolyVox //POLYVOX_ASSERT(itUnloadBlock->second.hasUncompressedData() == false, "This function should never flush blocks with uncompressed data."); - uMemoryToReclaim -= itUnloadBlock->second.calculateSizeInBytes(); + uMemoryToReclaim -= itUnloadBlock->second->calculateSizeInBytes(); eraseBlock(itUnloadBlock); } From 404f12a43e57ea4222bd1a515389cc54ce101a86 Mon Sep 17 00:00:00 2001 From: David Williams Date: Wed, 17 Jul 2013 16:33:46 +0200 Subject: [PATCH 23/55] Added typedefs to simplify code. --- .../include/PolyVoxCore/LargeVolume.h | 11 ++++++---- .../include/PolyVoxCore/LargeVolume.inl | 20 +++++++++---------- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h index 01781cc6..02e226a5 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h @@ -312,7 +312,10 @@ namespace PolyVox } return false; } - }; + }; + + typedef std::map*, BlockPositionCompare> CompressedBlockMap; + typedef std::map*, BlockPositionCompare> UncompressedBlockMap; uint32_t calculateBlockMemoryUsage(void) const; @@ -331,11 +334,11 @@ namespace PolyVox CompressedBlock* getCompressedBlock(int32_t uBlockX, int32_t uBlockY, int32_t uBlockZ) const; UncompressedBlock* getUncompressedBlock(int32_t uBlockX, int32_t uBlockY, int32_t uBlockZ) const; - void eraseBlock(typename std::map*, BlockPositionCompare>::iterator itBlock) const; + void eraseBlock(typename CompressedBlockMap::iterator itBlock) const; // The block data - mutable std::map*, BlockPositionCompare> m_pUncompressedBlockCache; - mutable std::map*, BlockPositionCompare> m_pBlocks; + mutable CompressedBlockMap m_pBlocks; + mutable UncompressedBlockMap m_pUncompressedBlockCache; mutable uint32_t m_uTimestamper; mutable Vector3DInt32 m_v3dLastAccessedBlockPos; diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl index 42692706..5ba67966 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl @@ -405,7 +405,7 @@ namespace PolyVox for(int32_t z = v3dStart.getZ(); z <= v3dEnd.getZ(); z++) { Vector3DInt32 pos(x,y,z); - typename std::map*, BlockPositionCompare>::iterator itBlock = m_pBlocks.find(pos); + typename CompressedBlockMap::iterator itBlock = m_pBlocks.find(pos); if(itBlock != m_pBlocks.end()) { @@ -436,7 +436,7 @@ namespace PolyVox template void LargeVolume::flushAll() { - typename std::map*, BlockPositionCompare>::iterator i; + typename CompressedBlockMap::iterator i; //Replaced the for loop here as the call to //eraseBlock was invalidating the iterator. while(m_pBlocks.size() > 0) @@ -470,7 +470,7 @@ namespace PolyVox for(int32_t z = v3dStart.getZ(); z <= v3dEnd.getZ(); z++) { Vector3DInt32 pos(x,y,z); - typename std::map*, BlockPositionCompare>::iterator itBlock = m_pBlocks.find(pos); + typename CompressedBlockMap::iterator itBlock = m_pBlocks.find(pos); if(itBlock == m_pBlocks.end()) { // not loaded, not unloading @@ -548,7 +548,7 @@ namespace PolyVox } template - void LargeVolume::eraseBlock(typename std::map*, BlockPositionCompare>::iterator itBlock) const + void LargeVolume::eraseBlock(typename CompressedBlockMap::iterator itBlock) const { POLYVOX_ASSERT(false, "This function has not been implemented properly"); @@ -591,7 +591,7 @@ namespace PolyVox { Vector3DInt32 v3dBlockPos(uBlockX, uBlockY, uBlockZ); - typename std::map*, BlockPositionCompare>::iterator itBlock = m_pBlocks.find(v3dBlockPos); + typename CompressedBlockMap::iterator itBlock = m_pBlocks.find(v3dBlockPos); // check whether the block is already loaded if(itBlock == m_pBlocks.end()) { @@ -635,7 +635,7 @@ namespace PolyVox CompressedBlock* block = getCompressedBlock(uBlockX, uBlockY, uBlockZ); - typename std::map*, BlockPositionCompare>::iterator itUncompressedBlock = m_pUncompressedBlockCache.find(v3dBlockPos); + typename UncompressedBlockMap::iterator itUncompressedBlock = m_pUncompressedBlockCache.find(v3dBlockPos); // check whether the block is already loaded if(itUncompressedBlock == m_pUncompressedBlockCache.end()) { @@ -681,7 +681,7 @@ namespace PolyVox uint32_t uSizeInBytes = sizeof(LargeVolume); //Memory used by the blocks - typename std::map*, BlockPositionCompare>::iterator i; + typename CompressedBlockMap::iterator i; for(i = m_pBlocks.begin(); i != m_pBlocks.end(); i++) { //Inaccurate - account for rest of loaded block. @@ -700,7 +700,7 @@ namespace PolyVox { uint32_t uMemoryUsage = 0; - typename std::map*, BlockPositionCompare>::iterator i; + typename CompressedBlockMap::iterator i; for(i = m_pBlocks.begin(); i != m_pBlocks.end(); i++) { //Inaccurate - account for rest of loaded block. @@ -720,8 +720,8 @@ namespace PolyVox while(calculateBlockMemoryUsage() > m_uCompressedBlockMemoryLimitInBytes) //FIXME - This calculation of size is slow and should be outside the loop. { // find the least recently used block - typename std::map*, BlockPositionCompare>::iterator i; - typename std::map*, BlockPositionCompare>::iterator itUnloadBlock = m_pBlocks.begin(); + typename CompressedBlockMap::iterator i; + typename CompressedBlockMap::iterator itUnloadBlock = m_pBlocks.begin(); for(i = m_pBlocks.begin(); i != m_pBlocks.end(); i++) { if(i->second->m_uBlockLastAccessed < itUnloadBlock->second->m_uBlockLastAccessed) From 1e986f9aa6f3e654d19f02e9f682e4d91e040e99 Mon Sep 17 00:00:00 2001 From: David Williams Date: Wed, 17 Jul 2013 17:00:12 +0200 Subject: [PATCH 24/55] Reimplemented eraseBlock for compressed blocks. --- .../include/PolyVoxCore/LargeVolume.h | 4 +- .../include/PolyVoxCore/LargeVolume.inl | 47 +++++++------------ 2 files changed, 21 insertions(+), 30 deletions(-) diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h index 02e226a5..809b9020 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h @@ -334,7 +334,9 @@ namespace PolyVox CompressedBlock* getCompressedBlock(int32_t uBlockX, int32_t uBlockY, int32_t uBlockZ) const; UncompressedBlock* getUncompressedBlock(int32_t uBlockX, int32_t uBlockY, int32_t uBlockZ) const; - void eraseBlock(typename CompressedBlockMap::iterator itBlock) const; + + void eraseBlock(typename CompressedBlockMap::iterator itCompressedBlock) const; + void eraseBlock(typename UncompressedBlockMap::iterator itUncompressedBlock) const; // The block data mutable CompressedBlockMap m_pBlocks; diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl index 5ba67966..c8dec3e3 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl @@ -548,42 +548,31 @@ namespace PolyVox } template - void LargeVolume::eraseBlock(typename CompressedBlockMap::iterator itBlock) const + void LargeVolume::eraseBlock(typename CompressedBlockMap::iterator itCompressedBlock) const { - POLYVOX_ASSERT(false, "This function has not been implemented properly"); - - /*if(itBlock->second.hasUncompressedData()) + // Before deleting the block we may need to page out it's data. We + // only do this if the data has been modified since it was paged in. + CompressedBlock* pCompressedBlock = itCompressedBlock->second; + if(pCompressedBlock->m_bDataModified) { - itBlock->second.destroyUncompressedData(); - } + // The position of the block within the volume. + Vector3DInt32 v3dBlockPos = itCompressedBlock->first; - if(m_pPager) - { - Vector3DInt32 v3dPos = itBlock->first; - Vector3DInt32 v3dLower(v3dPos.getX() << m_uBlockSideLengthPower, v3dPos.getY() << m_uBlockSideLengthPower, v3dPos.getZ() << m_uBlockSideLengthPower); + // From the coordinates of the block we deduce the coordinates of the contained voxels. + Vector3DInt32 v3dLower(v3dBlockPos.getX() << m_uBlockSideLengthPower, v3dBlockPos.getY() << m_uBlockSideLengthPower, v3dBlockPos.getZ() << m_uBlockSideLengthPower); Vector3DInt32 v3dUpper = v3dLower + Vector3DInt32(m_uBlockSideLength-1, m_uBlockSideLength-1, m_uBlockSideLength-1); - Region reg(v3dLower, v3dUpper); + // Page the data out + m_pPager->pageOut(Region(v3dLower, v3dUpper), pCompressedBlock); + } - m_pPager->pageOut(reg, &(itBlock->second)); - }*/ + // We can now remove the block data from memory. + m_pBlocks.erase(itCompressedBlock); + } - - // FIXME - the code below used to make sure the uncompressed version of the data was removed. Reinstate something similar. - /*for(uint32_t ct = 0; ct < m_vecBlocksWithUncompressedData.size(); ct++) - { - // find the block in the uncompressed cache - if(m_vecBlocksWithUncompressedData[ct] == &(itBlock->second)) - { - // put last object in cache here - m_vecBlocksWithUncompressedData[ct] = m_vecBlocksWithUncompressedData.back(); - // decrease cache size by one since last element is now in here twice - m_vecBlocksWithUncompressedData.resize(m_vecBlocksWithUncompressedData.size()-1); - break; - } - }*/ - - m_pBlocks.erase(itBlock); + template + void LargeVolume::eraseBlock(typename UncompressedBlockMap::iterator itUncompressedBlock) const + { } template From 88cbeb309c8297bf3489b0c8bc463e8307e61490 Mon Sep 17 00:00:00 2001 From: David Williams Date: Thu, 18 Jul 2013 16:08:18 +0200 Subject: [PATCH 25/55] Reimplemented eraseBlock for uncompressed blocks. --- .../include/PolyVoxCore/LargeVolume.inl | 76 ++++++++++++++++++- 1 file changed, 74 insertions(+), 2 deletions(-) diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl index c8dec3e3..99e75787 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl @@ -550,9 +550,10 @@ namespace PolyVox template void LargeVolume::eraseBlock(typename CompressedBlockMap::iterator itCompressedBlock) const { - // Before deleting the block we may need to page out it's data. We - // only do this if the data has been modified since it was paged in. CompressedBlock* pCompressedBlock = itCompressedBlock->second; + + // Before deleting the block we may need to page out its data. We + // only do this if the data has been modified since it was paged in. if(pCompressedBlock->m_bDataModified) { // The position of the block within the volume. @@ -573,6 +574,77 @@ namespace PolyVox template void LargeVolume::eraseBlock(typename UncompressedBlockMap::iterator itUncompressedBlock) const { + UncompressedBlock* pUncompressedBlock = itUncompressedBlock->second; + + // This should not often happen as blocks are normally deleted based on being least recently used. + // However, I can imagine that flushing a large number of blocks could cause this to occur. Just + // to be safe we handle it by invalidating the last accessed block pointer. + if(pUncompressedBlock == m_pLastAccessedBlock) + { + logWarning() << "The last accessed block is being erased from the uncompressed cache."; + m_pLastAccessedBlock = 0; + } + + // Before deleting the block we may need to recompress its data. We + // only do this if the data has been modified since it was decompressed. + if(pUncompressedBlock->m_bDataModified) + { + // Get the compressed block which we will copy the data back in to. + Vector3DInt32 v3dBlockPos = itCompressedBlock->first; + CompressedBlock* pCompressedBlock = getCompressedBlock(v3dBlockPos); + + void* pSrcData = reinterpret_cast(pUncompressedBlock->m_tData); + uint32_t uSrcLength = m_uSideLength * m_uSideLength * m_uSideLength * sizeof(VoxelType); + + uint8_t tempBuffer[10000]; + void* pDstData = reinterpret_cast( tempBuffer ); + uint32_t uDstLength = 10000; + + uint32_t uCompressedLength = 0; + + try + { + // Perform the compression + uCompressedLength = m_pCompressor->compress(pSrcData, uSrcLength, pDstData, uDstLength); + + // Copy the resulting compressed data into the compressed block + pCompressedBlock->setData(pDstData, uDstLength); + } + catch(std::exception&) + { + // It is possible for the compression to fail. A common cause for this would be if the destination + // buffer is not big enough. So now we try again using a buffer that is definitely big enough. + // Note that ideally we will choose our earlier buffer size so that this almost never happens. + logWarning() << "The compressor failed to compress the block, proabaly due to the buffer being too small."; + logWarning() << "The compression will be tried again with a larger buffer"; + uint32_t uMaxCompressedSize = m_pCompressor->getMaxCompressedSize(uSrcLength); + uint8_t* buffer = new uint8_t[ uMaxCompressedSize ]; + + pDstData = reinterpret_cast( buffer ); + uDstLength = uMaxCompressedSize; + + try + { + // Perform the compression + uCompressedLength = m_pCompressor->compress(pSrcData, uSrcLength, pDstData, uDstLength); + + // Copy the resulting compressed data into the compressed block + pCompressedBlock->setData(pDstData, uDstLength); + } + catch(std::exception&) + { + // At this point it didn't work even with a bigger buffer. + // Not much more we can do so just rethrow the exception. + delete[] buffer; + POLYVOX_THROW(std::runtime_error, "Failed to compress block data"); + } + + delete[] buffer; + } + } + + // We can now remove the block data from memory. + m_pUncompressedBlockCache.erase(itUncompressedBlock); } template From ef6bd31651f19af278e7ab21f1fcdbf84e8aed21 Mon Sep 17 00:00:00 2001 From: David Williams Date: Thu, 18 Jul 2013 17:03:14 +0200 Subject: [PATCH 26/55] Small fixes. --- .../PolyVoxCore/include/PolyVoxCore/LargeVolume.inl | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl index 99e75787..dae9d2d4 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl @@ -567,6 +567,8 @@ namespace PolyVox m_pPager->pageOut(Region(v3dLower, v3dUpper), pCompressedBlock); } + delete itCompressedBlock->second; + // We can now remove the block data from memory. m_pBlocks.erase(itCompressedBlock); } @@ -615,7 +617,7 @@ namespace PolyVox // It is possible for the compression to fail. A common cause for this would be if the destination // buffer is not big enough. So now we try again using a buffer that is definitely big enough. // Note that ideally we will choose our earlier buffer size so that this almost never happens. - logWarning() << "The compressor failed to compress the block, proabaly due to the buffer being too small."; + logWarning() << "The compressor failed to compress the block, probabaly due to the buffer being too small."; logWarning() << "The compression will be tried again with a larger buffer"; uint32_t uMaxCompressedSize = m_pCompressor->getMaxCompressedSize(uSrcLength); uint8_t* buffer = new uint8_t[ uMaxCompressedSize ]; @@ -658,6 +660,7 @@ namespace PolyVox { //The block is not in the map, so we will have to create a new block and add it. CompressedBlock* newBlock = new CompressedBlock; + newBlock->m_uBlockLastAccessed = ++m_uTimestamper; itBlock = m_pBlocks.insert(std::make_pair(v3dBlockPos, newBlock)).first; // Now use the pager to fill the block with it's initial data. @@ -692,15 +695,15 @@ namespace PolyVox return m_pLastAccessedBlock; } - //Get the block and mark that we accessed it + //Gets the block and marks that we accessed it CompressedBlock* block = getCompressedBlock(uBlockX, uBlockY, uBlockZ); - typename UncompressedBlockMap::iterator itUncompressedBlock = m_pUncompressedBlockCache.find(v3dBlockPos); // check whether the block is already loaded if(itUncompressedBlock == m_pUncompressedBlockCache.end()) { UncompressedBlock* pUncompressedBlock = new UncompressedBlock(m_uBlockSideLength); + pUncompressedBlock->m_uBlockLastAccessed = ++m_uTimestamper; const void* pSrcData = reinterpret_cast(block->getData()); void* pDstData = reinterpret_cast(pUncompressedBlock->m_tData); @@ -775,7 +778,7 @@ namespace PolyVox void LargeVolume::flushOldestExcessiveBlocks(void) const { const uint32_t uMemoryUsedForCompressedBlocks = calculateBlockMemoryUsage(); - uint32_t uMemoryToReclaim = uMemoryUsedForCompressedBlocks - m_uCompressedBlockMemoryLimitInBytes; + //uint32_t uMemoryToReclaim = uMemoryUsedForCompressedBlocks - m_uCompressedBlockMemoryLimitInBytes; //while(uMemoryToReclaim > 0) while(calculateBlockMemoryUsage() > m_uCompressedBlockMemoryLimitInBytes) //FIXME - This calculation of size is slow and should be outside the loop. @@ -793,7 +796,7 @@ namespace PolyVox //POLYVOX_ASSERT(itUnloadBlock->second.hasUncompressedData() == false, "This function should never flush blocks with uncompressed data."); - uMemoryToReclaim -= itUnloadBlock->second->calculateSizeInBytes(); + //uMemoryToReclaim -= itUnloadBlock->second->calculateSizeInBytes(); eraseBlock(itUnloadBlock); } From 7cd115b33d5b7f0d3ae23254441833fbc16e48bf Mon Sep 17 00:00:00 2001 From: David Williams Date: Fri, 19 Jul 2013 16:43:59 +0200 Subject: [PATCH 27/55] Work on code which computes how the memory should be split between compressed and uncompressed data. --- .../include/PolyVoxCore/LargeVolume.h | 2 +- .../include/PolyVoxCore/LargeVolume.inl | 60 +++++++++++++++---- 2 files changed, 50 insertions(+), 12 deletions(-) diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h index 809b9020..499a5f91 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h @@ -347,7 +347,7 @@ namespace PolyVox mutable UncompressedBlock* m_pLastAccessedBlock; uint32_t m_uMaxNumberOfUncompressedBlocks; - uint32_t m_uCompressedBlockMemoryLimitInBytes; + uint32_t m_uTargetMemoryLimitInBytes; // The size of the volume Region m_regValidRegionInBlocks; diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl index dae9d2d4..e174a578 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl @@ -246,23 +246,57 @@ namespace PolyVox template void LargeVolume::setTargetMemoryLimitInBytes(uint32_t uTargetMemoryLimitInBytes) { + // Set the limit, and then work out how big the uncompressed cache should be. + m_uTargetMemoryLimitInBytes = uTargetMemoryLimitInBytes; + // The size of a single uncompressed block of data. uint32_t uUncompressedBlockSizeInBytes = m_uBlockSideLength * m_uBlockSideLength * m_uBlockSideLength * sizeof(VoxelType); - // The idea number of uncompressed blocks is chosen by gut feeling as much as anything. Part of the - // rationale is that it should let us iterate along any edge without data being pushed out of the cache. - uint32_t uIdealNoOfUncompressedBlocks = m_regValidRegionInBlocks.getWidthInVoxels() + m_regValidRegionInBlocks.getHeightInVoxels() * m_regValidRegionInBlocks.getDepthInVoxels(); + // When deciding how to split the memory between compressed and uncompressed blocks it is useful to know the dimensions of the + // volume in blocks. After sorting, the array below will hold the smallest dimension in [0] and the largest dimension in [2]. + const uint32_t uNoOfDimensions = 3; + uint32_t uSortedDimensionsInBlocks[uNoOfDimensions]; + uSortedDimensionsInBlocks[0] = (getEnclosingRegion().getWidthInVoxels() >> m_uBlockSideLengthPower) + 1; + uSortedDimensionsInBlocks[1] = (getEnclosingRegion().getHeightInVoxels() >> m_uBlockSideLengthPower) + 1; + uSortedDimensionsInBlocks[2] = (getEnclosingRegion().getDepthInVoxels() >> m_uBlockSideLengthPower) + 1; + std::sort(uSortedDimensionsInBlocks, uSortedDimensionsInBlocks + uNoOfDimensions); - // Let's (arbitrarily?) say that we should never use more than half the available memory for the uncompressed block cache. - uint32_t uMaxMemoryForUncompressedBlocks = uTargetMemoryLimitInBytes / 2; + const uint32_t uLongestEdgeInBlocks = uSortedDimensionsInBlocks[2]; + const uint32_t uLargestFaceSizeInBlocks = uSortedDimensionsInBlocks[1] * uSortedDimensionsInBlocks[2]; - uint32_t uMaxFittableNoOfUncompressedBlocks = uMaxMemoryForUncompressedBlocks / uUncompressedBlockSizeInBytes; + // In the ideal situation we will be able to keep an entire face of blocks decompressed in the cache, and still have significant memory + // left for storing a large number of compressed block. Failing this, we would at least like to be able to keep enough blocks decompressed + // to iterate over the longest edge. Otherwise we will find that excessive compression and decompresion occurs. In both cases we multiply + // by an extra factor make sure there is space for storing some compressed data as well. + const uint32_t uFactor = 2; // Just to make sure we leave some space for compressed data as well + const uint32_t uIdealMemoryTarget = uLargestFaceSizeInBlocks * uUncompressedBlockSizeInBytes * uFactor; + const uint32_t uAcceptableMemoryTarget = uLongestEdgeInBlocks * uUncompressedBlockSizeInBytes * uFactor; - setMaxNumberOfUncompressedBlocks((std::min)(uIdealNoOfUncompressedBlocks, uMaxFittableNoOfUncompressedBlocks)); + if(uTargetMemoryLimitInBytes >= uIdealMemoryTarget) + { + logDebug() << "The target memory limit (" << uTargetMemoryLimitInBytes << "bytes) is plenty for a volume of this size)"; - uint32_t uUncompressedBlockCacheSizeInBytes = m_uMaxNumberOfUncompressedBlocks * uUncompressedBlockSizeInBytes; + // The added value below is just some extra 'grace' space. + setMaxNumberOfUncompressedBlocks(uLargestFaceSizeInBlocks + uSortedDimensionsInBlocks[1]); + } + else if(uTargetMemoryLimitInBytes >= uAcceptableMemoryTarget) + { + logDebug() << "The target memory limit (" << uTargetMemoryLimitInBytes << "bytes) should be acceptable for a volume of this size)"; - m_uCompressedBlockMemoryLimitInBytes = uTargetMemoryLimitInBytes - uUncompressedBlockCacheSizeInBytes; + // Let's (arbitrarily) use 1/4 of the available memory for the cache, while making sure that it is as long as the longest side. + uint32_t uMaxFittableUncompressedBlocks = uTargetMemoryLimitInBytes / uUncompressedBlockSizeInBytes; + setMaxNumberOfUncompressedBlocks((std::max)(uMaxFittableUncompressedBlocks / 4, uLongestEdgeInBlocks)); + } + else + { + logWarning() << "The target memory limit (" << uTargetMemoryLimitInBytes << "bytes) is too low for a volume of this size)"; + logWarning() << "Excessive block thrashing may occur and the target memory limit may not be honored"; + + // We still have to do something. Let's still use 1/4 of the memory for the cache, but also set a minimum size. + const uint32_t uMinNoOfUncompressedBlocks = 8; // Chosen because it just feels about right. + uint32_t uMaxFittableUncompressedBlocks = uTargetMemoryLimitInBytes / uUncompressedBlockSizeInBytes; + setMaxNumberOfUncompressedBlocks((std::max)(uMaxFittableUncompressedBlocks / 4, uMinNoOfUncompressedBlocks)); + } } //////////////////////////////////////////////////////////////////////////////// @@ -277,6 +311,10 @@ namespace PolyVox clearBlockCache(); m_uMaxNumberOfUncompressedBlocks = uMaxNumberOfUncompressedBlocks; + + uint32_t uUncompressedBlockSizeInBytes = m_uBlockSideLength * m_uBlockSideLength * m_uBlockSideLength * sizeof(VoxelType); + logDebug() << "The maximum number of uncompresed blocks has been set to " << m_uMaxNumberOfUncompressedBlocks + << ", which is " << m_uMaxNumberOfUncompressedBlocks * uUncompressedBlockSizeInBytes << " bytes"; } //////////////////////////////////////////////////////////////////////////////// @@ -781,7 +819,7 @@ namespace PolyVox //uint32_t uMemoryToReclaim = uMemoryUsedForCompressedBlocks - m_uCompressedBlockMemoryLimitInBytes; //while(uMemoryToReclaim > 0) - while(calculateBlockMemoryUsage() > m_uCompressedBlockMemoryLimitInBytes) //FIXME - This calculation of size is slow and should be outside the loop. + /*while(calculateBlockMemoryUsage() > m_uCompressedBlockMemoryLimitInBytes) //FIXME - This calculation of size is slow and should be outside the loop. { // find the least recently used block typename CompressedBlockMap::iterator i; @@ -799,7 +837,7 @@ namespace PolyVox //uMemoryToReclaim -= itUnloadBlock->second->calculateSizeInBytes(); eraseBlock(itUnloadBlock); - } + }*/ } template From 63dfaa5a13bad65c5f54210f5491ba1a94acc4f7 Mon Sep 17 00:00:00 2001 From: Matt Williams Date: Sun, 21 Jul 2013 10:31:24 +0100 Subject: [PATCH 28/55] Enable C# bindings even if Python wasn't found. --- library/bindings/CMakeLists.txt | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/library/bindings/CMakeLists.txt b/library/bindings/CMakeLists.txt index bb1f4c7e..2c806080 100644 --- a/library/bindings/CMakeLists.txt +++ b/library/bindings/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2009-2012 Matt Williams +# Copyright (c) 2009-2013 Matt Williams # # This software is provided 'as-is', without any express or implied # warranty. In no event will the authors be held liable for any damages @@ -19,7 +19,7 @@ # 3. This notice may not be removed or altered from any source # distribution. -option(ENABLE_BINDINGS "Build Python bindings" ON) +option(ENABLE_BINDINGS "Build bindings" ON) if(ENABLE_BINDINGS) find_package(SWIG) mark_as_advanced(SWIG_DIR SWIG_VERSION) @@ -31,24 +31,26 @@ if(ENABLE_BINDINGS) set_package_properties(SWIG PROPERTIES DESCRIPTION "Bindings generator" URL http://www.swig.org) set_package_properties(PythonLibs PROPERTIES DESCRIPTION "Programming language" URL http://www.python.org) endif() - if(SWIG_FOUND AND PYTHONLIBS_FOUND) - set(BUILD_BINDINGS ON CACHE BOOL "Will the bindings be built" FORCE ) + if(SWIG_FOUND) + set(BUILD_BINDINGS ON CACHE BOOL "Will the bindings be built" FORCE) include(${SWIG_USE_FILE}) - include_directories(${PYTHON_INCLUDE_PATH}) - include_directories(${PolyVoxCore_BINARY_DIR}/include ${PolyVoxCore_SOURCE_DIR}/include ${PolyVoxCore_SOURCE_DIR}/include/PolyVoxCore) - link_directories(${PolyVoxCore_BINARY_DIR}) - set(CMAKE_SWIG_FLAGS "") set_source_files_properties(PolyVoxCore.i PROPERTIES CPLUSPLUS ON) - #set_source_files_properties(PolyVoxCore.i PROPERTIES SWIG_FLAGS "-builtin") - set(SWIG_MODULE_PolyVoxCorePython_EXTRA_FLAGS "-py3") - swig_add_module(PolyVoxCorePython python PolyVoxCore.i) - swig_link_libraries(PolyVoxCorePython ${PYTHON_LIBRARIES} PolyVoxCore) - set_target_properties(${SWIG_MODULE_PolyVoxCorePython_REAL_NAME} PROPERTIES OUTPUT_NAME _PolyVoxCore) - #set_target_properties(${SWIG_MODULE_PolyVoxCore_REAL_NAME} PROPERTIES SUFFIX ".pyd") - SET_PROPERTY(TARGET ${SWIG_MODULE_PolyVoxCorePython_REAL_NAME} PROPERTY FOLDER "Bindings") + if(PYTHONLIBS_FOUND) + include_directories(${PYTHON_INCLUDE_PATH}) + include_directories(${PolyVoxCore_BINARY_DIR}/include ${PolyVoxCore_SOURCE_DIR}/include ${PolyVoxCore_SOURCE_DIR}/include/PolyVoxCore) + link_directories(${PolyVoxCore_BINARY_DIR}) + + #set_source_files_properties(PolyVoxCore.i PROPERTIES SWIG_FLAGS "-builtin") + set(SWIG_MODULE_PolyVoxCorePython_EXTRA_FLAGS "-py3") + swig_add_module(PolyVoxCorePython python PolyVoxCore.i) + swig_link_libraries(PolyVoxCorePython ${PYTHON_LIBRARIES} PolyVoxCore) + set_target_properties(${SWIG_MODULE_PolyVoxCorePython_REAL_NAME} PROPERTIES OUTPUT_NAME _PolyVoxCore) + #set_target_properties(${SWIG_MODULE_PolyVoxCore_REAL_NAME} PROPERTIES SUFFIX ".pyd") + SET_PROPERTY(TARGET ${SWIG_MODULE_PolyVoxCorePython_REAL_NAME} PROPERTY FOLDER "Bindings") + endif() set(SWIG_MODULE_PolyVoxCoreCSharp_EXTRA_FLAGS "-dllimport;PolyVoxCoreCSharp") #This _should_ be inside UseSWIG.cmake - http://www.cmake.org/Bug/view.php?id=13814 swig_add_module(PolyVoxCoreCSharp csharp PolyVoxCore.i) From 3971e6415c7e2c153943177e1494074faa1b187f Mon Sep 17 00:00:00 2001 From: David Williams Date: Mon, 22 Jul 2013 16:37:35 +0200 Subject: [PATCH 29/55] Added SWIG tests to stop build warning about operator<<. --- library/PolyVoxCore/include/PolyVoxCore/Region.h | 4 +++- library/PolyVoxCore/source/Region.cpp | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/library/PolyVoxCore/include/PolyVoxCore/Region.h b/library/PolyVoxCore/include/PolyVoxCore/Region.h index 4fe45a7e..74084c25 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/Region.h +++ b/library/PolyVoxCore/include/PolyVoxCore/Region.h @@ -204,7 +204,9 @@ namespace PolyVox // Non-member overloaded operators. /// Stream insertion operator. - std::ostream& operator<<(std::ostream& os, const Region& region); +#if !defined SWIG + std::ostream& operator<<(std::ostream& os, const Region& region); +#endif // Functions to be inlined to to be in the header rather than the .cpp. // 'inline' keyword is used for the definition rather than the declaration. diff --git a/library/PolyVoxCore/source/Region.cpp b/library/PolyVoxCore/source/Region.cpp index 2f61f63a..d86c55cb 100644 --- a/library/PolyVoxCore/source/Region.cpp +++ b/library/PolyVoxCore/source/Region.cpp @@ -494,10 +494,12 @@ namespace PolyVox * \param region The Region to write to the stream. * \return A reference to the output stream to allow chaining. */ +#if !defined SWIG std::ostream& operator<<(std::ostream& os, const Region& region) { os << "(" << region.getLowerX() << "," << region.getLowerY() << "," << region.getLowerZ() << ") to (" << region.getUpperX() << "," << region.getUpperY() << "," << region.getUpperZ() << ")"; return os; - } + } +#endif } \ No newline at end of file From 7ee913c8a82a75d0e6deca7cc32848fa2a5ed3e6 Mon Sep 17 00:00:00 2001 From: Matt Williams Date: Mon, 22 Jul 2013 16:10:31 +0100 Subject: [PATCH 30/55] Fix the SWIG warning in the SWIG interface files This keeps the .cpp and .h files clean. --- library/PolyVoxCore/include/PolyVoxCore/Region.h | 2 -- library/PolyVoxCore/source/Region.cpp | 4 +--- library/bindings/PolyVoxCore.i | 1 + library/bindings/Region.i | 4 ++++ 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/library/PolyVoxCore/include/PolyVoxCore/Region.h b/library/PolyVoxCore/include/PolyVoxCore/Region.h index 74084c25..7312bc67 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/Region.h +++ b/library/PolyVoxCore/include/PolyVoxCore/Region.h @@ -204,9 +204,7 @@ namespace PolyVox // Non-member overloaded operators. /// Stream insertion operator. -#if !defined SWIG std::ostream& operator<<(std::ostream& os, const Region& region); -#endif // Functions to be inlined to to be in the header rather than the .cpp. // 'inline' keyword is used for the definition rather than the declaration. diff --git a/library/PolyVoxCore/source/Region.cpp b/library/PolyVoxCore/source/Region.cpp index d86c55cb..4ac22dbd 100644 --- a/library/PolyVoxCore/source/Region.cpp +++ b/library/PolyVoxCore/source/Region.cpp @@ -494,12 +494,10 @@ namespace PolyVox * \param region The Region to write to the stream. * \return A reference to the output stream to allow chaining. */ -#if !defined SWIG std::ostream& operator<<(std::ostream& os, const Region& region) { os << "(" << region.getLowerX() << "," << region.getLowerY() << "," << region.getLowerZ() << ") to (" << region.getUpperX() << "," << region.getUpperY() << "," << region.getUpperZ() << ")"; return os; } -#endif -} \ No newline at end of file +} diff --git a/library/bindings/PolyVoxCore.i b/library/bindings/PolyVoxCore.i index 30a71407..0a814a59 100644 --- a/library/bindings/PolyVoxCore.i +++ b/library/bindings/PolyVoxCore.i @@ -68,6 +68,7 @@ EXTRACTOR(shortname, LargeVolume) %ignore *::operator-=; %ignore *::operator*=; %ignore *::operator/=; +%ignore *::operator<<; //This is covered by STR() #endif %include "stdint.i" diff --git a/library/bindings/Region.i b/library/bindings/Region.i index 0860bf09..e57043af 100644 --- a/library/bindings/Region.i +++ b/library/bindings/Region.i @@ -4,6 +4,10 @@ #include "Region.h" %} +%extend PolyVox::Region { + STR() +}; + %ignore depth; %ignore height; %ignore width; From 7146b5ecdb400dd44068b1ea3720b7c1aa068ac8 Mon Sep 17 00:00:00 2001 From: David Williams Date: Tue, 23 Jul 2013 13:15:01 +0200 Subject: [PATCH 31/55] The idea of a 'setTargetMemoryLimitInBytes' function was not really working out. It was too comp-lex and not really clear what the ideal size would be. I think user code needs to call setMaxNumberOfBlocksInMemory() and setMaxNumberOfUncompressedBlocks() directly, if they see from profiling toat too much camressing or paging is taking place. --- examples/Paging/main.cpp | 5 +- .../include/PolyVoxCore/LargeVolume.h | 7 +- .../include/PolyVoxCore/LargeVolume.inl | 71 ++++--------------- tests/testvolume.cpp | 5 +- 4 files changed, 22 insertions(+), 66 deletions(-) diff --git a/examples/Paging/main.cpp b/examples/Paging/main.cpp index 07877405..117145b7 100644 --- a/examples/Paging/main.cpp +++ b/examples/Paging/main.cpp @@ -153,9 +153,8 @@ int main(int argc, char *argv[]) RLECompressor* compressor = new RLECompressor(); PerlinNoisePager* pager = new PerlinNoisePager(); LargeVolume volData(PolyVox::Region::MaxRegion, compressor, pager, 256); - //volData.setMaxNumberOfBlocksInMemory(4096); - //volData.setMaxNumberOfUncompressedBlocks(64); - volData.setTargetMemoryLimitInBytes(128 * 1024 * 1024); + volData.setMaxNumberOfBlocksInMemory(4096); + volData.setMaxNumberOfUncompressedBlocks(64); //volData.setMaxNumberOfUncompressedBlocks(4096); //createSphereInVolume(volData, 30); diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h index 499a5f91..e1786609 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h @@ -263,10 +263,10 @@ namespace PolyVox /// Gets a voxel at the position given by a 3D vector POLYVOX_DEPRECATED VoxelType getVoxelAt(const Vector3DInt32& v3dPos) const; - void setTargetMemoryLimitInBytes(uint32_t uTargetMemoryLimitInBytes); - /// Sets the number of blocks for which uncompressed data is stored void setMaxNumberOfUncompressedBlocks(uint32_t uMaxNumberOfUncompressedBlocks); + /// Sets the number of blocks which can be in memory before the paging system starts unloading them + void setMaxNumberOfBlocksInMemory(uint32_t uMaxNumberOfBlocksInMemory); /// Sets the voxel at the position given by x,y,z coordinates void setVoxel(int32_t uXPos, int32_t uYPos, int32_t uZPos, VoxelType tValue, WrapMode eWrapMode = WrapModes::Validate); /// Sets the voxel at the position given by a 3D vector @@ -346,8 +346,7 @@ namespace PolyVox mutable Vector3DInt32 m_v3dLastAccessedBlockPos; mutable UncompressedBlock* m_pLastAccessedBlock; uint32_t m_uMaxNumberOfUncompressedBlocks; - - uint32_t m_uTargetMemoryLimitInBytes; + uint32_t m_uMaxNumberOfBlocksInMemory; // The size of the volume Region m_regValidRegionInBlocks; diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl index e174a578..04b68a6d 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl @@ -243,62 +243,6 @@ namespace PolyVox return getVoxelAt(v3dPos.getX(), v3dPos.getY(), v3dPos.getZ()); } - template - void LargeVolume::setTargetMemoryLimitInBytes(uint32_t uTargetMemoryLimitInBytes) - { - // Set the limit, and then work out how big the uncompressed cache should be. - m_uTargetMemoryLimitInBytes = uTargetMemoryLimitInBytes; - - // The size of a single uncompressed block of data. - uint32_t uUncompressedBlockSizeInBytes = m_uBlockSideLength * m_uBlockSideLength * m_uBlockSideLength * sizeof(VoxelType); - - // When deciding how to split the memory between compressed and uncompressed blocks it is useful to know the dimensions of the - // volume in blocks. After sorting, the array below will hold the smallest dimension in [0] and the largest dimension in [2]. - const uint32_t uNoOfDimensions = 3; - uint32_t uSortedDimensionsInBlocks[uNoOfDimensions]; - uSortedDimensionsInBlocks[0] = (getEnclosingRegion().getWidthInVoxels() >> m_uBlockSideLengthPower) + 1; - uSortedDimensionsInBlocks[1] = (getEnclosingRegion().getHeightInVoxels() >> m_uBlockSideLengthPower) + 1; - uSortedDimensionsInBlocks[2] = (getEnclosingRegion().getDepthInVoxels() >> m_uBlockSideLengthPower) + 1; - std::sort(uSortedDimensionsInBlocks, uSortedDimensionsInBlocks + uNoOfDimensions); - - const uint32_t uLongestEdgeInBlocks = uSortedDimensionsInBlocks[2]; - const uint32_t uLargestFaceSizeInBlocks = uSortedDimensionsInBlocks[1] * uSortedDimensionsInBlocks[2]; - - // In the ideal situation we will be able to keep an entire face of blocks decompressed in the cache, and still have significant memory - // left for storing a large number of compressed block. Failing this, we would at least like to be able to keep enough blocks decompressed - // to iterate over the longest edge. Otherwise we will find that excessive compression and decompresion occurs. In both cases we multiply - // by an extra factor make sure there is space for storing some compressed data as well. - const uint32_t uFactor = 2; // Just to make sure we leave some space for compressed data as well - const uint32_t uIdealMemoryTarget = uLargestFaceSizeInBlocks * uUncompressedBlockSizeInBytes * uFactor; - const uint32_t uAcceptableMemoryTarget = uLongestEdgeInBlocks * uUncompressedBlockSizeInBytes * uFactor; - - if(uTargetMemoryLimitInBytes >= uIdealMemoryTarget) - { - logDebug() << "The target memory limit (" << uTargetMemoryLimitInBytes << "bytes) is plenty for a volume of this size)"; - - // The added value below is just some extra 'grace' space. - setMaxNumberOfUncompressedBlocks(uLargestFaceSizeInBlocks + uSortedDimensionsInBlocks[1]); - } - else if(uTargetMemoryLimitInBytes >= uAcceptableMemoryTarget) - { - logDebug() << "The target memory limit (" << uTargetMemoryLimitInBytes << "bytes) should be acceptable for a volume of this size)"; - - // Let's (arbitrarily) use 1/4 of the available memory for the cache, while making sure that it is as long as the longest side. - uint32_t uMaxFittableUncompressedBlocks = uTargetMemoryLimitInBytes / uUncompressedBlockSizeInBytes; - setMaxNumberOfUncompressedBlocks((std::max)(uMaxFittableUncompressedBlocks / 4, uLongestEdgeInBlocks)); - } - else - { - logWarning() << "The target memory limit (" << uTargetMemoryLimitInBytes << "bytes) is too low for a volume of this size)"; - logWarning() << "Excessive block thrashing may occur and the target memory limit may not be honored"; - - // We still have to do something. Let's still use 1/4 of the memory for the cache, but also set a minimum size. - const uint32_t uMinNoOfUncompressedBlocks = 8; // Chosen because it just feels about right. - uint32_t uMaxFittableUncompressedBlocks = uTargetMemoryLimitInBytes / uUncompressedBlockSizeInBytes; - setMaxNumberOfUncompressedBlocks((std::max)(uMaxFittableUncompressedBlocks / 4, uMinNoOfUncompressedBlocks)); - } - } - //////////////////////////////////////////////////////////////////////////////// /// Increasing the size of the block cache will increase memory but may improve performance. /// You may want to set this to a large value (e.g. 1024) when you are first loading your @@ -317,6 +261,21 @@ namespace PolyVox << ", which is " << m_uMaxNumberOfUncompressedBlocks * uUncompressedBlockSizeInBytes << " bytes"; } + //////////////////////////////////////////////////////////////////////////////// + /// Increasing the number of blocks in memory causes less paging to occur + /// \param uMaxNumberOfBlocksInMemory The number of blocks + //////////////////////////////////////////////////////////////////////////////// + template + void LargeVolume::setMaxNumberOfBlocksInMemory(uint32_t uMaxNumberOfBlocksInMemory) + { + if(m_pBlocks.size() > uMaxNumberOfBlocksInMemory) + { + flushAll(); + } + + m_uMaxNumberOfBlocksInMemory = uMaxNumberOfBlocksInMemory; + } + //////////////////////////////////////////////////////////////////////////////// /// \param uXPos the \c x position of the voxel /// \param uYPos the \c y position of the voxel diff --git a/tests/testvolume.cpp b/tests/testvolume.cpp index 2063e8e3..77ad1063 100644 --- a/tests/testvolume.cpp +++ b/tests/testvolume.cpp @@ -281,9 +281,8 @@ TestVolume::TestVolume() m_pSimpleVolume = new SimpleVolume(region); m_pLargeVolume = new LargeVolume(region, m_pCompressor, m_pFilePager, 32); - //m_pLargeVolume->setMaxNumberOfBlocksInMemory(32); - //m_pLargeVolume->setMaxNumberOfUncompressedBlocks(16); - m_pLargeVolume->setTargetMemoryLimitInBytes(4 * 1024 * 1024); + m_pLargeVolume->setMaxNumberOfBlocksInMemory(32); + m_pLargeVolume->setMaxNumberOfUncompressedBlocks(16); //Fill the volume with some data for(int z = region.getLowerZ(); z <= region.getUpperZ(); z++) From 7c11a53484e37bcc34d92313aa9cb0965683052d Mon Sep 17 00:00:00 2001 From: David Williams Date: Tue, 23 Jul 2013 13:31:24 +0200 Subject: [PATCH 32/55] Fixed flushOldestExcessiveBlocks(). --- .../include/PolyVoxCore/LargeVolume.inl | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl index 04b68a6d..c77b2883 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl @@ -775,12 +775,11 @@ namespace PolyVox void LargeVolume::flushOldestExcessiveBlocks(void) const { const uint32_t uMemoryUsedForCompressedBlocks = calculateBlockMemoryUsage(); - //uint32_t uMemoryToReclaim = uMemoryUsedForCompressedBlocks - m_uCompressedBlockMemoryLimitInBytes; - //while(uMemoryToReclaim > 0) - /*while(calculateBlockMemoryUsage() > m_uCompressedBlockMemoryLimitInBytes) //FIXME - This calculation of size is slow and should be outside the loop. + while(m_pBlocks.size() > m_uMaxNumberOfBlocksInMemory) { - // find the least recently used block + // Find the least recently used block. This is somewhat inefficient as it searches through + // the map, so if it proves to be a bottleneck we may want some kind of sorted structure. typename CompressedBlockMap::iterator i; typename CompressedBlockMap::iterator itUnloadBlock = m_pBlocks.begin(); for(i = m_pBlocks.begin(); i != m_pBlocks.end(); i++) @@ -791,12 +790,9 @@ namespace PolyVox } } - //POLYVOX_ASSERT(itUnloadBlock->second.hasUncompressedData() == false, "This function should never flush blocks with uncompressed data."); - - //uMemoryToReclaim -= itUnloadBlock->second->calculateSizeInBytes(); - + // Erase the least recently used block eraseBlock(itUnloadBlock); - }*/ + } } template From 69066a068e6d6618b941ea4474f1dc8a879c442c Mon Sep 17 00:00:00 2001 From: David Williams Date: Tue, 23 Jul 2013 13:48:04 +0200 Subject: [PATCH 33/55] Implemented flushExcessiveCacheEntries() --- .../include/PolyVoxCore/LargeVolume.inl | 32 +++++++++++++++---- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl index c77b2883..e82bfc0f 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl @@ -589,14 +589,14 @@ namespace PolyVox if(pUncompressedBlock->m_bDataModified) { // Get the compressed block which we will copy the data back in to. - Vector3DInt32 v3dBlockPos = itCompressedBlock->first; - CompressedBlock* pCompressedBlock = getCompressedBlock(v3dBlockPos); + Vector3DInt32 v3dBlockPos = itUncompressedBlock->first; + CompressedBlock* pCompressedBlock = getCompressedBlock(v3dBlockPos.getX(), v3dBlockPos.getY(), v3dBlockPos.getZ()); void* pSrcData = reinterpret_cast(pUncompressedBlock->m_tData); - uint32_t uSrcLength = m_uSideLength * m_uSideLength * m_uSideLength * sizeof(VoxelType); + uint32_t uSrcLength = m_uBlockSideLength * m_uBlockSideLength * m_uBlockSideLength * sizeof(VoxelType); uint8_t tempBuffer[10000]; - void* pDstData = reinterpret_cast( tempBuffer ); + uint8_t* pDstData = reinterpret_cast( tempBuffer ); uint32_t uDstLength = 10000; uint32_t uCompressedLength = 0; @@ -619,7 +619,7 @@ namespace PolyVox uint32_t uMaxCompressedSize = m_pCompressor->getMaxCompressedSize(uSrcLength); uint8_t* buffer = new uint8_t[ uMaxCompressedSize ]; - pDstData = reinterpret_cast( buffer ); + pDstData = reinterpret_cast( buffer ); uDstLength = uMaxCompressedSize; try @@ -714,6 +714,9 @@ namespace PolyVox POLYVOX_ASSERT(uUncompressedLength == m_uBlockSideLength * m_uBlockSideLength * m_uBlockSideLength * sizeof(VoxelType), "Destination length has changed."); itUncompressedBlock = m_pUncompressedBlockCache.insert(std::make_pair(v3dBlockPos, pUncompressedBlock)).first; + + // Our block cache may now have grown too large. Fluch some entries is necessary. + flushExcessiveCacheEntries(); } m_pLastAccessedBlock = (*itUncompressedBlock).second; @@ -774,8 +777,6 @@ namespace PolyVox template void LargeVolume::flushOldestExcessiveBlocks(void) const { - const uint32_t uMemoryUsedForCompressedBlocks = calculateBlockMemoryUsage(); - while(m_pBlocks.size() > m_uMaxNumberOfBlocksInMemory) { // Find the least recently used block. This is somewhat inefficient as it searches through @@ -798,6 +799,23 @@ namespace PolyVox template void LargeVolume::flushExcessiveCacheEntries(void) const { + while(m_pUncompressedBlockCache.size() > m_uMaxNumberOfUncompressedBlocks) + { + // Find the least recently used block. The uncompressed block cache should be + // much smaller than the total number of blocks, so hopefully this isn't too slow. + typename UncompressedBlockMap::iterator i; + typename UncompressedBlockMap::iterator itUnloadBlock = m_pUncompressedBlockCache.begin(); + for(i = m_pUncompressedBlockCache.begin(); i != m_pUncompressedBlockCache.end(); i++) + { + if(i->second->m_uBlockLastAccessed < itUnloadBlock->second->m_uBlockLastAccessed) + { + itUnloadBlock = i; + } + } + + // Erase the least recently used block + eraseBlock(itUnloadBlock); + } } template From b07dafc9fa3fc4d58b527b8fb69deea644a06dd2 Mon Sep 17 00:00:00 2001 From: David Williams Date: Wed, 24 Jul 2013 16:39:31 +0200 Subject: [PATCH 34/55] Added code to initilise empty compressed blocks. --- .../include/PolyVoxCore/LargeVolume.h | 3 +++ .../include/PolyVoxCore/LargeVolume.inl | 26 ++++++++++++++++++- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h index e1786609..9f409571 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h @@ -359,6 +359,9 @@ namespace PolyVox Compressor* m_pCompressor; Pager* m_pPager; + // Compressed data for an empty block (sometimes needed for initialisation). + CompressedBlock* m_pCompressedEmptyBlock; + // Whether we created the compressor or whether it was provided // by the user. This controls whether we delete it on destruction. bool m_bIsOurCompressor; diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl index e82bfc0f..10069d49 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl @@ -109,6 +109,8 @@ namespace PolyVox { delete m_pCompressor; } + + delete m_pCompressedEmptyBlock; } //////////////////////////////////////////////////////////////////////////////// @@ -542,6 +544,21 @@ namespace PolyVox this->m_uLongestSideLength = (std::max)((std::max)(this->getWidth(),this->getHeight()),this->getDepth()); this->m_uShortestSideLength = (std::min)((std::min)(this->getWidth(),this->getHeight()),this->getDepth()); this->m_fDiagonalLength = sqrtf(static_cast(this->getWidth() * this->getWidth() + this->getHeight() * this->getHeight() + this->getDepth() * this->getDepth())); + + // This is used for initialising empty blocks. + // FIXME - Should probably build an UncompressedBlock and then call some 'fill()' method. Ideally we should set voxels to an 'empty' (default?) value rather than zeros. + VoxelType* pZeros = new VoxelType[m_uBlockSideLength * m_uBlockSideLength * m_uBlockSideLength]; + uint32_t uSrcLength = m_uBlockSideLength * m_uBlockSideLength * m_uBlockSideLength * sizeof(VoxelType); + memset(pZeros, 0, uSrcLength); + uint32_t uDstLength = 100000; + uint8_t* pCompressedZeros = new uint8_t[uDstLength]; //Should be plenty as zeros compress down very small. + uint32_t uCompressedSize = m_pCompressor->compress(reinterpret_cast(pZeros), uSrcLength, reinterpret_cast(pCompressedZeros), uDstLength); + + m_pCompressedEmptyBlock = new CompressedBlock; + m_pCompressedEmptyBlock->setData(pCompressedZeros, uCompressedSize); + + delete[] pZeros; + delete[] pCompressedZeros; } template @@ -664,7 +681,14 @@ namespace PolyVox Vector3DInt32 v3dLower(v3dBlockPos.getX() << m_uBlockSideLengthPower, v3dBlockPos.getY() << m_uBlockSideLengthPower, v3dBlockPos.getZ() << m_uBlockSideLengthPower); Vector3DInt32 v3dUpper = v3dLower + Vector3DInt32(m_uBlockSideLength-1, m_uBlockSideLength-1, m_uBlockSideLength-1); Region reg(v3dLower, v3dUpper); - m_pPager->pageIn(reg, itBlock->second); + m_pPager->pageIn(reg, newBlock); + + // The pager may not actually have given us any data (perhaps there wasn't any) so in that case we need to create some ourselves. + if(newBlock->m_bDataModified == false) + { + // Internally this performs a memcpy() of the data. + newBlock->setData(m_pCompressedEmptyBlock->getData(), m_pCompressedEmptyBlock->getDataSizeInBytes()); + } // Paging in this new block may mean we are now using too much memory. If necessary, flush some old blocks. flushOldestExcessiveBlocks(); From cf9b54e5ab231d99d31a66e0b564b157f038c38a Mon Sep 17 00:00:00 2001 From: David Williams Date: Wed, 24 Jul 2013 16:53:40 +0200 Subject: [PATCH 35/55] Properly deleting data (delete vs. delete[]). --- library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl | 2 +- library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl index dab2f35d..16927b4e 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl @@ -111,7 +111,7 @@ namespace PolyVox template UncompressedBlock::~UncompressedBlock() { - delete m_tData; + delete[] m_tData; m_tData = 0; } diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl index 10069d49..ad67d849 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl @@ -659,6 +659,8 @@ namespace PolyVox } } + delete itUncompressedBlock->second; + // We can now remove the block data from memory. m_pUncompressedBlockCache.erase(itUncompressedBlock); } From f4f85551c6179a1b8bb50feaaee4a6bf9b270629 Mon Sep 17 00:00:00 2001 From: David Williams Date: Thu, 25 Jul 2013 15:51:30 +0200 Subject: [PATCH 36/55] Starting some refactoring of the LargeVolume such that the uncompressed blocks are the 'main' representation, rather than the compressed block being the main version and the uncompressed blocks simply being a cached version. I hope this simplifies and improves the code. --- .../include/PolyVoxCore/LargeVolume.inl | 34 ++++++++++++++++--- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl index ad67d849..65bbbed3 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl @@ -713,13 +713,39 @@ namespace PolyVox //the time stamp. If we updated it everytime then that would be every time we touched //a voxel, which would overflow a uint32_t and require us to use a uint64_t instead. //This check should also provide a significant speed boost as usually it is true. - if((v3dBlockPos == m_v3dLastAccessedBlockPos) && (m_pLastAccessedBlock != 0)) + /*if((v3dBlockPos == m_v3dLastAccessedBlockPos) && (m_pLastAccessedBlock != 0)) { return m_pLastAccessedBlock; + }*/ + + UncompressedBlock* pUncompressedBlock = 0; + + typename UncompressedBlockMap::iterator itUncompressedBlock = m_pUncompressedBlockCache.find(v3dBlockPos); + // check whether the block is already loaded + if(itUncompressedBlock != m_pUncompressedBlockCache.end()) + { + pUncompressedBlock = itUncompressedBlock->second; + + } + //else if(check compressed list...) + //{ + //} + else + { + // At this point we just create a new block. + pUncompressedBlock = new UncompressedBlock(m_uBlockSideLength); + + m_pUncompressedBlockCache.insert(std::make_pair(v3dBlockPos, pUncompressedBlock)); + + // Our block cache may now have grown too large. Flush some entries is necessary. + // FIXME - Watch out for flushing the block we just created! + //flushExcessiveCacheEntries(); } + pUncompressedBlock->m_uBlockLastAccessed = ++m_uTimestamper; + //Gets the block and marks that we accessed it - CompressedBlock* block = getCompressedBlock(uBlockX, uBlockY, uBlockZ); + /*CompressedBlock* block = getCompressedBlock(uBlockX, uBlockY, uBlockZ); typename UncompressedBlockMap::iterator itUncompressedBlock = m_pUncompressedBlockCache.find(v3dBlockPos); // check whether the block is already loaded @@ -743,9 +769,9 @@ namespace PolyVox // Our block cache may now have grown too large. Fluch some entries is necessary. flushExcessiveCacheEntries(); - } + }*/ - m_pLastAccessedBlock = (*itUncompressedBlock).second; + m_pLastAccessedBlock = pUncompressedBlock; m_v3dLastAccessedBlockPos = v3dBlockPos; return m_pLastAccessedBlock; From b767d9b89661e84fa4462599a3fb8f9d9bd5b194 Mon Sep 17 00:00:00 2001 From: David Williams Date: Thu, 25 Jul 2013 17:07:50 +0200 Subject: [PATCH 37/55] Tidying up. --- .../include/PolyVoxCore/Impl/Block.h | 2 +- .../include/PolyVoxCore/LargeVolume.inl | 105 +++++++++++------- 2 files changed, 65 insertions(+), 42 deletions(-) diff --git a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h index 3d795e26..aeb309cc 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h +++ b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h @@ -42,7 +42,7 @@ namespace PolyVox public: Block() :m_uBlockLastAccessed(0) - ,m_bDataModified(false) + ,m_bDataModified(true) { } diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl index 65bbbed3..f0d15bdc 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl @@ -625,6 +625,9 @@ namespace PolyVox // Copy the resulting compressed data into the compressed block pCompressedBlock->setData(pDstData, uDstLength); + + // The compressed data has been updated, so the uncompressed data is no longer modified with respect to it. + pUncompressedBlock->m_bDataModified = false; } catch(std::exception&) { @@ -646,6 +649,9 @@ namespace PolyVox // Copy the resulting compressed data into the compressed block pCompressedBlock->setData(pDstData, uDstLength); + + // The compressed data has been updated, so the uncompressed data is no longer modified with respect to it. + pUncompressedBlock->m_bDataModified = false; } catch(std::exception&) { @@ -670,7 +676,28 @@ namespace PolyVox { Vector3DInt32 v3dBlockPos(uBlockX, uBlockY, uBlockZ); + CompressedBlock* pBlock = 0; + typename CompressedBlockMap::iterator itBlock = m_pBlocks.find(v3dBlockPos); + if(itBlock != m_pBlocks.end()) + { + pBlock = itBlock->second; + } + /*else if(check on disk) + { + }*/ + else + { + pBlock = new CompressedBlock; + pBlock->m_uBlockLastAccessed = ++m_uTimestamper; + itBlock = m_pBlocks.insert(std::make_pair(v3dBlockPos, pBlock)).first; + } + + pBlock->m_uBlockLastAccessed = ++m_uTimestamper; + + return pBlock; + + /*typename CompressedBlockMap::iterator itBlock = m_pBlocks.find(v3dBlockPos); // check whether the block is already loaded if(itBlock == m_pBlocks.end()) { @@ -701,7 +728,7 @@ namespace PolyVox block->m_uBlockLastAccessed = ++m_uTimestamper; //m_v3dLastAccessedBlockPos = v3dBlockPos; - return block; + return block;*/ } template @@ -718,63 +745,59 @@ namespace PolyVox return m_pLastAccessedBlock; }*/ - UncompressedBlock* pUncompressedBlock = 0; - typename UncompressedBlockMap::iterator itUncompressedBlock = m_pUncompressedBlockCache.find(v3dBlockPos); // check whether the block is already loaded if(itUncompressedBlock != m_pUncompressedBlockCache.end()) { - pUncompressedBlock = itUncompressedBlock->second; - + UncompressedBlock* pUncompressedBlock = itUncompressedBlock->second; + + pUncompressedBlock->m_uBlockLastAccessed = ++m_uTimestamper; + m_pLastAccessedBlock = pUncompressedBlock; + m_v3dLastAccessedBlockPos = v3dBlockPos; + + return pUncompressedBlock; } - //else if(check compressed list...) - //{ - //} else { // At this point we just create a new block. - pUncompressedBlock = new UncompressedBlock(m_uBlockSideLength); - - m_pUncompressedBlockCache.insert(std::make_pair(v3dBlockPos, pUncompressedBlock)); - - // Our block cache may now have grown too large. Flush some entries is necessary. - // FIXME - Watch out for flushing the block we just created! - //flushExcessiveCacheEntries(); - } - - pUncompressedBlock->m_uBlockLastAccessed = ++m_uTimestamper; - - //Gets the block and marks that we accessed it - /*CompressedBlock* block = getCompressedBlock(uBlockX, uBlockY, uBlockZ); - - typename UncompressedBlockMap::iterator itUncompressedBlock = m_pUncompressedBlockCache.find(v3dBlockPos); - // check whether the block is already loaded - if(itUncompressedBlock == m_pUncompressedBlockCache.end()) - { UncompressedBlock* pUncompressedBlock = new UncompressedBlock(m_uBlockSideLength); + + // It's important to set the timestamp before we flush later. pUncompressedBlock->m_uBlockLastAccessed = ++m_uTimestamper; - const void* pSrcData = reinterpret_cast(block->getData()); - void* pDstData = reinterpret_cast(pUncompressedBlock->m_tData); - uint32_t uSrcLength = block->getDataSizeInBytes(); - uint32_t uDstLength = m_uBlockSideLength * m_uBlockSideLength * m_uBlockSideLength * sizeof(VoxelType); + // We set these before flushing because the flush can cause block to be erased, and there + // is a test to make sure the block which is being erase is not the last accessed block. + m_pLastAccessedBlock = pUncompressedBlock; + m_v3dLastAccessedBlockPos = v3dBlockPos; - //MinizCompressor compressor; - //RLECompressor compressor; - uint32_t uUncompressedLength = m_pCompressor->decompress(pSrcData, uSrcLength, pDstData, uDstLength); + // An uncompressed bock is always backed by a compressed one, and this is created by getCompressedBlock() if it doesn't + // already exist. If it does already exist and has data then we bring this across into the ucompressed version. + if(getCompressedBlock(uBlockX, uBlockY, uBlockZ)->getData() != 0) + { + // FIXME - multiple getCompressedBlock() calls (including the one above) + CompressedBlock* pBlock = getCompressedBlock(uBlockX, uBlockY, uBlockZ); - POLYVOX_ASSERT(uUncompressedLength == m_uBlockSideLength * m_uBlockSideLength * m_uBlockSideLength * sizeof(VoxelType), "Destination length has changed."); + const void* pSrcData = reinterpret_cast(pBlock->getData()); + void* pDstData = reinterpret_cast(pUncompressedBlock->m_tData); + uint32_t uSrcLength = pBlock->getDataSizeInBytes(); + uint32_t uDstLength = m_uBlockSideLength * m_uBlockSideLength * m_uBlockSideLength * sizeof(VoxelType); - itUncompressedBlock = m_pUncompressedBlockCache.insert(std::make_pair(v3dBlockPos, pUncompressedBlock)).first; + //MinizCompressor compressor; + //RLECompressor compressor; + uint32_t uUncompressedLength = m_pCompressor->decompress(pSrcData, uSrcLength, pDstData, uDstLength); - // Our block cache may now have grown too large. Fluch some entries is necessary. + POLYVOX_ASSERT(uUncompressedLength == m_uBlockSideLength * m_uBlockSideLength * m_uBlockSideLength * sizeof(VoxelType), "Destination length has changed."); + } + + // Addd our new block to the map. + m_pUncompressedBlockCache.insert(std::make_pair(v3dBlockPos, pUncompressedBlock)); + + // Our block cache may now have grown too large. Flush some entries if necessary. + // FIXME - Watch out for flushing the block we just created! flushExcessiveCacheEntries(); - }*/ - m_pLastAccessedBlock = pUncompressedBlock; - m_v3dLastAccessedBlockPos = v3dBlockPos; - - return m_pLastAccessedBlock; + return pUncompressedBlock; + } } //////////////////////////////////////////////////////////////////////////////// From c3c4ead1f339427607a2526dd2ec0fb4fcc52b0b Mon Sep 17 00:00:00 2001 From: David Williams Date: Fri, 26 Jul 2013 15:38:03 +0200 Subject: [PATCH 38/55] Work on paging to files. --- .../include/PolyVoxCore/LargeVolume.inl | 36 ++++++++++++++----- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl index f0d15bdc..e40d3c8a 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl @@ -102,7 +102,7 @@ namespace PolyVox template LargeVolume::~LargeVolume() { - //flushAll(); + flushAll(); // Only delete the compressor if it was created by us (in the constructor), not by the user. if(m_bIsOurCompressor) @@ -579,6 +579,9 @@ namespace PolyVox // Page the data out m_pPager->pageOut(Region(v3dLower, v3dUpper), pCompressedBlock); + + // The compressed data is no longer modified with respect to the data on disk + pCompressedBlock->m_bDataModified = false; } delete itCompressedBlock->second; @@ -682,20 +685,35 @@ namespace PolyVox if(itBlock != m_pBlocks.end()) { pBlock = itBlock->second; + pBlock->m_uBlockLastAccessed = ++m_uTimestamper; + return pBlock; } - /*else if(check on disk) - { - }*/ else { + // The block wasn't found so we create a new one pBlock = new CompressedBlock; + + // It's important to set the timestamp before we flush later. pBlock->m_uBlockLastAccessed = ++m_uTimestamper; + + // Pass the block to the Pager to give it a chance to initialise it with any data + Vector3DInt32 v3dLower(v3dBlockPos.getX() << m_uBlockSideLengthPower, v3dBlockPos.getY() << m_uBlockSideLengthPower, v3dBlockPos.getZ() << m_uBlockSideLengthPower); + Vector3DInt32 v3dUpper = v3dLower + Vector3DInt32(m_uBlockSideLength-1, m_uBlockSideLength-1, m_uBlockSideLength-1); + Region reg(v3dLower, v3dUpper); + m_pPager->pageIn(reg, pBlock); + + // Add the block to the map itBlock = m_pBlocks.insert(std::make_pair(v3dBlockPos, pBlock)).first; + + // Paging in this new block may mean we are now using too much memory. If necessary, flush some old blocks. + flushOldestExcessiveBlocks(); + + return pBlock; } - pBlock->m_uBlockLastAccessed = ++m_uTimestamper; + - return pBlock; + /*typename CompressedBlockMap::iterator itBlock = m_pBlocks.find(v3dBlockPos); // check whether the block is already loaded @@ -740,10 +758,10 @@ namespace PolyVox //the time stamp. If we updated it everytime then that would be every time we touched //a voxel, which would overflow a uint32_t and require us to use a uint64_t instead. //This check should also provide a significant speed boost as usually it is true. - /*if((v3dBlockPos == m_v3dLastAccessedBlockPos) && (m_pLastAccessedBlock != 0)) + if((v3dBlockPos == m_v3dLastAccessedBlockPos) && (m_pLastAccessedBlock != 0)) { return m_pLastAccessedBlock; - }*/ + } typename UncompressedBlockMap::iterator itUncompressedBlock = m_pUncompressedBlockCache.find(v3dBlockPos); // check whether the block is already loaded @@ -789,7 +807,7 @@ namespace PolyVox POLYVOX_ASSERT(uUncompressedLength == m_uBlockSideLength * m_uBlockSideLength * m_uBlockSideLength * sizeof(VoxelType), "Destination length has changed."); } - // Addd our new block to the map. + // Add our new block to the map. m_pUncompressedBlockCache.insert(std::make_pair(v3dBlockPos, pUncompressedBlock)); // Our block cache may now have grown too large. Flush some entries if necessary. From f54532a9052870b09e7277581d91e3be02c49411 Mon Sep 17 00:00:00 2001 From: David Williams Date: Fri, 26 Jul 2013 16:00:29 +0200 Subject: [PATCH 39/55] Started moving compression code into separate class. --- library/PolyVoxCore/CMakeLists.txt | 2 + .../include/PolyVoxCore/Impl/Block.h | 3 + .../include/PolyVoxCore/Impl/Block.inl | 12 ++++ .../include/PolyVoxCore/LargeVolume.h | 2 + .../include/PolyVoxCore/LargeVolume.inl | 8 ++- .../PolyVoxCore/MinizBlockCompressor.h | 52 +++++++++++++++++ .../PolyVoxCore/MinizBlockCompressor.inl | 57 +++++++++++++++++++ 7 files changed, 134 insertions(+), 2 deletions(-) create mode 100644 library/PolyVoxCore/include/PolyVoxCore/MinizBlockCompressor.h create mode 100644 library/PolyVoxCore/include/PolyVoxCore/MinizBlockCompressor.inl diff --git a/library/PolyVoxCore/CMakeLists.txt b/library/PolyVoxCore/CMakeLists.txt index a8ce8a0c..f79df326 100644 --- a/library/PolyVoxCore/CMakeLists.txt +++ b/library/PolyVoxCore/CMakeLists.txt @@ -71,6 +71,8 @@ SET(CORE_INC_FILES include/PolyVoxCore/MarchingCubesSurfaceExtractor.inl include/PolyVoxCore/Material.h include/PolyVoxCore/MaterialDensityPair.h + include/PolyVoxCore/MinizBlockCompressor.h + include/PolyVoxCore/MinizBlockCompressor.inl include/PolyVoxCore/MinizCompressor.h include/PolyVoxCore/Pager.h include/PolyVoxCore/PolyVoxForwardDeclarations.h diff --git a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h index aeb309cc..554bc3c2 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h +++ b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h @@ -100,6 +100,9 @@ namespace PolyVox UncompressedBlock(uint16_t uSideLength); ~UncompressedBlock(); + VoxelType* getData(void) const; + uint32_t getDataSizeInBytes(void) const; + VoxelType getVoxel(uint16_t uXPos, uint16_t uYPos, uint16_t uZPos) const; VoxelType getVoxel(const Vector3DUint16& v3dPos) const; diff --git a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl index 16927b4e..ba156ef4 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl @@ -115,6 +115,18 @@ namespace PolyVox m_tData = 0; } + template + VoxelType* UncompressedBlock::getData(void) const + { + return m_tData; + } + + template + uint32_t UncompressedBlock::getDataSizeInBytes(void) const + { + return m_uSideLength * m_uSideLength * m_uSideLength * sizeof(VoxelType); + } + template VoxelType UncompressedBlock::getVoxel(uint16_t uXPos, uint16_t uYPos, uint16_t uZPos) const { diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h index 9f409571..e5095c3f 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h @@ -27,6 +27,7 @@ freely, subject to the following restrictions: #include "PolyVoxCore/BaseVolume.h" #include "Impl/Block.h" #include "PolyVoxCore/Compressor.h" +#include "PolyVoxCore/MinizBlockCompressor.h" //Shouldn't include the implementation here. #include "PolyVoxCore/Pager.h" #include "PolyVoxCore/Region.h" #include "PolyVoxCore/Vector.h" @@ -357,6 +358,7 @@ namespace PolyVox // The compressor used by the Blocks to compress their data if required. Compressor* m_pCompressor; + MinizBlockCompressor* m_pBlockCompressor; Pager* m_pPager; // Compressed data for an empty block (sometimes needed for initialisation). diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl index e40d3c8a..c6cb68c3 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl @@ -559,6 +559,8 @@ namespace PolyVox delete[] pZeros; delete[] pCompressedZeros; + + m_pBlockCompressor = new MinizBlockCompressor; } template @@ -795,7 +797,7 @@ namespace PolyVox // FIXME - multiple getCompressedBlock() calls (including the one above) CompressedBlock* pBlock = getCompressedBlock(uBlockX, uBlockY, uBlockZ); - const void* pSrcData = reinterpret_cast(pBlock->getData()); + /*const void* pSrcData = reinterpret_cast(pBlock->getData()); void* pDstData = reinterpret_cast(pUncompressedBlock->m_tData); uint32_t uSrcLength = pBlock->getDataSizeInBytes(); uint32_t uDstLength = m_uBlockSideLength * m_uBlockSideLength * m_uBlockSideLength * sizeof(VoxelType); @@ -804,7 +806,9 @@ namespace PolyVox //RLECompressor compressor; uint32_t uUncompressedLength = m_pCompressor->decompress(pSrcData, uSrcLength, pDstData, uDstLength); - POLYVOX_ASSERT(uUncompressedLength == m_uBlockSideLength * m_uBlockSideLength * m_uBlockSideLength * sizeof(VoxelType), "Destination length has changed."); + POLYVOX_ASSERT(uUncompressedLength == m_uBlockSideLength * m_uBlockSideLength * m_uBlockSideLength * sizeof(VoxelType), "Destination length has changed.");*/ + + m_pBlockCompressor->decompress(pBlock, pUncompressedBlock); } // Add our new block to the map. diff --git a/library/PolyVoxCore/include/PolyVoxCore/MinizBlockCompressor.h b/library/PolyVoxCore/include/PolyVoxCore/MinizBlockCompressor.h new file mode 100644 index 00000000..b6053fbc --- /dev/null +++ b/library/PolyVoxCore/include/PolyVoxCore/MinizBlockCompressor.h @@ -0,0 +1,52 @@ +/******************************************************************************* +Copyright (c) 2005-2013 David Williams and Matthew Williams + +This software is provided 'as-is', without any express or implied +warranty. In no event will the authors be held liable for any damages +arising from the use of this software. + +Permission is granted to anyone to use this software for any purpose, +including commercial applications, and to alter it and redistribute it +freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + + 3. This notice may not be removed or altered from any source + distribution. +*******************************************************************************/ + +#ifndef __PolyVox_MinizBlockCompressor_H__ +#define __PolyVox_MinizBlockCompressor_H__ + +#include "PolyVoxCore/Impl/Block.h" + +#include "PolyVoxCore/MinizCompressor.h" + +namespace PolyVox +{ + /** + * Provides an interface for performing paging of data. + */ + template + class MinizBlockCompressor + { + public: + MinizBlockCompressor(); + ~MinizBlockCompressor(); + + void compress(UncompressedBlock* pSrcBlock, CompressedBlock* pDstBlock); + void decompress(CompressedBlock* pSrcBlock, UncompressedBlock* pDstBlock); + + MinizCompressor* m_pCompressor; + }; +} + +#include "PolyVoxCore/MinizBlockCompressor.inl" + +#endif //__PolyVox_MinizBlockCompressor_H__ diff --git a/library/PolyVoxCore/include/PolyVoxCore/MinizBlockCompressor.inl b/library/PolyVoxCore/include/PolyVoxCore/MinizBlockCompressor.inl new file mode 100644 index 00000000..69f6b2e9 --- /dev/null +++ b/library/PolyVoxCore/include/PolyVoxCore/MinizBlockCompressor.inl @@ -0,0 +1,57 @@ +/******************************************************************************* +Copyright (c) 2005-2013 David Williams and Matthew Williams + +This software is provided 'as-is', without any express or implied +warranty. In no event will the authors be held liable for any damages +arising from the use of this software. + +Permission is granted to anyone to use this software for any purpose, +including commercial applications, and to alter it and redistribute it +freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + + 3. This notice may not be removed or altered from any source + distribution. +*******************************************************************************/ + +namespace PolyVox +{ + template + MinizBlockCompressor::MinizBlockCompressor() + { + m_pCompressor = new MinizCompressor; + } + + template + MinizBlockCompressor::~MinizBlockCompressor() + { + delete m_pCompressor; + } + + template + void MinizBlockCompressor::compress(UncompressedBlock* pSrcBlock, CompressedBlock* pDstBlock) + { + } + + template + void MinizBlockCompressor::decompress(CompressedBlock* pSrcBlock, UncompressedBlock* pDstBlock) + { + const void* pSrcData = reinterpret_cast(pSrcBlock->getData()); + void* pDstData = reinterpret_cast(pDstBlock->getData()); + uint32_t uSrcLength = pSrcBlock->getDataSizeInBytes(); + uint32_t uDstLength = pDstBlock->getDataSizeInBytes(); + + + //RLECompressor compressor; + uint32_t uUncompressedLength = m_pCompressor->decompress(pSrcData, uSrcLength, pDstData, uDstLength); + + POLYVOX_ASSERT(uUncompressedLength == pDstBlock->getDataSizeInBytes(), "Destination length has changed."); + } +} \ No newline at end of file From ef59f329b61de55dccdadcb8a61cf22cb0c9d6c3 Mon Sep 17 00:00:00 2001 From: David Williams Date: Fri, 26 Jul 2013 16:25:50 +0200 Subject: [PATCH 40/55] Reorganising compression code. --- .../include/PolyVoxCore/LargeVolume.inl | 57 +++---------------- .../PolyVoxCore/MinizBlockCompressor.inl | 48 ++++++++++++++++ 2 files changed, 55 insertions(+), 50 deletions(-) diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl index c6cb68c3..7534e8bb 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl @@ -614,7 +614,12 @@ namespace PolyVox Vector3DInt32 v3dBlockPos = itUncompressedBlock->first; CompressedBlock* pCompressedBlock = getCompressedBlock(v3dBlockPos.getX(), v3dBlockPos.getY(), v3dBlockPos.getZ()); - void* pSrcData = reinterpret_cast(pUncompressedBlock->m_tData); + m_pBlockCompressor->compress(pUncompressedBlock, pCompressedBlock); + + // The compressed data has been updated, so the uncompressed data is no longer modified with respect to it. + pUncompressedBlock->m_bDataModified = false; + + /*void* pSrcData = reinterpret_cast(pUncompressedBlock->m_tData); uint32_t uSrcLength = m_uBlockSideLength * m_uBlockSideLength * m_uBlockSideLength * sizeof(VoxelType); uint8_t tempBuffer[10000]; @@ -667,7 +672,7 @@ namespace PolyVox } delete[] buffer; - } + }*/ } delete itUncompressedBlock->second; @@ -712,43 +717,6 @@ namespace PolyVox return pBlock; } - - - - - - /*typename CompressedBlockMap::iterator itBlock = m_pBlocks.find(v3dBlockPos); - // check whether the block is already loaded - if(itBlock == m_pBlocks.end()) - { - //The block is not in the map, so we will have to create a new block and add it. - CompressedBlock* newBlock = new CompressedBlock; - newBlock->m_uBlockLastAccessed = ++m_uTimestamper; - itBlock = m_pBlocks.insert(std::make_pair(v3dBlockPos, newBlock)).first; - - // Now use the pager to fill the block with it's initial data. - Vector3DInt32 v3dLower(v3dBlockPos.getX() << m_uBlockSideLengthPower, v3dBlockPos.getY() << m_uBlockSideLengthPower, v3dBlockPos.getZ() << m_uBlockSideLengthPower); - Vector3DInt32 v3dUpper = v3dLower + Vector3DInt32(m_uBlockSideLength-1, m_uBlockSideLength-1, m_uBlockSideLength-1); - Region reg(v3dLower, v3dUpper); - m_pPager->pageIn(reg, newBlock); - - // The pager may not actually have given us any data (perhaps there wasn't any) so in that case we need to create some ourselves. - if(newBlock->m_bDataModified == false) - { - // Internally this performs a memcpy() of the data. - newBlock->setData(m_pCompressedEmptyBlock->getData(), m_pCompressedEmptyBlock->getDataSizeInBytes()); - } - - // Paging in this new block may mean we are now using too much memory. If necessary, flush some old blocks. - flushOldestExcessiveBlocks(); - } - - //Get the block and mark that we accessed it - CompressedBlock* block = itBlock->second; - block->m_uBlockLastAccessed = ++m_uTimestamper; - //m_v3dLastAccessedBlockPos = v3dBlockPos; - - return block;*/ } template @@ -797,17 +765,6 @@ namespace PolyVox // FIXME - multiple getCompressedBlock() calls (including the one above) CompressedBlock* pBlock = getCompressedBlock(uBlockX, uBlockY, uBlockZ); - /*const void* pSrcData = reinterpret_cast(pBlock->getData()); - void* pDstData = reinterpret_cast(pUncompressedBlock->m_tData); - uint32_t uSrcLength = pBlock->getDataSizeInBytes(); - uint32_t uDstLength = m_uBlockSideLength * m_uBlockSideLength * m_uBlockSideLength * sizeof(VoxelType); - - //MinizCompressor compressor; - //RLECompressor compressor; - uint32_t uUncompressedLength = m_pCompressor->decompress(pSrcData, uSrcLength, pDstData, uDstLength); - - POLYVOX_ASSERT(uUncompressedLength == m_uBlockSideLength * m_uBlockSideLength * m_uBlockSideLength * sizeof(VoxelType), "Destination length has changed.");*/ - m_pBlockCompressor->decompress(pBlock, pUncompressedBlock); } diff --git a/library/PolyVoxCore/include/PolyVoxCore/MinizBlockCompressor.inl b/library/PolyVoxCore/include/PolyVoxCore/MinizBlockCompressor.inl index 69f6b2e9..67501ea9 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/MinizBlockCompressor.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/MinizBlockCompressor.inl @@ -38,6 +38,54 @@ namespace PolyVox template void MinizBlockCompressor::compress(UncompressedBlock* pSrcBlock, CompressedBlock* pDstBlock) { + void* pSrcData = reinterpret_cast(pSrcBlock->getData()); + uint32_t uSrcLength = pSrcBlock->getDataSizeInBytes(); + + uint8_t tempBuffer[10000]; + uint8_t* pDstData = reinterpret_cast( tempBuffer ); + uint32_t uDstLength = 10000; + + uint32_t uCompressedLength = 0; + + try + { + // Perform the compression + uCompressedLength = m_pCompressor->compress(pSrcData, uSrcLength, pDstData, uDstLength); + + // Copy the resulting compressed data into the compressed block + pDstBlock->setData(pDstData, uDstLength); + } + catch(std::exception&) + { + // It is possible for the compression to fail. A common cause for this would be if the destination + // buffer is not big enough. So now we try again using a buffer that is definitely big enough. + // Note that ideally we will choose our earlier buffer size so that this almost never happens. + logWarning() << "The compressor failed to compress the block, probabaly due to the buffer being too small."; + logWarning() << "The compression will be tried again with a larger buffer"; + uint32_t uMaxCompressedSize = m_pCompressor->getMaxCompressedSize(uSrcLength); + uint8_t* buffer = new uint8_t[ uMaxCompressedSize ]; + + pDstData = reinterpret_cast( buffer ); + uDstLength = uMaxCompressedSize; + + try + { + // Perform the compression + uCompressedLength = m_pCompressor->compress(pSrcData, uSrcLength, pDstData, uDstLength); + + // Copy the resulting compressed data into the compressed block + pDstBlock->setData(pDstData, uDstLength); + } + catch(std::exception&) + { + // At this point it didn't work even with a bigger buffer. + // Not much more we can do so just rethrow the exception. + delete[] buffer; + POLYVOX_THROW(std::runtime_error, "Failed to compress block data"); + } + + delete[] buffer; + } } template From a4fc814a511409ac3de7f21a2bd0d9120a6a5bd5 Mon Sep 17 00:00:00 2001 From: David Williams Date: Mon, 29 Jul 2013 16:04:01 +0200 Subject: [PATCH 41/55] Rearranging and renaming some code. --- library/PolyVoxCore/CMakeLists.txt | 1 + .../include/PolyVoxCore/BlockCompressor.h | 46 ++++++++++ .../include/PolyVoxCore/LargeVolume.h | 12 ++- .../include/PolyVoxCore/LargeVolume.inl | 89 ++----------------- .../PolyVoxCore/MinizBlockCompressor.h | 4 +- .../PolyVoxCore/PolyVoxForwardDeclarations.h | 5 ++ tests/testvolume.cpp | 8 +- tests/testvolume.h | 2 +- 8 files changed, 72 insertions(+), 95 deletions(-) create mode 100644 library/PolyVoxCore/include/PolyVoxCore/BlockCompressor.h diff --git a/library/PolyVoxCore/CMakeLists.txt b/library/PolyVoxCore/CMakeLists.txt index f79df326..11951b0d 100644 --- a/library/PolyVoxCore/CMakeLists.txt +++ b/library/PolyVoxCore/CMakeLists.txt @@ -48,6 +48,7 @@ SET(CORE_INC_FILES include/PolyVoxCore/BaseVolume.h include/PolyVoxCore/BaseVolume.inl include/PolyVoxCore/BaseVolumeSampler.inl + include/PolyVoxCore/BlockCompressor.h include/PolyVoxCore/Compressor.h include/PolyVoxCore/CubicSurfaceExtractor.h include/PolyVoxCore/CubicSurfaceExtractor.inl diff --git a/library/PolyVoxCore/include/PolyVoxCore/BlockCompressor.h b/library/PolyVoxCore/include/PolyVoxCore/BlockCompressor.h new file mode 100644 index 00000000..21338de7 --- /dev/null +++ b/library/PolyVoxCore/include/PolyVoxCore/BlockCompressor.h @@ -0,0 +1,46 @@ +/******************************************************************************* +Copyright (c) 2005-2013 David Williams and Matthew Williams + +This software is provided 'as-is', without any express or implied +warranty. In no event will the authors be held liable for any damages +arising from the use of this software. + +Permission is granted to anyone to use this software for any purpose, +including commercial applications, and to alter it and redistribute it +freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + + 3. This notice may not be removed or altered from any source + distribution. +*******************************************************************************/ + +#ifndef __PolyVox_BlockCompressor_H__ +#define __PolyVox_BlockCompressor_H__ + +#include "PolyVoxCore/Impl/Block.h" + +namespace PolyVox +{ + /** + * Provides an interface for performing compression of blocks. + */ + template + class BlockCompressor + { + public: + BlockCompressor() {}; + virtual ~BlockCompressor() {}; + + virtual void compress(UncompressedBlock* pSrcBlock, CompressedBlock* pDstBlock) = 0; + virtual void decompress(CompressedBlock* pSrcBlock, UncompressedBlock* pDstBlock) = 0; + }; +} + +#endif //__PolyVox_BlockCompressor_H__ diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h index e5095c3f..986bcf2b 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h @@ -25,9 +25,8 @@ freely, subject to the following restrictions: #define __PolyVox_LargeVolume_H__ #include "PolyVoxCore/BaseVolume.h" -#include "Impl/Block.h" -#include "PolyVoxCore/Compressor.h" -#include "PolyVoxCore/MinizBlockCompressor.h" //Shouldn't include the implementation here. +#include "PolyVoxCore/Impl/Block.h" +#include "PolyVoxCore/BlockCompressor.h" #include "PolyVoxCore/Pager.h" #include "PolyVoxCore/Region.h" #include "PolyVoxCore/Vector.h" @@ -240,7 +239,7 @@ namespace PolyVox LargeVolume ( const Region& regValid, - Compressor* pCompressor, + BlockCompressor* pBlockCompressor, Pager* pPager , uint16_t uBlockSideLength = 32 ); @@ -357,12 +356,11 @@ namespace PolyVox uint8_t m_uBlockSideLengthPower; // The compressor used by the Blocks to compress their data if required. - Compressor* m_pCompressor; - MinizBlockCompressor* m_pBlockCompressor; + BlockCompressor* m_pBlockCompressor; Pager* m_pPager; // Compressed data for an empty block (sometimes needed for initialisation). - CompressedBlock* m_pCompressedEmptyBlock; + //CompressedBlock* m_pCompressedEmptyBlock; // Whether we created the compressor or whether it was provided // by the user. This controls whether we delete it on destruction. diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl index 7534e8bb..7be743fd 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl @@ -23,7 +23,7 @@ freely, subject to the following restrictions: #include "PolyVoxCore/Impl/ErrorHandling.h" -#include "PolyVoxCore/MinizCompressor.h" +#include "PolyVoxCore/MinizBlockCompressor.h" // For creating a default compressor when none is provided. #include @@ -45,7 +45,7 @@ namespace PolyVox { m_uBlockSideLength = uBlockSideLength; - m_pCompressor = new MinizCompressor(); + m_pBlockCompressor = new MinizBlockCompressor(); m_bIsOurCompressor = true; m_pPager = 0; @@ -56,7 +56,7 @@ namespace PolyVox //////////////////////////////////////////////////////////////////////////////// /// This constructor creates a volume with a fixed size which is specified as a parameter. By default this constructor will not enable paging but you can override this if desired. If you do wish to enable paging then you are required to provide the call back function (see the other LargeVolume constructor). /// \param regValid Specifies the minimum and maximum valid voxel positions. - /// \param pCompressor An implementation of the Compressor interface which is used to compress blocks in memory. + /// \param pBlockCompressor An implementation of the Compressor interface which is used to compress blocks in memory. /// \param dataRequiredHandler The callback function which will be called when PolyVox tries to use data which is not currently in momory. /// \param dataOverflowHandler The callback function which will be called when PolyVox has too much data and needs to remove some from memory. /// \param bPagingEnabled Controls whether or not paging is enabled for this LargeVolume. @@ -66,16 +66,15 @@ namespace PolyVox LargeVolume::LargeVolume ( const Region& regValid, - Compressor* pCompressor, + BlockCompressor* pBlockCompressor, Pager* pPager, uint16_t uBlockSideLength ) :BaseVolume(regValid) { - m_uBlockSideLength = uBlockSideLength; - m_pCompressor = pCompressor; + m_pBlockCompressor = pBlockCompressor; m_bIsOurCompressor = false; m_pPager = pPager; @@ -107,10 +106,10 @@ namespace PolyVox // Only delete the compressor if it was created by us (in the constructor), not by the user. if(m_bIsOurCompressor) { - delete m_pCompressor; + delete m_pBlockCompressor; } - delete m_pCompressedEmptyBlock; + //delete m_pCompressedEmptyBlock; } //////////////////////////////////////////////////////////////////////////////// @@ -511,7 +510,7 @@ namespace PolyVox POLYVOX_THROW(std::invalid_argument, "Block side length must be a power of two."); } - if(!m_pCompressor) + if(!m_pBlockCompressor) { POLYVOX_THROW(std::invalid_argument, "You must provide a valid compressor for the LargeVolume to use."); } @@ -544,23 +543,6 @@ namespace PolyVox this->m_uLongestSideLength = (std::max)((std::max)(this->getWidth(),this->getHeight()),this->getDepth()); this->m_uShortestSideLength = (std::min)((std::min)(this->getWidth(),this->getHeight()),this->getDepth()); this->m_fDiagonalLength = sqrtf(static_cast(this->getWidth() * this->getWidth() + this->getHeight() * this->getHeight() + this->getDepth() * this->getDepth())); - - // This is used for initialising empty blocks. - // FIXME - Should probably build an UncompressedBlock and then call some 'fill()' method. Ideally we should set voxels to an 'empty' (default?) value rather than zeros. - VoxelType* pZeros = new VoxelType[m_uBlockSideLength * m_uBlockSideLength * m_uBlockSideLength]; - uint32_t uSrcLength = m_uBlockSideLength * m_uBlockSideLength * m_uBlockSideLength * sizeof(VoxelType); - memset(pZeros, 0, uSrcLength); - uint32_t uDstLength = 100000; - uint8_t* pCompressedZeros = new uint8_t[uDstLength]; //Should be plenty as zeros compress down very small. - uint32_t uCompressedSize = m_pCompressor->compress(reinterpret_cast(pZeros), uSrcLength, reinterpret_cast(pCompressedZeros), uDstLength); - - m_pCompressedEmptyBlock = new CompressedBlock; - m_pCompressedEmptyBlock->setData(pCompressedZeros, uCompressedSize); - - delete[] pZeros; - delete[] pCompressedZeros; - - m_pBlockCompressor = new MinizBlockCompressor; } template @@ -618,61 +600,6 @@ namespace PolyVox // The compressed data has been updated, so the uncompressed data is no longer modified with respect to it. pUncompressedBlock->m_bDataModified = false; - - /*void* pSrcData = reinterpret_cast(pUncompressedBlock->m_tData); - uint32_t uSrcLength = m_uBlockSideLength * m_uBlockSideLength * m_uBlockSideLength * sizeof(VoxelType); - - uint8_t tempBuffer[10000]; - uint8_t* pDstData = reinterpret_cast( tempBuffer ); - uint32_t uDstLength = 10000; - - uint32_t uCompressedLength = 0; - - try - { - // Perform the compression - uCompressedLength = m_pCompressor->compress(pSrcData, uSrcLength, pDstData, uDstLength); - - // Copy the resulting compressed data into the compressed block - pCompressedBlock->setData(pDstData, uDstLength); - - // The compressed data has been updated, so the uncompressed data is no longer modified with respect to it. - pUncompressedBlock->m_bDataModified = false; - } - catch(std::exception&) - { - // It is possible for the compression to fail. A common cause for this would be if the destination - // buffer is not big enough. So now we try again using a buffer that is definitely big enough. - // Note that ideally we will choose our earlier buffer size so that this almost never happens. - logWarning() << "The compressor failed to compress the block, probabaly due to the buffer being too small."; - logWarning() << "The compression will be tried again with a larger buffer"; - uint32_t uMaxCompressedSize = m_pCompressor->getMaxCompressedSize(uSrcLength); - uint8_t* buffer = new uint8_t[ uMaxCompressedSize ]; - - pDstData = reinterpret_cast( buffer ); - uDstLength = uMaxCompressedSize; - - try - { - // Perform the compression - uCompressedLength = m_pCompressor->compress(pSrcData, uSrcLength, pDstData, uDstLength); - - // Copy the resulting compressed data into the compressed block - pCompressedBlock->setData(pDstData, uDstLength); - - // The compressed data has been updated, so the uncompressed data is no longer modified with respect to it. - pUncompressedBlock->m_bDataModified = false; - } - catch(std::exception&) - { - // At this point it didn't work even with a bigger buffer. - // Not much more we can do so just rethrow the exception. - delete[] buffer; - POLYVOX_THROW(std::runtime_error, "Failed to compress block data"); - } - - delete[] buffer; - }*/ } delete itUncompressedBlock->second; diff --git a/library/PolyVoxCore/include/PolyVoxCore/MinizBlockCompressor.h b/library/PolyVoxCore/include/PolyVoxCore/MinizBlockCompressor.h index b6053fbc..e4b57cf6 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/MinizBlockCompressor.h +++ b/library/PolyVoxCore/include/PolyVoxCore/MinizBlockCompressor.h @@ -24,7 +24,7 @@ freely, subject to the following restrictions: #ifndef __PolyVox_MinizBlockCompressor_H__ #define __PolyVox_MinizBlockCompressor_H__ -#include "PolyVoxCore/Impl/Block.h" +#include "PolyVoxCore/BlockCompressor.h" #include "PolyVoxCore/MinizCompressor.h" @@ -34,7 +34,7 @@ namespace PolyVox * Provides an interface for performing paging of data. */ template - class MinizBlockCompressor + class MinizBlockCompressor : public BlockCompressor { public: MinizBlockCompressor(); diff --git a/library/PolyVoxCore/include/PolyVoxCore/PolyVoxForwardDeclarations.h b/library/PolyVoxCore/include/PolyVoxCore/PolyVoxForwardDeclarations.h index 05092ab1..2eef3077 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/PolyVoxForwardDeclarations.h +++ b/library/PolyVoxCore/include/PolyVoxCore/PolyVoxForwardDeclarations.h @@ -60,6 +60,11 @@ namespace PolyVox typedef Array<3,int32_t> Array3DInt32; typedef Array<3,uint32_t> Array3DUint32; + //////////////////////////////////////////////////////////////////////////////// + // BlockCompressor + //////////////////////////////////////////////////////////////////////////////// + template class BlockCompressor; + //////////////////////////////////////////////////////////////////////////////// // Compressor //////////////////////////////////////////////////////////////////////////////// diff --git a/tests/testvolume.cpp b/tests/testvolume.cpp index 77ad1063..d15e1fcc 100644 --- a/tests/testvolume.cpp +++ b/tests/testvolume.cpp @@ -25,7 +25,7 @@ freely, subject to the following restrictions: #include "PolyVoxCore/FilePager.h" #include "PolyVoxCore/LargeVolume.h" -#include "PolyVoxCore/MinizCompressor.h" +#include "PolyVoxCore/MinizBlockCompressor.h" #include "PolyVoxCore/RawVolume.h" #include "PolyVoxCore/RLECompressor.h" #include "PolyVoxCore/SimpleVolume.h" @@ -273,13 +273,13 @@ TestVolume::TestVolume() Region region(-57, -31, 12, 64, 96, 131); // Deliberatly awkward size //m_pCompressor = new RLECompressor; - m_pCompressor = new MinizCompressor; + m_pBlockCompressor = new MinizBlockCompressor; m_pFilePager = new FilePager("./"); //Create the volumes m_pRawVolume = new RawVolume(region); m_pSimpleVolume = new SimpleVolume(region); - m_pLargeVolume = new LargeVolume(region, m_pCompressor, m_pFilePager, 32); + m_pLargeVolume = new LargeVolume(region, m_pBlockCompressor, m_pFilePager, 32); m_pLargeVolume->setMaxNumberOfBlocksInMemory(32); m_pLargeVolume->setMaxNumberOfUncompressedBlocks(16); @@ -307,7 +307,7 @@ TestVolume::~TestVolume() delete m_pLargeVolume; delete m_pFilePager; - delete m_pCompressor; + delete m_pBlockCompressor; } /* diff --git a/tests/testvolume.h b/tests/testvolume.h index 8a3e46d7..9db5f7a0 100644 --- a/tests/testvolume.h +++ b/tests/testvolume.h @@ -65,7 +65,7 @@ private slots: void testLargeVolumeSamplersWithExternalBackwards(); private: - PolyVox::Compressor* m_pCompressor; + PolyVox::BlockCompressor* m_pBlockCompressor; PolyVox::FilePager* m_pFilePager; PolyVox::RawVolume* m_pRawVolume; From 020ac9655a008de379b624076d57d54265218295 Mon Sep 17 00:00:00 2001 From: David Williams Date: Tue, 30 Jul 2013 15:42:51 +0200 Subject: [PATCH 42/55] Added random prefix to filename to prevent name conflicts. --- .../include/PolyVoxCore/FilePager.h | 23 ++++++++++++++----- 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/library/PolyVoxCore/include/PolyVoxCore/FilePager.h b/library/PolyVoxCore/include/PolyVoxCore/FilePager.h index 66091d9c..4f8f15c8 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/FilePager.h +++ b/library/PolyVoxCore/include/PolyVoxCore/FilePager.h @@ -29,6 +29,8 @@ freely, subject to the following restrictions: #include "PolyVoxCore/Pager.h" #include "PolyVoxCore/Region.h" +#include +#include #include #include #include @@ -47,6 +49,12 @@ namespace PolyVox :Pager() ,m_strFolderName(strFolderName) { + srand(time(0)); + int iRandomValue = rand(); + + std::stringstream ss; + ss << std::hex << iRandomValue; + m_strRandomPrefix = ss.str(); } /// Destructor @@ -57,11 +65,12 @@ namespace PolyVox POLYVOX_ASSERT(pBlockData, "Attempting to page in NULL block"); //POLYVOX_ASSERT(pBlockData->hasUncompressedData() == false, "Block should not have uncompressed data"); - std::stringstream ss; - ss << region.getLowerX() << "_" << region.getLowerY() << "_" << region.getLowerZ() << "_" + std::stringstream ssFilename; + ssFilename << m_strFolderName << "/" << m_strRandomPrefix << "-" + << region.getLowerX() << "_" << region.getLowerY() << "_" << region.getLowerZ() << "_" << region.getUpperX() << "_" << region.getUpperY() << "_" << region.getUpperZ(); - std::string filename = m_strFolderName + ss.str(); + std::string filename = ssFilename.str(); // FIXME - This should be replaced by C++ style IO, but currently this causes problems with // the gameplay-cubiquity integration. See: https://github.com/blackberry/GamePlay/issues/919 @@ -100,11 +109,12 @@ namespace PolyVox logTrace() << "Paging out data for " << region; - std::stringstream ss; - ss << region.getLowerX() << "_" << region.getLowerY() << "_" << region.getLowerZ() << "_" + std::stringstream ssFilename; + ssFilename << m_strFolderName << "/" << m_strRandomPrefix << "-" + << region.getLowerX() << "_" << region.getLowerY() << "_" << region.getLowerZ() << "_" << region.getUpperX() << "_" << region.getUpperY() << "_" << region.getUpperZ(); - std::string filename = m_strFolderName + ss.str(); + std::string filename = ssFilename.str(); // FIXME - This should be replaced by C++ style IO, but currently this causes problems with // the gameplay-cubiquity integration. See: https://github.com/blackberry/GamePlay/issues/919 @@ -127,6 +137,7 @@ namespace PolyVox protected: std::string m_strFolderName; + std::string m_strRandomPrefix; }; } From d1138dcdb12d35824b49b05e61cc39ee1ff5c396 Mon Sep 17 00:00:00 2001 From: David Williams Date: Tue, 30 Jul 2013 16:01:03 +0200 Subject: [PATCH 43/55] Some new block compression code as I rework the previous code... --- library/PolyVoxCore/CMakeLists.txt | 2 + .../include/PolyVoxCore/RLEBlockCompressor.h | 61 +++++++ .../PolyVoxCore/RLEBlockCompressor.inl | 169 ++++++++++++++++++ 3 files changed, 232 insertions(+) create mode 100644 library/PolyVoxCore/include/PolyVoxCore/RLEBlockCompressor.h create mode 100644 library/PolyVoxCore/include/PolyVoxCore/RLEBlockCompressor.inl diff --git a/library/PolyVoxCore/CMakeLists.txt b/library/PolyVoxCore/CMakeLists.txt index 11951b0d..018efe2d 100644 --- a/library/PolyVoxCore/CMakeLists.txt +++ b/library/PolyVoxCore/CMakeLists.txt @@ -85,6 +85,8 @@ SET(CORE_INC_FILES include/PolyVoxCore/Raycast.h include/PolyVoxCore/Raycast.inl include/PolyVoxCore/Region.h + include/PolyVoxCore/RLEBlockCompressor.h + include/PolyVoxCore/RLEBlockCompressor.inl include/PolyVoxCore/RLECompressor.h include/PolyVoxCore/RLECompressor.inl include/PolyVoxCore/SimpleVolume.h diff --git a/library/PolyVoxCore/include/PolyVoxCore/RLEBlockCompressor.h b/library/PolyVoxCore/include/PolyVoxCore/RLEBlockCompressor.h new file mode 100644 index 00000000..fd77522f --- /dev/null +++ b/library/PolyVoxCore/include/PolyVoxCore/RLEBlockCompressor.h @@ -0,0 +1,61 @@ +/******************************************************************************* +Copyright (c) 2005-2013 David Williams and Matthew Williams + +This software is provided 'as-is', without any express or implied +warranty. In no event will the authors be held liable for any damages +arising from the use of this software. + +Permission is granted to anyone to use this software for any purpose, +including commercial applications, and to alter it and redistribute it +freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + + 3. This notice may not be removed or altered from any source + distribution. +*******************************************************************************/ + +#ifndef __PolyVox_RLEBlockCompressor_H__ +#define __PolyVox_RLEBlockCompressor_H__ + +#include "PolyVoxCore/BlockCompressor.h" + +#include "PolyVoxCore/MinizCompressor.h" + +namespace PolyVox +{ + template + class Run + { + public: + typedef uint16_t LengthType; + VoxelType value; + LengthType length; + }; + + /** + * Provides an interface for performing paging of data. + */ + template + class RLEBlockCompressor : public BlockCompressor + { + + + public: + RLEBlockCompressor(); + ~RLEBlockCompressor(); + + void compress(UncompressedBlock* pSrcBlock, CompressedBlock* pDstBlock); + void decompress(CompressedBlock* pSrcBlock, UncompressedBlock* pDstBlock); + }; +} + +#include "PolyVoxCore/RLEBlockCompressor.inl" + +#endif //__PolyVox_RLEBlockCompressor_H__ diff --git a/library/PolyVoxCore/include/PolyVoxCore/RLEBlockCompressor.inl b/library/PolyVoxCore/include/PolyVoxCore/RLEBlockCompressor.inl new file mode 100644 index 00000000..4fc90b2e --- /dev/null +++ b/library/PolyVoxCore/include/PolyVoxCore/RLEBlockCompressor.inl @@ -0,0 +1,169 @@ +/******************************************************************************* +Copyright (c) 2005-2013 David Williams and Matthew Williams + +This software is provided 'as-is', without any express or implied +warranty. In no event will the authors be held liable for any damages +arising from the use of this software. + +Permission is granted to anyone to use this software for any purpose, +including commercial applications, and to alter it and redistribute it +freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + + 3. This notice may not be removed or altered from any source + distribution. +*******************************************************************************/ + +#include + +#include "RLECompressor.h" + +namespace PolyVox +{ + template + RLEBlockCompressor::RLEBlockCompressor() + { + } + + template + RLEBlockCompressor::~RLEBlockCompressor() + { + } + + template + void RLEBlockCompressor::compress(UncompressedBlock* pSrcBlock, CompressedBlock* pDstBlock) + { + void* pSrcData = reinterpret_cast(pSrcBlock->getData()); + uint32_t uSrcLength = pSrcBlock->getDataSizeInBytes(); + + if(uSrcLength % sizeof(VoxelType) != 0) + { + POLYVOX_THROW(std::length_error, "Source length must be a integer multiple of the ValueType size"); + } + + // Lengths provided are in bytes, so convert them to be in terms of our types. + uSrcLength /= sizeof(VoxelType); + //uDstLength /= sizeof(Run); + + // Get data pointers in the appropriate type + const VoxelType* pSrcDataAsType = reinterpret_cast(pSrcData); + //Run* pDstDataAsRun = reinterpret_cast(pDstData); + std::vector< Run > vecDstDataAsRuns; + + // Pointers to just past the end of the data + const VoxelType* pSrcDataEnd = pSrcDataAsType + uSrcLength; + //Run* pDstDataEnd = pDstDataAsRun + uDstLength; + + //Counter for the output length + //uint32_t uDstLengthInBytes = 0; + + // Read the first element of the source and set up the first run based on it. + /*pDstDataAsRun->value = *pSrcDataAsType; + pSrcDataAsType++; + pDstDataAsRun->length = 1; + uDstLengthInBytes += sizeof(Run);*/ + Run firstRun; + firstRun.value = *pSrcDataAsType; + firstRun.length = 1; + vecDstDataAsRuns.push_back(firstRun); + + pSrcDataAsType++; + + //Now process all remaining elements of the source. + while(pSrcDataAsType < pSrcDataEnd) + { + // If the value is the same as the current run (and we have not + // reached the maximum run length) then extend the current run. + std::vector< Run >::iterator currentRun = (vecDstDataAsRuns.end() - 1); + if((*pSrcDataAsType == currentRun->value) && (currentRun->length < (std::numeric_limits::LengthType>::max)())) + { + currentRun->length++; + } + // Otherwise we need to start a new Run. + else + { + /*pDstDataAsRun++; + + // Check if we have enough space in the destination buffer. + if(pDstDataAsRun >= pDstDataEnd) + { + POLYVOX_THROW(std::runtime_error, "Insufficient space in destination buffer."); + }*/ + + // Create the new run. + /*pDstDataAsRun->value = *pSrcDataAsType; + pDstDataAsRun->length = 1; + uDstLengthInBytes += sizeof(Run);*/ + + Run newRun; + newRun.value = *pSrcDataAsType; + newRun.length = 1; + vecDstDataAsRuns.push_back(newRun); + } + + pSrcDataAsType++; + } + + //Now copy the data into the compressed block. + + pDstBlock->setData(reinterpret_cast(&(vecDstDataAsRuns[0])), vecDstDataAsRuns.size() * sizeof(Run)); + } + + template + void RLEBlockCompressor::decompress(CompressedBlock* pSrcBlock, UncompressedBlock* pDstBlock) + { + const void* pSrcData = reinterpret_cast(pSrcBlock->getData()); + uint32_t uSrcLength = pSrcBlock->getDataSizeInBytes(); + + void* pDstData = pDstBlock->getData(); + uint32_t uDstLength = pDstBlock->getDataSizeInBytes(); + + if(uSrcLength % sizeof(Run) != 0) + { + POLYVOX_THROW(std::length_error, "Source length must be a integer multiple of the Run size"); + } + + // Lengths provided are in bytes, so convert them to be in terms of our types. + uSrcLength /= sizeof(Run); + uDstLength /= sizeof(VoxelType); + + // Get data pointers in the appropriate type + const Run* pSrcDataAsRun = reinterpret_cast*>(pSrcData); + VoxelType* pDstDataAsType = reinterpret_cast(pDstData); + + // Pointers to just past the end of the data + const Run* pSrcDataEnd = pSrcDataAsRun + uSrcLength; + VoxelType* pDstDataEnd = pDstDataAsType + uDstLength; + + //Counter for the output length + uint32_t uDstLengthInBytes = 0; + + while(pSrcDataAsRun < pSrcDataEnd) + { + // Check if we have enough space in the destination buffer. + if(pDstDataAsType + pSrcDataAsRun->length > pDstDataEnd) + { + POLYVOX_THROW(std::runtime_error, "Insufficient space in destination buffer."); + } + else + { + + // Write the run into the destination + std::fill(pDstDataAsType, pDstDataAsType + pSrcDataAsRun->length, pSrcDataAsRun->value); + pDstDataAsType += pSrcDataAsRun->length; + + uDstLengthInBytes += pSrcDataAsRun->length * sizeof(VoxelType); + } + pSrcDataAsRun++; + } + + POLYVOX_ASSERT(uDstLengthInBytes == pDstBlock->getDataSizeInBytes(), "Uncompressed data does not have the correct length."); + } +} \ No newline at end of file From fa8813ba86cfdb3752daf8c69c8dc8eb48fee657 Mon Sep 17 00:00:00 2001 From: David Williams Date: Tue, 30 Jul 2013 16:34:40 +0200 Subject: [PATCH 44/55] FilePager now cleans up after itself. --- .../include/PolyVoxCore/FilePager.h | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/library/PolyVoxCore/include/PolyVoxCore/FilePager.h b/library/PolyVoxCore/include/PolyVoxCore/FilePager.h index 4f8f15c8..3e087b5f 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/FilePager.h +++ b/library/PolyVoxCore/include/PolyVoxCore/FilePager.h @@ -58,7 +58,18 @@ namespace PolyVox } /// Destructor - virtual ~FilePager() {}; + virtual ~FilePager() + { + for(std::vector::iterator iter = m_vecCreatedFiles.begin(); iter < m_vecCreatedFiles.end(); iter++) + { + if(!std::remove(iter->c_str())) + { + logWarning() << "Failed to delete '" << *iter << "' when destroying FilePager"; + } + } + + m_vecCreatedFiles.clear(); + } virtual void pageIn(const Region& region, CompressedBlock* pBlockData) { @@ -125,6 +136,9 @@ namespace PolyVox POLYVOX_THROW(std::runtime_error, "Unable to open file to write out block data."); } + //The file has been created, so add it to the list to delete on shutdown. + m_vecCreatedFiles.push_back(filename); + fwrite(pBlockData->getData(), sizeof(uint8_t), pBlockData->getDataSizeInBytes(), pFile); if(ferror(pFile)) @@ -138,6 +152,8 @@ namespace PolyVox protected: std::string m_strFolderName; std::string m_strRandomPrefix; + + std::vector m_vecCreatedFiles; }; } From 4478e365c9c6d3d4533f0198842c8cc61df6ddf2 Mon Sep 17 00:00:00 2001 From: David Williams Date: Tue, 30 Jul 2013 17:01:27 +0200 Subject: [PATCH 45/55] Removed RLECompressor as the code is now in RLEBlockCompressor. Things are starting to get back under control. All tests pass, and all examples except PagingExample work. --- examples/OpenGL/main.cpp | 8 +- examples/Paging/main.cpp | 4 +- library/PolyVoxCore/CMakeLists.txt | 2 - .../PolyVoxCore/RLEBlockCompressor.inl | 2 - .../include/PolyVoxCore/RLECompressor.h | 40 ------ .../include/PolyVoxCore/RLECompressor.inl | 126 ------------------ tests/testvolume.cpp | 1 - 7 files changed, 7 insertions(+), 176 deletions(-) delete mode 100644 library/PolyVoxCore/include/PolyVoxCore/RLECompressor.h delete mode 100644 library/PolyVoxCore/include/PolyVoxCore/RLECompressor.inl diff --git a/examples/OpenGL/main.cpp b/examples/OpenGL/main.cpp index ccae9941..e6ae33d7 100644 --- a/examples/OpenGL/main.cpp +++ b/examples/OpenGL/main.cpp @@ -21,11 +21,12 @@ freely, subject to the following restrictions: distribution. *******************************************************************************/ +#include "PolyVoxCore/FilePager.h" #include "PolyVoxCore/MaterialDensityPair.h" #include "PolyVoxCore/LargeVolume.h" #include "PolyVoxCore/LowPassFilter.h" #include "PolyVoxCore/RawVolume.h" -#include "PolyVoxCore/RLECompressor.h" +#include "PolyVoxCore/RLEBlockCompressor.h" #include "PolyVoxCore/SurfaceMesh.h" #include "PolyVoxCore/Impl/Utility.h" @@ -49,8 +50,9 @@ using namespace std; int main(int argc, char *argv[]) { - RLECompressor* compressor = new RLECompressor(); - LargeVolume volData(PolyVox::Region(Vector3DInt32(0,0,0), Vector3DInt32(g_uVolumeSideLength-1, g_uVolumeSideLength-1, g_uVolumeSideLength-1)), compressor, 0); + RLEBlockCompressor* compressor = new RLEBlockCompressor(); + FilePager* pager = new FilePager("./"); + LargeVolume volData(PolyVox::Region(Vector3DInt32(0,0,0), Vector3DInt32(g_uVolumeSideLength-1, g_uVolumeSideLength-1, g_uVolumeSideLength-1)), compressor, pager); //Make our volume contain a sphere in the center. int32_t minPos = 0; diff --git a/examples/Paging/main.cpp b/examples/Paging/main.cpp index 117145b7..f28c54e6 100644 --- a/examples/Paging/main.cpp +++ b/examples/Paging/main.cpp @@ -28,7 +28,7 @@ freely, subject to the following restrictions: #include "PolyVoxCore/CubicSurfaceExtractorWithNormals.h" #include "PolyVoxCore/MarchingCubesSurfaceExtractor.h" #include "PolyVoxCore/Pager.h" -#include "PolyVoxCore/RLECompressor.h" +#include "PolyVoxCore/RLEBlockCompressor.h" #include "PolyVoxCore/SurfaceMesh.h" #include "PolyVoxCore/LargeVolume.h" @@ -150,7 +150,7 @@ int main(int argc, char *argv[]) OpenGLWidget openGLWidget(0); openGLWidget.show(); - RLECompressor* compressor = new RLECompressor(); + RLEBlockCompressor* compressor = new RLEBlockCompressor(); PerlinNoisePager* pager = new PerlinNoisePager(); LargeVolume volData(PolyVox::Region::MaxRegion, compressor, pager, 256); volData.setMaxNumberOfBlocksInMemory(4096); diff --git a/library/PolyVoxCore/CMakeLists.txt b/library/PolyVoxCore/CMakeLists.txt index 018efe2d..f618c9ae 100644 --- a/library/PolyVoxCore/CMakeLists.txt +++ b/library/PolyVoxCore/CMakeLists.txt @@ -87,8 +87,6 @@ SET(CORE_INC_FILES include/PolyVoxCore/Region.h include/PolyVoxCore/RLEBlockCompressor.h include/PolyVoxCore/RLEBlockCompressor.inl - include/PolyVoxCore/RLECompressor.h - include/PolyVoxCore/RLECompressor.inl include/PolyVoxCore/SimpleVolume.h include/PolyVoxCore/SimpleVolume.inl include/PolyVoxCore/SimpleVolumeBlock.inl diff --git a/library/PolyVoxCore/include/PolyVoxCore/RLEBlockCompressor.inl b/library/PolyVoxCore/include/PolyVoxCore/RLEBlockCompressor.inl index 4fc90b2e..aba87809 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/RLEBlockCompressor.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/RLEBlockCompressor.inl @@ -23,8 +23,6 @@ freely, subject to the following restrictions: #include -#include "RLECompressor.h" - namespace PolyVox { template diff --git a/library/PolyVoxCore/include/PolyVoxCore/RLECompressor.h b/library/PolyVoxCore/include/PolyVoxCore/RLECompressor.h deleted file mode 100644 index 03fd3318..00000000 --- a/library/PolyVoxCore/include/PolyVoxCore/RLECompressor.h +++ /dev/null @@ -1,40 +0,0 @@ -#ifndef __PolyVox_RLECompressor_H__ -#define __PolyVox_RLECompressor_H__ - -#include "PolyVoxCore/Compressor.h" - -namespace PolyVox -{ - /** - * Performs compression of data using Run Length Encoding (RLE). - * - * This compressor is designed for voxel data which contains long runs of the same value. Minecraft-style terrain and other - * cubic-style terrains are likely to fall under this category, whereas density fields for Marching Cubes terrain will not. Please - * see the following article if you want more details of how RLE compression works: http://en.wikipedia.org/wiki/Run-length_encoding - * - * \sa MinizCompressor - */ - template - class RLECompressor : public Compressor - { - struct Run - { - ValueType value; - LengthType length; - }; - public: - /// Constructor - RLECompressor(); - /// Destructor - ~RLECompressor(); - - // API documentation is in base class and gets inherited by Doxygen. - uint32_t getMaxCompressedSize(uint32_t uUncompressedInputSize); - uint32_t compress(const void* pSrcData, uint32_t uSrcLength, void* pDstData, uint32_t uDstLength); - uint32_t decompress(const void* pSrcData, uint32_t uSrcLength, void* pDstData, uint32_t uDstLength); - }; -} - -#include "RLECompressor.inl" - -#endif //__PolyVox_RLECompressor_H__ diff --git a/library/PolyVoxCore/include/PolyVoxCore/RLECompressor.inl b/library/PolyVoxCore/include/PolyVoxCore/RLECompressor.inl deleted file mode 100644 index cf65bc03..00000000 --- a/library/PolyVoxCore/include/PolyVoxCore/RLECompressor.inl +++ /dev/null @@ -1,126 +0,0 @@ -#include -#include -#include - -namespace PolyVox -{ - template - RLECompressor::RLECompressor() - { - } - - template - RLECompressor::~RLECompressor() - { - } - - template - uint32_t RLECompressor::getMaxCompressedSize(uint32_t uUncompressedInputSize) - { - // In the worst case we will have a seperate Run (of length one) for each element of the input data. - return uUncompressedInputSize * sizeof(Run); - } - - template - uint32_t RLECompressor::compress(const void* pSrcData, uint32_t uSrcLength, void* pDstData, uint32_t uDstLength) - { - if(uSrcLength % sizeof(ValueType) != 0) - { - POLYVOX_THROW(std::length_error, "Source length must be a integer multiple of the ValueType size"); - } - - // Lengths provided are in bytes, so convert them to be in terms of our types. - uSrcLength /= sizeof(ValueType); - uDstLength /= sizeof(Run); - - // Get data pointers in the appropriate type - const ValueType* pSrcDataAsType = reinterpret_cast(pSrcData); - Run* pDstDataAsRun = reinterpret_cast(pDstData); - - // Pointers to just past the end of the data - const ValueType* pSrcDataEnd = pSrcDataAsType + uSrcLength; - Run* pDstDataEnd = pDstDataAsRun + uDstLength; - - //Counter for the output length - uint32_t uDstLengthInBytes = 0; - - // Read the first element of the source and set up the first run based on it. - pDstDataAsRun->value = *pSrcDataAsType; - pSrcDataAsType++; - pDstDataAsRun->length = 1; - uDstLengthInBytes += sizeof(Run); - - //Now process all remaining elements of the source. - while(pSrcDataAsType < pSrcDataEnd) - { - // If the value is the same as the current run (and we have not - // reached the maximum run length) then extend the current run. - if((*pSrcDataAsType == pDstDataAsRun->value) && (pDstDataAsRun->length < (std::numeric_limits::max)())) - { - pDstDataAsRun->length++; - } - // Otherwise we need to start a new Run. - else - { - pDstDataAsRun++; - - // Check if we have enough space in the destination buffer. - if(pDstDataAsRun >= pDstDataEnd) - { - POLYVOX_THROW(std::runtime_error, "Insufficient space in destination buffer."); - } - - // Create the new run. - pDstDataAsRun->value = *pSrcDataAsType; - pDstDataAsRun->length = 1; - uDstLengthInBytes += sizeof(Run); - } - - pSrcDataAsType++; - } - - return uDstLengthInBytes; - } - - template - uint32_t RLECompressor::decompress(const void* pSrcData, uint32_t uSrcLength, void* pDstData, uint32_t uDstLength) - { - if(uSrcLength % sizeof(Run) != 0) - { - POLYVOX_THROW(std::length_error, "Source length must be a integer multiple of the Run size"); - } - - // Lengths provided are in bytes, so convert them to be in terms of our types. - uSrcLength /= sizeof(Run); - uDstLength /= sizeof(ValueType); - - // Get data pointers in the appropriate type - const Run* pSrcDataAsRun = reinterpret_cast(pSrcData); - ValueType* pDstDataAsType = reinterpret_cast(pDstData); - - // Pointers to just past the end of the data - const Run* pSrcDataEnd = pSrcDataAsRun + uSrcLength; - ValueType* pDstDataEnd = pDstDataAsType + uDstLength; - - //Counter for the output length - uint32_t uDstLengthInBytes = 0; - - while(pSrcDataAsRun < pSrcDataEnd) - { - // Check if we have enough space in the destination buffer. - if(pDstDataAsType + pSrcDataAsRun->length > pDstDataEnd) - { - POLYVOX_THROW(std::runtime_error, "Insufficient space in destination buffer."); - } - - // Write the run into the destination - std::fill(pDstDataAsType, pDstDataAsType + pSrcDataAsRun->length, pSrcDataAsRun->value); - pDstDataAsType += pSrcDataAsRun->length; - - uDstLengthInBytes += pSrcDataAsRun->length * sizeof(ValueType); - pSrcDataAsRun++; - } - - return uDstLengthInBytes; - } -} diff --git a/tests/testvolume.cpp b/tests/testvolume.cpp index d15e1fcc..f879f1bf 100644 --- a/tests/testvolume.cpp +++ b/tests/testvolume.cpp @@ -27,7 +27,6 @@ freely, subject to the following restrictions: #include "PolyVoxCore/LargeVolume.h" #include "PolyVoxCore/MinizBlockCompressor.h" #include "PolyVoxCore/RawVolume.h" -#include "PolyVoxCore/RLECompressor.h" #include "PolyVoxCore/SimpleVolume.h" #include From e35b58ba188335613481f623bb6bd8bfecff7765 Mon Sep 17 00:00:00 2001 From: David Williams Date: Wed, 31 Jul 2013 15:49:00 +0200 Subject: [PATCH 46/55] Fixed paging example. --- examples/Paging/main.cpp | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/examples/Paging/main.cpp b/examples/Paging/main.cpp index f28c54e6..27236afd 100644 --- a/examples/Paging/main.cpp +++ b/examples/Paging/main.cpp @@ -93,7 +93,9 @@ public: virtual void pageIn(const PolyVox::Region& region, CompressedBlock* pBlockData) { - /*pBlockData->createUncompressedData(); + // FIXME - this isn't a great example... it's a shame we have to hard clode the block size and also create/destroy + // a compressor each time. These could at least be moved outside somewhere if we can't fix it in a better way... + UncompressedBlock block(256); Perlin perlin(2,2,1,234); @@ -131,10 +133,15 @@ public: // Voxel position within a block always start from zero. So if a block represents region (4, 8, 12) to (11, 19, 15) // then the valid block voxels are from (0, 0, 0) to (7, 11, 3). Hence we subtract the lower corner position of the // region from the volume space position in order to get the block space position. - pBlockData->setVoxelAt(x - region.getLowerX(), y - region.getLowerY(), z - region.getLowerZ(), voxel); + block.setVoxelAt(x - region.getLowerX(), y - region.getLowerY(), z - region.getLowerZ(), voxel); } } - }*/ + } + + // Now compress the computed data into the provided block. + RLEBlockCompressor* compressor = new RLEBlockCompressor(); + compressor->compress(&block, pBlockData); + delete compressor; } virtual void pageOut(const PolyVox::Region& region, CompressedBlock* /*pBlockData*/) From 9503d975c1b5b4f797e321194229bd463ec9058e Mon Sep 17 00:00:00 2001 From: David Williams Date: Wed, 31 Jul 2013 16:37:39 +0200 Subject: [PATCH 47/55] GCC compile fixes. --- library/PolyVoxCore/include/PolyVoxCore/BlockCompressor.h | 1 + library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h | 6 +++--- library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl | 4 ++-- library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl | 2 +- .../PolyVoxCore/include/PolyVoxCore/RLEBlockCompressor.inl | 4 ++-- 5 files changed, 9 insertions(+), 8 deletions(-) diff --git a/library/PolyVoxCore/include/PolyVoxCore/BlockCompressor.h b/library/PolyVoxCore/include/PolyVoxCore/BlockCompressor.h index 21338de7..dbef7a02 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/BlockCompressor.h +++ b/library/PolyVoxCore/include/PolyVoxCore/BlockCompressor.h @@ -24,6 +24,7 @@ freely, subject to the following restrictions: #ifndef __PolyVox_BlockCompressor_H__ #define __PolyVox_BlockCompressor_H__ +#include "PolyVoxCore/PolyVoxForwardDeclarations.h" #include "PolyVoxCore/Impl/Block.h" namespace PolyVox diff --git a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h index 554bc3c2..718ee9db 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h +++ b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h @@ -37,7 +37,7 @@ namespace PolyVox template class Block { - friend LargeVolume; + friend class LargeVolume; public: Block() @@ -65,7 +65,7 @@ namespace PolyVox template class CompressedBlock : public Block { - friend LargeVolume; + friend class LargeVolume; public: CompressedBlock(); @@ -94,7 +94,7 @@ namespace PolyVox template class UncompressedBlock : public Block { - friend LargeVolume; + friend class LargeVolume; public: UncompressedBlock(uint16_t uSideLength); diff --git a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl index ba156ef4..279dbf0e 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl @@ -80,7 +80,7 @@ namespace PolyVox memcpy(m_pData, pData, uDataSizeInBytes); // Flag as modified - m_bDataModified = true; + this->m_bDataModified = true; } template @@ -168,7 +168,7 @@ namespace PolyVox uZPos * m_uSideLength * m_uSideLength ] = tValue; - m_bDataModified = true; + this->m_bDataModified = true; } template diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl index 7be743fd..e18568bd 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl @@ -45,7 +45,7 @@ namespace PolyVox { m_uBlockSideLength = uBlockSideLength; - m_pBlockCompressor = new MinizBlockCompressor(); + m_pBlockCompressor = new MinizBlockCompressor(); m_bIsOurCompressor = true; m_pPager = 0; diff --git a/library/PolyVoxCore/include/PolyVoxCore/RLEBlockCompressor.inl b/library/PolyVoxCore/include/PolyVoxCore/RLEBlockCompressor.inl index aba87809..369fb912 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/RLEBlockCompressor.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/RLEBlockCompressor.inl @@ -79,8 +79,8 @@ namespace PolyVox { // If the value is the same as the current run (and we have not // reached the maximum run length) then extend the current run. - std::vector< Run >::iterator currentRun = (vecDstDataAsRuns.end() - 1); - if((*pSrcDataAsType == currentRun->value) && (currentRun->length < (std::numeric_limits::LengthType>::max)())) + typename std::vector< Run< VoxelType> >::iterator currentRun = (vecDstDataAsRuns.end() - 1); + if((*pSrcDataAsType == currentRun->value) && (currentRun->length < (std::numeric_limits::LengthType>::max)())) { currentRun->length++; } From 6f17e0faa67f53d6f4ac8b0093474dbe3d50fb1f Mon Sep 17 00:00:00 2001 From: David Williams Date: Thu, 1 Aug 2013 15:19:10 +0200 Subject: [PATCH 48/55] The 'Block' class is no longer considered just to be an implementation details of LargeVolume. Users may need to interact with them directly of they implement their own paging or compression systems. --- library/PolyVoxCore/CMakeLists.txt | 4 ++-- library/PolyVoxCore/include/PolyVoxCore/{Impl => }/Block.h | 2 +- library/PolyVoxCore/include/PolyVoxCore/{Impl => }/Block.inl | 0 library/PolyVoxCore/include/PolyVoxCore/BlockCompressor.h | 2 +- library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h | 2 +- library/PolyVoxCore/include/PolyVoxCore/Pager.h | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) rename library/PolyVoxCore/include/PolyVoxCore/{Impl => }/Block.h (99%) rename library/PolyVoxCore/include/PolyVoxCore/{Impl => }/Block.inl (100%) diff --git a/library/PolyVoxCore/CMakeLists.txt b/library/PolyVoxCore/CMakeLists.txt index f618c9ae..5f2d49b3 100644 --- a/library/PolyVoxCore/CMakeLists.txt +++ b/library/PolyVoxCore/CMakeLists.txt @@ -48,6 +48,8 @@ SET(CORE_INC_FILES include/PolyVoxCore/BaseVolume.h include/PolyVoxCore/BaseVolume.inl include/PolyVoxCore/BaseVolumeSampler.inl + include/PolyVoxCore/Block.h + include/PolyVoxCore/Block.inl include/PolyVoxCore/BlockCompressor.h include/PolyVoxCore/Compressor.h include/PolyVoxCore/CubicSurfaceExtractor.h @@ -114,8 +116,6 @@ SET(IMPL_INC_FILES include/PolyVoxCore/Impl/ArraySizesImpl.h include/PolyVoxCore/Impl/ArraySizesImpl.inl include/PolyVoxCore/Impl/AStarPathfinderImpl.h - include/PolyVoxCore/Impl/Block.h - include/PolyVoxCore/Impl/Block.inl include/PolyVoxCore/Impl/CompilerCapabilities.h include/PolyVoxCore/Impl/Config.h include/PolyVoxCore/Impl/ErrorHandling.h diff --git a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h b/library/PolyVoxCore/include/PolyVoxCore/Block.h similarity index 99% rename from library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h rename to library/PolyVoxCore/include/PolyVoxCore/Block.h index 718ee9db..a1567552 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h +++ b/library/PolyVoxCore/include/PolyVoxCore/Block.h @@ -126,6 +126,6 @@ namespace PolyVox }; } -#include "PolyVoxCore/Impl/Block.inl" +#include "PolyVoxCore/Block.inl" #endif diff --git a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl b/library/PolyVoxCore/include/PolyVoxCore/Block.inl similarity index 100% rename from library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl rename to library/PolyVoxCore/include/PolyVoxCore/Block.inl diff --git a/library/PolyVoxCore/include/PolyVoxCore/BlockCompressor.h b/library/PolyVoxCore/include/PolyVoxCore/BlockCompressor.h index dbef7a02..2646e94c 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/BlockCompressor.h +++ b/library/PolyVoxCore/include/PolyVoxCore/BlockCompressor.h @@ -25,7 +25,7 @@ freely, subject to the following restrictions: #define __PolyVox_BlockCompressor_H__ #include "PolyVoxCore/PolyVoxForwardDeclarations.h" -#include "PolyVoxCore/Impl/Block.h" +#include "PolyVoxCore/Block.h" namespace PolyVox { diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h index 986bcf2b..a26bc29b 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h @@ -25,7 +25,7 @@ freely, subject to the following restrictions: #define __PolyVox_LargeVolume_H__ #include "PolyVoxCore/BaseVolume.h" -#include "PolyVoxCore/Impl/Block.h" +#include "PolyVoxCore/Block.h" #include "PolyVoxCore/BlockCompressor.h" #include "PolyVoxCore/Pager.h" #include "PolyVoxCore/Region.h" diff --git a/library/PolyVoxCore/include/PolyVoxCore/Pager.h b/library/PolyVoxCore/include/PolyVoxCore/Pager.h index af1e3387..6189566a 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/Pager.h +++ b/library/PolyVoxCore/include/PolyVoxCore/Pager.h @@ -24,7 +24,7 @@ freely, subject to the following restrictions: #ifndef __PolyVox_Pager_H__ #define __PolyVox_Pager_H__ -#include "PolyVoxCore/Impl/Block.h" +#include "PolyVoxCore/Block.h" #include "PolyVoxCore/Impl/TypeDef.h" namespace PolyVox From dcf86a905c64cc755790054a6565062c51479b4c Mon Sep 17 00:00:00 2001 From: David Williams Date: Thu, 1 Aug 2013 15:32:12 +0200 Subject: [PATCH 49/55] Split Block, CompressedBlock and UncompressedBlock into separate files. It's tidier, and maybe better for SWIG? --- library/PolyVoxCore/CMakeLists.txt | 5 +- .../PolyVoxCore/include/PolyVoxCore/Block.h | 67 +------------- .../include/PolyVoxCore/BlockCompressor.h | 3 +- .../include/PolyVoxCore/CompressedBlock.h | 63 +++++++++++++ .../include/PolyVoxCore/CompressedBlock.inl | 79 +++++++++++++++++ .../PolyVoxCore/include/PolyVoxCore/Pager.h | 2 +- .../include/PolyVoxCore/UncompressedBlock.h | 68 ++++++++++++++ .../{Block.inl => UncompressedBlock.inl} | 88 ++----------------- 8 files changed, 227 insertions(+), 148 deletions(-) create mode 100644 library/PolyVoxCore/include/PolyVoxCore/CompressedBlock.h create mode 100644 library/PolyVoxCore/include/PolyVoxCore/CompressedBlock.inl create mode 100644 library/PolyVoxCore/include/PolyVoxCore/UncompressedBlock.h rename library/PolyVoxCore/include/PolyVoxCore/{Block.inl => UncompressedBlock.inl} (62%) diff --git a/library/PolyVoxCore/CMakeLists.txt b/library/PolyVoxCore/CMakeLists.txt index 5f2d49b3..bb28c01e 100644 --- a/library/PolyVoxCore/CMakeLists.txt +++ b/library/PolyVoxCore/CMakeLists.txt @@ -49,8 +49,9 @@ SET(CORE_INC_FILES include/PolyVoxCore/BaseVolume.inl include/PolyVoxCore/BaseVolumeSampler.inl include/PolyVoxCore/Block.h - include/PolyVoxCore/Block.inl include/PolyVoxCore/BlockCompressor.h + include/PolyVoxCore/CompressedBlock.h + include/PolyVoxCore/CompressedBlock.inl include/PolyVoxCore/Compressor.h include/PolyVoxCore/CubicSurfaceExtractor.h include/PolyVoxCore/CubicSurfaceExtractor.inl @@ -95,6 +96,8 @@ SET(CORE_INC_FILES include/PolyVoxCore/SimpleVolumeSampler.inl include/PolyVoxCore/SurfaceMesh.h include/PolyVoxCore/SurfaceMesh.inl + include/PolyVoxCore/UncompressedBlock.h + include/PolyVoxCore/UncompressedBlock.inl include/PolyVoxCore/Vector.h include/PolyVoxCore/Vector.inl include/PolyVoxCore/VertexTypes.h diff --git a/library/PolyVoxCore/include/PolyVoxCore/Block.h b/library/PolyVoxCore/include/PolyVoxCore/Block.h index a1567552..e2bb16cb 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/Block.h +++ b/library/PolyVoxCore/include/PolyVoxCore/Block.h @@ -1,5 +1,5 @@ /******************************************************************************* -Copyright (c) 2005-2009 David Williams +Copyright (c) 2005-2013 David Williams and Matthew Williams This software is provided 'as-is', without any express or implied warranty. In no event will the authors be held liable for any damages @@ -61,71 +61,6 @@ namespace PolyVox /// Private assignment operator to prevent accisdental copying Block& operator=(const Block& /*rhs*/) {}; }; - - template - class CompressedBlock : public Block - { - friend class LargeVolume; - - public: - CompressedBlock(); - ~CompressedBlock(); - - const uint8_t* getData(void) const; - uint32_t getDataSizeInBytes(void) const; - - void setData(const uint8_t* const pData, uint32_t uDataSizeInBytes); - - private: - /// Private copy constructor to prevent accisdental copying - CompressedBlock(const CompressedBlock& /*rhs*/) {}; - - /// Private assignment operator to prevent accisdental copying - CompressedBlock& operator=(const CompressedBlock& /*rhs*/) {}; - - // Made this private to avoid any confusion with getDataSizeInBytes(). - // Users shouldn't really need this for CompressedBlock anyway. - uint32_t calculateSizeInBytes(void); - - uint8_t* m_pData; - uint32_t m_uDataSizeInBytes; - }; - - template - class UncompressedBlock : public Block - { - friend class LargeVolume; - - public: - UncompressedBlock(uint16_t uSideLength); - ~UncompressedBlock(); - - VoxelType* getData(void) const; - uint32_t getDataSizeInBytes(void) const; - - VoxelType getVoxel(uint16_t uXPos, uint16_t uYPos, uint16_t uZPos) const; - VoxelType getVoxel(const Vector3DUint16& v3dPos) const; - - void setVoxelAt(uint16_t uXPos, uint16_t uYPos, uint16_t uZPos, VoxelType tValue); - void setVoxelAt(const Vector3DUint16& v3dPos, VoxelType tValue); - - private: - /// Private copy constructor to prevent accisdental copying - UncompressedBlock(const UncompressedBlock& /*rhs*/) {}; - - /// Private assignment operator to prevent accisdental copying - UncompressedBlock& operator=(const UncompressedBlock& /*rhs*/) {}; - - // Made this private for consistancy with CompressedBlock. - // Users shouldn't really need this for UncompressedBlock anyway. - uint32_t calculateSizeInBytes(void); - - VoxelType* m_tData; - uint16_t m_uSideLength; - uint8_t m_uSideLengthPower; - }; } -#include "PolyVoxCore/Block.inl" - #endif diff --git a/library/PolyVoxCore/include/PolyVoxCore/BlockCompressor.h b/library/PolyVoxCore/include/PolyVoxCore/BlockCompressor.h index 2646e94c..5af6dbfb 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/BlockCompressor.h +++ b/library/PolyVoxCore/include/PolyVoxCore/BlockCompressor.h @@ -25,7 +25,8 @@ freely, subject to the following restrictions: #define __PolyVox_BlockCompressor_H__ #include "PolyVoxCore/PolyVoxForwardDeclarations.h" -#include "PolyVoxCore/Block.h" +#include "PolyVoxCore/CompressedBlock.h" +#include "PolyVoxCore/UncompressedBlock.h" namespace PolyVox { diff --git a/library/PolyVoxCore/include/PolyVoxCore/CompressedBlock.h b/library/PolyVoxCore/include/PolyVoxCore/CompressedBlock.h new file mode 100644 index 00000000..5a8ba89a --- /dev/null +++ b/library/PolyVoxCore/include/PolyVoxCore/CompressedBlock.h @@ -0,0 +1,63 @@ +/******************************************************************************* +Copyright (c) 2005-2013 David Williams and Matthew Williams + +This software is provided 'as-is', without any express or implied +warranty. In no event will the authors be held liable for any damages +arising from the use of this software. + +Permission is granted to anyone to use this software for any purpose, +including commercial applications, and to alter it and redistribute it +freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + + 3. This notice may not be removed or altered from any source + distribution. +*******************************************************************************/ + +#ifndef __PolyVox_CompressedBlock_H__ +#define __PolyVox_CompressedBlock_H__ + +#include "PolyVoxCore/Block.h" + +namespace PolyVox +{ + template + class CompressedBlock : public Block + { + friend class LargeVolume; + + public: + CompressedBlock(); + ~CompressedBlock(); + + const uint8_t* getData(void) const; + uint32_t getDataSizeInBytes(void) const; + + void setData(const uint8_t* const pData, uint32_t uDataSizeInBytes); + + private: + /// Private copy constructor to prevent accisdental copying + CompressedBlock(const CompressedBlock& /*rhs*/) {}; + + /// Private assignment operator to prevent accisdental copying + CompressedBlock& operator=(const CompressedBlock& /*rhs*/) {}; + + // Made this private to avoid any confusion with getDataSizeInBytes(). + // Users shouldn't really need this for CompressedBlock anyway. + uint32_t calculateSizeInBytes(void); + + uint8_t* m_pData; + uint32_t m_uDataSizeInBytes; + }; +} + +#include "PolyVoxCore/CompressedBlock.inl" + +#endif diff --git a/library/PolyVoxCore/include/PolyVoxCore/CompressedBlock.inl b/library/PolyVoxCore/include/PolyVoxCore/CompressedBlock.inl new file mode 100644 index 00000000..e98df8e3 --- /dev/null +++ b/library/PolyVoxCore/include/PolyVoxCore/CompressedBlock.inl @@ -0,0 +1,79 @@ +/******************************************************************************* +Copyright (c) 2005-2013 David Williams and Matthew Williams + +This software is provided 'as-is', without any express or implied +warranty. In no event will the authors be held liable for any damages +arising from the use of this software. + +Permission is granted to anyone to use this software for any purpose, +including commercial applications, and to alter it and redistribute it +freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + + 3. This notice may not be removed or altered from any source + distribution. +*******************************************************************************/ + +namespace PolyVox +{ + template + CompressedBlock::CompressedBlock() + :m_pData(0) + ,m_uDataSizeInBytes(0) + { + } + + template + CompressedBlock::~CompressedBlock() + { + delete[] m_pData; + m_pData = 0; + } + + template + const uint8_t* CompressedBlock::getData(void) const + { + return m_pData; + } + + template + uint32_t CompressedBlock::getDataSizeInBytes(void) const + { + return m_uDataSizeInBytes; + } + + template + void CompressedBlock::setData(const uint8_t* const pData, uint32_t uDataSizeInBytes) + { + POLYVOX_THROW_IF(pData == 0, std::invalid_argument, "Pointer to data cannot be null"); + POLYVOX_THROW_IF(m_pData == pData, std::invalid_argument, "Attempting to copy data onto itself"); + + // Delete any existing data + delete[] m_pData; + + // Allocate new data + m_uDataSizeInBytes = uDataSizeInBytes; + m_pData = new uint8_t[uDataSizeInBytes]; + + // Copy the data across + memcpy(m_pData, pData, uDataSizeInBytes); + + // Flag as modified + this->m_bDataModified = true; + } + + template + uint32_t CompressedBlock::calculateSizeInBytes(void) + { + // Returns the size of this class plus the size of the compressed data. + uint32_t uSizeInBytes = sizeof(CompressedBlock) + m_uDataSizeInBytes; + return uSizeInBytes; + } +} diff --git a/library/PolyVoxCore/include/PolyVoxCore/Pager.h b/library/PolyVoxCore/include/PolyVoxCore/Pager.h index 6189566a..eb37e4f7 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/Pager.h +++ b/library/PolyVoxCore/include/PolyVoxCore/Pager.h @@ -24,7 +24,7 @@ freely, subject to the following restrictions: #ifndef __PolyVox_Pager_H__ #define __PolyVox_Pager_H__ -#include "PolyVoxCore/Block.h" +#include "PolyVoxCore/CompressedBlock.h" #include "PolyVoxCore/Impl/TypeDef.h" namespace PolyVox diff --git a/library/PolyVoxCore/include/PolyVoxCore/UncompressedBlock.h b/library/PolyVoxCore/include/PolyVoxCore/UncompressedBlock.h new file mode 100644 index 00000000..54a1aacb --- /dev/null +++ b/library/PolyVoxCore/include/PolyVoxCore/UncompressedBlock.h @@ -0,0 +1,68 @@ +/******************************************************************************* +Copyright (c) 2005-2013 David Williams and Matthew Williams + +This software is provided 'as-is', without any express or implied +warranty. In no event will the authors be held liable for any damages +arising from the use of this software. + +Permission is granted to anyone to use this software for any purpose, +including commercial applications, and to alter it and redistribute it +freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + + 3. This notice may not be removed or altered from any source + distribution. +*******************************************************************************/ + +#ifndef __PolyVox_UncompressedBlock_H__ +#define __PolyVox_UncompressedBlock_H__ + +#include "PolyVoxCore/Block.h" + +namespace PolyVox +{ + template + class UncompressedBlock : public Block + { + friend class LargeVolume; + + public: + UncompressedBlock(uint16_t uSideLength); + ~UncompressedBlock(); + + VoxelType* getData(void) const; + uint32_t getDataSizeInBytes(void) const; + + VoxelType getVoxel(uint16_t uXPos, uint16_t uYPos, uint16_t uZPos) const; + VoxelType getVoxel(const Vector3DUint16& v3dPos) const; + + void setVoxelAt(uint16_t uXPos, uint16_t uYPos, uint16_t uZPos, VoxelType tValue); + void setVoxelAt(const Vector3DUint16& v3dPos, VoxelType tValue); + + private: + /// Private copy constructor to prevent accisdental copying + UncompressedBlock(const UncompressedBlock& /*rhs*/) {}; + + /// Private assignment operator to prevent accisdental copying + UncompressedBlock& operator=(const UncompressedBlock& /*rhs*/) {}; + + // Made this private for consistancy with CompressedBlock. + // Users shouldn't really need this for UncompressedBlock anyway. + uint32_t calculateSizeInBytes(void); + + VoxelType* m_tData; + uint16_t m_uSideLength; + uint8_t m_uSideLengthPower; + }; +} + +#include "PolyVoxCore/UncompressedBlock.inl" + +#endif diff --git a/library/PolyVoxCore/include/PolyVoxCore/Block.inl b/library/PolyVoxCore/include/PolyVoxCore/UncompressedBlock.inl similarity index 62% rename from library/PolyVoxCore/include/PolyVoxCore/Block.inl rename to library/PolyVoxCore/include/PolyVoxCore/UncompressedBlock.inl index 279dbf0e..4d506d74 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/Block.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/UncompressedBlock.inl @@ -1,5 +1,5 @@ /******************************************************************************* -Copyright (c) 2005-2009 David Williams +Copyright (c) 2005-2013 David Williams and Matthew Williams This software is provided 'as-is', without any express or implied warranty. In no event will the authors be held liable for any damages @@ -9,90 +9,20 @@ Permission is granted to anyone to use this software for any purpose, including commercial applications, and to alter it and redistribute it freely, subject to the following restrictions: -1. The origin of this software must not be misrepresented; you must not -claim that you wrote the original software. If you use this software -in a product, an acknowledgment in the product documentation would be -appreciated but is not required. + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. -2. Altered source versions must be plainly marked as such, and must not be -misrepresented as being the original software. + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. -3. This notice may not be removed or altered from any source -distribution. + 3. This notice may not be removed or altered from any source + distribution. *******************************************************************************/ -#include "PolyVoxCore/Impl/ErrorHandling.h" -#include "PolyVoxCore/Impl/Utility.h" - -#include "PolyVoxCore/Compressor.h" -#include "PolyVoxCore/Vector.h" - -#include "PolyVoxCore/Impl/ErrorHandling.h" - -#include //For memcpy -#include -#include //for std::invalid_argument - namespace PolyVox { - //////////////////////////////////////////////////////////////////////////////// - - template - CompressedBlock::CompressedBlock() - :m_pData(0) - ,m_uDataSizeInBytes(0) - { - } - - template - CompressedBlock::~CompressedBlock() - { - delete[] m_pData; - m_pData = 0; - } - - template - const uint8_t* CompressedBlock::getData(void) const - { - return m_pData; - } - - template - uint32_t CompressedBlock::getDataSizeInBytes(void) const - { - return m_uDataSizeInBytes; - } - - template - void CompressedBlock::setData(const uint8_t* const pData, uint32_t uDataSizeInBytes) - { - POLYVOX_THROW_IF(pData == 0, std::invalid_argument, "Pointer to data cannot be null"); - POLYVOX_THROW_IF(m_pData == pData, std::invalid_argument, "Attempting to copy data onto itself"); - - // Delete any existing data - delete[] m_pData; - - // Allocate new data - m_uDataSizeInBytes = uDataSizeInBytes; - m_pData = new uint8_t[uDataSizeInBytes]; - - // Copy the data across - memcpy(m_pData, pData, uDataSizeInBytes); - - // Flag as modified - this->m_bDataModified = true; - } - - template - uint32_t CompressedBlock::calculateSizeInBytes(void) - { - // Returns the size of this class plus the size of the compressed data. - uint32_t uSizeInBytes = sizeof(CompressedBlock) + m_uDataSizeInBytes; - return uSizeInBytes; - } - - //////////////////////////////////////////////////////////////////////////////// - template UncompressedBlock::UncompressedBlock(uint16_t uSideLength) :m_tData(0) From e485265dd85050df6dc7afd2f7fbdf2e2a5bf9da Mon Sep 17 00:00:00 2001 From: David Williams Date: Thu, 1 Aug 2013 16:08:44 +0200 Subject: [PATCH 50/55] Fixes to allow the SWIG/Python bindings to build again. --- library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl | 2 +- library/bindings/Block.i | 9 +++++++++ library/bindings/BlockCompressor.i | 8 ++++++++ library/bindings/CompressedBlock.i | 8 ++++++++ library/bindings/MinizBlockCompressor.i | 6 ++++++ library/bindings/MinizCompressor.i | 6 ------ library/bindings/PolyVoxCore.i | 8 ++++++-- library/bindings/RLEBlockCompressor.i | 6 ++++++ library/bindings/RLECompressor.i | 6 ------ library/bindings/UncompressedBlock.i | 8 ++++++++ 10 files changed, 52 insertions(+), 15 deletions(-) create mode 100644 library/bindings/Block.i create mode 100644 library/bindings/BlockCompressor.i create mode 100644 library/bindings/CompressedBlock.i create mode 100644 library/bindings/MinizBlockCompressor.i delete mode 100644 library/bindings/MinizCompressor.i create mode 100644 library/bindings/RLEBlockCompressor.i delete mode 100644 library/bindings/RLECompressor.i create mode 100644 library/bindings/UncompressedBlock.i diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl index e18568bd..c9fe8fa6 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl @@ -311,7 +311,7 @@ namespace PolyVox const uint16_t zOffset = static_cast(uZPos - (blockZ << m_uBlockSideLengthPower)); UncompressedBlock* pUncompressedBlock = getUncompressedBlock(blockX, blockY, blockZ); - pUncompressedBlock->setVoxelAt(xOffset, yOffset, zOffset); + pUncompressedBlock->setVoxelAt(xOffset, yOffset, zOffset, tValue); } //////////////////////////////////////////////////////////////////////////////// diff --git a/library/bindings/Block.i b/library/bindings/Block.i new file mode 100644 index 00000000..d5aa8da7 --- /dev/null +++ b/library/bindings/Block.i @@ -0,0 +1,9 @@ +%module Block +%{ +#include "Block.h" +%} + +%include "Block.h" + +VOLUMETYPES(Block) + \ No newline at end of file diff --git a/library/bindings/BlockCompressor.i b/library/bindings/BlockCompressor.i new file mode 100644 index 00000000..ddb6eb41 --- /dev/null +++ b/library/bindings/BlockCompressor.i @@ -0,0 +1,8 @@ +%module BlockCompressor +%{ +#include "BlockCompressor.h" +%} + +%include "BlockCompressor.h" + +VOLUMETYPES(BlockCompressor) \ No newline at end of file diff --git a/library/bindings/CompressedBlock.i b/library/bindings/CompressedBlock.i new file mode 100644 index 00000000..c27ef617 --- /dev/null +++ b/library/bindings/CompressedBlock.i @@ -0,0 +1,8 @@ +%module CompressedBlock +%{ +#include "CompressedBlock.h" +%} + +%include "CompressedBlock.h" + +VOLUMETYPES(CompressedBlock) diff --git a/library/bindings/MinizBlockCompressor.i b/library/bindings/MinizBlockCompressor.i new file mode 100644 index 00000000..7f2f0c9a --- /dev/null +++ b/library/bindings/MinizBlockCompressor.i @@ -0,0 +1,6 @@ +%module MinizBlockCompressor +%{ +#include "MinizBlockCompressor.h" +%} + +%include "MinizBlockCompressor.h" diff --git a/library/bindings/MinizCompressor.i b/library/bindings/MinizCompressor.i deleted file mode 100644 index f846265f..00000000 --- a/library/bindings/MinizCompressor.i +++ /dev/null @@ -1,6 +0,0 @@ -%module MinizCompressor -%{ -#include "MinizCompressor.h" -%} - -%include "MinizCompressor.h" diff --git a/library/bindings/PolyVoxCore.i b/library/bindings/PolyVoxCore.i index 30a71407..7e6bf245 100644 --- a/library/bindings/PolyVoxCore.i +++ b/library/bindings/PolyVoxCore.i @@ -75,11 +75,15 @@ EXTRACTOR(shortname, LargeVolume) %include "Vector.i" %include "DefaultMarchingCubesController.i" %include "Region.i" +%include "Block.i" +%include "CompressedBlock.i" +%include "UncompressedBlock.i" %include "Compressor.i" +%include "BlockCompressor.i" %include "Pager.i" %include "FilePager.i" -%include "MinizCompressor.i" -%include "RLECompressor.i" +%include "MinizBlockCompressor.i" +%include "RLEBlockCompressor.i" %include "BaseVolume.i" %include "SimpleVolume.i" %include "RawVolume.i" diff --git a/library/bindings/RLEBlockCompressor.i b/library/bindings/RLEBlockCompressor.i new file mode 100644 index 00000000..b1ea1810 --- /dev/null +++ b/library/bindings/RLEBlockCompressor.i @@ -0,0 +1,6 @@ +%module RLEBlockCompressor +%{ +#include "RLEBlockCompressor.h" +%} + +%include "RLEBlockCompressor.h" diff --git a/library/bindings/RLECompressor.i b/library/bindings/RLECompressor.i deleted file mode 100644 index 01023c27..00000000 --- a/library/bindings/RLECompressor.i +++ /dev/null @@ -1,6 +0,0 @@ -%module RLECompressor -%{ -#include "RLECompressor.h" -%} - -%include "RLECompressor.h" diff --git a/library/bindings/UncompressedBlock.i b/library/bindings/UncompressedBlock.i new file mode 100644 index 00000000..26598f31 --- /dev/null +++ b/library/bindings/UncompressedBlock.i @@ -0,0 +1,8 @@ +%module UncompressedBlock +%{ +#include "UncompressedBlock.h" +%} + +%include "UncompressedBlock.h" + +VOLUMETYPES(UncompressedBlock) From a3081025853ed144b97cc3c8aa33b4c48bd57f43 Mon Sep 17 00:00:00 2001 From: David Williams Date: Wed, 7 Aug 2013 14:53:08 +0200 Subject: [PATCH 51/55] VS2012 compile warning fix. --- library/PolyVoxCore/include/PolyVoxCore/FilePager.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/library/PolyVoxCore/include/PolyVoxCore/FilePager.h b/library/PolyVoxCore/include/PolyVoxCore/FilePager.h index 3e087b5f..b8bb7b97 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/FilePager.h +++ b/library/PolyVoxCore/include/PolyVoxCore/FilePager.h @@ -49,7 +49,7 @@ namespace PolyVox :Pager() ,m_strFolderName(strFolderName) { - srand(time(0)); + srand(static_cast(time(0))); int iRandomValue = rand(); std::stringstream ss; From 9ad4c3fcf7b2a61080461984d1c609cae17bec46 Mon Sep 17 00:00:00 2001 From: David Williams Date: Wed, 7 Aug 2013 23:07:26 +0200 Subject: [PATCH 52/55] Initial checkin of Timer class. --- examples/Basic/main.cpp | 3 + library/PolyVoxCore/CMakeLists.txt | 2 + .../PolyVoxCore/CubicSurfaceExtractor.inl | 7 ++ .../CubicSurfaceExtractorWithNormals.inl | 7 ++ .../MarchingCubesSurfaceExtractor.inl | 7 ++ library/PolyVoxCore/source/Impl/Timer.cpp | 92 +++++++++++++++++++ .../include/PolyVoxCore/Impl/Timer.h | 64 +++++++++++++ 7 files changed, 182 insertions(+) create mode 100644 library/PolyVoxCore/source/Impl/Timer.cpp create mode 100644 library/polyvoxcore/include/PolyVoxCore/Impl/Timer.h diff --git a/examples/Basic/main.cpp b/examples/Basic/main.cpp index 64097731..b8164e31 100644 --- a/examples/Basic/main.cpp +++ b/examples/Basic/main.cpp @@ -68,6 +68,9 @@ void createSphereInVolume(SimpleVolume& volData, float fRadius) int main(int argc, char *argv[]) { + // Show logs + setTraceStream(&(std::cout)); + //Create and show the Qt OpenGL window QApplication app(argc, argv); OpenGLWidget openGLWidget(0); diff --git a/library/PolyVoxCore/CMakeLists.txt b/library/PolyVoxCore/CMakeLists.txt index bb28c01e..5521f1dc 100644 --- a/library/PolyVoxCore/CMakeLists.txt +++ b/library/PolyVoxCore/CMakeLists.txt @@ -112,6 +112,7 @@ SET(IMPL_SRC_FILES source/Impl/MarchingCubesTables.cpp source/Impl/RandomUnitVectors.cpp source/Impl/RandomVectors.cpp + source/Impl/Timer.cpp source/Impl/Utility.cpp ) @@ -127,6 +128,7 @@ SET(IMPL_INC_FILES include/PolyVoxCore/Impl/RandomVectors.h include/PolyVoxCore/Impl/SubArray.h include/PolyVoxCore/Impl/SubArray.inl + include/PolyVoxCore/Impl/Timer.h include/PolyVoxCore/Impl/TypeDef.h include/PolyVoxCore/Impl/Utility.h ) diff --git a/library/PolyVoxCore/include/PolyVoxCore/CubicSurfaceExtractor.inl b/library/PolyVoxCore/include/PolyVoxCore/CubicSurfaceExtractor.inl index 96692bdb..79f8792a 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/CubicSurfaceExtractor.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/CubicSurfaceExtractor.inl @@ -21,6 +21,8 @@ freely, subject to the following restrictions: distribution. *******************************************************************************/ +#include "PolyVoxCore/Impl/Timer.h" + namespace PolyVox { // We try to avoid duplicate vertices by checking whether a vertex has already been added at a given position. @@ -48,6 +50,7 @@ namespace PolyVox template void CubicSurfaceExtractor::execute() { + Timer timer; m_meshCurrent->clear(); uint32_t uArrayWidth = m_regSizeInVoxels.getUpperX() - m_regSizeInVoxels.getLowerX() + 2; @@ -195,6 +198,10 @@ namespace PolyVox lodRecord.beginIndex = 0; lodRecord.endIndex = m_meshCurrent->getNoOfIndices(); m_meshCurrent->m_vecLodRecords.push_back(lodRecord); + + logTrace() << "Cubic surface extraction took " << timer.elapsedTimeInMilliSeconds() + << "ms (Region size = " << m_regSizeInVoxels.getWidthInVoxels() << "x" << m_regSizeInVoxels.getHeightInVoxels() + << "x" << m_regSizeInVoxels.getDepthInVoxels() << ")"; } template diff --git a/library/PolyVoxCore/include/PolyVoxCore/CubicSurfaceExtractorWithNormals.inl b/library/PolyVoxCore/include/PolyVoxCore/CubicSurfaceExtractorWithNormals.inl index 3956858c..df347b32 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/CubicSurfaceExtractorWithNormals.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/CubicSurfaceExtractorWithNormals.inl @@ -21,6 +21,8 @@ freely, subject to the following restrictions: distribution. *******************************************************************************/ +#include "PolyVoxCore/Impl/Timer.h" + namespace PolyVox { template @@ -38,6 +40,7 @@ namespace PolyVox template void CubicSurfaceExtractorWithNormals::execute() { + Timer timer; m_meshCurrent->clear(); for(int32_t z = m_regSizeInVoxels.getLowerZ(); z < m_regSizeInVoxels.getUpperZ(); z++) @@ -126,5 +129,9 @@ namespace PolyVox lodRecord.beginIndex = 0; lodRecord.endIndex = m_meshCurrent->getNoOfIndices(); m_meshCurrent->m_vecLodRecords.push_back(lodRecord); + + logTrace() << "Cubic surface extraction took " << timer.elapsedTimeInMilliSeconds() + << "ms (Region size = " << m_regSizeInVoxels.getWidthInVoxels() << "x" << m_regSizeInVoxels.getHeightInVoxels() + << "x" << m_regSizeInVoxels.getDepthInVoxels() << ")"; } } diff --git a/library/PolyVoxCore/include/PolyVoxCore/MarchingCubesSurfaceExtractor.inl b/library/PolyVoxCore/include/PolyVoxCore/MarchingCubesSurfaceExtractor.inl index 22e1373d..be43387c 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/MarchingCubesSurfaceExtractor.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/MarchingCubesSurfaceExtractor.inl @@ -21,6 +21,8 @@ freely, subject to the following restrictions: distribution. *******************************************************************************/ +#include "PolyVoxCore/Impl/Timer.h" + namespace PolyVox { template @@ -42,6 +44,7 @@ namespace PolyVox template void MarchingCubesSurfaceExtractor::execute() { + Timer timer; m_meshCurrent->clear(); const uint32_t uArrayWidth = m_regSizeInVoxels.getUpperX() - m_regSizeInVoxels.getLowerX() + 1; @@ -126,6 +129,10 @@ namespace PolyVox lodRecord.beginIndex = 0; lodRecord.endIndex = m_meshCurrent->getNoOfIndices(); m_meshCurrent->m_vecLodRecords.push_back(lodRecord); + + logTrace() << "Marching cubes surface extraction took " << timer.elapsedTimeInMilliSeconds() + << "ms (Region size = " << m_regSizeInVoxels.getWidthInVoxels() << "x" << m_regSizeInVoxels.getHeightInVoxels() + << "x" << m_regSizeInVoxels.getDepthInVoxels() << ")"; } template diff --git a/library/PolyVoxCore/source/Impl/Timer.cpp b/library/PolyVoxCore/source/Impl/Timer.cpp new file mode 100644 index 00000000..d8039348 --- /dev/null +++ b/library/PolyVoxCore/source/Impl/Timer.cpp @@ -0,0 +1,92 @@ +/******************************************************************************* +Copyright (c) 2005-20013 David Williams and Matthew Williams + +This software is provided 'as-is', without any express or implied +warranty. In no event will the authors be held liable for any damages +arising from the use of this software. + +Permission is granted to anyone to use this software for any purpose, +including commercial applications, and to alter it and redistribute it +freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + + 3. This notice may not be removed or altered from any source + distribution. +*******************************************************************************/ + +#include "PolyVoxCore/Impl/Timer.h" + +#include "PolyVoxCore/Impl/ErrorHandling.h" + +#include + +namespace PolyVox +{ +#if defined(_MSC_VER) + Timer::Timer(bool bAutoStart) + :m_fPCFreq(0.0) + ,m_iStartTime(0) + { + if(bAutoStart) + { + start(); + } + } + + void Timer::start(void) + { + LARGE_INTEGER li; + if(!QueryPerformanceFrequency(&li)) + { + logWarning() << "QueryPerformanceFrequency failed!"; + m_fPCFreq = 1.0f; + } + + m_fPCFreq = double(li.QuadPart)/1000.0; + + QueryPerformanceCounter(&li); + m_iStartTime = li.QuadPart; + } + + float Timer::elapsedTimeInSeconds(void) + { + return static_cast(elapsedTimeInMilliSeconds()) / 1000.0f; + } + + uint32_t Timer::elapsedTimeInMilliSeconds(void) + { + LARGE_INTEGER li; + QueryPerformanceCounter(&li); + return static_cast(double(li.QuadPart-m_iStartTime)/m_fPCFreq); + } +#else //_MSC_VER + Timer::Timer(bool bAutoStart) + { + if(bAutoStart) + { + start(); + } + } + + void Timer::start(void) + { + } + + float Timer::elapsedTimeInSeconds(void) + { + return 0.0f + } + + uint32_t Timer::elapsedTimeInMilliSeconds(void) + { + return 0; + } +#endif //_MSC_VER +} diff --git a/library/polyvoxcore/include/PolyVoxCore/Impl/Timer.h b/library/polyvoxcore/include/PolyVoxCore/Impl/Timer.h new file mode 100644 index 00000000..6ce9c328 --- /dev/null +++ b/library/polyvoxcore/include/PolyVoxCore/Impl/Timer.h @@ -0,0 +1,64 @@ +/******************************************************************************* +Copyright (c) 2005-20013 David Williams and Matthew Williams + +This software is provided 'as-is', without any express or implied +warranty. In no event will the authors be held liable for any damages +arising from the use of this software. + +Permission is granted to anyone to use this software for any purpose, +including commercial applications, and to alter it and redistribute it +freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + + 3. This notice may not be removed or altered from any source + distribution. +*******************************************************************************/ + +#ifndef __PolyVox_Timer_H__ +#define __PolyVox_Timer_H__ + +#include + +#ifdef _MSC_VER // Don't worry about the exact version, as long as this is defied. +#include +#endif //_MSC_VER + +namespace PolyVox +{ +#if defined(_MSC_VER) + class Timer + { + public: + Timer(bool bAutoStart = true); + + void start(void); + + float elapsedTimeInSeconds(void); + uint32_t elapsedTimeInMilliSeconds(void); + + private: + double m_fPCFreq; + __int64 m_iStartTime; + }; +#else //_MSC_VER + class Timer + { + public: + Timer(bool bAutoStart = true); + + void start(void); + + float elapsedTimeInSeconds(void); + uint32_t elapsedTimeInMilliSeconds(void); + }; +#endif //_MSC_VER +} + +#endif //__PolyVox_Timer_H__ \ No newline at end of file From 2f999646062e9489c259d4b982d76e084621abbd Mon Sep 17 00:00:00 2001 From: David Williams Date: Wed, 7 Aug 2013 23:24:20 +0200 Subject: [PATCH 53/55] Tidying timer. --- library/PolyVoxCore/source/Impl/Timer.cpp | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/library/PolyVoxCore/source/Impl/Timer.cpp b/library/PolyVoxCore/source/Impl/Timer.cpp index d8039348..bad32baa 100644 --- a/library/PolyVoxCore/source/Impl/Timer.cpp +++ b/library/PolyVoxCore/source/Impl/Timer.cpp @@ -49,7 +49,7 @@ namespace PolyVox m_fPCFreq = 1.0f; } - m_fPCFreq = double(li.QuadPart)/1000.0; + m_fPCFreq = double(li.QuadPart); QueryPerformanceCounter(&li); m_iStartTime = li.QuadPart; @@ -57,14 +57,15 @@ namespace PolyVox float Timer::elapsedTimeInSeconds(void) { - return static_cast(elapsedTimeInMilliSeconds()) / 1000.0f; + LARGE_INTEGER li; + QueryPerformanceCounter(&li); + double fDifference = static_cast(li.QuadPart-m_iStartTime); + return static_cast(fDifference / m_fPCFreq); } uint32_t Timer::elapsedTimeInMilliSeconds(void) { - LARGE_INTEGER li; - QueryPerformanceCounter(&li); - return static_cast(double(li.QuadPart-m_iStartTime)/m_fPCFreq); + return static_cast(elapsedTimeInSeconds() * 1000.0f); } #else //_MSC_VER Timer::Timer(bool bAutoStart) From 7c74e1faff18592dcfe66ae629858fb1c9b69520 Mon Sep 17 00:00:00 2001 From: Matt Williams Date: Thu, 8 Aug 2013 10:42:57 +0100 Subject: [PATCH 54/55] Move Timer.h to correct location Also add a missing semi-colon. --- .../include/PolyVoxCore/Impl/Timer.h | 0 library/PolyVoxCore/source/Impl/Timer.cpp | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename library/{polyvoxcore => PolyVoxCore}/include/PolyVoxCore/Impl/Timer.h (100%) diff --git a/library/polyvoxcore/include/PolyVoxCore/Impl/Timer.h b/library/PolyVoxCore/include/PolyVoxCore/Impl/Timer.h similarity index 100% rename from library/polyvoxcore/include/PolyVoxCore/Impl/Timer.h rename to library/PolyVoxCore/include/PolyVoxCore/Impl/Timer.h diff --git a/library/PolyVoxCore/source/Impl/Timer.cpp b/library/PolyVoxCore/source/Impl/Timer.cpp index bad32baa..49ec9a11 100644 --- a/library/PolyVoxCore/source/Impl/Timer.cpp +++ b/library/PolyVoxCore/source/Impl/Timer.cpp @@ -82,7 +82,7 @@ namespace PolyVox float Timer::elapsedTimeInSeconds(void) { - return 0.0f + return 0.0f; } uint32_t Timer::elapsedTimeInMilliSeconds(void) From b9546ddcfaf793f622c18a085112e4e632d9ae00 Mon Sep 17 00:00:00 2001 From: David Williams Date: Fri, 9 Aug 2013 14:09:59 +0200 Subject: [PATCH 55/55] Extra checks in during marching cubes to try and catch corrupt data. --- .../MarchingCubesSurfaceExtractor.inl | 26 ++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/library/PolyVoxCore/include/PolyVoxCore/MarchingCubesSurfaceExtractor.inl b/library/PolyVoxCore/include/PolyVoxCore/MarchingCubesSurfaceExtractor.inl index be43387c..53a6cf96 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/MarchingCubesSurfaceExtractor.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/MarchingCubesSurfaceExtractor.inl @@ -440,6 +440,7 @@ namespace PolyVox { m_sampVolume.movePositiveX(); const typename VolumeType::VoxelType v100 = m_sampVolume.getVoxel(); + POLYVOX_ASSERT(v000 != v100, "Attempting to insert vertex between two voxels with the same value"); const Vector3DFloat n100 = computeCentralDifferenceGradient(m_sampVolume); const float fInterp = static_cast(m_tThreshold - m_controller.convertToDensity(v000)) / static_cast(m_controller.convertToDensity(v100) - m_controller.convertToDensity(v000)); @@ -447,7 +448,13 @@ namespace PolyVox const Vector3DFloat v3dPosition(static_cast(iXVolSpace - m_regSizeInVoxels.getLowerX()) + fInterp, static_cast(iYVolSpace - m_regSizeInVoxels.getLowerY()), static_cast(iZVolSpace - m_regSizeInCells.getLowerZ())); Vector3DFloat v3dNormal = (n100*fInterp) + (n000*(1-fInterp)); - v3dNormal.normalise(); + + // The gradient for a voxel can be zero (e.g. solid voxel surrounded by empty ones) and so + // the interpolated normal can also be zero (e.g. a grid of alternating solid and empty voxels). + if(v3dNormal.lengthSquared() > 0.000001f) + { + v3dNormal.normalise(); + } //Choose one of the two materials to use for the vertex (we don't interpolate as interpolation of //material IDs does not make sense). We take the largest, so that if we are working on a material-only @@ -466,6 +473,7 @@ namespace PolyVox { m_sampVolume.movePositiveY(); const typename VolumeType::VoxelType v010 = m_sampVolume.getVoxel(); + POLYVOX_ASSERT(v000 != v010, "Attempting to insert vertex between two voxels with the same value"); const Vector3DFloat n010 = computeCentralDifferenceGradient(m_sampVolume); const float fInterp = static_cast(m_tThreshold - m_controller.convertToDensity(v000)) / static_cast(m_controller.convertToDensity(v010) - m_controller.convertToDensity(v000)); @@ -473,7 +481,13 @@ namespace PolyVox const Vector3DFloat v3dPosition(static_cast(iXVolSpace - m_regSizeInVoxels.getLowerX()), static_cast(iYVolSpace - m_regSizeInVoxels.getLowerY()) + fInterp, static_cast(iZVolSpace - m_regSizeInVoxels.getLowerZ())); Vector3DFloat v3dNormal = (n010*fInterp) + (n000*(1-fInterp)); - v3dNormal.normalise(); + + // The gradient for a voxel can be zero (e.g. solid voxel surrounded by empty ones) and so + // the interpolated normal can also be zero (e.g. a grid of alternating solid and empty voxels). + if(v3dNormal.lengthSquared() > 0.000001f) + { + v3dNormal.normalise(); + } //Choose one of the two materials to use for the vertex (we don't interpolate as interpolation of //material IDs does not make sense). We take the largest, so that if we are working on a material-only @@ -492,6 +506,7 @@ namespace PolyVox { m_sampVolume.movePositiveZ(); const typename VolumeType::VoxelType v001 = m_sampVolume.getVoxel(); + POLYVOX_ASSERT(v000 != v001, "Attempting to insert vertex between two voxels with the same value"); const Vector3DFloat n001 = computeCentralDifferenceGradient(m_sampVolume); const float fInterp = static_cast(m_tThreshold - m_controller.convertToDensity(v000)) / static_cast(m_controller.convertToDensity(v001) - m_controller.convertToDensity(v000)); @@ -499,7 +514,12 @@ namespace PolyVox const Vector3DFloat v3dPosition(static_cast(iXVolSpace - m_regSizeInVoxels.getLowerX()), static_cast(iYVolSpace - m_regSizeInVoxels.getLowerY()), static_cast(iZVolSpace - m_regSizeInVoxels.getLowerZ()) + fInterp); Vector3DFloat v3dNormal = (n001*fInterp) + (n000*(1-fInterp)); - v3dNormal.normalise(); + // The gradient for a voxel can be zero (e.g. solid voxel surrounded by empty ones) and so + // the interpolated normal can also be zero (e.g. a grid of alternating solid and empty voxels). + if(v3dNormal.lengthSquared() > 0.000001f) + { + v3dNormal.normalise(); + } //Choose one of the two materials to use for the vertex (we don't interpolate as interpolation of //material IDs does not make sense). We take the largest, so that if we are working on a material-only