From bd60f34bd71bb724ae04b3921b46530c0e19f88b Mon Sep 17 00:00:00 2001 From: Daviw Williams Date: Thu, 4 Jul 2013 16:23:58 +0200 Subject: [PATCH] Removing explicit functions to control the number of compressed and uncompressed blocks in memory, and letting the user set a memory limit instead. --- examples/Paging/main.cpp | 5 +- .../include/PolyVoxCore/Impl/Block.inl | 5 +- .../include/PolyVoxCore/LargeVolume.h | 9 ++- .../include/PolyVoxCore/LargeVolume.inl | 67 +++++++++++++------ tests/testvolume.cpp | 5 +- 5 files changed, 61 insertions(+), 30 deletions(-) diff --git a/examples/Paging/main.cpp b/examples/Paging/main.cpp index cac31ff3..f172a214 100644 --- a/examples/Paging/main.cpp +++ b/examples/Paging/main.cpp @@ -153,8 +153,9 @@ int main(int argc, char *argv[]) RLECompressor* compressor = new RLECompressor(); PerlinNoisePager* pager = new PerlinNoisePager(); LargeVolume volData(PolyVox::Region::MaxRegion, compressor, pager, 256); - volData.setMaxNumberOfBlocksInMemory(4096); - volData.setMaxNumberOfUncompressedBlocks(64); + //volData.setMaxNumberOfBlocksInMemory(4096); + //volData.setMaxNumberOfUncompressedBlocks(64); + volData.setTargetMemoryLimitInBytes(128 * 1024 * 1024); //volData.setMaxNumberOfUncompressedBlocks(4096); //createSphereInVolume(volData, 30); diff --git a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl index cfeb00c1..bebcfb35 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl @@ -269,8 +269,9 @@ namespace PolyVox template uint32_t Block::calculateSizeInBytes(void) { - //FIXME - This function is incomplete. - uint32_t uSizeInBytes = sizeof(Block); + // Returns the size of this class plus the size of the compressed data. It + // doesn't include the uncompressed data cache as that is owned by the volume. + uint32_t uSizeInBytes = sizeof(Block) + m_uCompressedDataLength; return uSizeInBytes; } } diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h index 103d7ca8..f46562d7 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h @@ -263,10 +263,10 @@ namespace PolyVox /// Gets a voxel at the position given by a 3D vector POLYVOX_DEPRECATED VoxelType getVoxelAt(const Vector3DInt32& v3dPos) const; + void setTargetMemoryLimitInBytes(uint32_t uTargetMemoryLimitInBytes); + /// Sets the number of blocks for which uncompressed data is stored void setMaxNumberOfUncompressedBlocks(uint32_t uMaxNumberOfUncompressedBlocks); - /// Sets the number of blocks which can be in memory before the paging system starts unloading them - void setMaxNumberOfBlocksInMemory(uint32_t uMaxNumberOfBlocksInMemory); /// Sets the voxel at the position given by x,y,z coordinates void setVoxel(int32_t uXPos, int32_t uYPos, int32_t uZPos, VoxelType tValue, WrapMode eWrapMode = WrapModes::Validate); /// Sets the voxel at the position given by a 3D vector @@ -315,6 +315,8 @@ namespace PolyVox }; void initialise(); + uint32_t calculateBlockMemoryUsage(void) const; + // A trick to implement specialization of template member functions in template classes. See http://stackoverflow.com/a/4951057 template VoxelType getVoxelImpl(int32_t uXPos, int32_t uYPos, int32_t uZPos, WrapModeType, VoxelType tBorder) const; @@ -338,7 +340,8 @@ namespace PolyVox mutable Vector3DInt32 m_v3dLastAccessedBlockPos; mutable Block* m_pLastAccessedBlock; uint32_t m_uMaxNumberOfUncompressedBlocks; - uint32_t m_uMaxNumberOfBlocksInMemory; + + uint32_t m_uCompressedBlockMemoryLimitInBytes; // The size of the volume Region m_regValidRegionInBlocks; diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl index 19b7d114..b50ef2bf 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl @@ -25,6 +25,8 @@ freely, subject to the following restrictions: #include "PolyVoxCore/MinizCompressor.h" +#include + namespace PolyVox { //////////////////////////////////////////////////////////////////////////////// @@ -241,6 +243,25 @@ namespace PolyVox return getVoxelAt(v3dPos.getX(), v3dPos.getY(), v3dPos.getZ()); } + template + void LargeVolume::setTargetMemoryLimitInBytes(uint32_t uTargetMemoryLimitInBytes) + { + uint32_t uUncompressedBlockSizeInBytes = m_uBlockSideLength * m_uBlockSideLength * m_uBlockSideLength * sizeof(VoxelType); + + uint32_t uIdealNoOfUncompressedBlocks = m_regValidRegionInBlocks.getWidthInVoxels() + m_regValidRegionInBlocks.getHeightInVoxels() * m_regValidRegionInBlocks.getDepthInVoxels(); + + // Let's say that we should never use more than half the available memory for the uncompressed block cache. + uint32_t uMaxMemoryForUncompressedBlocks = uTargetMemoryLimitInBytes / 2; + + uint32_t uMaxFittableNoOfUncompressedBlocks = uMaxMemoryForUncompressedBlocks / uUncompressedBlockSizeInBytes; + + setMaxNumberOfUncompressedBlocks((std::min)(uIdealNoOfUncompressedBlocks, uMaxFittableNoOfUncompressedBlocks)); + + uint32_t uUncompressedBlockCacheSizeInBytes = m_uMaxNumberOfUncompressedBlocks * uUncompressedBlockSizeInBytes; + + m_uCompressedBlockMemoryLimitInBytes = uTargetMemoryLimitInBytes - uUncompressedBlockCacheSizeInBytes; + } + //////////////////////////////////////////////////////////////////////////////// /// Increasing the size of the block cache will increase memory but may improve performance. /// You may want to set this to a large value (e.g. 1024) when you are first loading your @@ -255,20 +276,6 @@ namespace PolyVox m_uMaxNumberOfUncompressedBlocks = uMaxNumberOfUncompressedBlocks; } - //////////////////////////////////////////////////////////////////////////////// - /// Increasing the number of blocks in memory causes fewer calls to dataRequiredHandler()/dataOverflowHandler() - /// \param uMaxNumberOfBlocksInMemory The number of blocks - //////////////////////////////////////////////////////////////////////////////// - template - void LargeVolume::setMaxNumberOfBlocksInMemory(uint32_t uMaxNumberOfBlocksInMemory) - { - if(m_pBlocks.size() > uMaxNumberOfBlocksInMemory) - { - flushAll(); - } - m_uMaxNumberOfBlocksInMemory = uMaxNumberOfBlocksInMemory; - } - //////////////////////////////////////////////////////////////////////////////// /// \param uXPos the \c x position of the voxel /// \param uYPos the \c y position of the voxel @@ -361,7 +368,7 @@ namespace PolyVox //////////////////////////////////////////////////////////////////////////////// - /// Note that if MaxNumberOfBlocksInMemory is not large enough to support the region this function will only load part of the region. In this case it is undefined which parts will actually be loaded. If all the voxels in the given region are already loaded, this function will not do anything. Other voxels might be unloaded to make space for the new voxels. + /// Note that if *NOTE - update docs - MaxNumberOfBlocksInMemory no longer exists* MaxNumberOfBlocksInMemory is not large enough to support the region this function will only load part of the region. In this case it is undefined which parts will actually be loaded. If all the voxels in the given region are already loaded, this function will not do anything. Other voxels might be unloaded to make space for the new voxels. /// \param regPrefetch The Region of voxels to prefetch into memory. //////////////////////////////////////////////////////////////////////////////// template @@ -381,11 +388,14 @@ namespace PolyVox Vector3DInt32 v3dSize = v3dEnd - v3dStart + Vector3DInt32(1,1,1); uint32_t numblocks = static_cast(v3dSize.getX() * v3dSize.getY() * v3dSize.getZ()); - if(numblocks > m_uMaxNumberOfBlocksInMemory) + + // FIXME - reinstate some logic to handle when the prefetched region is too large. + + /*if(numblocks > m_uMaxNumberOfBlocksInMemory) { // cannot support the amount of blocks... so only load the maximum possible numblocks = m_uMaxNumberOfBlocksInMemory; - } + }*/ for(int32_t x = v3dStart.getX(); x <= v3dEnd.getX(); x++) { for(int32_t y = v3dStart.getY(); y <= v3dEnd.getY(); y++) @@ -511,8 +521,8 @@ namespace PolyVox } m_uTimestamper = 0; - m_uMaxNumberOfUncompressedBlocks = 16; - m_uMaxNumberOfBlocksInMemory = 1024; + //m_uMaxNumberOfUncompressedBlocks = 16; + //m_uMaxNumberOfBlocksInMemory = 1024; m_v3dLastAccessedBlockPos = Vector3DInt32(0,0,0); //There are no invalid positions, but initially the m_pLastAccessedBlock pointer will be null; m_pLastAccessedBlock = 0; @@ -526,7 +536,7 @@ namespace PolyVox m_regValidRegionInBlocks.setUpperY(this->m_regValidRegion.getUpperY() >> m_uBlockSideLengthPower); m_regValidRegionInBlocks.setUpperZ(this->m_regValidRegion.getUpperZ() >> m_uBlockSideLengthPower); - setMaxNumberOfUncompressedBlocks(m_uMaxNumberOfUncompressedBlocks); + //setMaxNumberOfUncompressedBlocks(m_uMaxNumberOfUncompressedBlocks); //Clear the previous data m_pBlocks.clear(); @@ -600,7 +610,7 @@ namespace PolyVox if(m_pPager) { // check wether another block needs to be unloaded before this one can be loaded - if(m_pBlocks.size() == m_uMaxNumberOfBlocksInMemory) + while(calculateBlockMemoryUsage() > m_uCompressedBlockMemoryLimitInBytes) //FIXME - This calculation of size is slow and should be outside the loop. { // find the least recently used block typename std::map, BlockPositionCompare>::iterator i; @@ -721,6 +731,21 @@ namespace PolyVox return uSizeInBytes; } + template + uint32_t LargeVolume::calculateBlockMemoryUsage(void) const + { + uint32_t uMemoryUsage = 0; + + typename std::map, BlockPositionCompare>::iterator i; + for(i = m_pBlocks.begin(); i != m_pBlocks.end(); i++) + { + //Inaccurate - account for rest of loaded block. + uMemoryUsage += i->second.calculateSizeInBytes(); + } + + return uMemoryUsage; + } + template template VoxelType LargeVolume::getVoxelImpl(int32_t uXPos, int32_t uYPos, int32_t uZPos, WrapModeType, VoxelType tBorder) const diff --git a/tests/testvolume.cpp b/tests/testvolume.cpp index 77ad1063..2063e8e3 100644 --- a/tests/testvolume.cpp +++ b/tests/testvolume.cpp @@ -281,8 +281,9 @@ TestVolume::TestVolume() m_pSimpleVolume = new SimpleVolume(region); m_pLargeVolume = new LargeVolume(region, m_pCompressor, m_pFilePager, 32); - m_pLargeVolume->setMaxNumberOfBlocksInMemory(32); - m_pLargeVolume->setMaxNumberOfUncompressedBlocks(16); + //m_pLargeVolume->setMaxNumberOfBlocksInMemory(32); + //m_pLargeVolume->setMaxNumberOfUncompressedBlocks(16); + m_pLargeVolume->setTargetMemoryLimitInBytes(4 * 1024 * 1024); //Fill the volume with some data for(int z = region.getLowerZ(); z <= region.getUpperZ(); z++)