From b08974c197a77da83ec828ece17a2881b65bd156 Mon Sep 17 00:00:00 2001 From: David Williams Date: Thu, 18 Sep 2014 16:26:47 +0200 Subject: [PATCH] Work on limiting memory usage. --- examples/Paging/main.cpp | 4 +- .../include/PolyVoxCore/LargeVolume.h | 5 ++ .../include/PolyVoxCore/LargeVolume.inl | 64 +++++++++++++------ .../include/PolyVoxCore/UncompressedBlock.h | 1 + .../include/PolyVoxCore/UncompressedBlock.inl | 9 ++- 5 files changed, 59 insertions(+), 24 deletions(-) diff --git a/examples/Paging/main.cpp b/examples/Paging/main.cpp index 886764d3..d9359087 100644 --- a/examples/Paging/main.cpp +++ b/examples/Paging/main.cpp @@ -157,8 +157,8 @@ int main(int argc, char *argv[]) openGLWidget.show(); PerlinNoisePager* pager = new PerlinNoisePager(); - LargeVolume volData(PolyVox::Region::MaxRegion, pager, 256); - volData.setMemoryUsageLimit(2 * 1024 * 1024); // 2Mb + LargeVolume volData(PolyVox::Region::MaxRegion, pager, 64); + volData.setMemoryUsageLimit(8 * 1024 * 1024); // 8Mb //createSphereInVolume(volData, 30); //createPerlinTerrain(volData); diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h index fc8f94c2..1fa9a5b6 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h @@ -334,6 +334,11 @@ namespace PolyVox uint8_t m_uBlockSideLengthPower; Pager* m_pPager; + + // Enough to make sure a blocks and it's neighbours can be loaded, with a few to spare. + static const uint32_t uMinPracticalNoOfBlocks = 32; + // Should preent multi-gigabyte volumes with reasonable block sizes. + static const uint32_t uMaxPracticalNoOfBlocks = 32768; }; } diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl index 425fac7c..b4ee5feb 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl @@ -51,8 +51,25 @@ namespace PolyVox if (m_pPager) { - // If a pager is available then we can set a sensible limit on our memory usage. - m_uBlockCountLimit = 256; + // If the user is creating a vast (almost infinite) volume then we can bet they will be + // expecting a high memory usage and will want a fair number of blocks to play around with. + if (regValid == Region::MaxRegion) + { + m_uBlockCountLimit = 1024; + } + else + { + // Otherwise we try to choose a block count to avoid too much thrashing, particularly when iterating + // over the whole volume. This means at least enough blocks to cover one edge of the volume, and ideally + // enough for a whole face. Which face? Longest edge by shortest edge seems like a reasonable guess. + uint32_t longestSide = (std::max)(regValid.getWidthInVoxels(), (std::max)(regValid.getHeightInVoxels(), regValid.getDepthInVoxels())); + uint32_t shortestSide = (std::min)(regValid.getWidthInVoxels(), (std::min)(regValid.getHeightInVoxels(), regValid.getDepthInVoxels())); + + longestSide /= m_uBlockSideLength; + shortestSide /= m_uBlockSideLength; + + m_uBlockCountLimit = longestSide * shortestSide; + } } else { @@ -61,6 +78,14 @@ namespace PolyVox m_uBlockCountLimit = (std::numeric_limits::max)(); } + // Make sure the calculated block limit is within practical bounds + m_uBlockCountLimit = (std::max)(m_uBlockCountLimit, uMinPracticalNoOfBlocks); + m_uBlockCountLimit = (std::min)(m_uBlockCountLimit, uMaxPracticalNoOfBlocks); + + uint32_t uUncompressedBlockSizeInBytes = UncompressedBlock::calculateSizeInBytes(m_uBlockSideLength); + POLYVOX_LOG_DEBUG("Memory usage limit for volume initially set to " << (m_uBlockCountLimit * uUncompressedBlockSizeInBytes) / (1024 * 1024) + << "Mb (" << m_uBlockCountLimit << " blocks of " << uUncompressedBlockSizeInBytes / 1024 << "Kb each)."); + initialise(); } @@ -229,22 +254,24 @@ namespace PolyVox { POLYVOX_THROW_IF(!m_pPager, invalid_operation, "You cannot limit the memory usage of the volume because it was created without a pager attached."); - uint32_t uUncompressedBlockSizeInBytes = m_uBlockSideLength * m_uBlockSideLength * m_uBlockSideLength * sizeof(VoxelType); - + // Calculate the number of blocks based on the memory limit and the size of each block. + uint32_t uUncompressedBlockSizeInBytes = UncompressedBlock::calculateSizeInBytes(m_uBlockSideLength); m_uBlockCountLimit = uMemoryUsageInBytes / uUncompressedBlockSizeInBytes; - const uint32_t uMinPracticalNoOfBlocks = 4; - POLYVOX_LOG_WARNING_IF(m_uBlockCountLimit < uMinPracticalNoOfBlocks, "The memory usage limit is set too low and cannot be adhered to."); + // We need at least a few blocks available to avoid thrashing, and in pratice there will probably be hundreds. + POLYVOX_LOG_WARNING_IF(m_uBlockCountLimit < uMinPracticalNoOfBlocks, "Requested memory usage limit of " + << uMemoryUsageInBytes / (1024 * 1024) << "Mb is too low and cannot be adhered to."); m_uBlockCountLimit = (std::max)(m_uBlockCountLimit, uMinPracticalNoOfBlocks); + m_uBlockCountLimit = (std::min)(m_uBlockCountLimit, uMaxPracticalNoOfBlocks); - + // If the new limit is less than the number of blocks already loaded then the easiest solution is to flush and start loading again. if (m_pRecentlyUsedBlocks.size() > m_uBlockCountLimit) { flushAll(); } - POLYVOX_LOG_DEBUG("Target memory usage for volume set to " << uMemoryUsageInBytes << "bytes (" - << m_uBlockCountLimit << " blocks of " << uUncompressedBlockSizeInBytes << "bytes each)."); + POLYVOX_LOG_DEBUG("Memory usage limit for volume now set to " << (m_uBlockCountLimit * uUncompressedBlockSizeInBytes) / (1024 * 1024) + << "Mb (" << m_uBlockCountLimit << " blocks of " << uUncompressedBlockSizeInBytes / 1024 << "Kb each)."); } //////////////////////////////////////////////////////////////////////////////// @@ -579,24 +606,19 @@ namespace PolyVox } //////////////////////////////////////////////////////////////////////////////// - /// Note: This function needs reviewing for accuracy... + /// Calculate the memory usage of the volume. //////////////////////////////////////////////////////////////////////////////// template uint32_t LargeVolume::calculateSizeInBytes(void) { + // Purge null blocks so we know that all blocks are used. + purgeNullPtrsFromAllBlocks(); + + // We include the size of the LargeVolume class but it should be insignificant. uint32_t uSizeInBytes = sizeof(LargeVolume); - //Memory used by the blocks - typename SharedPtrBlockMap::iterator i; - for (i = m_pRecentlyUsedBlocks.begin(); i != m_pRecentlyUsedBlocks.end(); i++) - { - //Inaccurate - account for rest of loaded block. - uSizeInBytes += i->second->calculateSizeInBytes(); - } - - //Memory used by the block cache. - //uSizeInBytes += m_vecBlocksWithUncompressedData.capacity() * sizeof(Block); - //uSizeInBytes += m_vecBlocksWithUncompressedData.size() * m_uBlockSideLength * m_uBlockSideLength * m_uBlockSideLength * sizeof(VoxelType); + // Memory used by the blocks + uSizeInBytes += UncompressedBlock::calculateSizeInBytes(m_uBlockSideLength) * m_pAllBlocks.size(); return uSizeInBytes; } diff --git a/library/PolyVoxCore/include/PolyVoxCore/UncompressedBlock.h b/library/PolyVoxCore/include/PolyVoxCore/UncompressedBlock.h index c8479322..f65b0dc2 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/UncompressedBlock.h +++ b/library/PolyVoxCore/include/PolyVoxCore/UncompressedBlock.h @@ -64,6 +64,7 @@ namespace PolyVox // Made this private for consistancy with CompressedBlock. // Users shouldn't really need this for UncompressedBlock anyway. uint32_t calculateSizeInBytes(void); + static uint32_t calculateSizeInBytes(uint32_t uSideLength); VoxelType* m_tData; uint16_t m_uSideLength; diff --git a/library/PolyVoxCore/include/PolyVoxCore/UncompressedBlock.inl b/library/PolyVoxCore/include/PolyVoxCore/UncompressedBlock.inl index 401ff40e..6f73d2f1 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/UncompressedBlock.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/UncompressedBlock.inl @@ -140,9 +140,16 @@ namespace PolyVox template uint32_t UncompressedBlock::calculateSizeInBytes(void) + { + // Call through to the static version + return calculateSizeInBytes(m_uSideLength); + } + + template + uint32_t UncompressedBlock::calculateSizeInBytes(uint32_t uSideLength) { // Returns the size of this class plus the size of the uncompressed data. - uint32_t uSizeInBytes = sizeof(UncompressedBlock) + (m_uSideLength * m_uSideLength * m_uSideLength * sizeof(VoxelType)); + uint32_t uSizeInBytes = sizeof(UncompressedBlock) + (uSideLength * uSideLength * uSideLength * sizeof(VoxelType)); return uSizeInBytes; } }