Work on limiting memory usage.

examples/Paging/main.cpp

@@ -157,8 +157,8 @@ int main(int argc, char *argv[])
 	openGLWidget.show();
 
 	PerlinNoisePager* pager = new PerlinNoisePager();
-	LargeVolume<MaterialDensityPair44> volData(PolyVox::Region::MaxRegion, pager, 256);
+	LargeVolume<MaterialDensityPair44> volData(PolyVox::Region::MaxRegion, pager, 64);
-	volData.setMemoryUsageLimit(2 * 1024 * 1024); // 2Mb
+	volData.setMemoryUsageLimit(8 * 1024 * 1024); // 8Mb
 
 	//createSphereInVolume(volData, 30);
 	//createPerlinTerrain(volData);

library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h

@@ -334,6 +334,11 @@ namespace PolyVox
 		uint8_t m_uBlockSideLengthPower;
 
 		Pager<VoxelType>* m_pPager;
+
+		// Enough to make sure a blocks and it's neighbours can be loaded, with a few to spare.
+		static const uint32_t uMinPracticalNoOfBlocks = 32;
+		// Should preent multi-gigabyte volumes with reasonable block sizes.
+		static const uint32_t uMaxPracticalNoOfBlocks = 32768;
 	};
 }
 

library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl

@@ -51,8 +51,25 @@ namespace PolyVox
 
 		if (m_pPager)
 		{
-			// If a pager is available then we can set a sensible limit on our memory usage.
+			// If the user is creating a vast (almost infinite) volume then we can bet they will be
-			m_uBlockCountLimit = 256;
+			// expecting a high memory usage and will want a fair number of blocks to play around with.
+			if (regValid == Region::MaxRegion)
+			{
+				m_uBlockCountLimit = 1024;
+			}
+			else
+			{
+				// Otherwise we try to choose a block count to avoid too much thrashing, particularly when iterating
+				// over the whole volume. This means at least enough blocks to cover one edge of the volume, and ideally 
+				// enough for a whole face. Which face? Longest edge by shortest edge seems like a reasonable guess.
+				uint32_t longestSide = (std::max)(regValid.getWidthInVoxels(), (std::max)(regValid.getHeightInVoxels(), regValid.getDepthInVoxels()));
+				uint32_t shortestSide = (std::min)(regValid.getWidthInVoxels(), (std::min)(regValid.getHeightInVoxels(), regValid.getDepthInVoxels()));
+
+				longestSide /= m_uBlockSideLength;
+				shortestSide /= m_uBlockSideLength;
+
+				m_uBlockCountLimit = longestSide * shortestSide;
+			}
 		}
 		else
 		{
@@ -61,6 +78,14 @@ namespace PolyVox
 			m_uBlockCountLimit = (std::numeric_limits<uint32_t>::max)();
 		}
 
+		// Make sure the calculated block limit is within practical bounds
+		m_uBlockCountLimit = (std::max)(m_uBlockCountLimit, uMinPracticalNoOfBlocks);
+		m_uBlockCountLimit = (std::min)(m_uBlockCountLimit, uMaxPracticalNoOfBlocks);
+
+		uint32_t uUncompressedBlockSizeInBytes = UncompressedBlock<VoxelType>::calculateSizeInBytes(m_uBlockSideLength);
+		POLYVOX_LOG_DEBUG("Memory usage limit for volume initially set to " << (m_uBlockCountLimit * uUncompressedBlockSizeInBytes) / (1024 * 1024)
+			<< "Mb (" << m_uBlockCountLimit << " blocks of " << uUncompressedBlockSizeInBytes / 1024 << "Kb each).");
+
 		initialise();
 	}
 
@@ -229,22 +254,24 @@ namespace PolyVox
 	{
 		POLYVOX_THROW_IF(!m_pPager, invalid_operation, "You cannot limit the memory usage of the volume because it was created without a pager attached.");
 
-		uint32_t uUncompressedBlockSizeInBytes = m_uBlockSideLength * m_uBlockSideLength * m_uBlockSideLength * sizeof(VoxelType);
+		// Calculate the number of blocks based on the memory limit and the size of each block.
-
+		uint32_t uUncompressedBlockSizeInBytes = UncompressedBlock<VoxelType>::calculateSizeInBytes(m_uBlockSideLength);
 		m_uBlockCountLimit = uMemoryUsageInBytes / uUncompressedBlockSizeInBytes;
 
-		const uint32_t uMinPracticalNoOfBlocks = 4;
+		// We need at least a few blocks available to avoid thrashing, and in pratice there will probably be hundreds.
-		POLYVOX_LOG_WARNING_IF(m_uBlockCountLimit < uMinPracticalNoOfBlocks, "The memory usage limit is set too low and cannot be adhered to.");
+		POLYVOX_LOG_WARNING_IF(m_uBlockCountLimit < uMinPracticalNoOfBlocks, "Requested memory usage limit of " 
+			<< uMemoryUsageInBytes / (1024 * 1024) << "Mb is too low and cannot be adhered to.");
 		m_uBlockCountLimit = (std::max)(m_uBlockCountLimit, uMinPracticalNoOfBlocks);
+		m_uBlockCountLimit = (std::min)(m_uBlockCountLimit, uMaxPracticalNoOfBlocks);
 
-
+		// If the new limit is less than the number of blocks already loaded then the easiest solution is to flush and start loading again.
 		if (m_pRecentlyUsedBlocks.size() > m_uBlockCountLimit)
 		{
 			flushAll();
 		}
 
-		POLYVOX_LOG_DEBUG("Target memory usage for volume set to " << uMemoryUsageInBytes << "bytes ("
+		POLYVOX_LOG_DEBUG("Memory usage limit for volume now set to " << (m_uBlockCountLimit * uUncompressedBlockSizeInBytes) / (1024 * 1024)
-			<< m_uBlockCountLimit << " blocks of " << uUncompressedBlockSizeInBytes << "bytes each).");
+			<< "Mb (" << m_uBlockCountLimit << " blocks of " << uUncompressedBlockSizeInBytes / 1024 << "Kb each).");
 	}
 
 	////////////////////////////////////////////////////////////////////////////////
@@ -579,24 +606,19 @@ namespace PolyVox
 	}
 
 	////////////////////////////////////////////////////////////////////////////////
-	/// Note: This function needs reviewing for accuracy...
+	/// Calculate the memory usage of the volume.
 	////////////////////////////////////////////////////////////////////////////////
 	template <typename VoxelType>
 	uint32_t LargeVolume<VoxelType>::calculateSizeInBytes(void)
 	{
+		// Purge null blocks so we know that all blocks are used.
+		purgeNullPtrsFromAllBlocks();
+
+		// We include the size of the LargeVolume class but it should be insignificant.
 		uint32_t uSizeInBytes = sizeof(LargeVolume);
 
-		//Memory used by the blocks
+		// Memory used by the blocks
-		typename SharedPtrBlockMap::iterator i;
+		uSizeInBytes += UncompressedBlock<VoxelType>::calculateSizeInBytes(m_uBlockSideLength) * m_pAllBlocks.size();
-		for (i = m_pRecentlyUsedBlocks.begin(); i != m_pRecentlyUsedBlocks.end(); i++)
-		{
-			//Inaccurate - account for rest of loaded block.
-			uSizeInBytes += i->second->calculateSizeInBytes();
-		}
-
-		//Memory used by the block cache.
-		//uSizeInBytes += m_vecBlocksWithUncompressedData.capacity() * sizeof(Block<VoxelType>);
-		//uSizeInBytes += m_vecBlocksWithUncompressedData.size() * m_uBlockSideLength * m_uBlockSideLength * m_uBlockSideLength * sizeof(VoxelType);
 
 		return uSizeInBytes;
 	}

library/PolyVoxCore/include/PolyVoxCore/UncompressedBlock.h

@@ -64,6 +64,7 @@ namespace PolyVox
 		// Made this private for consistancy with CompressedBlock.
 		// Users shouldn't really need this for UncompressedBlock anyway.
 		uint32_t calculateSizeInBytes(void);
+		static uint32_t calculateSizeInBytes(uint32_t uSideLength);
 
         VoxelType* m_tData;
         uint16_t m_uSideLength;

library/PolyVoxCore/include/PolyVoxCore/UncompressedBlock.inl

@@ -140,9 +140,16 @@ namespace PolyVox
 
 	template <typename VoxelType>
 	uint32_t UncompressedBlock<VoxelType>::calculateSizeInBytes(void)
+	{
+		// Call through to the static version
+		return calculateSizeInBytes(m_uSideLength);
+	}
+
+	template <typename VoxelType>
+	uint32_t UncompressedBlock<VoxelType>::calculateSizeInBytes(uint32_t uSideLength)
 	{
 		// Returns the size of this class plus the size of the uncompressed data.
-		uint32_t uSizeInBytes = sizeof(UncompressedBlock<VoxelType>) + (m_uSideLength * m_uSideLength * m_uSideLength * sizeof(VoxelType));
+		uint32_t uSizeInBytes = sizeof(UncompressedBlock<VoxelType>) + (uSideLength * uSideLength * uSideLength * sizeof(VoxelType));
 		return  uSizeInBytes;
 	}
 }