AnotherFoxGuy/polyvox
1
0
Fork 0
Code Issues Pull Requests Activity

More work eliminating compression from inside LargeVolume.

Browse Source
This commit is contained in:
David Williams 2014-09-13 11:59:51 +02:00
parent bfe03142b7
commit 302c6e688b
6 changed files with 28 additions and 213 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
examples/OpenGL/main.cpp
View File

@ -53,7 +53,7 @@ int main(int argc, char *argv[])
{
RLEBlockCompressor<MaterialDensityPair88>* compressor = new RLEBlockCompressor<MaterialDensityPair88>();
FilePager<MaterialDensityPair88>* pager = new FilePager<MaterialDensityPair88>("./");
LargeVolume<MaterialDensityPair88> volData(PolyVox::Region(Vector3DInt32(0, 0, 0), Vector3DInt32(g_uVolumeSideLength - 1, g_uVolumeSideLength - 1, g_uVolumeSideLength - 1)), compressor, pager);
LargeVolume<MaterialDensityPair88> volData(PolyVox::Region(Vector3DInt32(0, 0, 0), Vector3DInt32(g_uVolumeSideLength - 1, g_uVolumeSideLength - 1, g_uVolumeSideLength - 1)), pager);
//Make our volume contain a sphere in the center.
int32_t minPos = 0;

12
examples/Paging/main.cpp
View File

@ -159,8 +159,8 @@ int main(int argc, char *argv[])
RLEBlockCompressor<MaterialDensityPair44>* compressor = new RLEBlockCompressor<MaterialDensityPair44>();
PerlinNoisePager* pager = new PerlinNoisePager();
LargeVolume<MaterialDensityPair44> volData(PolyVox::Region::MaxRegion, compressor, pager, 256);
volData.setMaxNumberOfBlocksInMemory(4096);
LargeVolume<MaterialDensityPair44> volData(PolyVox::Region::MaxRegion, pager, 256);
//volData.setMaxNumberOfBlocksInMemory(4096);
volData.setMaxNumberOfUncompressedBlocks(64);
//volData.setMaxNumberOfUncompressedBlocks(4096);
@ -168,22 +168,22 @@ int main(int argc, char *argv[])
//createPerlinTerrain(volData);
//createPerlinVolumeSlow(volData);
std::cout << "Memory usage: " << (volData.calculateSizeInBytes()/1024.0/1024.0) << "MB" << std::endl;
std::cout << "Compression ratio: 1 to " << (1.0/(volData.calculateCompressionRatio())) << std::endl;
//std::cout << "Compression ratio: 1 to " << (1.0/(volData.calculateCompressionRatio())) << std::endl;
//volData.setBlockCacheSize(64);
PolyVox::Region reg(Vector3DInt32(-255,0,0), Vector3DInt32(255,255,255));
std::cout << "Prefetching region: " << reg.getLowerCorner() << " -> " << reg.getUpperCorner() << std::endl;
volData.prefetch(reg);
std::cout << "Memory usage: " << (volData.calculateSizeInBytes()/1024.0/1024.0) << "MB" << std::endl;
std::cout << "Compression ratio: 1 to " << (1.0/(volData.calculateCompressionRatio())) << std::endl;
//std::cout << "Compression ratio: 1 to " << (1.0/(volData.calculateCompressionRatio())) << std::endl;
PolyVox::Region reg2(Vector3DInt32(0,0,0), Vector3DInt32(255,255,255));
std::cout << "Flushing region: " << reg2.getLowerCorner() << " -> " << reg2.getUpperCorner() << std::endl;
volData.flush(reg2);
std::cout << "Memory usage: " << (volData.calculateSizeInBytes()/1024.0/1024.0) << "MB" << std::endl;
std::cout << "Compression ratio: 1 to " << (1.0/(volData.calculateCompressionRatio())) << std::endl;
//std::cout << "Compression ratio: 1 to " << (1.0/(volData.calculateCompressionRatio())) << std::endl;
std::cout << "Flushing entire volume" << std::endl;
volData.flushAll();
std::cout << "Memory usage: " << (volData.calculateSizeInBytes()/1024.0/1024.0) << "MB" << std::endl;
std::cout << "Compression ratio: 1 to " << (1.0/(volData.calculateCompressionRatio())) << std::endl;
//std::cout << "Compression ratio: 1 to " << (1.0/(volData.calculateCompressionRatio())) << std::endl;
//Extract the surface
auto mesh = extractCubicMesh(&volData, reg2);

27
library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h
View File

@ -232,15 +232,8 @@ namespace PolyVox
/// Constructor for creating a fixed size volume.
LargeVolume
(
const Region& regValid,
uint16_t uBlockSideLength = 32
);
/// Constructor for creating a fixed size volume.
LargeVolume
(
const Region& regValid,
BlockCompressor<VoxelType>* pBlockCompressor,
Pager<VoxelType>* pPager ,
const Region& regValid,
Pager<VoxelType>* pPager = nullptr,
uint16_t uBlockSideLength = 32
);
/// Destructor
@ -265,8 +258,6 @@ namespace PolyVox
/// Sets the number of blocks for which uncompressed data is stored
void setMaxNumberOfUncompressedBlocks(uint32_t uMaxNumberOfUncompressedBlocks);
/// Sets the number of blocks which can be in memory before the paging system starts unloading them
void setMaxNumberOfBlocksInMemory(uint32_t uMaxNumberOfBlocksInMemory);
/// Sets the voxel at the position given by <tt>x,y,z</tt> coordinates
void setVoxel(int32_t uXPos, int32_t uYPos, int32_t uZPos, VoxelType tValue, WrapMode eWrapMode = WrapModes::Validate);
/// Sets the voxel at the position given by a 3D vector
@ -282,10 +273,6 @@ namespace PolyVox
/// Removes all voxels from memory
void flushAll();
/// Empties the cache of uncompressed blocks
void clearBlockCache(void);
/// Calculates the approximate compression ratio of the store volume data
float calculateCompressionRatio(void);
/// Calculates approximatly how many bytes of memory the volume is currently using.
uint32_t calculateSizeInBytes(void);
@ -314,12 +301,10 @@ namespace PolyVox
}
};
typedef std::map<Vector3DInt32, CompressedBlock<VoxelType>*, BlockPositionCompare> CompressedBlockMap;
typedef std::map<Vector3DInt32, UncompressedBlock<VoxelType>*, BlockPositionCompare> UncompressedBlockMap;
uint32_t calculateBlockMemoryUsage(void) const;
void ensureCompressedBlockMapHasFreeSpace(void) const;
void ensureUncompressedBlockMapHasFreeSpace(void) const;
void initialise();
@ -332,21 +317,17 @@ namespace PolyVox
VoxelType getVoxelImpl(int32_t uXPos, int32_t uYPos, int32_t uZPos, WrapModeType<WrapModes::Border>, VoxelType tBorder) const;
VoxelType getVoxelImpl(int32_t uXPos, int32_t uYPos, int32_t uZPos, WrapModeType<WrapModes::AssumeValid>, VoxelType tBorder) const;
CompressedBlock<VoxelType>* getCompressedBlock(int32_t uBlockX, int32_t uBlockY, int32_t uBlockZ) const;
UncompressedBlock<VoxelType>* getUncompressedBlock(int32_t uBlockX, int32_t uBlockY, int32_t uBlockZ) const;
void eraseBlock(typename CompressedBlockMap::iterator itCompressedBlock) const;
void eraseBlock(typename UncompressedBlockMap::iterator itUncompressedBlock) const;
// The block data
mutable CompressedBlockMap m_pBlocks;
mutable UncompressedBlockMap m_pUncompressedBlockCache;
mutable uint32_t m_uTimestamper;
mutable Vector3DInt32 m_v3dLastAccessedBlockPos;
mutable UncompressedBlock<VoxelType>* m_pLastAccessedBlock;
uint32_t m_uMaxNumberOfUncompressedBlocks;
uint32_t m_uMaxNumberOfBlocksInMemory;
// The size of the volume
Region m_regValidRegionInBlocks;
@ -355,8 +336,6 @@ namespace PolyVox
uint16_t m_uBlockSideLength;
uint8_t m_uBlockSideLengthPower;
// The compressor used by the Blocks to compress their data if required.
BlockCompressor<VoxelType>* m_pBlockCompressor;
Pager<VoxelType>* m_pPager;
// Compressed data for an empty block (sometimes needed for initialisation).
@ -364,7 +343,7 @@ namespace PolyVox
// Whether we created the compressor or whether it was provided
// by the user. This controls whether we delete it on destruction.
bool m_bIsOurCompressor;
//bool m_bIsOurCompressor;
};
}

190
library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl
View File

@ -29,30 +29,6 @@ freely, subject to the following restrictions:
namespace PolyVox
{
////////////////////////////////////////////////////////////////////////////////
/// When construncting a very large volume you need to be prepared to handle the scenario where there is so much data that PolyVox cannot fit it all in memory. When PolyVox runs out of memory, it identifies the least recently used data and hands it back to the application via a callback function. It is then the responsibility of the application to store this data until PolyVox needs it again (as signalled by another callback function). Please see the LargeVolume class documentation for a full description of this process and the required function signatures. If you really don't want to handle these events then you can provide null pointers here, in which case the data will be discarded and/or filled with default values.
/// \param dataRequiredHandler The callback function which will be called when PolyVox tries to use data which is not currently in memory.
/// \param dataOverflowHandler The callback function which will be called when PolyVox has too much data and needs to remove some from memory.
/// \param uBlockSideLength The size of the blocks making up the volume. Small blocks will compress/decompress faster, but there will also be more of them meaning voxel access could be slower.
////////////////////////////////////////////////////////////////////////////////
template <typename VoxelType>
LargeVolume<VoxelType>::LargeVolume
(
const Region& regValid,
uint16_t uBlockSideLength
)
:BaseVolume<VoxelType>(regValid)
{
m_uBlockSideLength = uBlockSideLength;
m_pBlockCompressor = new MinizBlockCompressor<VoxelType>();
m_bIsOurCompressor = true;
m_pPager = 0;
initialise();
}
////////////////////////////////////////////////////////////////////////////////
/// This constructor creates a volume with a fixed size which is specified as a parameter. By default this constructor will not enable paging but you can override this if desired. If you do wish to enable paging then you are required to provide the call back function (see the other LargeVolume constructor).
/// \param regValid Specifies the minimum and maximum valid voxel positions.
@ -66,7 +42,6 @@ namespace PolyVox
LargeVolume<VoxelType>::LargeVolume
(
const Region& regValid,
BlockCompressor<VoxelType>* pBlockCompressor,
Pager<VoxelType>* pPager,
uint16_t uBlockSideLength
)
@ -74,9 +49,6 @@ namespace PolyVox
{
m_uBlockSideLength = uBlockSideLength;
m_pBlockCompressor = pBlockCompressor;
m_bIsOurCompressor = false;
m_pPager = pPager;
initialise();
@ -102,14 +74,6 @@ namespace PolyVox
LargeVolume<VoxelType>::~LargeVolume()
{
flushAll();
// Only delete the compressor if it was created by us (in the constructor), not by the user.
if(m_bIsOurCompressor)
{
delete m_pBlockCompressor;
}
//delete m_pCompressedEmptyBlock;
}
////////////////////////////////////////////////////////////////////////////////
@ -253,7 +217,12 @@ namespace PolyVox
template <typename VoxelType>
void LargeVolume<VoxelType>::setMaxNumberOfUncompressedBlocks(uint32_t uMaxNumberOfUncompressedBlocks)
{
clearBlockCache();
//clearBlockCache();
/*if (m_pBlocks.size() > uMaxNumberOfBlocksInMemory)
{
flushAll();
}*/
m_uMaxNumberOfUncompressedBlocks = uMaxNumberOfUncompressedBlocks;
@ -262,21 +231,6 @@ namespace PolyVox
<< ", which is " << m_uMaxNumberOfUncompressedBlocks * uUncompressedBlockSizeInBytes << " bytes");
}
////////////////////////////////////////////////////////////////////////////////
/// Increasing the number of blocks in memory causes less paging to occur
/// \param uMaxNumberOfBlocksInMemory The number of blocks
////////////////////////////////////////////////////////////////////////////////
template <typename VoxelType>
void LargeVolume<VoxelType>::setMaxNumberOfBlocksInMemory(uint32_t uMaxNumberOfBlocksInMemory)
{
if(m_pBlocks.size() > uMaxNumberOfBlocksInMemory)
{
flushAll();
}
m_uMaxNumberOfBlocksInMemory = uMaxNumberOfBlocksInMemory;
}
////////////////////////////////////////////////////////////////////////////////
/// \param uXPos the \c x position of the voxel
/// \param uYPos the \c y position of the voxel
@ -403,7 +357,7 @@ namespace PolyVox
for(int32_t z = v3dStart.getZ(); z <= v3dEnd.getZ(); z++)
{
Vector3DInt32 pos(x,y,z);
typename CompressedBlockMap::iterator itBlock = m_pBlocks.find(pos);
/*typename CompressedBlockMap::iterator itBlock = m_pBlocks.find(pos);
if(itBlock != m_pBlocks.end())
{
@ -412,7 +366,7 @@ namespace PolyVox
// This might be nice, but on the prefetch region could be bigger than the uncompressed cache size.
// This would limit the amount of prefetching we could do.
continue;
}
}*/
if(numblocks == 0)
{
@ -434,8 +388,6 @@ namespace PolyVox
template <typename VoxelType>
void LargeVolume<VoxelType>::flushAll()
{
typename CompressedBlockMap::iterator i;
// Flushing will remove the most accessed block, so invalidate the pointer.
m_pLastAccessedBlock = 0;
@ -445,11 +397,6 @@ namespace PolyVox
{
eraseBlock(m_pUncompressedBlockCache.begin());
}
while(m_pBlocks.size() > 0)
{
eraseBlock(m_pBlocks.begin());
}
}
////////////////////////////////////////////////////////////////////////////////
@ -477,8 +424,8 @@ namespace PolyVox
for(int32_t z = v3dStart.getZ(); z <= v3dEnd.getZ(); z++)
{
Vector3DInt32 pos(x,y,z);
typename CompressedBlockMap::iterator itBlock = m_pBlocks.find(pos);
if(itBlock == m_pBlocks.end())
typename UncompressedBlockMap::iterator itBlock = m_pUncompressedBlockCache.find(pos);
if (itBlock == m_pUncompressedBlockCache.end())
{
// not loaded, not unloading
continue;
@ -494,14 +441,6 @@ namespace PolyVox
} // for x
}
////////////////////////////////////////////////////////////////////////////////
///
////////////////////////////////////////////////////////////////////////////////
template <typename VoxelType>
void LargeVolume<VoxelType>::clearBlockCache(void)
{
}
////////////////////////////////////////////////////////////////////////////////
/// This function should probably be made internal...
////////////////////////////////////////////////////////////////////////////////
@ -519,11 +458,6 @@ namespace PolyVox
POLYVOX_THROW(std::invalid_argument, "Block side length must be a power of two.");
}
if(!m_pBlockCompressor)
{
POLYVOX_THROW(std::invalid_argument, "You must provide a valid compressor for the LargeVolume to use.");
}
m_uTimestamper = 0;
//m_uMaxNumberOfUncompressedBlocks = 16;
//m_uMaxNumberOfBlocksInMemory = 1024;
@ -543,10 +477,7 @@ namespace PolyVox
//setMaxNumberOfUncompressedBlocks(m_uMaxNumberOfUncompressedBlocks);
//Clear the previous data
m_pBlocks.clear();
//Clear the previous data
m_pBlocks.clear();
m_pUncompressedBlockCache.clear();
//Other properties we might find useful later
this->m_uLongestSideLength = (std::max)((std::max)(this->getWidth(),this->getHeight()),this->getDepth());
@ -554,35 +485,6 @@ namespace PolyVox
this->m_fDiagonalLength = sqrtf(static_cast<float>(this->getWidth() * this->getWidth() + this->getHeight() * this->getHeight() + this->getDepth() * this->getDepth()));
}
template <typename VoxelType>
void LargeVolume<VoxelType>::eraseBlock(typename CompressedBlockMap::iterator itCompressedBlock) const
{
CompressedBlock<VoxelType>* pCompressedBlock = itCompressedBlock->second;
// Before deleting the block we may need to page out its data. We
// only do this if the data has been modified since it was paged in.
if(pCompressedBlock->m_bDataModified)
{
// The position of the block within the volume.
Vector3DInt32 v3dBlockPos = itCompressedBlock->first;
// From the coordinates of the block we deduce the coordinates of the contained voxels.
Vector3DInt32 v3dLower(v3dBlockPos.getX() << m_uBlockSideLengthPower, v3dBlockPos.getY() << m_uBlockSideLengthPower, v3dBlockPos.getZ() << m_uBlockSideLengthPower);
Vector3DInt32 v3dUpper = v3dLower + Vector3DInt32(m_uBlockSideLength-1, m_uBlockSideLength-1, m_uBlockSideLength-1);
// Page the data out
//m_pPager->pageOut(Region(v3dLower, v3dUpper), pCompressedBlock);
// The compressed data is no longer modified with respect to the data on disk
pCompressedBlock->m_bDataModified = false;
}
delete itCompressedBlock->second;
// We can now remove the block data from memory.
m_pBlocks.erase(itCompressedBlock);
}
template <typename VoxelType>
void LargeVolume<VoxelType>::eraseBlock(typename UncompressedBlockMap::iterator itUncompressedBlock) const
{
@ -617,39 +519,6 @@ namespace PolyVox
m_pUncompressedBlockCache.erase(itUncompressedBlock);
}
template <typename VoxelType>
CompressedBlock<VoxelType>* LargeVolume<VoxelType>::getCompressedBlock(int32_t uBlockX, int32_t uBlockY, int32_t uBlockZ) const
{
Vector3DInt32 v3dBlockPos(uBlockX, uBlockY, uBlockZ);
CompressedBlock<VoxelType>* pBlock = 0;
typename CompressedBlockMap::iterator itBlock = m_pBlocks.find(v3dBlockPos);
if(itBlock != m_pBlocks.end())
{
pBlock = itBlock->second;
}
else
{
ensureCompressedBlockMapHasFreeSpace();
// The block wasn't found so we create a new one
pBlock = new CompressedBlock<VoxelType>;
// Pass the block to the Pager to give it a chance to initialise it with any data
Vector3DInt32 v3dLower(v3dBlockPos.getX() << m_uBlockSideLengthPower, v3dBlockPos.getY() << m_uBlockSideLengthPower, v3dBlockPos.getZ() << m_uBlockSideLengthPower);
Vector3DInt32 v3dUpper = v3dLower + Vector3DInt32(m_uBlockSideLength-1, m_uBlockSideLength-1, m_uBlockSideLength-1);
Region reg(v3dLower, v3dUpper);
m_pPager->pageIn(reg, pBlock);
// Add the block to the map
itBlock = m_pBlocks.insert(std::make_pair(v3dBlockPos, pBlock)).first;
}
pBlock->m_uBlockLastAccessed = ++m_uTimestamper;
return pBlock;
}
template <typename VoxelType>
UncompressedBlock<VoxelType>* LargeVolume<VoxelType>::getUncompressedBlock(int32_t uBlockX, int32_t uBlockY, int32_t uBlockZ) const
{
@ -708,17 +577,6 @@ namespace PolyVox
return pUncompressedBlock;
}
////////////////////////////////////////////////////////////////////////////////
/// Note: This function needs reviewing for accuracy...
////////////////////////////////////////////////////////////////////////////////
template <typename VoxelType>
float LargeVolume<VoxelType>::calculateCompressionRatio(void)
{
float fRawSize = static_cast<float>(m_pBlocks.size() * m_uBlockSideLength * m_uBlockSideLength* m_uBlockSideLength * sizeof(VoxelType));
float fCompressedSize = static_cast<float>(calculateSizeInBytes());
return fCompressedSize/fRawSize;
}
////////////////////////////////////////////////////////////////////////////////
/// Note: This function needs reviewing for accuracy...
////////////////////////////////////////////////////////////////////////////////
@ -728,8 +586,8 @@ namespace PolyVox
uint32_t uSizeInBytes = sizeof(LargeVolume);
//Memory used by the blocks
typename CompressedBlockMap::iterator i;
for(i = m_pBlocks.begin(); i != m_pBlocks.end(); i++)
typename UncompressedBlockMap::iterator i;
for (i = m_pUncompressedBlockCache.begin(); i != m_pUncompressedBlockCache.end(); i++)
{
//Inaccurate - account for rest of loaded block.
uSizeInBytes += i->second->calculateSizeInBytes();
@ -757,28 +615,6 @@ namespace PolyVox
return uMemoryUsage;
}
template <typename VoxelType>
void LargeVolume<VoxelType>::ensureCompressedBlockMapHasFreeSpace(void) const
{
while(m_pBlocks.size() > m_uMaxNumberOfBlocksInMemory)
{
// Find the least recently used block. This is somewhat inefficient as it searches through
// the map, so if it proves to be a bottleneck we may want some kind of sorted structure.
typename CompressedBlockMap::iterator i;
typename CompressedBlockMap::iterator itUnloadBlock = m_pBlocks.begin();
for(i = m_pBlocks.begin(); i != m_pBlocks.end(); i++)
{
if(i->second->m_uBlockLastAccessed < itUnloadBlock->second->m_uBlockLastAccessed)
{
itUnloadBlock = i;
}
}
// Erase the least recently used block
eraseBlock(itUnloadBlock);
}
}
template <typename VoxelType>
void LargeVolume<VoxelType>::ensureUncompressedBlockMapHasFreeSpace(void) const
{

4
tests/TestSurfaceExtractor.cpp
View File

@ -107,7 +107,7 @@ VolumeType* createAndFillVolume(void)
FilePager<VolumeType::VoxelType>* pager = new FilePager<VolumeType::VoxelType>("./");
//Create empty volume
VolumeType* volData = new VolumeType(Region(Vector3DInt32(0, 0, 0), Vector3DInt32(uVolumeSideLength - 1, uVolumeSideLength - 1, uVolumeSideLength - 1)), compressor, pager);
VolumeType* volData = new VolumeType(Region(Vector3DInt32(0, 0, 0), Vector3DInt32(uVolumeSideLength - 1, uVolumeSideLength - 1, uVolumeSideLength - 1)), pager);
// Fill
for (int32_t z = 0; z < uVolumeSideLength; z++)
@ -145,7 +145,7 @@ VolumeType* createAndFillVolumeWithNoise(int32_t iVolumeSideLength, float minVal
FilePager<float>* pager = new FilePager<float>("./");
//Create empty volume
VolumeType* volData = new VolumeType(Region(Vector3DInt32(0, 0, 0), Vector3DInt32(iVolumeSideLength - 1, iVolumeSideLength - 1, iVolumeSideLength - 1)), compressor, pager);
VolumeType* volData = new VolumeType(Region(Vector3DInt32(0, 0, 0), Vector3DInt32(iVolumeSideLength - 1, iVolumeSideLength - 1, iVolumeSideLength - 1)), pager);
// Seed generator for consistency between runs.
srand(12345);

6
tests/testvolume.cpp
View File

@ -278,10 +278,10 @@ TestVolume::TestVolume()
//Create the volumes
m_pRawVolume = new RawVolume<int32_t>(region);
m_pSimpleVolume = new SimpleVolume<int32_t>(region);
m_pLargeVolume = new LargeVolume<int32_t>(region, m_pBlockCompressor, m_pFilePager, 32);
m_pLargeVolume = new LargeVolume<int32_t>(region, m_pFilePager, 32);
m_pLargeVolume->setMaxNumberOfBlocksInMemory(32);
m_pLargeVolume->setMaxNumberOfUncompressedBlocks(16);
//m_pLargeVolume->setMaxNumberOfBlocksInMemory(32);
m_pLargeVolume->setMaxNumberOfUncompressedBlocks(64);
//Fill the volume with some data
for(int z = region.getLowerZ(); z <= region.getUpperZ(); z++)