From ad0e92341360214df08907c8fe161948b2ecc7a0 Mon Sep 17 00:00:00 2001 From: David Williams Date: Sun, 13 Feb 2011 00:07:12 +0000 Subject: [PATCH] Documentation and tidying. --- examples/Basic/main.cpp | 11 +- .../PolyVoxCore/include/PolyVoxImpl/Block.h | 2 +- .../PolyVoxCore/include/PolyVoxImpl/Block.inl | 2 +- library/PolyVoxCore/include/Volume.h | 189 +++++++++--------- library/PolyVoxCore/include/Volume.inl | 34 +++- 5 files changed, 124 insertions(+), 114 deletions(-) diff --git a/examples/Basic/main.cpp b/examples/Basic/main.cpp index 3d71bce9..8315d3be 100644 --- a/examples/Basic/main.cpp +++ b/examples/Basic/main.cpp @@ -382,7 +382,7 @@ void createPerlinVolumeSlow(Volume& volData) } } -void createPerlinVolumeFast(Volume& volData) +/*void createPerlinVolumeFast(Volume& volData) { Perlin perlin(2,8,1,234); @@ -428,7 +428,7 @@ void createPerlinVolumeFast(Volume& volData) } } } -} +}*/ void createPerlinTerrain(Volume& volData) { @@ -510,13 +510,14 @@ int main(int argc, char *argv[]) openGLWidget.show(); //Create an empty volume and then place a sphere in it - Volume volData(2048, 2048, 256); + Volume volData(256, 256, 256); //createSphereInVolume(volData, 30); createPerlinTerrain(volData); //createPerlinVolumeSlow(volData); - std::cout << "Memory usage: " << volData.sizeInBytes() << std::endl; + std::cout << "Memory usage: " << volData.calculateSizeInBytes() << std::endl; volData.setBlockCacheSize(8); - std::cout << "Memory usage: " << volData.sizeInBytes() << std::endl; + std::cout << "Memory usage: " << volData.calculateSizeInBytes() << std::endl; + std::cout << "Compression ratio: " << volData.calculateCompressionRatio() << std::endl; /*srand(12345); for(int ct = 0; ct < 1000; ct++) diff --git a/library/PolyVoxCore/include/PolyVoxImpl/Block.h b/library/PolyVoxCore/include/PolyVoxImpl/Block.h index 111e5870..1e9dfeb7 100644 --- a/library/PolyVoxCore/include/PolyVoxImpl/Block.h +++ b/library/PolyVoxCore/include/PolyVoxImpl/Block.h @@ -58,7 +58,7 @@ namespace PolyVox void fill(VoxelType tValue); void initialise(uint16_t uSideLength); - uint32_t sizeInBytes(void); + uint32_t calculateSizeInBytes(void); public: void compress(void); diff --git a/library/PolyVoxCore/include/PolyVoxImpl/Block.inl b/library/PolyVoxCore/include/PolyVoxImpl/Block.inl index 0eea8867..405e9132 100644 --- a/library/PolyVoxCore/include/PolyVoxImpl/Block.inl +++ b/library/PolyVoxCore/include/PolyVoxImpl/Block.inl @@ -131,7 +131,7 @@ namespace PolyVox } template - uint32_t Block::sizeInBytes(void) + uint32_t Block::calculateSizeInBytes(void) { uint32_t uSizeInBytes = sizeof(Block); uSizeInBytes += m_vecCompressedData.capacity() * sizeof(RunlengthEntry); diff --git a/library/PolyVoxCore/include/Volume.h b/library/PolyVoxCore/include/Volume.h index 43f026e5..fee73b26 100644 --- a/library/PolyVoxCore/include/Volume.h +++ b/library/PolyVoxCore/include/Volume.h @@ -36,86 +36,81 @@ freely, subject to the following restrictions: namespace PolyVox { ///The Volume class provides a memory efficient method of storing voxel data while also allowing fast access and modification. - //////////////////////////////////////////////////////////////////////////////// - /// A Volume is essentially a '3D image' in which each element (voxel) is identified - /// by a three dimensional (x,y,z) coordinate, rather than the two dimensional (x,y) - /// coordinate which is used to identify an element (pixel) in a normal image. Within - /// PolyVox, the Volume class is used to store and manipulate our data before we extract - /// our SurfaceMeshs from it. + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + /// A Volume is essentially a '3D image' in which each element (voxel) is identified by a three dimensional (x,y,z) coordinate, + /// rather than the two dimensional (x,y) coordinate which is used to identify an element (pixel) in a normal image. Within PolyVox, + /// the Volume class is used to store and manipulate our data before we extract our SurfaceMeshs from it. /// - /// Data Representaion - feel free to skip - /// If stored carelessly, volume data can take up a huge amount of memory. For example, a - /// volume of dimensions 1024x1024x1024 with 1 byte per voxel will require 1GB of memory - /// if stored in an uncompressed form. Natuarally our Volume class is much more efficient - /// than this and it is worth understanding (at least at a high level) the approach - /// which is used. + /// Data Representaion + /// If stored carelessly, volume data can take up a huge amount of memory. For example, a volume of dimensions 1024x1024x1024 with + /// 1 byte per voxel will require 1GB of memory if stored in an uncompressed form. Natuarally our Volume class is much more efficient + /// than this and it is worth understanding (at least at a high level) the approach which is used. + /// + /// Essentially, the Volume class stores its data as a collection of blocks. Each of these block is much smaller than the whole volume, + /// for example a typical size might be 32x32x32 voxels (though is is configurable by the user). In this case, a 256x512x1024 volume + /// would contain 8x16x32 = 4096 blocks. The data for each block is stored in a compressed form, which uses only a small amout of + /// memory but it is hard to modify the data. Therefore, before any given voxel can be modified, its corresponding block must be uncompressed. + /// + /// The compression and decompression of block is a relatively slow process and so we aim to do this as rarely as possible. In order + /// to achive this, the volume class store a cache of recently used blocks and their associated uncompressed data. Each time a voxel + /// is touched a timestamp is updated on the corresponding block, when the cache becomes full the block with the oldest timestamp is + /// recompressed and moved out of the cache. + /// + /// Achieving high compression rates + /// Note: This section is theorectical and not well tested. Please let us know if you find the tips below do or do not work. + /// + /// The compression rates which can be achieved can vary significantly depending the nature of the data you are storing, but you can + /// encourage high compression rates by making your data as homogenous as possible. If you are simply storing a material with each + /// voxel then this will probably happen naturally. Games such as Minecraft which use this approach will typically involve large ares + /// of th same material which will compress down well. + /// + /// However, if you are storing density values then you may want to take some care. The advantage of storing smoothly changing values + /// is that you can get smooth surfaces extracted, but storing smoothly changing values inside or outside objects (rather than just + /// on the boundary) does not benefit the surface and is very hard to compress effectively. You should apply some thresholding to your + /// density values to reduce this problem (this threasholding should only be applied to voxels who don't contribute to the surface). /// - /// Essentially, the Volume class stores its data as a collection of blocks. Each - /// of these block is much smaller than the whole volume, for example a typical size - /// might be 32x32x32 voxels (though is is configurable by the user). In this case, - /// a 256x512x1024 volume would contain 8x16x32 = 4096 blocks. However, it is unlikely that - /// all these blocks actually have to be stored because usually there are duplicates - /// in which case common data can be shared. - /// - /// Identifying duplicate blocks is in general a difficult task which involves looking at pairs - /// of blocks and comparing all the voxels. This is a time consuming task which is not amiable - /// to being performed when the volume is being modified in real time. However, there are two - /// specific scenarios which are easily spotted and which PolyVox uses to identify block - /// sharing opportunities. - /// - /// -# Homogeneous blocks (those which contain just a single voxel value) are easy to - /// spot and fairly common becuase volumes often contain large homogeous regions. Any time - /// you change the value of a voxel you have potentially made the block which contains - /// it homogeneous. PolyVox does not check the homogeneity immediately as this would slow - /// down the process of modifying voxels, but you can use the tidyUpMemory() function - /// to check for and remove duplicate homogeneous regions whenever you have spare - /// processing time. - /// - /// -# Copying a volume naturally means that all the voxels in the second volume are - /// the same as the first. Therefore volume copying is a relatively fast operation in - /// which all the blocks in the second volume simply reference the first volume. Future - /// modifications to either volume will, of course, cause the blocks to become unshared. - /// - /// Other advantages of breaking the volume down into blocks include enhancing data locality - /// (i.e. voxels which are spatially near to each other are also likely to be near in - /// memory) and the ability to load larger volumes as no large contiguous areas of - /// memory are needed. However, these advantages are more transparent to user code - /// so we will not dwell on them here. - /// - /// Usage - /// Volumes are constructed by passing the desired width, height and depth to the - /// constructor. Note that for speed reasons only values which are a power of two - /// are permitted for these sidelengths. - /// - /// \code - /// Volume volData(g_uVolumeSideLength, g_uVolumeSideLength, g_uVolumeSideLength); - /// \endcode - /// - /// Access to specific voxels is provided by the getVoxelAt() and setVoxelAt() fuctions. - /// Each of these has two forms so that voxels can be identified by integer triples - /// or by Vector3DUint16%s. - /// - /// \code - /// volData.setVoxelAt(12, 186, 281, 3); - /// uint8_t voxelValue = volData.getVoxelAt(12, 186, 281); - /// //voxelValue is now 3 - /// \endcode - /// - /// The tidyUpMemory() function should normally be called after you first populate - /// the volume with data, and then at periodic intervals as the volume is modified. - /// However, you don't actually have to call it at all. See the function's - /// documentation for further details. - /// - /// One further important point of note is that this class is templatised on the voxel - /// type. This allows you to store volumes of data types you might not normally expect, - /// for example the OpenGL example 'abuses' this class to store a 3D grid of pointers. - /// However, it is not guaranteed that all functionality works correctly with non-integer - /// voxel types. - //////////////////////////////////////////////////////////////////////////////// + /// For example, suppose you are using layers of 3D Perlin noise to create a 3D terrain (not a heightmap). If you store the raw Perlin + /// noise value at each voxel then a slice through the volume might look like the following: + /// + /// + /// + /// However, by setting high values to be fixed to one and low values to be fixed to zero you can make a slice through your volume look more like this: + /// + /// + /// + /// The boundary is in the same place and is still smooth, but the large homogenous regions mean the data should compress much more effectively. + /// Although it may look like you have lost some precision in this process this is only because the images above are constrained to 256 + /// greyscale values, where as true Perlin noise will give you floating point values. + /// + /// Cache-aware traversal + /// You might be suprised at just how many cache misses can occur when you traverse the volume in a naive manner. Consider a 1024x1024x1024 volume + /// with blocks of size 32x32x32. And imagine you iterate over this volume with a simple three-level for loop which iterates over x, the y, then z. + /// If you start at position (0,0,0) then ny the time you reach position (1023,0,0) you have touched 1024 voxels along one edge of the volume and + /// have pulled 32 blocks into the cache. By the time you reach (1023,1023,0) you have hit 1024x1024 voxels and pulled 32x32 blocks into the cache. + /// You are now ready to touch voxel (0,0,1) which is right nect to where you started, but unless your cache is at least 32x32 blocks large then this + /// initial block has already been cleared from the cache. + /// + /// Ensuring you have a large enough cache size can obviously help the above situation, but you might also consider iterating over the voxels in a + /// different order. For example, if you replace your three-level loop with a six-level loop then you can first process all the voxels between (0,0,0) + /// and (31,31,31), then process all the voxels between (32,0,0) and (63,0,0), and so forth. Using this approach you will have no cache misses even + /// is your cache sise is only one. Of course the logic is more complex, but writing code in such a cache-aware manner may be beneficial in some situations. + /// + /// Threading + /// The volume class does not provide any thread safety constructs and can therefore not be assumed to be thread safe. To be safe you should only allow + /// one thread to access the volume at a time. Even if you have several threads just reading data from the volume they can cause blocks to be pushed + /// out of the cache, potentially invalidating any pointers other threads might be using. + /// + /// That said, we believe that if care is taken then multiple threads can be used, and are currently experimenting with this. + /// + /// Use of templates + /// Although this class is templatised on the voxel type it is not expected that you can use any primative type to represent your voxels. It is only + /// intended for PolyVox's voxel types such as Material, Density, and MarterialDensityPair. If you need to store 3D grids of ints, floats, or pointers + /// you should look ar the Array class instead. + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// template class Volume { - //Make VolumeSampler a friend + // Make VolumeSampler a friend friend class VolumeSampler; struct UncompressedBlock @@ -126,49 +121,49 @@ namespace PolyVox }; public: - ///Constructor + /// Constructor Volume(uint16_t uWidth, uint16_t uHeight, uint16_t uDepth, uint16_t uBlockSideLength = 32); - ///Destructor + /// Destructor ~Volume(); - ///Gets the value used for voxels which are outside the volume + /// Gets the value used for voxels which are outside the volume VoxelType getBorderValue(void) const; - ///Gets a Region representing the extents of the Volume. + /// Gets a Region representing the extents of the Volume. Region getEnclosingRegion(void) const; - ///Gets the width of the volume in voxels. + /// Gets the width of the volume in voxels. uint16_t getWidth(void) const; - ///Gets the height of the volume in voxels. + /// Gets the height of the volume in voxels. uint16_t getHeight(void) const; - ///Gets the depth of the volume in voxels. + /// Gets the depth of the volume in voxels. uint16_t getDepth(void) const; - ///Gets the length of the longest side in voxels + /// Gets the length of the longest side in voxels uint16_t getLongestSideLength(void) const; - ///Gets the length of the shortest side in voxels + /// Gets the length of the shortest side in voxels uint16_t getShortestSideLength(void) const; - ///Gets the length of the diagonal in voxels + /// Gets the length of the diagonal in voxels float getDiagonalLength(void) const; - ///Gets a voxel by x,y,z position + /// Gets a voxel by x,y,z position VoxelType getVoxelAt(uint16_t uXPos, uint16_t uYPos, uint16_t uZPos) const; - ///Gets a voxel by 3D vector position + /// Gets a voxel by 3D vector position VoxelType getVoxelAt(const Vector3DUint16& v3dPos) const; - ///Sets the value used for voxels which are outside the volume + /// Sets the number of blocks for which uncompressed data is stored. + void setBlockCacheSize(uint16_t uBlockCacheSize); + /// Sets the value used for voxels which are outside the volume void setBorderValue(const VoxelType& tBorder); - ///Sets the voxel at an x,y,z position + /// Sets the voxel at an x,y,z position bool setVoxelAt(uint16_t uXPos, uint16_t uYPos, uint16_t uZPos, VoxelType tValue); - ///Sets the voxel at a 3D vector position + /// Sets the voxel at a 3D vector position bool setVoxelAt(const Vector3DUint16& v3dPos, VoxelType tValue); - ///Resizes the volume to the specified dimensions - void resize(uint16_t uWidth, uint16_t uHeight, uint16_t uDepth, uint16_t uBlockSideLength = 32); - - void setBlockCacheSize(uint16_t uBlockCacheSize); void clearBlockCache(void); + float calculateCompressionRatio(void); + uint32_t calculateSizeInBytes(void); - uint32_t sizeInBytes(void); - - public: + private: Block* getUncompressedBlock(uint16_t uBlockX, uint16_t uBlockY, uint16_t uBlockZ) const; + /// Resizes the volume to the specified dimensions + void resize(uint16_t uWidth, uint16_t uHeight, uint16_t uDepth, uint16_t uBlockSideLength = 32); //The block data mutable std::vector< Block > m_pBlocks; diff --git a/library/PolyVoxCore/include/Volume.inl b/library/PolyVoxCore/include/Volume.inl index ec3feffb..f257c350 100644 --- a/library/PolyVoxCore/include/Volume.inl +++ b/library/PolyVoxCore/include/Volume.inl @@ -198,6 +198,20 @@ namespace PolyVox return getVoxelAt(v3dPos.getX(), v3dPos.getY(), v3dPos.getZ()); } + //////////////////////////////////////////////////////////////////////////////// + /// Increasing the size of the block cache will increase memory but may improve performance. + /// You may want to set this to a large value (e.g. 1024) when you are first loading your + /// volume data and then set it to a smaller value (e.g.64) for general processing. + /// \param uBlockCacheSize The number of blocks for which uncompressed data can be cached. + //////////////////////////////////////////////////////////////////////////////// + template + void Volume::setBlockCacheSize(uint16_t uBlockCacheSize) + { + clearBlockCache(); + + m_uMaxUncompressedBlockCacheSize = uBlockCacheSize; + } + //////////////////////////////////////////////////////////////////////////////// /// \param tBorder The value to use for voxels outside the volume. //////////////////////////////////////////////////////////////////////////////// @@ -352,14 +366,6 @@ namespace PolyVox m_fDiagonalLength = sqrtf(static_cast(m_uWidth * m_uWidth + m_uHeight * m_uHeight + m_uDepth * m_uDepth)); } - template - void Volume::setBlockCacheSize(uint16_t uBlockCacheSize) - { - clearBlockCache(); - - m_uMaxUncompressedBlockCacheSize = uBlockCacheSize; - } - template Block* Volume::getUncompressedBlock(uint16_t uBlockX, uint16_t uBlockY, uint16_t uBlockZ) const { @@ -424,14 +430,22 @@ namespace PolyVox } template - uint32_t Volume::sizeInBytes(void) + float Volume::calculateCompressionRatio(void) + { + float fRawSize = m_uWidth * m_uHeight * m_uDepth * sizeof(VoxelType); + float fCompressedSize = calculateSizeInBytes(); + return fCompressedSize/fRawSize; + } + + template + uint32_t Volume::calculateSizeInBytes(void) { uint32_t uSizeInBytes = sizeof(Volume); //Memory used by the blocks for(uint32_t i = 0; i < m_uNoOfBlocksInVolume; ++i) { - uSizeInBytes += m_pBlocks[i].sizeInBytes(); + uSizeInBytes += m_pBlocks[i].calculateSizeInBytes(); } //Memory used by the block cache.