From 882798ccb8f17c5749eb5ed7cc7c4bd6be6c3a1e Mon Sep 17 00:00:00 2001 From: David Williams Date: Fri, 12 Sep 2014 00:02:06 +0200 Subject: [PATCH 01/42] Modifying surface extractor test to work with LargeVolume. --- tests/TestSurfaceExtractor.cpp | 37 +++++++++++++++++++++------------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/tests/TestSurfaceExtractor.cpp b/tests/TestSurfaceExtractor.cpp index 496c39ac..440e8a73 100644 --- a/tests/TestSurfaceExtractor.cpp +++ b/tests/TestSurfaceExtractor.cpp @@ -24,9 +24,12 @@ freely, subject to the following restrictions: #include "TestSurfaceExtractor.h" #include "PolyVoxCore/Density.h" +#include "PolyVoxCore/FilePager.h" +#include "PolyVoxCore/MinizBlockCompressor.h" #include "PolyVoxCore/MaterialDensityPair.h" #include "PolyVoxCore/RawVolume.h" #include "PolyVoxCore/SimpleVolume.h" +#include "PolyVoxCore/LargeVolume.h" #include "PolyVoxCore/MarchingCubesSurfaceExtractor.h" #include @@ -95,13 +98,16 @@ void writeMaterialValueToVoxel(int valueToWrite, MaterialDensityPair88& voxel) voxel.setMaterial(valueToWrite); } -template -SimpleVolume* createAndFillVolume(void) +template +VolumeType* createAndFillVolume(void) { const int32_t uVolumeSideLength = 64; + MinizBlockCompressor* compressor = new MinizBlockCompressor(); + FilePager* pager = new FilePager("./"); + //Create empty volume - SimpleVolume* volData = new SimpleVolume(Region(Vector3DInt32(0, 0, 0), Vector3DInt32(uVolumeSideLength - 1, uVolumeSideLength - 1, uVolumeSideLength - 1))); + VolumeType* volData = new VolumeType(Region(Vector3DInt32(0, 0, 0), Vector3DInt32(uVolumeSideLength - 1, uVolumeSideLength - 1, uVolumeSideLength - 1)), compressor, pager); // Fill for (int32_t z = 0; z < uVolumeSideLength; z++) @@ -112,9 +118,9 @@ SimpleVolume* createAndFillVolume(void) { // Create a density field which changes throughout the volume. It's // zero in the lower corner and increasing as the coordinates increase. - VoxelType voxelValue; - writeDensityValueToVoxel(x + y + z, voxelValue); - writeMaterialValueToVoxel(z > uVolumeSideLength / 2 ? 42 : 79, voxelValue); + VolumeType::VoxelType voxelValue; + writeDensityValueToVoxel(x + y + z, voxelValue); + writeMaterialValueToVoxel(z > uVolumeSideLength / 2 ? 42 : 79, voxelValue); volData->setVoxelAt(x, y, z, voxelValue); } } @@ -135,8 +141,11 @@ float randomFloat(float a, float b) template VolumeType* createAndFillVolumeWithNoise(int32_t iVolumeSideLength, float minValue, float maxValue) { + MinizBlockCompressor* compressor = new MinizBlockCompressor(); + FilePager* pager = new FilePager("./"); + //Create empty volume - VolumeType* volData = new VolumeType(Region(Vector3DInt32(0, 0, 0), Vector3DInt32(iVolumeSideLength - 1, iVolumeSideLength - 1, iVolumeSideLength - 1))); + VolumeType* volData = new VolumeType(Region(Vector3DInt32(0, 0, 0), Vector3DInt32(iVolumeSideLength - 1, iVolumeSideLength - 1, iVolumeSideLength - 1)), compressor, pager); // Seed generator for consistency between runs. srand(12345); @@ -167,7 +176,7 @@ void TestSurfaceExtractor::testBehaviour() // Of course, the use of a custom controller will also make a significant diference, but this probably does need investigating further in the future. // This basic test just uses the default controller and automatically generates a mesh of the appropriate type. - auto uintVol = createAndFillVolume(); + auto uintVol = createAndFillVolume< LargeVolume >(); auto uintMesh = extractMarchingCubesMesh(uintVol, uintVol->getEnclosingRegion()); QCOMPARE(uintMesh.getNoOfVertices(), uint32_t(12096)); // Verifies size of mesh and that we have 32-bit indices QCOMPARE(uintMesh.getNoOfIndices(), uint32_t(35157)); // Verifies size of mesh @@ -175,7 +184,7 @@ void TestSurfaceExtractor::testBehaviour() QCOMPARE(uintMesh.getVertex(100).data, uint8_t(1)); // Not really meaningful for a primative type // This test makes use of a custom controller - auto floatVol = createAndFillVolume(); + auto floatVol = createAndFillVolume< LargeVolume >(); CustomMarchingCubesController floatCustomController; auto floatMesh = extractMarchingCubesMesh(floatVol, floatVol->getEnclosingRegion(), floatCustomController); QCOMPARE(floatMesh.getNoOfVertices(), uint32_t(16113)); // Verifies size of mesh and that we have 32-bit indices @@ -185,7 +194,7 @@ void TestSurfaceExtractor::testBehaviour() // This test makes use of a user provided mesh. It uses the default controller, but we have to explicitly provide this because C++ won't let us // use a default for the second-to-last parameter but noot use a default for the last parameter. - auto intVol = createAndFillVolume(); + auto intVol = createAndFillVolume< LargeVolume >(); Mesh< MarchingCubesVertex< int8_t >, uint16_t > intMesh; extractMarchingCubesMeshCustom(intVol, intVol->getEnclosingRegion(), &intMesh); QCOMPARE(intMesh.getNoOfVertices(), uint16_t(11718)); // Verifies size of mesh and that we have 16-bit indices @@ -194,7 +203,7 @@ void TestSurfaceExtractor::testBehaviour() QCOMPARE(intMesh.getVertex(100).data, int8_t(1)); // Not really meaningful for a primative type // This test makes use of a user-provided mesh and also a custom controller. - auto doubleVol = createAndFillVolume(); + auto doubleVol = createAndFillVolume< LargeVolume >(); CustomMarchingCubesController doubleCustomController; Mesh< MarchingCubesVertex< double >, uint16_t > doubleMesh; extractMarchingCubesMeshCustom(doubleVol, doubleVol->getEnclosingRegion(), &doubleMesh, doubleCustomController); @@ -204,7 +213,7 @@ void TestSurfaceExtractor::testBehaviour() QCOMPARE(doubleMesh.getVertex(100).data, double(1.0f)); // Not really meaningful for a primative type // This test ensures the extractor works on a non-primitive voxel type. - auto materialVol = createAndFillVolume(); + auto materialVol = createAndFillVolume< LargeVolume >(); auto materialMesh = extractMarchingCubesMesh(materialVol, materialVol->getEnclosingRegion()); QCOMPARE(materialMesh.getNoOfVertices(), uint32_t(12096)); // Verifies size of mesh and that we have 32-bit indices QCOMPARE(materialMesh.getNoOfIndices(), uint32_t(35157)); // Verifies size of mesh @@ -214,7 +223,7 @@ void TestSurfaceExtractor::testBehaviour() void TestSurfaceExtractor::testEmptyVolumePerformance() { - auto emptyVol = createAndFillVolumeWithNoise< SimpleVolume >(128, -2.0f, -1.0f); + auto emptyVol = createAndFillVolumeWithNoise< LargeVolume >(128, -2.0f, -1.0f); Mesh< MarchingCubesVertex< float >, uint16_t > emptyMesh; QBENCHMARK{ extractMarchingCubesMeshCustom(emptyVol, Region(32, 32, 32, 63, 63, 63), &emptyMesh); } QCOMPARE(emptyMesh.getNoOfVertices(), uint16_t(0)); @@ -222,7 +231,7 @@ void TestSurfaceExtractor::testEmptyVolumePerformance() void TestSurfaceExtractor::testNoiseVolumePerformance() { - auto noiseVol = createAndFillVolumeWithNoise< SimpleVolume >(128, -1.0f, 1.0f); + auto noiseVol = createAndFillVolumeWithNoise< LargeVolume >(128, -1.0f, 1.0f); Mesh< MarchingCubesVertex< float >, uint16_t > noiseMesh; QBENCHMARK{ extractMarchingCubesMeshCustom(noiseVol, Region(32, 32, 32, 63, 63, 63), &noiseMesh); } QCOMPARE(noiseMesh.getNoOfVertices(), uint16_t(48967)); From 4781ca5c42c42faec1be7ab0f9d3e3133f4453ff Mon Sep 17 00:00:00 2001 From: David Williams Date: Fri, 12 Sep 2014 15:36:28 +0200 Subject: [PATCH 02/42] Work on removing compression from LargeVolume. --- examples/Paging/main.cpp | 14 +++++------ .../include/PolyVoxCore/FilePager.h | 10 ++++---- .../include/PolyVoxCore/LargeVolume.inl | 24 +++++++++++++------ .../PolyVoxCore/include/PolyVoxCore/Pager.h | 6 ++--- tests/testvolume.cpp | 8 +++---- tests/testvolume.h | 4 ++-- 6 files changed, 39 insertions(+), 27 deletions(-) diff --git a/examples/Paging/main.cpp b/examples/Paging/main.cpp index 58e94b5d..53c3f161 100644 --- a/examples/Paging/main.cpp +++ b/examples/Paging/main.cpp @@ -91,11 +91,11 @@ public: /// Destructor virtual ~PerlinNoisePager() {}; - virtual void pageIn(const PolyVox::Region& region, CompressedBlock* pBlockData) + virtual void pageIn(const PolyVox::Region& region, UncompressedBlock* pBlockData) { // FIXME - this isn't a great example... it's a shame we have to hard clode the block size and also create/destroy // a compressor each time. These could at least be moved outside somewhere if we can't fix it in a better way... - UncompressedBlock block(256); + //UncompressedBlock block(256); Perlin perlin(2,2,1,234); @@ -133,18 +133,18 @@ public: // Voxel position within a block always start from zero. So if a block represents region (4, 8, 12) to (11, 19, 15) // then the valid block voxels are from (0, 0, 0) to (7, 11, 3). Hence we subtract the lower corner position of the // region from the volume space position in order to get the block space position. - block.setVoxelAt(x - region.getLowerX(), y - region.getLowerY(), z - region.getLowerZ(), voxel); + pBlockData->setVoxelAt(x - region.getLowerX(), y - region.getLowerY(), z - region.getLowerZ(), voxel); } } } // Now compress the computed data into the provided block. - RLEBlockCompressor* compressor = new RLEBlockCompressor(); - compressor->compress(&block, pBlockData); - delete compressor; + //RLEBlockCompressor* compressor = new RLEBlockCompressor(); + //compressor->compress(&block, pBlockData); + //delete compressor; } - virtual void pageOut(const PolyVox::Region& region, CompressedBlock* /*pBlockData*/) + virtual void pageOut(const PolyVox::Region& region, UncompressedBlock* /*pBlockData*/) { std::cout << "warning unloading region: " << region.getLowerCorner() << " -> " << region.getUpperCorner() << std::endl; } diff --git a/library/PolyVoxCore/include/PolyVoxCore/FilePager.h b/library/PolyVoxCore/include/PolyVoxCore/FilePager.h index 830e0374..5e0a6081 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/FilePager.h +++ b/library/PolyVoxCore/include/PolyVoxCore/FilePager.h @@ -68,7 +68,7 @@ namespace PolyVox m_vecCreatedFiles.clear(); } - virtual void pageIn(const Region& region, CompressedBlock* pBlockData) + virtual void pageIn(const Region& region, UncompressedBlock* pBlockData) { POLYVOX_ASSERT(pBlockData, "Attempting to page in NULL block"); //POLYVOX_ASSERT(pBlockData->hasUncompressedData() == false, "Block should not have uncompressed data"); @@ -88,14 +88,16 @@ namespace PolyVox { POLYVOX_LOG_TRACE("Paging in data for " << region); - fseek(pFile, 0L, SEEK_END); + /*fseek(pFile, 0L, SEEK_END); size_t fileSizeInBytes = ftell(pFile); fseek(pFile, 0L, SEEK_SET); uint8_t* buffer = new uint8_t[fileSizeInBytes]; fread(buffer, sizeof(uint8_t), fileSizeInBytes, pFile); pBlockData->setData(buffer, fileSizeInBytes); - delete[] buffer; + delete[] buffer;*/ + + fread(pBlockData->getData(), sizeof(uint8_t), pBlockData->getDataSizeInBytes(), pFile); if(ferror(pFile)) { @@ -110,7 +112,7 @@ namespace PolyVox } } - virtual void pageOut(const Region& region, CompressedBlock* pBlockData) + virtual void pageOut(const Region& region, UncompressedBlock* pBlockData) { POLYVOX_ASSERT(pBlockData, "Attempting to page out NULL block"); //POLYVOX_ASSERT(pBlockData->hasUncompressedData() == false, "Block should not have uncompressed data"); diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl index 8c12aa45..b17eb128 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl @@ -571,7 +571,7 @@ namespace PolyVox Vector3DInt32 v3dUpper = v3dLower + Vector3DInt32(m_uBlockSideLength-1, m_uBlockSideLength-1, m_uBlockSideLength-1); // Page the data out - m_pPager->pageOut(Region(v3dLower, v3dUpper), pCompressedBlock); + //m_pPager->pageOut(Region(v3dLower, v3dUpper), pCompressedBlock); // The compressed data is no longer modified with respect to the data on disk pCompressedBlock->m_bDataModified = false; @@ -591,7 +591,7 @@ namespace PolyVox // This should never happen as blocks are deleted based on being least recently used. // I the case that we are flushing we delete all blocks, but the flush function will // reset the 'm_pLastAccessedBlock' anyway to prevent it being accidentally reused. - POLYVOX_ASSERT(pUncompressedBlock != m_pLastAccessedBlock, "Attempted to delete last accessed block."); + //POLYVOX_ASSERT(pUncompressedBlock != m_pLastAccessedBlock, "Attempted to delete last accessed block."); // Before deleting the block we may need to recompress its data. We // only do this if the data has been modified since it was decompressed. @@ -599,9 +599,13 @@ namespace PolyVox { // Get the compressed block which we will copy the data back in to. Vector3DInt32 v3dBlockPos = itUncompressedBlock->first; - CompressedBlock* pCompressedBlock = getCompressedBlock(v3dBlockPos.getX(), v3dBlockPos.getY(), v3dBlockPos.getZ()); - m_pBlockCompressor->compress(pUncompressedBlock, pCompressedBlock); + // From the coordinates of the block we deduce the coordinates of the contained voxels. + Vector3DInt32 v3dLower(v3dBlockPos.getX() << m_uBlockSideLengthPower, v3dBlockPos.getY() << m_uBlockSideLengthPower, v3dBlockPos.getZ() << m_uBlockSideLengthPower); + Vector3DInt32 v3dUpper = v3dLower + Vector3DInt32(m_uBlockSideLength - 1, m_uBlockSideLength - 1, m_uBlockSideLength - 1); + + // Page the data out + m_pPager->pageOut(Region(v3dLower, v3dUpper), itUncompressedBlock->second); // The compressed data has been updated, so the uncompressed data is no longer modified with respect to it. pUncompressedBlock->m_bDataModified = false; @@ -679,13 +683,19 @@ namespace PolyVox // An uncompressed bock is always backed by a compressed one, and this is created by getCompressedBlock() if it doesn't // already exist. If it does already exist and has data then we bring this across into the ucompressed version. - if(getCompressedBlock(uBlockX, uBlockY, uBlockZ)->getData() != 0) + /*if(getCompressedBlock(uBlockX, uBlockY, uBlockZ)->getData() != 0) { // FIXME - multiple getCompressedBlock() calls (including the one above) - CompressedBlock* pBlock = getCompressedBlock(uBlockX, uBlockY, uBlockZ); + CompressedBlock* pBlock = getCompressedBlock(uBlockX, uBlockY, ); m_pBlockCompressor->decompress(pBlock, pUncompressedBlock); - } + }*/ + + // Pass the block to the Pager to give it a chance to initialise it with any data + Vector3DInt32 v3dLower(uBlockX << m_uBlockSideLengthPower, uBlockY << m_uBlockSideLengthPower, uBlockZ << m_uBlockSideLengthPower); + Vector3DInt32 v3dUpper = v3dLower + Vector3DInt32(m_uBlockSideLength - 1, m_uBlockSideLength - 1, m_uBlockSideLength - 1); + Region reg(v3dLower, v3dUpper); + m_pPager->pageIn(reg, pUncompressedBlock); // Add our new block to the map. m_pUncompressedBlockCache.insert(std::make_pair(v3dBlockPos, pUncompressedBlock)); diff --git a/library/PolyVoxCore/include/PolyVoxCore/Pager.h b/library/PolyVoxCore/include/PolyVoxCore/Pager.h index eb37e4f7..c11d8afd 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/Pager.h +++ b/library/PolyVoxCore/include/PolyVoxCore/Pager.h @@ -24,7 +24,7 @@ freely, subject to the following restrictions: #ifndef __PolyVox_Pager_H__ #define __PolyVox_Pager_H__ -#include "PolyVoxCore/CompressedBlock.h" +#include "PolyVoxCore/UncompressedBlock.h" #include "PolyVoxCore/Impl/TypeDef.h" namespace PolyVox @@ -41,8 +41,8 @@ namespace PolyVox /// Destructor virtual ~Pager() {}; - virtual void pageIn(const Region& region, CompressedBlock* pBlockData) = 0; - virtual void pageOut(const Region& region, CompressedBlock* pBlockData) = 0; + virtual void pageIn(const Region& region, UncompressedBlock* pBlockData) = 0; + virtual void pageOut(const Region& region, UncompressedBlock* pBlockData) = 0; }; } diff --git a/tests/testvolume.cpp b/tests/testvolume.cpp index f879f1bf..4a0d65e1 100644 --- a/tests/testvolume.cpp +++ b/tests/testvolume.cpp @@ -313,7 +313,7 @@ TestVolume::~TestVolume() * RawVolume Tests */ -void TestVolume::testRawVolumeDirectAccessAllInternalForwards() +/*void TestVolume::testRawVolumeDirectAccessAllInternalForwards() { int32_t result = 0; @@ -399,13 +399,13 @@ void TestVolume::testRawVolumeSamplersWithExternalBackwards() result = testSamplersWithWrappingBackwards(m_pRawVolume, -1, -3, -2, 2, 5, 4); } QCOMPARE(result, static_cast(-769775893)); -} +}*/ /* * SimpleVolume Tests */ -void TestVolume::testSimpleVolumeDirectAccessAllInternalForwards() +/*void TestVolume::testSimpleVolumeDirectAccessAllInternalForwards() { int32_t result = 0; QBENCHMARK @@ -483,7 +483,7 @@ void TestVolume::testSimpleVolumeSamplersWithExternalBackwards() result = testSamplersWithWrappingBackwards(m_pSimpleVolume, -1, -3, -2, 2, 5, 4); } QCOMPARE(result, static_cast(-769775893)); -} +}*/ /* * LargeVolume Tests diff --git a/tests/testvolume.h b/tests/testvolume.h index 9db5f7a0..57390641 100644 --- a/tests/testvolume.h +++ b/tests/testvolume.h @@ -37,7 +37,7 @@ public: ~TestVolume(); private slots: - void testRawVolumeDirectAccessAllInternalForwards(); + /*void testRawVolumeDirectAccessAllInternalForwards(); void testRawVolumeSamplersAllInternalForwards(); void testRawVolumeDirectAccessWithExternalForwards(); void testRawVolumeSamplersWithExternalForwards(); @@ -53,7 +53,7 @@ private slots: void testSimpleVolumeDirectAccessAllInternalBackwards(); void testSimpleVolumeSamplersAllInternalBackwards(); void testSimpleVolumeDirectAccessWithExternalBackwards(); - void testSimpleVolumeSamplersWithExternalBackwards(); + void testSimpleVolumeSamplersWithExternalBackwards();*/ void testLargeVolumeDirectAccessAllInternalForwards(); void testLargeVolumeSamplersAllInternalForwards(); From bfe03142b7e7669e97905ea29abf514d5bb677f0 Mon Sep 17 00:00:00 2001 From: David Williams Date: Sat, 13 Sep 2014 11:30:52 +0200 Subject: [PATCH 03/42] Carefully dismanteling compression related code. --- .../include/PolyVoxCore/BlockCompressor.h | 2 +- .../include/PolyVoxCore/Impl/MinizWrapper.h | 12 ++++++------ .../include/PolyVoxCore/MinizBlockCompressor.h | 4 ++-- .../include/PolyVoxCore/MinizBlockCompressor.inl | 2 +- .../include/PolyVoxCore/RLEBlockCompressor.h | 2 +- .../include/PolyVoxCore/RLEBlockCompressor.inl | 2 +- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/library/PolyVoxCore/include/PolyVoxCore/BlockCompressor.h b/library/PolyVoxCore/include/PolyVoxCore/BlockCompressor.h index 5af6dbfb..da4a8c51 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/BlockCompressor.h +++ b/library/PolyVoxCore/include/PolyVoxCore/BlockCompressor.h @@ -40,7 +40,7 @@ namespace PolyVox BlockCompressor() {}; virtual ~BlockCompressor() {}; - virtual void compress(UncompressedBlock* pSrcBlock, CompressedBlock* pDstBlock) = 0; + virtual void compressData(UncompressedBlock* pSrcBlock, CompressedBlock* pDstBlock) = 0; virtual void decompress(CompressedBlock* pSrcBlock, UncompressedBlock* pDstBlock) = 0; }; } diff --git a/library/PolyVoxCore/include/PolyVoxCore/Impl/MinizWrapper.h b/library/PolyVoxCore/include/PolyVoxCore/Impl/MinizWrapper.h index fbb52250..04659ac1 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/Impl/MinizWrapper.h +++ b/library/PolyVoxCore/include/PolyVoxCore/Impl/MinizWrapper.h @@ -33,12 +33,12 @@ freely, subject to the following restrictions: // Diable things we don't need, and in particular the zlib compatible names which // would cause conflicts if a user application is using both PolyVox and zlib. -#define MINIZ_NO_STDIO -#define MINIZ_NO_ARCHIVE_APIS -#define MINIZ_NO_TIME -#define MINIZ_NO_ZLIB_APIS -#define MINIZ_NO_ZLIB_COMPATIBLE_NAMES -#define MINIZ_NO_MALLOC +//#define MINIZ_NO_STDIO +//#define MINIZ_NO_ARCHIVE_APIS +//#define MINIZ_NO_TIME +//#define MINIZ_NO_ZLIB_APIS +//#define MINIZ_NO_ZLIB_COMPATIBLE_NAMES +//#define MINIZ_NO_MALLOC // Include only the declarations of the functions in miniz.c. Don't include // the actual definitions, as this 'MinizWrapper.h' may be included from multiple diff --git a/library/PolyVoxCore/include/PolyVoxCore/MinizBlockCompressor.h b/library/PolyVoxCore/include/PolyVoxCore/MinizBlockCompressor.h index fc6bdda5..f701101c 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/MinizBlockCompressor.h +++ b/library/PolyVoxCore/include/PolyVoxCore/MinizBlockCompressor.h @@ -40,10 +40,10 @@ namespace PolyVox MinizBlockCompressor(int iCompressionLevel = 6); // Miniz defines MZ_DEFAULT_LEVEL = 6 so we use the same here ~MinizBlockCompressor(); - void compress(UncompressedBlock* pSrcBlock, CompressedBlock* pDstBlock); + void compressData(UncompressedBlock* pSrcBlock, CompressedBlock* pDstBlock); void decompress(CompressedBlock* pSrcBlock, UncompressedBlock* pDstBlock); - private: + public: uint32_t getExpectedCompressedSize(uint32_t uUncompressedInputSize); uint32_t getMaxCompressedSize(uint32_t uUncompressedInputSize); uint32_t compressWithMiniz(const void* pSrcData, size_t uSrcLength, void* pDstData, size_t uDstLength); diff --git a/library/PolyVoxCore/include/PolyVoxCore/MinizBlockCompressor.inl b/library/PolyVoxCore/include/PolyVoxCore/MinizBlockCompressor.inl index 59befd51..a4e41e72 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/MinizBlockCompressor.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/MinizBlockCompressor.inl @@ -57,7 +57,7 @@ namespace PolyVox } template - void MinizBlockCompressor::compress(UncompressedBlock* pSrcBlock, CompressedBlock* pDstBlock) + void MinizBlockCompressor::compressData(UncompressedBlock* pSrcBlock, CompressedBlock* pDstBlock) { // The uncompressed data will be read straight out of the block void* pSrcData = reinterpret_cast(pSrcBlock->getData()); diff --git a/library/PolyVoxCore/include/PolyVoxCore/RLEBlockCompressor.h b/library/PolyVoxCore/include/PolyVoxCore/RLEBlockCompressor.h index ad388b6e..cb4fc38d 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/RLEBlockCompressor.h +++ b/library/PolyVoxCore/include/PolyVoxCore/RLEBlockCompressor.h @@ -49,7 +49,7 @@ namespace PolyVox RLEBlockCompressor(); ~RLEBlockCompressor(); - void compress(UncompressedBlock* pSrcBlock, CompressedBlock* pDstBlock); + void compressData(UncompressedBlock* pSrcBlock, CompressedBlock* pDstBlock); void decompress(CompressedBlock* pSrcBlock, UncompressedBlock* pDstBlock); }; } diff --git a/library/PolyVoxCore/include/PolyVoxCore/RLEBlockCompressor.inl b/library/PolyVoxCore/include/PolyVoxCore/RLEBlockCompressor.inl index 369fb912..a8408059 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/RLEBlockCompressor.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/RLEBlockCompressor.inl @@ -36,7 +36,7 @@ namespace PolyVox } template - void RLEBlockCompressor::compress(UncompressedBlock* pSrcBlock, CompressedBlock* pDstBlock) + void RLEBlockCompressor::compressData(UncompressedBlock* pSrcBlock, CompressedBlock* pDstBlock) { void* pSrcData = reinterpret_cast(pSrcBlock->getData()); uint32_t uSrcLength = pSrcBlock->getDataSizeInBytes(); From 302c6e688b0f4b3adaff3bdc0af3fb9e371f4087 Mon Sep 17 00:00:00 2001 From: David Williams Date: Sat, 13 Sep 2014 11:59:51 +0200 Subject: [PATCH 04/42] More work eliminating compression from inside LargeVolume. --- examples/OpenGL/main.cpp | 2 +- examples/Paging/main.cpp | 12 +- .../include/PolyVoxCore/LargeVolume.h | 27 +-- .../include/PolyVoxCore/LargeVolume.inl | 190 ++---------------- tests/TestSurfaceExtractor.cpp | 4 +- tests/testvolume.cpp | 6 +- 6 files changed, 28 insertions(+), 213 deletions(-) diff --git a/examples/OpenGL/main.cpp b/examples/OpenGL/main.cpp index bdf700e0..580448ee 100644 --- a/examples/OpenGL/main.cpp +++ b/examples/OpenGL/main.cpp @@ -53,7 +53,7 @@ int main(int argc, char *argv[]) { RLEBlockCompressor* compressor = new RLEBlockCompressor(); FilePager* pager = new FilePager("./"); - LargeVolume volData(PolyVox::Region(Vector3DInt32(0, 0, 0), Vector3DInt32(g_uVolumeSideLength - 1, g_uVolumeSideLength - 1, g_uVolumeSideLength - 1)), compressor, pager); + LargeVolume volData(PolyVox::Region(Vector3DInt32(0, 0, 0), Vector3DInt32(g_uVolumeSideLength - 1, g_uVolumeSideLength - 1, g_uVolumeSideLength - 1)), pager); //Make our volume contain a sphere in the center. int32_t minPos = 0; diff --git a/examples/Paging/main.cpp b/examples/Paging/main.cpp index 53c3f161..862ede7b 100644 --- a/examples/Paging/main.cpp +++ b/examples/Paging/main.cpp @@ -159,8 +159,8 @@ int main(int argc, char *argv[]) RLEBlockCompressor* compressor = new RLEBlockCompressor(); PerlinNoisePager* pager = new PerlinNoisePager(); - LargeVolume volData(PolyVox::Region::MaxRegion, compressor, pager, 256); - volData.setMaxNumberOfBlocksInMemory(4096); + LargeVolume volData(PolyVox::Region::MaxRegion, pager, 256); + //volData.setMaxNumberOfBlocksInMemory(4096); volData.setMaxNumberOfUncompressedBlocks(64); //volData.setMaxNumberOfUncompressedBlocks(4096); @@ -168,22 +168,22 @@ int main(int argc, char *argv[]) //createPerlinTerrain(volData); //createPerlinVolumeSlow(volData); std::cout << "Memory usage: " << (volData.calculateSizeInBytes()/1024.0/1024.0) << "MB" << std::endl; - std::cout << "Compression ratio: 1 to " << (1.0/(volData.calculateCompressionRatio())) << std::endl; + //std::cout << "Compression ratio: 1 to " << (1.0/(volData.calculateCompressionRatio())) << std::endl; //volData.setBlockCacheSize(64); PolyVox::Region reg(Vector3DInt32(-255,0,0), Vector3DInt32(255,255,255)); std::cout << "Prefetching region: " << reg.getLowerCorner() << " -> " << reg.getUpperCorner() << std::endl; volData.prefetch(reg); std::cout << "Memory usage: " << (volData.calculateSizeInBytes()/1024.0/1024.0) << "MB" << std::endl; - std::cout << "Compression ratio: 1 to " << (1.0/(volData.calculateCompressionRatio())) << std::endl; + //std::cout << "Compression ratio: 1 to " << (1.0/(volData.calculateCompressionRatio())) << std::endl; PolyVox::Region reg2(Vector3DInt32(0,0,0), Vector3DInt32(255,255,255)); std::cout << "Flushing region: " << reg2.getLowerCorner() << " -> " << reg2.getUpperCorner() << std::endl; volData.flush(reg2); std::cout << "Memory usage: " << (volData.calculateSizeInBytes()/1024.0/1024.0) << "MB" << std::endl; - std::cout << "Compression ratio: 1 to " << (1.0/(volData.calculateCompressionRatio())) << std::endl; + //std::cout << "Compression ratio: 1 to " << (1.0/(volData.calculateCompressionRatio())) << std::endl; std::cout << "Flushing entire volume" << std::endl; volData.flushAll(); std::cout << "Memory usage: " << (volData.calculateSizeInBytes()/1024.0/1024.0) << "MB" << std::endl; - std::cout << "Compression ratio: 1 to " << (1.0/(volData.calculateCompressionRatio())) << std::endl; + //std::cout << "Compression ratio: 1 to " << (1.0/(volData.calculateCompressionRatio())) << std::endl; //Extract the surface auto mesh = extractCubicMesh(&volData, reg2); diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h index 5d02efea..01741c2b 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h @@ -232,15 +232,8 @@ namespace PolyVox /// Constructor for creating a fixed size volume. LargeVolume ( - const Region& regValid, - uint16_t uBlockSideLength = 32 - ); - /// Constructor for creating a fixed size volume. - LargeVolume - ( - const Region& regValid, - BlockCompressor* pBlockCompressor, - Pager* pPager , + const Region& regValid, + Pager* pPager = nullptr, uint16_t uBlockSideLength = 32 ); /// Destructor @@ -265,8 +258,6 @@ namespace PolyVox /// Sets the number of blocks for which uncompressed data is stored void setMaxNumberOfUncompressedBlocks(uint32_t uMaxNumberOfUncompressedBlocks); - /// Sets the number of blocks which can be in memory before the paging system starts unloading them - void setMaxNumberOfBlocksInMemory(uint32_t uMaxNumberOfBlocksInMemory); /// Sets the voxel at the position given by x,y,z coordinates void setVoxel(int32_t uXPos, int32_t uYPos, int32_t uZPos, VoxelType tValue, WrapMode eWrapMode = WrapModes::Validate); /// Sets the voxel at the position given by a 3D vector @@ -282,10 +273,6 @@ namespace PolyVox /// Removes all voxels from memory void flushAll(); - /// Empties the cache of uncompressed blocks - void clearBlockCache(void); - /// Calculates the approximate compression ratio of the store volume data - float calculateCompressionRatio(void); /// Calculates approximatly how many bytes of memory the volume is currently using. uint32_t calculateSizeInBytes(void); @@ -314,12 +301,10 @@ namespace PolyVox } }; - typedef std::map*, BlockPositionCompare> CompressedBlockMap; typedef std::map*, BlockPositionCompare> UncompressedBlockMap; uint32_t calculateBlockMemoryUsage(void) const; - void ensureCompressedBlockMapHasFreeSpace(void) const; void ensureUncompressedBlockMapHasFreeSpace(void) const; void initialise(); @@ -332,21 +317,17 @@ namespace PolyVox VoxelType getVoxelImpl(int32_t uXPos, int32_t uYPos, int32_t uZPos, WrapModeType, VoxelType tBorder) const; VoxelType getVoxelImpl(int32_t uXPos, int32_t uYPos, int32_t uZPos, WrapModeType, VoxelType tBorder) const; - CompressedBlock* getCompressedBlock(int32_t uBlockX, int32_t uBlockY, int32_t uBlockZ) const; UncompressedBlock* getUncompressedBlock(int32_t uBlockX, int32_t uBlockY, int32_t uBlockZ) const; - void eraseBlock(typename CompressedBlockMap::iterator itCompressedBlock) const; void eraseBlock(typename UncompressedBlockMap::iterator itUncompressedBlock) const; // The block data - mutable CompressedBlockMap m_pBlocks; mutable UncompressedBlockMap m_pUncompressedBlockCache; mutable uint32_t m_uTimestamper; mutable Vector3DInt32 m_v3dLastAccessedBlockPos; mutable UncompressedBlock* m_pLastAccessedBlock; uint32_t m_uMaxNumberOfUncompressedBlocks; - uint32_t m_uMaxNumberOfBlocksInMemory; // The size of the volume Region m_regValidRegionInBlocks; @@ -355,8 +336,6 @@ namespace PolyVox uint16_t m_uBlockSideLength; uint8_t m_uBlockSideLengthPower; - // The compressor used by the Blocks to compress their data if required. - BlockCompressor* m_pBlockCompressor; Pager* m_pPager; // Compressed data for an empty block (sometimes needed for initialisation). @@ -364,7 +343,7 @@ namespace PolyVox // Whether we created the compressor or whether it was provided // by the user. This controls whether we delete it on destruction. - bool m_bIsOurCompressor; + //bool m_bIsOurCompressor; }; } diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl index b17eb128..f3914a64 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl @@ -29,30 +29,6 @@ freely, subject to the following restrictions: namespace PolyVox { - //////////////////////////////////////////////////////////////////////////////// - /// When construncting a very large volume you need to be prepared to handle the scenario where there is so much data that PolyVox cannot fit it all in memory. When PolyVox runs out of memory, it identifies the least recently used data and hands it back to the application via a callback function. It is then the responsibility of the application to store this data until PolyVox needs it again (as signalled by another callback function). Please see the LargeVolume class documentation for a full description of this process and the required function signatures. If you really don't want to handle these events then you can provide null pointers here, in which case the data will be discarded and/or filled with default values. - /// \param dataRequiredHandler The callback function which will be called when PolyVox tries to use data which is not currently in memory. - /// \param dataOverflowHandler The callback function which will be called when PolyVox has too much data and needs to remove some from memory. - /// \param uBlockSideLength The size of the blocks making up the volume. Small blocks will compress/decompress faster, but there will also be more of them meaning voxel access could be slower. - //////////////////////////////////////////////////////////////////////////////// - template - LargeVolume::LargeVolume - ( - const Region& regValid, - uint16_t uBlockSideLength - ) - :BaseVolume(regValid) - { - m_uBlockSideLength = uBlockSideLength; - - m_pBlockCompressor = new MinizBlockCompressor(); - m_bIsOurCompressor = true; - - m_pPager = 0; - - initialise(); - } - //////////////////////////////////////////////////////////////////////////////// /// This constructor creates a volume with a fixed size which is specified as a parameter. By default this constructor will not enable paging but you can override this if desired. If you do wish to enable paging then you are required to provide the call back function (see the other LargeVolume constructor). /// \param regValid Specifies the minimum and maximum valid voxel positions. @@ -66,7 +42,6 @@ namespace PolyVox LargeVolume::LargeVolume ( const Region& regValid, - BlockCompressor* pBlockCompressor, Pager* pPager, uint16_t uBlockSideLength ) @@ -74,9 +49,6 @@ namespace PolyVox { m_uBlockSideLength = uBlockSideLength; - m_pBlockCompressor = pBlockCompressor; - m_bIsOurCompressor = false; - m_pPager = pPager; initialise(); @@ -102,14 +74,6 @@ namespace PolyVox LargeVolume::~LargeVolume() { flushAll(); - - // Only delete the compressor if it was created by us (in the constructor), not by the user. - if(m_bIsOurCompressor) - { - delete m_pBlockCompressor; - } - - //delete m_pCompressedEmptyBlock; } //////////////////////////////////////////////////////////////////////////////// @@ -253,7 +217,12 @@ namespace PolyVox template void LargeVolume::setMaxNumberOfUncompressedBlocks(uint32_t uMaxNumberOfUncompressedBlocks) { - clearBlockCache(); + //clearBlockCache(); + + /*if (m_pBlocks.size() > uMaxNumberOfBlocksInMemory) + { + flushAll(); + }*/ m_uMaxNumberOfUncompressedBlocks = uMaxNumberOfUncompressedBlocks; @@ -262,21 +231,6 @@ namespace PolyVox << ", which is " << m_uMaxNumberOfUncompressedBlocks * uUncompressedBlockSizeInBytes << " bytes"); } - //////////////////////////////////////////////////////////////////////////////// - /// Increasing the number of blocks in memory causes less paging to occur - /// \param uMaxNumberOfBlocksInMemory The number of blocks - //////////////////////////////////////////////////////////////////////////////// - template - void LargeVolume::setMaxNumberOfBlocksInMemory(uint32_t uMaxNumberOfBlocksInMemory) - { - if(m_pBlocks.size() > uMaxNumberOfBlocksInMemory) - { - flushAll(); - } - - m_uMaxNumberOfBlocksInMemory = uMaxNumberOfBlocksInMemory; - } - //////////////////////////////////////////////////////////////////////////////// /// \param uXPos the \c x position of the voxel /// \param uYPos the \c y position of the voxel @@ -403,7 +357,7 @@ namespace PolyVox for(int32_t z = v3dStart.getZ(); z <= v3dEnd.getZ(); z++) { Vector3DInt32 pos(x,y,z); - typename CompressedBlockMap::iterator itBlock = m_pBlocks.find(pos); + /*typename CompressedBlockMap::iterator itBlock = m_pBlocks.find(pos); if(itBlock != m_pBlocks.end()) { @@ -412,7 +366,7 @@ namespace PolyVox // This might be nice, but on the prefetch region could be bigger than the uncompressed cache size. // This would limit the amount of prefetching we could do. continue; - } + }*/ if(numblocks == 0) { @@ -434,8 +388,6 @@ namespace PolyVox template void LargeVolume::flushAll() { - typename CompressedBlockMap::iterator i; - // Flushing will remove the most accessed block, so invalidate the pointer. m_pLastAccessedBlock = 0; @@ -445,11 +397,6 @@ namespace PolyVox { eraseBlock(m_pUncompressedBlockCache.begin()); } - - while(m_pBlocks.size() > 0) - { - eraseBlock(m_pBlocks.begin()); - } } //////////////////////////////////////////////////////////////////////////////// @@ -477,8 +424,8 @@ namespace PolyVox for(int32_t z = v3dStart.getZ(); z <= v3dEnd.getZ(); z++) { Vector3DInt32 pos(x,y,z); - typename CompressedBlockMap::iterator itBlock = m_pBlocks.find(pos); - if(itBlock == m_pBlocks.end()) + typename UncompressedBlockMap::iterator itBlock = m_pUncompressedBlockCache.find(pos); + if (itBlock == m_pUncompressedBlockCache.end()) { // not loaded, not unloading continue; @@ -494,14 +441,6 @@ namespace PolyVox } // for x } - //////////////////////////////////////////////////////////////////////////////// - /// - //////////////////////////////////////////////////////////////////////////////// - template - void LargeVolume::clearBlockCache(void) - { - } - //////////////////////////////////////////////////////////////////////////////// /// This function should probably be made internal... //////////////////////////////////////////////////////////////////////////////// @@ -519,11 +458,6 @@ namespace PolyVox POLYVOX_THROW(std::invalid_argument, "Block side length must be a power of two."); } - if(!m_pBlockCompressor) - { - POLYVOX_THROW(std::invalid_argument, "You must provide a valid compressor for the LargeVolume to use."); - } - m_uTimestamper = 0; //m_uMaxNumberOfUncompressedBlocks = 16; //m_uMaxNumberOfBlocksInMemory = 1024; @@ -543,10 +477,7 @@ namespace PolyVox //setMaxNumberOfUncompressedBlocks(m_uMaxNumberOfUncompressedBlocks); //Clear the previous data - m_pBlocks.clear(); - - //Clear the previous data - m_pBlocks.clear(); + m_pUncompressedBlockCache.clear(); //Other properties we might find useful later this->m_uLongestSideLength = (std::max)((std::max)(this->getWidth(),this->getHeight()),this->getDepth()); @@ -554,35 +485,6 @@ namespace PolyVox this->m_fDiagonalLength = sqrtf(static_cast(this->getWidth() * this->getWidth() + this->getHeight() * this->getHeight() + this->getDepth() * this->getDepth())); } - template - void LargeVolume::eraseBlock(typename CompressedBlockMap::iterator itCompressedBlock) const - { - CompressedBlock* pCompressedBlock = itCompressedBlock->second; - - // Before deleting the block we may need to page out its data. We - // only do this if the data has been modified since it was paged in. - if(pCompressedBlock->m_bDataModified) - { - // The position of the block within the volume. - Vector3DInt32 v3dBlockPos = itCompressedBlock->first; - - // From the coordinates of the block we deduce the coordinates of the contained voxels. - Vector3DInt32 v3dLower(v3dBlockPos.getX() << m_uBlockSideLengthPower, v3dBlockPos.getY() << m_uBlockSideLengthPower, v3dBlockPos.getZ() << m_uBlockSideLengthPower); - Vector3DInt32 v3dUpper = v3dLower + Vector3DInt32(m_uBlockSideLength-1, m_uBlockSideLength-1, m_uBlockSideLength-1); - - // Page the data out - //m_pPager->pageOut(Region(v3dLower, v3dUpper), pCompressedBlock); - - // The compressed data is no longer modified with respect to the data on disk - pCompressedBlock->m_bDataModified = false; - } - - delete itCompressedBlock->second; - - // We can now remove the block data from memory. - m_pBlocks.erase(itCompressedBlock); - } - template void LargeVolume::eraseBlock(typename UncompressedBlockMap::iterator itUncompressedBlock) const { @@ -617,39 +519,6 @@ namespace PolyVox m_pUncompressedBlockCache.erase(itUncompressedBlock); } - template - CompressedBlock* LargeVolume::getCompressedBlock(int32_t uBlockX, int32_t uBlockY, int32_t uBlockZ) const - { - Vector3DInt32 v3dBlockPos(uBlockX, uBlockY, uBlockZ); - - CompressedBlock* pBlock = 0; - - typename CompressedBlockMap::iterator itBlock = m_pBlocks.find(v3dBlockPos); - if(itBlock != m_pBlocks.end()) - { - pBlock = itBlock->second; - } - else - { - ensureCompressedBlockMapHasFreeSpace(); - - // The block wasn't found so we create a new one - pBlock = new CompressedBlock; - - // Pass the block to the Pager to give it a chance to initialise it with any data - Vector3DInt32 v3dLower(v3dBlockPos.getX() << m_uBlockSideLengthPower, v3dBlockPos.getY() << m_uBlockSideLengthPower, v3dBlockPos.getZ() << m_uBlockSideLengthPower); - Vector3DInt32 v3dUpper = v3dLower + Vector3DInt32(m_uBlockSideLength-1, m_uBlockSideLength-1, m_uBlockSideLength-1); - Region reg(v3dLower, v3dUpper); - m_pPager->pageIn(reg, pBlock); - - // Add the block to the map - itBlock = m_pBlocks.insert(std::make_pair(v3dBlockPos, pBlock)).first; - } - - pBlock->m_uBlockLastAccessed = ++m_uTimestamper; - return pBlock; - } - template UncompressedBlock* LargeVolume::getUncompressedBlock(int32_t uBlockX, int32_t uBlockY, int32_t uBlockZ) const { @@ -708,17 +577,6 @@ namespace PolyVox return pUncompressedBlock; } - //////////////////////////////////////////////////////////////////////////////// - /// Note: This function needs reviewing for accuracy... - //////////////////////////////////////////////////////////////////////////////// - template - float LargeVolume::calculateCompressionRatio(void) - { - float fRawSize = static_cast(m_pBlocks.size() * m_uBlockSideLength * m_uBlockSideLength* m_uBlockSideLength * sizeof(VoxelType)); - float fCompressedSize = static_cast(calculateSizeInBytes()); - return fCompressedSize/fRawSize; - } - //////////////////////////////////////////////////////////////////////////////// /// Note: This function needs reviewing for accuracy... //////////////////////////////////////////////////////////////////////////////// @@ -728,8 +586,8 @@ namespace PolyVox uint32_t uSizeInBytes = sizeof(LargeVolume); //Memory used by the blocks - typename CompressedBlockMap::iterator i; - for(i = m_pBlocks.begin(); i != m_pBlocks.end(); i++) + typename UncompressedBlockMap::iterator i; + for (i = m_pUncompressedBlockCache.begin(); i != m_pUncompressedBlockCache.end(); i++) { //Inaccurate - account for rest of loaded block. uSizeInBytes += i->second->calculateSizeInBytes(); @@ -757,28 +615,6 @@ namespace PolyVox return uMemoryUsage; } - template - void LargeVolume::ensureCompressedBlockMapHasFreeSpace(void) const - { - while(m_pBlocks.size() > m_uMaxNumberOfBlocksInMemory) - { - // Find the least recently used block. This is somewhat inefficient as it searches through - // the map, so if it proves to be a bottleneck we may want some kind of sorted structure. - typename CompressedBlockMap::iterator i; - typename CompressedBlockMap::iterator itUnloadBlock = m_pBlocks.begin(); - for(i = m_pBlocks.begin(); i != m_pBlocks.end(); i++) - { - if(i->second->m_uBlockLastAccessed < itUnloadBlock->second->m_uBlockLastAccessed) - { - itUnloadBlock = i; - } - } - - // Erase the least recently used block - eraseBlock(itUnloadBlock); - } - } - template void LargeVolume::ensureUncompressedBlockMapHasFreeSpace(void) const { diff --git a/tests/TestSurfaceExtractor.cpp b/tests/TestSurfaceExtractor.cpp index 440e8a73..71e74d70 100644 --- a/tests/TestSurfaceExtractor.cpp +++ b/tests/TestSurfaceExtractor.cpp @@ -107,7 +107,7 @@ VolumeType* createAndFillVolume(void) FilePager* pager = new FilePager("./"); //Create empty volume - VolumeType* volData = new VolumeType(Region(Vector3DInt32(0, 0, 0), Vector3DInt32(uVolumeSideLength - 1, uVolumeSideLength - 1, uVolumeSideLength - 1)), compressor, pager); + VolumeType* volData = new VolumeType(Region(Vector3DInt32(0, 0, 0), Vector3DInt32(uVolumeSideLength - 1, uVolumeSideLength - 1, uVolumeSideLength - 1)), pager); // Fill for (int32_t z = 0; z < uVolumeSideLength; z++) @@ -145,7 +145,7 @@ VolumeType* createAndFillVolumeWithNoise(int32_t iVolumeSideLength, float minVal FilePager* pager = new FilePager("./"); //Create empty volume - VolumeType* volData = new VolumeType(Region(Vector3DInt32(0, 0, 0), Vector3DInt32(iVolumeSideLength - 1, iVolumeSideLength - 1, iVolumeSideLength - 1)), compressor, pager); + VolumeType* volData = new VolumeType(Region(Vector3DInt32(0, 0, 0), Vector3DInt32(iVolumeSideLength - 1, iVolumeSideLength - 1, iVolumeSideLength - 1)), pager); // Seed generator for consistency between runs. srand(12345); diff --git a/tests/testvolume.cpp b/tests/testvolume.cpp index 4a0d65e1..c3090f2c 100644 --- a/tests/testvolume.cpp +++ b/tests/testvolume.cpp @@ -278,10 +278,10 @@ TestVolume::TestVolume() //Create the volumes m_pRawVolume = new RawVolume(region); m_pSimpleVolume = new SimpleVolume(region); - m_pLargeVolume = new LargeVolume(region, m_pBlockCompressor, m_pFilePager, 32); + m_pLargeVolume = new LargeVolume(region, m_pFilePager, 32); - m_pLargeVolume->setMaxNumberOfBlocksInMemory(32); - m_pLargeVolume->setMaxNumberOfUncompressedBlocks(16); + //m_pLargeVolume->setMaxNumberOfBlocksInMemory(32); + m_pLargeVolume->setMaxNumberOfUncompressedBlocks(64); //Fill the volume with some data for(int z = region.getLowerZ(); z <= region.getUpperZ(); z++) From c0a22de2df945f793696c2f0ad7ea595dba68580 Mon Sep 17 00:00:00 2001 From: David Williams Date: Sat, 13 Sep 2014 12:20:56 +0200 Subject: [PATCH 05/42] Removed a lot of compression-related code. --- examples/OpenGL/main.cpp | 2 - examples/Paging/main.cpp | 2 - library/PolyVoxCore/CMakeLists.txt | 9 - .../include/PolyVoxCore/BlockCompressor.h | 48 - .../include/PolyVoxCore/CompressedBlock.h | 63 - .../include/PolyVoxCore/CompressedBlock.inl | 79 - .../include/PolyVoxCore/Impl/MinizWrapper.h | 49 - .../include/PolyVoxCore/Impl/miniz.c | 4777 ----------------- .../include/PolyVoxCore/LargeVolume.h | 1 - .../include/PolyVoxCore/LargeVolume.inl | 2 - .../PolyVoxCore/MinizBlockCompressor.h | 65 - .../PolyVoxCore/MinizBlockCompressor.inl | 210 - .../include/PolyVoxCore/RLEBlockCompressor.h | 59 - .../PolyVoxCore/RLEBlockCompressor.inl | 167 - .../PolyVoxCore/source/Impl/MinizWrapper.cpp | 29 - tests/TestSurfaceExtractor.cpp | 7 +- tests/testvolume.cpp | 3 - 17 files changed, 2 insertions(+), 5570 deletions(-) delete mode 100644 library/PolyVoxCore/include/PolyVoxCore/BlockCompressor.h delete mode 100644 library/PolyVoxCore/include/PolyVoxCore/CompressedBlock.h delete mode 100644 library/PolyVoxCore/include/PolyVoxCore/CompressedBlock.inl delete mode 100644 library/PolyVoxCore/include/PolyVoxCore/Impl/MinizWrapper.h delete mode 100644 library/PolyVoxCore/include/PolyVoxCore/Impl/miniz.c delete mode 100644 library/PolyVoxCore/include/PolyVoxCore/MinizBlockCompressor.h delete mode 100644 library/PolyVoxCore/include/PolyVoxCore/MinizBlockCompressor.inl delete mode 100644 library/PolyVoxCore/include/PolyVoxCore/RLEBlockCompressor.h delete mode 100644 library/PolyVoxCore/include/PolyVoxCore/RLEBlockCompressor.inl delete mode 100644 library/PolyVoxCore/source/Impl/MinizWrapper.cpp diff --git a/examples/OpenGL/main.cpp b/examples/OpenGL/main.cpp index 580448ee..1c51f511 100644 --- a/examples/OpenGL/main.cpp +++ b/examples/OpenGL/main.cpp @@ -27,7 +27,6 @@ freely, subject to the following restrictions: #include "PolyVoxCore/LargeVolume.h" #include "PolyVoxCore/LowPassFilter.h" #include "PolyVoxCore/RawVolume.h" -#include "PolyVoxCore/RLEBlockCompressor.h" #include "PolyVoxCore/Mesh.h" #include "PolyVoxCore/Impl/Utility.h" @@ -51,7 +50,6 @@ const int32_t g_uVolumeSideLength = 128; int main(int argc, char *argv[]) { - RLEBlockCompressor* compressor = new RLEBlockCompressor(); FilePager* pager = new FilePager("./"); LargeVolume volData(PolyVox::Region(Vector3DInt32(0, 0, 0), Vector3DInt32(g_uVolumeSideLength - 1, g_uVolumeSideLength - 1, g_uVolumeSideLength - 1)), pager); diff --git a/examples/Paging/main.cpp b/examples/Paging/main.cpp index 862ede7b..a01779fb 100644 --- a/examples/Paging/main.cpp +++ b/examples/Paging/main.cpp @@ -28,7 +28,6 @@ freely, subject to the following restrictions: #include "PolyVoxCore/CubicSurfaceExtractor.h" #include "PolyVoxCore/MarchingCubesSurfaceExtractor.h" #include "PolyVoxCore/Pager.h" -#include "PolyVoxCore/RLEBlockCompressor.h" #include "PolyVoxCore/Mesh.h" #include "PolyVoxCore/LargeVolume.h" @@ -157,7 +156,6 @@ int main(int argc, char *argv[]) OpenGLWidget openGLWidget(0); openGLWidget.show(); - RLEBlockCompressor* compressor = new RLEBlockCompressor(); PerlinNoisePager* pager = new PerlinNoisePager(); LargeVolume volData(PolyVox::Region::MaxRegion, pager, 256); //volData.setMaxNumberOfBlocksInMemory(4096); diff --git a/library/PolyVoxCore/CMakeLists.txt b/library/PolyVoxCore/CMakeLists.txt index 2f115630..c8b13071 100644 --- a/library/PolyVoxCore/CMakeLists.txt +++ b/library/PolyVoxCore/CMakeLists.txt @@ -39,9 +39,6 @@ SET(CORE_INC_FILES include/PolyVoxCore/BaseVolume.inl include/PolyVoxCore/BaseVolumeSampler.inl include/PolyVoxCore/Block.h - include/PolyVoxCore/BlockCompressor.h - include/PolyVoxCore/CompressedBlock.h - include/PolyVoxCore/CompressedBlock.inl include/PolyVoxCore/CubicSurfaceExtractor.h include/PolyVoxCore/CubicSurfaceExtractor.inl include/PolyVoxCore/DefaultIsQuadNeeded.h @@ -64,8 +61,6 @@ SET(CORE_INC_FILES include/PolyVoxCore/MaterialDensityPair.h include/PolyVoxCore/Mesh.h include/PolyVoxCore/Mesh.inl - include/PolyVoxCore/MinizBlockCompressor.h - include/PolyVoxCore/MinizBlockCompressor.inl include/PolyVoxCore/Pager.h include/PolyVoxCore/PolyVoxForwardDeclarations.h include/PolyVoxCore/Picking.h @@ -76,8 +71,6 @@ SET(CORE_INC_FILES include/PolyVoxCore/Raycast.h include/PolyVoxCore/Raycast.inl include/PolyVoxCore/Region.h - include/PolyVoxCore/RLEBlockCompressor.h - include/PolyVoxCore/RLEBlockCompressor.inl include/PolyVoxCore/SimpleVolume.h include/PolyVoxCore/SimpleVolume.inl include/PolyVoxCore/SimpleVolumeBlock.inl @@ -97,7 +90,6 @@ SET(IMPL_SRC_FILES source/Impl/ErrorHandling.cpp source/Impl/Logging.cpp source/Impl/MarchingCubesTables.cpp - source/Impl/MinizWrapper.cpp source/Impl/RandomUnitVectors.cpp source/Impl/RandomVectors.cpp source/Impl/Timer.cpp @@ -110,7 +102,6 @@ SET(IMPL_INC_FILES include/PolyVoxCore/Impl/ErrorHandling.h include/PolyVoxCore/Impl/Logging.h include/PolyVoxCore/Impl/MarchingCubesTables.h - include/PolyVoxCore/Impl/MinizWrapper.h include/PolyVoxCore/Impl/RandomUnitVectors.h include/PolyVoxCore/Impl/RandomVectors.h include/PolyVoxCore/Impl/Timer.h diff --git a/library/PolyVoxCore/include/PolyVoxCore/BlockCompressor.h b/library/PolyVoxCore/include/PolyVoxCore/BlockCompressor.h deleted file mode 100644 index da4a8c51..00000000 --- a/library/PolyVoxCore/include/PolyVoxCore/BlockCompressor.h +++ /dev/null @@ -1,48 +0,0 @@ -/******************************************************************************* -Copyright (c) 2005-2013 David Williams and Matthew Williams - -This software is provided 'as-is', without any express or implied -warranty. In no event will the authors be held liable for any damages -arising from the use of this software. - -Permission is granted to anyone to use this software for any purpose, -including commercial applications, and to alter it and redistribute it -freely, subject to the following restrictions: - - 1. The origin of this software must not be misrepresented; you must not - claim that you wrote the original software. If you use this software - in a product, an acknowledgment in the product documentation would be - appreciated but is not required. - - 2. Altered source versions must be plainly marked as such, and must not be - misrepresented as being the original software. - - 3. This notice may not be removed or altered from any source - distribution. -*******************************************************************************/ - -#ifndef __PolyVox_BlockCompressor_H__ -#define __PolyVox_BlockCompressor_H__ - -#include "PolyVoxCore/PolyVoxForwardDeclarations.h" -#include "PolyVoxCore/CompressedBlock.h" -#include "PolyVoxCore/UncompressedBlock.h" - -namespace PolyVox -{ - /** - * Provides an interface for performing compression of blocks. - */ - template - class BlockCompressor - { - public: - BlockCompressor() {}; - virtual ~BlockCompressor() {}; - - virtual void compressData(UncompressedBlock* pSrcBlock, CompressedBlock* pDstBlock) = 0; - virtual void decompress(CompressedBlock* pSrcBlock, UncompressedBlock* pDstBlock) = 0; - }; -} - -#endif //__PolyVox_BlockCompressor_H__ diff --git a/library/PolyVoxCore/include/PolyVoxCore/CompressedBlock.h b/library/PolyVoxCore/include/PolyVoxCore/CompressedBlock.h deleted file mode 100644 index 5a8ba89a..00000000 --- a/library/PolyVoxCore/include/PolyVoxCore/CompressedBlock.h +++ /dev/null @@ -1,63 +0,0 @@ -/******************************************************************************* -Copyright (c) 2005-2013 David Williams and Matthew Williams - -This software is provided 'as-is', without any express or implied -warranty. In no event will the authors be held liable for any damages -arising from the use of this software. - -Permission is granted to anyone to use this software for any purpose, -including commercial applications, and to alter it and redistribute it -freely, subject to the following restrictions: - - 1. The origin of this software must not be misrepresented; you must not - claim that you wrote the original software. If you use this software - in a product, an acknowledgment in the product documentation would be - appreciated but is not required. - - 2. Altered source versions must be plainly marked as such, and must not be - misrepresented as being the original software. - - 3. This notice may not be removed or altered from any source - distribution. -*******************************************************************************/ - -#ifndef __PolyVox_CompressedBlock_H__ -#define __PolyVox_CompressedBlock_H__ - -#include "PolyVoxCore/Block.h" - -namespace PolyVox -{ - template - class CompressedBlock : public Block - { - friend class LargeVolume; - - public: - CompressedBlock(); - ~CompressedBlock(); - - const uint8_t* getData(void) const; - uint32_t getDataSizeInBytes(void) const; - - void setData(const uint8_t* const pData, uint32_t uDataSizeInBytes); - - private: - /// Private copy constructor to prevent accisdental copying - CompressedBlock(const CompressedBlock& /*rhs*/) {}; - - /// Private assignment operator to prevent accisdental copying - CompressedBlock& operator=(const CompressedBlock& /*rhs*/) {}; - - // Made this private to avoid any confusion with getDataSizeInBytes(). - // Users shouldn't really need this for CompressedBlock anyway. - uint32_t calculateSizeInBytes(void); - - uint8_t* m_pData; - uint32_t m_uDataSizeInBytes; - }; -} - -#include "PolyVoxCore/CompressedBlock.inl" - -#endif diff --git a/library/PolyVoxCore/include/PolyVoxCore/CompressedBlock.inl b/library/PolyVoxCore/include/PolyVoxCore/CompressedBlock.inl deleted file mode 100644 index e98df8e3..00000000 --- a/library/PolyVoxCore/include/PolyVoxCore/CompressedBlock.inl +++ /dev/null @@ -1,79 +0,0 @@ -/******************************************************************************* -Copyright (c) 2005-2013 David Williams and Matthew Williams - -This software is provided 'as-is', without any express or implied -warranty. In no event will the authors be held liable for any damages -arising from the use of this software. - -Permission is granted to anyone to use this software for any purpose, -including commercial applications, and to alter it and redistribute it -freely, subject to the following restrictions: - - 1. The origin of this software must not be misrepresented; you must not - claim that you wrote the original software. If you use this software - in a product, an acknowledgment in the product documentation would be - appreciated but is not required. - - 2. Altered source versions must be plainly marked as such, and must not be - misrepresented as being the original software. - - 3. This notice may not be removed or altered from any source - distribution. -*******************************************************************************/ - -namespace PolyVox -{ - template - CompressedBlock::CompressedBlock() - :m_pData(0) - ,m_uDataSizeInBytes(0) - { - } - - template - CompressedBlock::~CompressedBlock() - { - delete[] m_pData; - m_pData = 0; - } - - template - const uint8_t* CompressedBlock::getData(void) const - { - return m_pData; - } - - template - uint32_t CompressedBlock::getDataSizeInBytes(void) const - { - return m_uDataSizeInBytes; - } - - template - void CompressedBlock::setData(const uint8_t* const pData, uint32_t uDataSizeInBytes) - { - POLYVOX_THROW_IF(pData == 0, std::invalid_argument, "Pointer to data cannot be null"); - POLYVOX_THROW_IF(m_pData == pData, std::invalid_argument, "Attempting to copy data onto itself"); - - // Delete any existing data - delete[] m_pData; - - // Allocate new data - m_uDataSizeInBytes = uDataSizeInBytes; - m_pData = new uint8_t[uDataSizeInBytes]; - - // Copy the data across - memcpy(m_pData, pData, uDataSizeInBytes); - - // Flag as modified - this->m_bDataModified = true; - } - - template - uint32_t CompressedBlock::calculateSizeInBytes(void) - { - // Returns the size of this class plus the size of the compressed data. - uint32_t uSizeInBytes = sizeof(CompressedBlock) + m_uDataSizeInBytes; - return uSizeInBytes; - } -} diff --git a/library/PolyVoxCore/include/PolyVoxCore/Impl/MinizWrapper.h b/library/PolyVoxCore/include/PolyVoxCore/Impl/MinizWrapper.h deleted file mode 100644 index 04659ac1..00000000 --- a/library/PolyVoxCore/include/PolyVoxCore/Impl/MinizWrapper.h +++ /dev/null @@ -1,49 +0,0 @@ -/******************************************************************************* -Copyright (c) 2005-2013 David Williams - -This software is provided 'as-is', without any express or implied -warranty. In no event will the authors be held liable for any damages -arising from the use of this software. - -Permission is granted to anyone to use this software for any purpose, -including commercial applications, and to alter it and redistribute it -freely, subject to the following restrictions: - - 1. The origin of this software must not be misrepresented; you must not - claim that you wrote the original software. If you use this software - in a product, an acknowledgment in the product documentation would be - appreciated but is not required. - - 2. Altered source versions must be plainly marked as such, and must not be - misrepresented as being the original software. - - 3. This notice may not be removed or altered from any source - distribution. -*******************************************************************************/ - -#ifndef __PolyVox_MinizWrapper_H__ -#define __PolyVox_MinizWrapper_H__ - -// The miniz library is suplied as a single '.c' file, but this is messy for a project like PolyVox -// because it consists mostly of headers. Many of our headers may want to make use of Miniz code which -// means we need to be careful about #including 'miniz.c' multiple times and getting linker errors. -// We simplify this situation with this 'MinizWrapper.h' and 'MinizWrapper.c' which include 'miniz.c' -// with only declarations and definitions respectively. 'MinizWrapper.cpp' only gets compiled once, -// and all other parts of PolyVox can simply include 'MinizWrapper.h' to get the declarations. - -// Diable things we don't need, and in particular the zlib compatible names which -// would cause conflicts if a user application is using both PolyVox and zlib. -//#define MINIZ_NO_STDIO -//#define MINIZ_NO_ARCHIVE_APIS -//#define MINIZ_NO_TIME -//#define MINIZ_NO_ZLIB_APIS -//#define MINIZ_NO_ZLIB_COMPATIBLE_NAMES -//#define MINIZ_NO_MALLOC - -// Include only the declarations of the functions in miniz.c. Don't include -// the actual definitions, as this 'MinizWrapper.h' may be included from multiple -// locations and we want to avoid linker errors regarding multiple definitions. -#define MINIZ_HEADER_FILE_ONLY -#include "PolyVoxCore/Impl/miniz.c" - -#endif //__PolyVox_MinizWrapper_H__ \ No newline at end of file diff --git a/library/PolyVoxCore/include/PolyVoxCore/Impl/miniz.c b/library/PolyVoxCore/include/PolyVoxCore/Impl/miniz.c deleted file mode 100644 index 4b2608e1..00000000 --- a/library/PolyVoxCore/include/PolyVoxCore/Impl/miniz.c +++ /dev/null @@ -1,4777 +0,0 @@ -/* - * CHANGES FOR POLYVOX - * ------------------- - * This file gave compiler warnings on certain versions of GCC (at least version 4.3.5 used by out build machine) - * and I did not want to risk tampering with the code to fix them.Therefore the only difference between this file - * and the official 'miniz.c' is the pragma below which disables warnings for this file in GCC. - */ -#ifdef __GNUC__ -#pragma GCC system_header -#endif - -/* miniz.c v1.14 - public domain deflate/inflate, zlib-subset, ZIP reading/writing/appending, PNG writing - See "unlicense" statement at the end of this file. - Rich Geldreich , last updated May 20, 2012 - Implements RFC 1950: http://www.ietf.org/rfc/rfc1950.txt and RFC 1951: http://www.ietf.org/rfc/rfc1951.txt - - Most API's defined in miniz.c are optional. For example, to disable the archive related functions just define - MINIZ_NO_ARCHIVE_APIS, or to get rid of all stdio usage define MINIZ_NO_STDIO (see the list below for more macros). - - * Change History - 5/20/12 v1.14 - MinGW32/64 GCC 4.6.1 compiler fixes: added MZ_FORCEINLINE, #include (thanks fermtect). - 5/19/12 v1.13 - From jason@cornsyrup.org and kelwert@mtu.edu - Fix mz_crc32() so it doesn't compute the wrong CRC-32's when mz_ulong is 64-bit. - Temporarily/locally slammed in "typedef unsigned long mz_ulong" and re-ran a randomized regression test on ~500k files. - Eliminated a bunch of warnings when compiling with GCC 32-bit/64. - Ran all examples, miniz.c, and tinfl.c through MSVC 2008's /analyze (static analysis) option and fixed all warnings (except for the silly - "Use of the comma-operator in a tested expression.." analysis warning, which I purposely use to work around a MSVC compiler warning). - Created 32-bit and 64-bit Codeblocks projects/workspace. Built and tested Linux executables. The codeblocks workspace is compatible with Linux+Win32/x64. - Added miniz_tester solution/project, which is a useful little app derived from LZHAM's tester app that I use as part of the regression test. - Ran miniz.c and tinfl.c through another series of regression testing on ~500,000 files and archives. - Modified example5.c so it purposely disables a bunch of high-level functionality (MINIZ_NO_STDIO, etc.). (Thanks to corysama for the MINIZ_NO_STDIO bug report.) - Fix ftell() usage in examples so they exit with an error on files which are too large (a limitation of the examples, not miniz itself). - 4/12/12 v1.12 - More comments, added low-level example5.c, fixed a couple minor level_and_flags issues in the archive API's. - level_and_flags can now be set to MZ_DEFAULT_COMPRESSION. Thanks to Bruce Dawson for the feedback/bug report. - 5/28/11 v1.11 - Added statement from unlicense.org - 5/27/11 v1.10 - Substantial compressor optimizations: - Level 1 is now ~4x faster than before. The L1 compressor's throughput now varies between 70-110MB/sec. on a - Core i7 (actual throughput varies depending on the type of data, and x64 vs. x86). - Improved baseline L2-L9 compression perf. Also, greatly improved compression perf. issues on some file types. - Refactored the compression code for better readability and maintainability. - Added level 10 compression level (L10 has slightly better ratio than level 9, but could have a potentially large - drop in throughput on some files). - 5/15/11 v1.09 - Initial stable release. - - * Low-level Deflate/Inflate implementation notes: - - Compression: Use the "tdefl" API's. The compressor supports raw, static, and dynamic blocks, lazy or - greedy parsing, match length filtering, RLE-only, and Huffman-only streams. It performs and compresses - approximately as well as zlib. - - Decompression: Use the "tinfl" API's. The entire decompressor is implemented as a single function - coroutine: see tinfl_decompress(). It supports decompression into a 32KB (or larger power of 2) wrapping buffer, or into a memory - block large enough to hold the entire file. - - The low-level tdefl/tinfl API's do not make any use of dynamic memory allocation. - - * zlib-style API notes: - - miniz.c implements a fairly large subset of zlib. There's enough functionality present for it to be a drop-in - zlib replacement in many apps: - The z_stream struct, optional memory allocation callbacks - deflateInit/deflateInit2/deflate/deflateReset/deflateEnd/deflateBound - inflateInit/inflateInit2/inflate/inflateEnd - compress, compress2, compressBound, uncompress - CRC-32, Adler-32 - Using modern, minimal code size, CPU cache friendly routines. - Supports raw deflate streams or standard zlib streams with adler-32 checking. - - Limitations: - The callback API's are not implemented yet. No support for gzip headers or zlib static dictionaries. - I've tried to closely emulate zlib's various flavors of stream flushing and return status codes, but - there are no guarantees that miniz.c pulls this off perfectly. - - * PNG writing: See the tdefl_write_image_to_png_file_in_memory() function, originally written by - Alex Evans. Supports 1-4 bytes/pixel images. - - * ZIP archive API notes: - - The ZIP archive API's where designed with simplicity and efficiency in mind, with just enough abstraction to - get the job done with minimal fuss. There are simple API's to retrieve file information, read files from - existing archives, create new archives, append new files to existing archives, or clone archive data from - one archive to another. It supports archives located in memory or the heap, on disk (using stdio.h), - or you can specify custom file read/write callbacks. - - - Archive reading: Just call this function to read a single file from a disk archive: - - void *mz_zip_extract_archive_file_to_heap(const char *pZip_filename, const char *pArchive_name, - size_t *pSize, mz_uint zip_flags); - - For more complex cases, use the "mz_zip_reader" functions. Upon opening an archive, the entire central - directory is located and read as-is into memory, and subsequent file access only occurs when reading individual files. - - - Archives file scanning: The simple way is to use this function to scan a loaded archive for a specific file: - - int mz_zip_reader_locate_file(mz_zip_archive *pZip, const char *pName, const char *pComment, mz_uint flags); - - The locate operation can optionally check file comments too, which (as one example) can be used to identify - multiple versions of the same file in an archive. This function uses a simple linear search through the central - directory, so it's not very fast. - - Alternately, you can iterate through all the files in an archive (using mz_zip_reader_get_num_files()) and - retrieve detailed info on each file by calling mz_zip_reader_file_stat(). - - - Archive creation: Use the "mz_zip_writer" functions. The ZIP writer immediately writes compressed file data - to disk and builds an exact image of the central directory in memory. The central directory image is written - all at once at the end of the archive file when the archive is finalized. - - The archive writer can optionally align each file's local header and file data to any power of 2 alignment, - which can be useful when the archive will be read from optical media. Also, the writer supports placing - arbitrary data blobs at the very beginning of ZIP archives. Archives written using either feature are still - readable by any ZIP tool. - - - Archive appending: The simple way to add a single file to an archive is to call this function: - - mz_bool mz_zip_add_mem_to_archive_file_in_place(const char *pZip_filename, const char *pArchive_name, - const void *pBuf, size_t buf_size, const void *pComment, mz_uint16 comment_size, mz_uint level_and_flags); - - The archive will be created if it doesn't already exist, otherwise it'll be appended to. - Note the appending is done in-place and is not an atomic operation, so if something goes wrong - during the operation it's possible the archive could be left without a central directory (although the local - file headers and file data will be fine, so the archive will be recoverable). - - For more complex archive modification scenarios: - 1. The safest way is to use a mz_zip_reader to read the existing archive, cloning only those bits you want to - preserve into a new archive using using the mz_zip_writer_add_from_zip_reader() function (which compiles the - compressed file data as-is). When you're done, delete the old archive and rename the newly written archive, and - you're done. This is safe but requires a bunch of temporary disk space or heap memory. - - 2. Or, you can convert an mz_zip_reader in-place to an mz_zip_writer using mz_zip_writer_init_from_reader(), - append new files as needed, then finalize the archive which will write an updated central directory to the - original archive. (This is basically what mz_zip_add_mem_to_archive_file_in_place() does.) There's a - possibility that the archive's central directory could be lost with this method if anything goes wrong, though. - - - ZIP archive support limitations: - No zip64 or spanning support. Extraction functions can only handle unencrypted, stored or deflated files. - Requires streams capable of seeking. - - * This is a header file library, like stb_image.c. To get only a header file, either cut and paste the - below header, or create miniz.h, #define MINIZ_HEADER_FILE_ONLY, and then include miniz.c from it. - - * Important: For best perf. be sure to customize the below macros for your target platform: - #define MINIZ_USE_UNALIGNED_LOADS_AND_STORES 1 - #define MINIZ_LITTLE_ENDIAN 1 - #define MINIZ_HAS_64BIT_REGISTERS 1 -*/ - -#ifndef MINIZ_HEADER_INCLUDED -#define MINIZ_HEADER_INCLUDED - -#include - -#if !defined(MINIZ_NO_TIME) && !defined(MINIZ_NO_ARCHIVE_APIS) -#include -#endif - -// Defines to completely disable specific portions of miniz.c: -// If all macros here are defined the only functionality remaining will be CRC-32, adler-32, tinfl, and tdefl. - -// Define MINIZ_NO_STDIO to disable all usage and any functions which rely on stdio for file I/O. -//#define MINIZ_NO_STDIO - -// If MINIZ_NO_TIME is specified then the ZIP archive functions will not be able to get the current time, or -// get/set file times. -//#define MINIZ_NO_TIME - -// Define MINIZ_NO_ARCHIVE_APIS to disable all ZIP archive API's. -//#define MINIZ_NO_ARCHIVE_APIS - -// Define MINIZ_NO_ARCHIVE_APIS to disable all writing related ZIP archive API's. -//#define MINIZ_NO_ARCHIVE_WRITING_APIS - -// Define MINIZ_NO_ZLIB_APIS to remove all ZLIB-style compression/decompression API's. -//#define MINIZ_NO_ZLIB_APIS - -// Define MINIZ_NO_ZLIB_COMPATIBLE_NAME to disable zlib names, to prevent conflicts against stock zlib. -//#define MINIZ_NO_ZLIB_COMPATIBLE_NAMES - -// Define MINIZ_NO_MALLOC to disable all calls to malloc, free, and realloc. -// Note if MINIZ_NO_MALLOC is defined then the user must always provide custom user alloc/free/realloc -// callbacks to the zlib and archive API's, and a few stand-alone helper API's which don't provide custom user -// functions (such as tdefl_compress_mem_to_heap() and tinfl_decompress_mem_to_heap()) won't work. -//#define MINIZ_NO_MALLOC - -#if defined(_M_IX86) || defined(_M_X64) || defined(__i386__) || defined(__i386) || defined(__i486__) || defined(__i486) || defined(i386) || defined(__ia64__) || defined(__x86_64__) -// MINIZ_X86_OR_X64_CPU is only used to help set the below macros. -#define MINIZ_X86_OR_X64_CPU 1 -#endif - -#if (__BYTE_ORDER__==__ORDER_LITTLE_ENDIAN__) || MINIZ_X86_OR_X64_CPU -// Set MINIZ_LITTLE_ENDIAN to 1 if the processor is little endian. -#define MINIZ_LITTLE_ENDIAN 1 -#endif - -#if MINIZ_X86_OR_X64_CPU -// Set MINIZ_USE_UNALIGNED_LOADS_AND_STORES to 1 on CPU's that permit efficient integer loads and stores from unaligned addresses. -#define MINIZ_USE_UNALIGNED_LOADS_AND_STORES 1 -#endif - -#if defined(_M_X64) || defined(_WIN64) || defined(__MINGW64__) || defined(_LP64) || defined(__LP64__) || defined(__ia64__) || defined(__x86_64__) -// Set MINIZ_HAS_64BIT_REGISTERS to 1 if operations on 64-bit integers are reasonably fast (and don't involve compiler generated calls to helper functions). -#define MINIZ_HAS_64BIT_REGISTERS 1 -#endif - -#ifdef __cplusplus -extern "C" { -#endif - -// ------------------- zlib-style API Definitions. - -// For more compatibility with zlib, miniz.c uses unsigned long for some parameters/struct members. Beware: mz_ulong can be either 32 or 64-bits! -typedef unsigned long mz_ulong; - -// Heap allocation callbacks. -// Note that mz_alloc_func parameter types purpsosely differ from zlib's: items/size is size_t, not unsigned long. -typedef void *(*mz_alloc_func)(void *opaque, size_t items, size_t size); -typedef void (*mz_free_func)(void *opaque, void *address); -typedef void *(*mz_realloc_func)(void *opaque, void *address, size_t items, size_t size); - -#define MZ_ADLER32_INIT (1) -// mz_adler32() returns the initial adler-32 value to use when called with ptr==NULL. -mz_ulong mz_adler32(mz_ulong adler, const unsigned char *ptr, size_t buf_len); - -#define MZ_CRC32_INIT (0) -// mz_crc32() returns the initial CRC-32 value to use when called with ptr==NULL. -mz_ulong mz_crc32(mz_ulong crc, const unsigned char *ptr, size_t buf_len); - -// Compression strategies. -enum { MZ_DEFAULT_STRATEGY = 0, MZ_FILTERED = 1, MZ_HUFFMAN_ONLY = 2, MZ_RLE = 3, MZ_FIXED = 4 }; - -// Method -#define MZ_DEFLATED 8 - -#ifndef MINIZ_NO_ZLIB_APIS - -#define MZ_VERSION "9.1.14" -#define MZ_VERNUM 0x91E0 -#define MZ_VER_MAJOR 9 -#define MZ_VER_MINOR 1 -#define MZ_VER_REVISION 14 -#define MZ_VER_SUBREVISION 0 - -// Flush values. For typical usage you only need MZ_NO_FLUSH and MZ_FINISH. The other values are for advanced use (refer to the zlib docs). -enum { MZ_NO_FLUSH = 0, MZ_PARTIAL_FLUSH = 1, MZ_SYNC_FLUSH = 2, MZ_FULL_FLUSH = 3, MZ_FINISH = 4, MZ_BLOCK = 5 }; - -// Return status codes. MZ_PARAM_ERROR is non-standard. -enum { MZ_OK = 0, MZ_STREAM_END = 1, MZ_NEED_DICT = 2, MZ_ERRNO = -1, MZ_STREAM_ERROR = -2, MZ_DATA_ERROR = -3, MZ_MEM_ERROR = -4, MZ_BUF_ERROR = -5, MZ_VERSION_ERROR = -6, MZ_PARAM_ERROR = -10000 }; - -// Compression levels: 0-9 are the standard zlib-style levels, 10 is best possible compression (not zlib compatible, and may be very slow), MZ_DEFAULT_COMPRESSION=MZ_DEFAULT_LEVEL. -enum { MZ_NO_COMPRESSION = 0, MZ_BEST_SPEED = 1, MZ_BEST_COMPRESSION = 9, MZ_UBER_COMPRESSION = 10, MZ_DEFAULT_LEVEL = 6, MZ_DEFAULT_COMPRESSION = -1 }; - -// Window bits -#define MZ_DEFAULT_WINDOW_BITS 15 - -struct mz_internal_state; - -// Compression/decompression stream struct. -typedef struct mz_stream_s -{ - const unsigned char *next_in; // pointer to next byte to read - unsigned int avail_in; // number of bytes available at next_in - mz_ulong total_in; // total number of bytes consumed so far - - unsigned char *next_out; // pointer to next byte to write - unsigned int avail_out; // number of bytes that can be written to next_out - mz_ulong total_out; // total number of bytes produced so far - - char *msg; // error msg (unused) - struct mz_internal_state *state; // internal state, allocated by zalloc/zfree - - mz_alloc_func zalloc; // optional heap allocation function (defaults to malloc) - mz_free_func zfree; // optional heap free function (defaults to free) - void *opaque; // heap alloc function user pointer - - int data_type; // data_type (unused) - mz_ulong adler; // adler32 of the source or uncompressed data - mz_ulong reserved; // not used -} mz_stream; - -typedef mz_stream *mz_streamp; - -// Returns the version string of miniz.c. -const char *mz_version(void); - -// mz_deflateInit() initializes a compressor with default options: -// Parameters: -// pStream must point to an initialized mz_stream struct. -// level must be between [MZ_NO_COMPRESSION, MZ_BEST_COMPRESSION]. -// level 1 enables a specially optimized compression function that's been optimized purely for performance, not ratio. -// (This special func. is currently only enabled when MINIZ_USE_UNALIGNED_LOADS_AND_STORES and MINIZ_LITTLE_ENDIAN are defined.) -// Return values: -// MZ_OK on success. -// MZ_STREAM_ERROR if the stream is bogus. -// MZ_PARAM_ERROR if the input parameters are bogus. -// MZ_MEM_ERROR on out of memory. -int mz_deflateInit(mz_streamp pStream, int level); - -// mz_deflateInit2() is like mz_deflate(), except with more control: -// Additional parameters: -// method must be MZ_DEFLATED -// window_bits must be MZ_DEFAULT_WINDOW_BITS (to wrap the deflate stream with zlib header/adler-32 footer) or -MZ_DEFAULT_WINDOW_BITS (raw deflate/no header or footer) -// mem_level must be between [1, 9] (it's checked but ignored by miniz.c) -int mz_deflateInit2(mz_streamp pStream, int level, int method, int window_bits, int mem_level, int strategy); - -// Quickly resets a compressor without having to reallocate anything. Same as calling mz_deflateEnd() followed by mz_deflateInit()/mz_deflateInit2(). -int mz_deflateReset(mz_streamp pStream); - -// mz_deflate() compresses the input to output, consuming as much of the input and producing as much output as possible. -// Parameters: -// pStream is the stream to read from and write to. You must initialize/update the next_in, avail_in, next_out, and avail_out members. -// flush may be MZ_NO_FLUSH, MZ_PARTIAL_FLUSH/MZ_SYNC_FLUSH, MZ_FULL_FLUSH, or MZ_FINISH. -// Return values: -// MZ_OK on success (when flushing, or if more input is needed but not available, and/or there's more output to be written but the output buffer is full). -// MZ_STREAM_END if all input has been consumed and all output bytes have been written. Don't call mz_deflate() on the stream anymore. -// MZ_STREAM_ERROR if the stream is bogus. -// MZ_PARAM_ERROR if one of the parameters is invalid. -// MZ_BUF_ERROR if no forward progress is possible because the input and/or output buffers are empty. (Fill up the input buffer or free up some output space and try again.) -int mz_deflate(mz_streamp pStream, int flush); - -// mz_deflateEnd() deinitializes a compressor: -// Return values: -// MZ_OK on success. -// MZ_STREAM_ERROR if the stream is bogus. -int mz_deflateEnd(mz_streamp pStream); - -// mz_deflateBound() returns a (very) conservative upper bound on the amount of data that could be generated by deflate(), assuming flush is set to only MZ_NO_FLUSH or MZ_FINISH. -mz_ulong mz_deflateBound(mz_streamp pStream, mz_ulong source_len); - -// Single-call compression functions mz_compress() and mz_compress2(): -// Returns MZ_OK on success, or one of the error codes from mz_deflate() on failure. -int mz_compress(unsigned char *pDest, mz_ulong *pDest_len, const unsigned char *pSource, mz_ulong source_len); -int mz_compress2(unsigned char *pDest, mz_ulong *pDest_len, const unsigned char *pSource, mz_ulong source_len, int level); - -// mz_compressBound() returns a (very) conservative upper bound on the amount of data that could be generated by calling mz_compress(). -mz_ulong mz_compressBound(mz_ulong source_len); - -// Initializes a decompressor. -int mz_inflateInit(mz_streamp pStream); - -// mz_inflateInit2() is like mz_inflateInit() with an additional option that controls the window size and whether or not the stream has been wrapped with a zlib header/footer: -// window_bits must be MZ_DEFAULT_WINDOW_BITS (to parse zlib header/footer) or -MZ_DEFAULT_WINDOW_BITS (raw deflate). -int mz_inflateInit2(mz_streamp pStream, int window_bits); - -// Decompresses the input stream to the output, consuming only as much of the input as needed, and writing as much to the output as possible. -// Parameters: -// pStream is the stream to read from and write to. You must initialize/update the next_in, avail_in, next_out, and avail_out members. -// flush may be MZ_NO_FLUSH, MZ_SYNC_FLUSH, or MZ_FINISH. -// On the first call, if flush is MZ_FINISH it's assumed the input and output buffers are both sized large enough to decompress the entire stream in a single call (this is slightly faster). -// MZ_FINISH implies that there are no more source bytes available beside what's already in the input buffer, and that the output buffer is large enough to hold the rest of the decompressed data. -// Return values: -// MZ_OK on success. Either more input is needed but not available, and/or there's more output to be written but the output buffer is full. -// MZ_STREAM_END if all needed input has been consumed and all output bytes have been written. For zlib streams, the adler-32 of the decompressed data has also been verified. -// MZ_STREAM_ERROR if the stream is bogus. -// MZ_DATA_ERROR if the deflate stream is invalid. -// MZ_PARAM_ERROR if one of the parameters is invalid. -// MZ_BUF_ERROR if no forward progress is possible because the input buffer is empty but the inflater needs more input to continue, or if the output buffer is not large enough. Call mz_inflate() again -// with more input data, or with more room in the output buffer (except when using single call decompression, described above). -int mz_inflate(mz_streamp pStream, int flush); - -// Deinitializes a decompressor. -int mz_inflateEnd(mz_streamp pStream); - -// Single-call decompression. -// Returns MZ_OK on success, or one of the error codes from mz_inflate() on failure. -int mz_uncompress(unsigned char *pDest, mz_ulong *pDest_len, const unsigned char *pSource, mz_ulong source_len); - -// Returns a string description of the specified error code, or NULL if the error code is invalid. -const char *mz_error(int err); - -// Redefine zlib-compatible names to miniz equivalents, so miniz.c can be used as a drop-in replacement for the subset of zlib that miniz.c supports. -// Define MINIZ_NO_ZLIB_COMPATIBLE_NAMES to disable zlib-compatibility if you use zlib in the same project. -#ifndef MINIZ_NO_ZLIB_COMPATIBLE_NAMES - typedef unsigned char Byte; - typedef unsigned int uInt; - typedef mz_ulong uLong; - typedef Byte Bytef; - typedef uInt uIntf; - typedef char charf; - typedef int intf; - typedef void *voidpf; - typedef uLong uLongf; - typedef void *voidp; - typedef void *const voidpc; - #define Z_NULL 0 - #define Z_NO_FLUSH MZ_NO_FLUSH - #define Z_PARTIAL_FLUSH MZ_PARTIAL_FLUSH - #define Z_SYNC_FLUSH MZ_SYNC_FLUSH - #define Z_FULL_FLUSH MZ_FULL_FLUSH - #define Z_FINISH MZ_FINISH - #define Z_BLOCK MZ_BLOCK - #define Z_OK MZ_OK - #define Z_STREAM_END MZ_STREAM_END - #define Z_NEED_DICT MZ_NEED_DICT - #define Z_ERRNO MZ_ERRNO - #define Z_STREAM_ERROR MZ_STREAM_ERROR - #define Z_DATA_ERROR MZ_DATA_ERROR - #define Z_MEM_ERROR MZ_MEM_ERROR - #define Z_BUF_ERROR MZ_BUF_ERROR - #define Z_VERSION_ERROR MZ_VERSION_ERROR - #define Z_PARAM_ERROR MZ_PARAM_ERROR - #define Z_NO_COMPRESSION MZ_NO_COMPRESSION - #define Z_BEST_SPEED MZ_BEST_SPEED - #define Z_BEST_COMPRESSION MZ_BEST_COMPRESSION - #define Z_DEFAULT_COMPRESSION MZ_DEFAULT_COMPRESSION - #define Z_DEFAULT_STRATEGY MZ_DEFAULT_STRATEGY - #define Z_FILTERED MZ_FILTERED - #define Z_HUFFMAN_ONLY MZ_HUFFMAN_ONLY - #define Z_RLE MZ_RLE - #define Z_FIXED MZ_FIXED - #define Z_DEFLATED MZ_DEFLATED - #define Z_DEFAULT_WINDOW_BITS MZ_DEFAULT_WINDOW_BITS - #define alloc_func mz_alloc_func - #define free_func mz_free_func - #define internal_state mz_internal_state - #define z_stream mz_stream - #define deflateInit mz_deflateInit - #define deflateInit2 mz_deflateInit2 - #define deflateReset mz_deflateReset - #define deflate mz_deflate - #define deflateEnd mz_deflateEnd - #define deflateBound mz_deflateBound - #define compress mz_compress - #define compress2 mz_compress2 - #define compressBound mz_compressBound - #define inflateInit mz_inflateInit - #define inflateInit2 mz_inflateInit2 - #define inflate mz_inflate - #define inflateEnd mz_inflateEnd - #define uncompress mz_uncompress - #define crc32 mz_crc32 - #define adler32 mz_adler32 - #define MAX_WBITS 15 - #define MAX_MEM_LEVEL 9 - #define zError mz_error - #define ZLIB_VERSION MZ_VERSION - #define ZLIB_VERNUM MZ_VERNUM - #define ZLIB_VER_MAJOR MZ_VER_MAJOR - #define ZLIB_VER_MINOR MZ_VER_MINOR - #define ZLIB_VER_REVISION MZ_VER_REVISION - #define ZLIB_VER_SUBREVISION MZ_VER_SUBREVISION - #define zlibVersion mz_version - #define zlib_version mz_version() -#endif // #ifndef MINIZ_NO_ZLIB_COMPATIBLE_NAMES - -#endif // MINIZ_NO_ZLIB_APIS - -// ------------------- Types and macros - -typedef unsigned char mz_uint8; -typedef signed short mz_int16; -typedef unsigned short mz_uint16; -typedef unsigned int mz_uint32; -typedef unsigned int mz_uint; -typedef long long mz_int64; -typedef unsigned long long mz_uint64; -typedef int mz_bool; - -#define MZ_FALSE (0) -#define MZ_TRUE (1) - -// Works around MSVC's spammy "warning C4127: conditional expression is constant" message. -#ifdef _MSC_VER - #define MZ_MACRO_END while (0, 0) -#else - #define MZ_MACRO_END while (0) -#endif - -// ------------------- ZIP archive reading/writing - -#ifndef MINIZ_NO_ARCHIVE_APIS - -enum -{ - MZ_ZIP_MAX_IO_BUF_SIZE = 64*1024, - MZ_ZIP_MAX_ARCHIVE_FILENAME_SIZE = 260, - MZ_ZIP_MAX_ARCHIVE_FILE_COMMENT_SIZE = 256 -}; - -typedef struct -{ - mz_uint32 m_file_index; - mz_uint32 m_central_dir_ofs; - mz_uint16 m_version_made_by; - mz_uint16 m_version_needed; - mz_uint16 m_bit_flag; - mz_uint16 m_method; -#ifndef MINIZ_NO_TIME - time_t m_time; -#endif - mz_uint32 m_crc32; - mz_uint64 m_comp_size; - mz_uint64 m_uncomp_size; - mz_uint16 m_internal_attr; - mz_uint32 m_external_attr; - mz_uint64 m_local_header_ofs; - mz_uint32 m_comment_size; - char m_filename[MZ_ZIP_MAX_ARCHIVE_FILENAME_SIZE]; - char m_comment[MZ_ZIP_MAX_ARCHIVE_FILE_COMMENT_SIZE]; -} mz_zip_archive_file_stat; - -typedef size_t (*mz_file_read_func)(void *pOpaque, mz_uint64 file_ofs, void *pBuf, size_t n); -typedef size_t (*mz_file_write_func)(void *pOpaque, mz_uint64 file_ofs, const void *pBuf, size_t n); - -struct mz_zip_internal_state_tag; -typedef struct mz_zip_internal_state_tag mz_zip_internal_state; - -typedef enum -{ - MZ_ZIP_MODE_INVALID = 0, - MZ_ZIP_MODE_READING = 1, - MZ_ZIP_MODE_WRITING = 2, - MZ_ZIP_MODE_WRITING_HAS_BEEN_FINALIZED = 3 -} mz_zip_mode; - -typedef struct -{ - mz_uint64 m_archive_size; - mz_uint64 m_central_directory_file_ofs; - mz_uint m_total_files; - mz_zip_mode m_zip_mode; - - mz_uint m_file_offset_alignment; - - mz_alloc_func m_pAlloc; - mz_free_func m_pFree; - mz_realloc_func m_pRealloc; - void *m_pAlloc_opaque; - - mz_file_read_func m_pRead; - mz_file_write_func m_pWrite; - void *m_pIO_opaque; - - mz_zip_internal_state *m_pState; - -} mz_zip_archive; - -typedef enum -{ - MZ_ZIP_FLAG_CASE_SENSITIVE = 0x0100, - MZ_ZIP_FLAG_IGNORE_PATH = 0x0200, - MZ_ZIP_FLAG_COMPRESSED_DATA = 0x0400, - MZ_ZIP_FLAG_DO_NOT_SORT_CENTRAL_DIRECTORY = 0x0800 -} mz_zip_flags; - -// ZIP archive reading - -// Inits a ZIP archive reader. -// These functions read and validate the archive's central directory. -mz_bool mz_zip_reader_init(mz_zip_archive *pZip, mz_uint64 size, mz_uint32 flags); -mz_bool mz_zip_reader_init_mem(mz_zip_archive *pZip, const void *pMem, size_t size, mz_uint32 flags); - -#ifndef MINIZ_NO_STDIO -mz_bool mz_zip_reader_init_file(mz_zip_archive *pZip, const char *pFilename, mz_uint32 flags); -#endif - -// Returns the total number of files in the archive. -mz_uint mz_zip_reader_get_num_files(mz_zip_archive *pZip); - -// Returns detailed information about an archive file entry. -mz_bool mz_zip_reader_file_stat(mz_zip_archive *pZip, mz_uint file_index, mz_zip_archive_file_stat *pStat); - -// Determines if an archive file entry is a directory entry. -mz_bool mz_zip_reader_is_file_a_directory(mz_zip_archive *pZip, mz_uint file_index); -mz_bool mz_zip_reader_is_file_encrypted(mz_zip_archive *pZip, mz_uint file_index); - -// Retrieves the filename of an archive file entry. -// Returns the number of bytes written to pFilename, or if filename_buf_size is 0 this function returns the number of bytes needed to fully store the filename. -mz_uint mz_zip_reader_get_filename(mz_zip_archive *pZip, mz_uint file_index, char *pFilename, mz_uint filename_buf_size); - -// Attempts to locates a file in the archive's central directory. -// Valid flags: MZ_ZIP_FLAG_CASE_SENSITIVE, MZ_ZIP_FLAG_IGNORE_PATH -// Returns -1 if the file cannot be found. -int mz_zip_reader_locate_file(mz_zip_archive *pZip, const char *pName, const char *pComment, mz_uint flags); - -// Extracts a archive file to a memory buffer using no memory allocation. -mz_bool mz_zip_reader_extract_to_mem_no_alloc(mz_zip_archive *pZip, mz_uint file_index, void *pBuf, size_t buf_size, mz_uint flags, void *pUser_read_buf, size_t user_read_buf_size); -mz_bool mz_zip_reader_extract_file_to_mem_no_alloc(mz_zip_archive *pZip, const char *pFilename, void *pBuf, size_t buf_size, mz_uint flags, void *pUser_read_buf, size_t user_read_buf_size); - -// Extracts a archive file to a memory buffer. -mz_bool mz_zip_reader_extract_to_mem(mz_zip_archive *pZip, mz_uint file_index, void *pBuf, size_t buf_size, mz_uint flags); -mz_bool mz_zip_reader_extract_file_to_mem(mz_zip_archive *pZip, const char *pFilename, void *pBuf, size_t buf_size, mz_uint flags); - -// Extracts a archive file to a dynamically allocated heap buffer. -void *mz_zip_reader_extract_to_heap(mz_zip_archive *pZip, mz_uint file_index, size_t *pSize, mz_uint flags); -void *mz_zip_reader_extract_file_to_heap(mz_zip_archive *pZip, const char *pFilename, size_t *pSize, mz_uint flags); - -// Extracts a archive file using a callback function to output the file's data. -mz_bool mz_zip_reader_extract_to_callback(mz_zip_archive *pZip, mz_uint file_index, mz_file_write_func pCallback, void *pOpaque, mz_uint flags); -mz_bool mz_zip_reader_extract_file_to_callback(mz_zip_archive *pZip, const char *pFilename, mz_file_write_func pCallback, void *pOpaque, mz_uint flags); - -#ifndef MINIZ_NO_STDIO -// Extracts a archive file to a disk file and sets its last accessed and modified times. -// This function only extracts files, not archive directory records. -mz_bool mz_zip_reader_extract_to_file(mz_zip_archive *pZip, mz_uint file_index, const char *pDst_filename, mz_uint flags); -mz_bool mz_zip_reader_extract_file_to_file(mz_zip_archive *pZip, const char *pArchive_filename, const char *pDst_filename, mz_uint flags); -#endif - -// Ends archive reading, freeing all allocations, and closing the input archive file if mz_zip_reader_init_file() was used. -mz_bool mz_zip_reader_end(mz_zip_archive *pZip); - -// ZIP archive writing - -#ifndef MINIZ_NO_ARCHIVE_WRITING_APIS - -// Inits a ZIP archive writer. -mz_bool mz_zip_writer_init(mz_zip_archive *pZip, mz_uint64 existing_size); -mz_bool mz_zip_writer_init_heap(mz_zip_archive *pZip, size_t size_to_reserve_at_beginning, size_t initial_allocation_size); - -#ifndef MINIZ_NO_STDIO -mz_bool mz_zip_writer_init_file(mz_zip_archive *pZip, const char *pFilename, mz_uint64 size_to_reserve_at_beginning); -#endif - -// Converts a ZIP archive reader object into a writer object, to allow efficient in-place file appends to occur on an existing archive. -// For archives opened using mz_zip_reader_init_file, pFilename must be the archive's filename so it can be reopened for writing. If the file can't be reopened, mz_zip_reader_end() will be called. -// For archives opened using mz_zip_reader_init_mem, the memory block must be growable using the realloc callback (which defaults to realloc unless you've overridden it). -// Finally, for archives opened using mz_zip_reader_init, the mz_zip_archive's user provided m_pWrite function cannot be NULL. -// Note: In-place archive modification is not recommended unless you know what you're doing, because if execution stops or something goes wrong before -// the archive is finalized the file's central directory will be hosed. -mz_bool mz_zip_writer_init_from_reader(mz_zip_archive *pZip, const char *pFilename); - -// Adds the contents of a memory buffer to an archive. These functions record the current local time into the archive. -// To add a directory entry, call this method with an archive name ending in a forwardslash with empty buffer. -// level_and_flags - compression level (0-10, see MZ_BEST_SPEED, MZ_BEST_COMPRESSION, etc.) logically OR'd with zero or more mz_zip_flags, or just set to MZ_DEFAULT_COMPRESSION. -mz_bool mz_zip_writer_add_mem(mz_zip_archive *pZip, const char *pArchive_name, const void *pBuf, size_t buf_size, mz_uint level_and_flags); -mz_bool mz_zip_writer_add_mem_ex(mz_zip_archive *pZip, const char *pArchive_name, const void *pBuf, size_t buf_size, const void *pComment, mz_uint16 comment_size, mz_uint level_and_flags, mz_uint64 uncomp_size, mz_uint32 uncomp_crc32); - -#ifndef MINIZ_NO_STDIO -// Adds the contents of a disk file to an archive. This function also records the disk file's modified time into the archive. -// level_and_flags - compression level (0-10, see MZ_BEST_SPEED, MZ_BEST_COMPRESSION, etc.) logically OR'd with zero or more mz_zip_flags, or just set to MZ_DEFAULT_COMPRESSION. -mz_bool mz_zip_writer_add_file(mz_zip_archive *pZip, const char *pArchive_name, const char *pSrc_filename, const void *pComment, mz_uint16 comment_size, mz_uint level_and_flags); -#endif - -// Adds a file to an archive by fully cloning the data from another archive. -// This function fully clones the source file's compressed data (no recompression), along with its full filename, extra data, and comment fields. -mz_bool mz_zip_writer_add_from_zip_reader(mz_zip_archive *pZip, mz_zip_archive *pSource_zip, mz_uint file_index); - -// Finalizes the archive by writing the central directory records followed by the end of central directory record. -// After an archive is finalized, the only valid call on the mz_zip_archive struct is mz_zip_writer_end(). -// An archive must be manually finalized by calling this function for it to be valid. -mz_bool mz_zip_writer_finalize_archive(mz_zip_archive *pZip); -mz_bool mz_zip_writer_finalize_heap_archive(mz_zip_archive *pZip, void **pBuf, size_t *pSize); - -// Ends archive writing, freeing all allocations, and closing the output file if mz_zip_writer_init_file() was used. -// Note for the archive to be valid, it must have been finalized before ending. -mz_bool mz_zip_writer_end(mz_zip_archive *pZip); - -// Misc. high-level helper functions: - -// mz_zip_add_mem_to_archive_file_in_place() efficiently (but not atomically) appends a memory blob to a ZIP archive. -// level_and_flags - compression level (0-10, see MZ_BEST_SPEED, MZ_BEST_COMPRESSION, etc.) logically OR'd with zero or more mz_zip_flags, or just set to MZ_DEFAULT_COMPRESSION. -mz_bool mz_zip_add_mem_to_archive_file_in_place(const char *pZip_filename, const char *pArchive_name, const void *pBuf, size_t buf_size, const void *pComment, mz_uint16 comment_size, mz_uint level_and_flags); - -// Reads a single file from an archive into a heap block. -// Returns NULL on failure. -void *mz_zip_extract_archive_file_to_heap(const char *pZip_filename, const char *pArchive_name, size_t *pSize, mz_uint zip_flags); - -#endif // #ifndef MINIZ_NO_ARCHIVE_WRITING_APIS - -#endif // #ifndef MINIZ_NO_ARCHIVE_APIS - -// ------------------- Low-level Decompression API Definitions - -// Decompression flags used by tinfl_decompress(). -// TINFL_FLAG_PARSE_ZLIB_HEADER: If set, the input has a valid zlib header and ends with an adler32 checksum (it's a valid zlib stream). Otherwise, the input is a raw deflate stream. -// TINFL_FLAG_HAS_MORE_INPUT: If set, there are more input bytes available beyond the end of the supplied input buffer. If clear, the input buffer contains all remaining input. -// TINFL_FLAG_USING_NON_WRAPPING_OUTPUT_BUF: If set, the output buffer is large enough to hold the entire decompressed stream. If clear, the output buffer is at least the size of the dictionary (typically 32KB). -// TINFL_FLAG_COMPUTE_ADLER32: Force adler-32 checksum computation of the decompressed bytes. -enum -{ - TINFL_FLAG_PARSE_ZLIB_HEADER = 1, - TINFL_FLAG_HAS_MORE_INPUT = 2, - TINFL_FLAG_USING_NON_WRAPPING_OUTPUT_BUF = 4, - TINFL_FLAG_COMPUTE_ADLER32 = 8 -}; - -// High level decompression functions: -// tinfl_decompress_mem_to_heap() decompresses a block in memory to a heap block allocated via malloc(). -// On entry: -// pSrc_buf, src_buf_len: Pointer and size of the Deflate or zlib source data to decompress. -// On return: -// Function returns a pointer to the decompressed data, or NULL on failure. -// *pOut_len will be set to the decompressed data's size, which could be larger than src_buf_len on uncompressible data. -// The caller must free() the returned block when it's no longer needed. -void *tinfl_decompress_mem_to_heap(const void *pSrc_buf, size_t src_buf_len, size_t *pOut_len, int flags); - -// tinfl_decompress_mem_to_mem() decompresses a block in memory to another block in memory. -// Returns TINFL_DECOMPRESS_MEM_TO_MEM_FAILED on failure, or the number of bytes written on success. -#define TINFL_DECOMPRESS_MEM_TO_MEM_FAILED ((size_t)(-1)) -size_t tinfl_decompress_mem_to_mem(void *pOut_buf, size_t out_buf_len, const void *pSrc_buf, size_t src_buf_len, int flags); - -// tinfl_decompress_mem_to_callback() decompresses a block in memory to an internal 32KB buffer, and a user provided callback function will be called to flush the buffer. -// Returns 1 on success or 0 on failure. -typedef int (*tinfl_put_buf_func_ptr)(const void* pBuf, int len, void *pUser); -int tinfl_decompress_mem_to_callback(const void *pIn_buf, size_t *pIn_buf_size, tinfl_put_buf_func_ptr pPut_buf_func, void *pPut_buf_user, int flags); - -struct tinfl_decompressor_tag; typedef struct tinfl_decompressor_tag tinfl_decompressor; - -// Max size of LZ dictionary. -#define TINFL_LZ_DICT_SIZE 32768 - -// Return status. -typedef enum -{ - TINFL_STATUS_BAD_PARAM = -3, - TINFL_STATUS_ADLER32_MISMATCH = -2, - TINFL_STATUS_FAILED = -1, - TINFL_STATUS_DONE = 0, - TINFL_STATUS_NEEDS_MORE_INPUT = 1, - TINFL_STATUS_HAS_MORE_OUTPUT = 2 -} tinfl_status; - -// Initializes the decompressor to its initial state. -#define tinfl_init(r) do { (r)->m_state = 0; } MZ_MACRO_END -#define tinfl_get_adler32(r) (r)->m_check_adler32 - -// Main low-level decompressor coroutine function. This is the only function actually needed for decompression. All the other functions are just high-level helpers for improved usability. -// This is a universal API, i.e. it can be used as a building block to build any desired higher level decompression API. In the limit case, it can be called once per every byte input or output. -tinfl_status tinfl_decompress(tinfl_decompressor *r, const mz_uint8 *pIn_buf_next, size_t *pIn_buf_size, mz_uint8 *pOut_buf_start, mz_uint8 *pOut_buf_next, size_t *pOut_buf_size, const mz_uint32 decomp_flags); - -// Internal/private bits follow. -enum -{ - TINFL_MAX_HUFF_TABLES = 3, TINFL_MAX_HUFF_SYMBOLS_0 = 288, TINFL_MAX_HUFF_SYMBOLS_1 = 32, TINFL_MAX_HUFF_SYMBOLS_2 = 19, - TINFL_FAST_LOOKUP_BITS = 10, TINFL_FAST_LOOKUP_SIZE = 1 << TINFL_FAST_LOOKUP_BITS -}; - -typedef struct -{ - mz_uint8 m_code_size[TINFL_MAX_HUFF_SYMBOLS_0]; - mz_int16 m_look_up[TINFL_FAST_LOOKUP_SIZE], m_tree[TINFL_MAX_HUFF_SYMBOLS_0 * 2]; -} tinfl_huff_table; - -#if MINIZ_HAS_64BIT_REGISTERS - #define TINFL_USE_64BIT_BITBUF 1 -#endif - -#if TINFL_USE_64BIT_BITBUF - typedef mz_uint64 tinfl_bit_buf_t; - #define TINFL_BITBUF_SIZE (64) -#else - typedef mz_uint32 tinfl_bit_buf_t; - #define TINFL_BITBUF_SIZE (32) -#endif - -struct tinfl_decompressor_tag -{ - mz_uint32 m_state, m_num_bits, m_zhdr0, m_zhdr1, m_z_adler32, m_final, m_type, m_check_adler32, m_dist, m_counter, m_num_extra, m_table_sizes[TINFL_MAX_HUFF_TABLES]; - tinfl_bit_buf_t m_bit_buf; - size_t m_dist_from_out_buf_start; - tinfl_huff_table m_tables[TINFL_MAX_HUFF_TABLES]; - mz_uint8 m_raw_header[4], m_len_codes[TINFL_MAX_HUFF_SYMBOLS_0 + TINFL_MAX_HUFF_SYMBOLS_1 + 137]; -}; - -// ------------------- Low-level Compression API Definitions - -// Set TDEFL_LESS_MEMORY to 1 to use less memory (compression will be slightly slower, and raw/dynamic blocks will be output more frequently). -#define TDEFL_LESS_MEMORY 0 - -// tdefl_init() compression flags logically OR'd together (low 12 bits contain the max. number of probes per dictionary search): -// TDEFL_DEFAULT_MAX_PROBES: The compressor defaults to 128 dictionary probes per dictionary search. 0=Huffman only, 1=Huffman+LZ (fastest/crap compression), 4095=Huffman+LZ (slowest/best compression). -enum -{ - TDEFL_HUFFMAN_ONLY = 0, TDEFL_DEFAULT_MAX_PROBES = 128, TDEFL_MAX_PROBES_MASK = 0xFFF -}; - -// TDEFL_WRITE_ZLIB_HEADER: If set, the compressor outputs a zlib header before the deflate data, and the Adler-32 of the source data at the end. Otherwise, you'll get raw deflate data. -// TDEFL_COMPUTE_ADLER32: Always compute the adler-32 of the input data (even when not writing zlib headers). -// TDEFL_GREEDY_PARSING_FLAG: Set to use faster greedy parsing, instead of more efficient lazy parsing. -// TDEFL_NONDETERMINISTIC_PARSING_FLAG: Enable to decrease the compressor's initialization time to the minimum, but the output may vary from run to run given the same input (depending on the contents of memory). -// TDEFL_RLE_MATCHES: Only look for RLE matches (matches with a distance of 1) -// TDEFL_FILTER_MATCHES: Discards matches <= 5 chars if enabled. -// TDEFL_FORCE_ALL_STATIC_BLOCKS: Disable usage of optimized Huffman tables. -// TDEFL_FORCE_ALL_RAW_BLOCKS: Only use raw (uncompressed) deflate blocks. -enum -{ - TDEFL_WRITE_ZLIB_HEADER = 0x01000, - TDEFL_COMPUTE_ADLER32 = 0x02000, - TDEFL_GREEDY_PARSING_FLAG = 0x04000, - TDEFL_NONDETERMINISTIC_PARSING_FLAG = 0x08000, - TDEFL_RLE_MATCHES = 0x10000, - TDEFL_FILTER_MATCHES = 0x20000, - TDEFL_FORCE_ALL_STATIC_BLOCKS = 0x40000, - TDEFL_FORCE_ALL_RAW_BLOCKS = 0x80000 -}; - -// High level compression functions: -// tdefl_compress_mem_to_heap() compresses a block in memory to a heap block allocated via malloc(). -// On entry: -// pSrc_buf, src_buf_len: Pointer and size of source block to compress. -// flags: The max match finder probes (default is 128) logically OR'd against the above flags. Higher probes are slower but improve compression. -// On return: -// Function returns a pointer to the compressed data, or NULL on failure. -// *pOut_len will be set to the compressed data's size, which could be larger than src_buf_len on uncompressible data. -// The caller must free() the returned block when it's no longer needed. -void *tdefl_compress_mem_to_heap(const void *pSrc_buf, size_t src_buf_len, size_t *pOut_len, int flags); - -// tdefl_compress_mem_to_mem() compresses a block in memory to another block in memory. -// Returns 0 on failure. -size_t tdefl_compress_mem_to_mem(void *pOut_buf, size_t out_buf_len, const void *pSrc_buf, size_t src_buf_len, int flags); - -// Compresses an image to a compressed PNG file in memory. -// On entry: -// pImage, w, h, and num_chans describe the image to compress. num_chans may be 1, 2, 3, or 4. -// On return: -// Function returns a pointer to the compressed data, or NULL on failure. -// *pLen_out will be set to the size of the PNG image file. -// The caller must free() the returned heap block (which will typically be larger than *pLen_out) when it's no longer needed. -void *tdefl_write_image_to_png_file_in_memory(const void *pImage, int w, int h, int num_chans, size_t *pLen_out); - -// Output stream interface. The compressor uses this interface to write compressed data. It'll typically be called TDEFL_OUT_BUF_SIZE at a time. -typedef mz_bool (*tdefl_put_buf_func_ptr)(const void* pBuf, int len, void *pUser); - -// tdefl_compress_mem_to_output() compresses a block to an output stream. The above helpers use this function internally. -mz_bool tdefl_compress_mem_to_output(const void *pBuf, size_t buf_len, tdefl_put_buf_func_ptr pPut_buf_func, void *pPut_buf_user, int flags); - -enum { TDEFL_MAX_HUFF_TABLES = 3, TDEFL_MAX_HUFF_SYMBOLS_0 = 288, TDEFL_MAX_HUFF_SYMBOLS_1 = 32, TDEFL_MAX_HUFF_SYMBOLS_2 = 19, TDEFL_LZ_DICT_SIZE = 32768, TDEFL_LZ_DICT_SIZE_MASK = TDEFL_LZ_DICT_SIZE - 1, TDEFL_MIN_MATCH_LEN = 3, TDEFL_MAX_MATCH_LEN = 258 }; - -// TDEFL_OUT_BUF_SIZE MUST be large enough to hold a single entire compressed output block (using static/fixed Huffman codes). -#if TDEFL_LESS_MEMORY -enum { TDEFL_LZ_CODE_BUF_SIZE = 24 * 1024, TDEFL_OUT_BUF_SIZE = (TDEFL_LZ_CODE_BUF_SIZE * 13 ) / 10, TDEFL_MAX_HUFF_SYMBOLS = 288, TDEFL_LZ_HASH_BITS = 12, TDEFL_LEVEL1_HASH_SIZE_MASK = 4095, TDEFL_LZ_HASH_SHIFT = (TDEFL_LZ_HASH_BITS + 2) / 3, TDEFL_LZ_HASH_SIZE = 1 << TDEFL_LZ_HASH_BITS }; -#else -enum { TDEFL_LZ_CODE_BUF_SIZE = 64 * 1024, TDEFL_OUT_BUF_SIZE = (TDEFL_LZ_CODE_BUF_SIZE * 13 ) / 10, TDEFL_MAX_HUFF_SYMBOLS = 288, TDEFL_LZ_HASH_BITS = 15, TDEFL_LEVEL1_HASH_SIZE_MASK = 4095, TDEFL_LZ_HASH_SHIFT = (TDEFL_LZ_HASH_BITS + 2) / 3, TDEFL_LZ_HASH_SIZE = 1 << TDEFL_LZ_HASH_BITS }; -#endif - -// The low-level tdefl functions below may be used directly if the above helper functions aren't flexible enough. The low-level functions don't make any heap allocations, unlike the above helper functions. -typedef enum -{ - TDEFL_STATUS_BAD_PARAM = -2, - TDEFL_STATUS_PUT_BUF_FAILED = -1, - TDEFL_STATUS_OKAY = 0, - TDEFL_STATUS_DONE = 1, -} tdefl_status; - -// Must map to MZ_NO_FLUSH, MZ_SYNC_FLUSH, etc. enums -typedef enum -{ - TDEFL_NO_FLUSH = 0, - TDEFL_SYNC_FLUSH = 2, - TDEFL_FULL_FLUSH = 3, - TDEFL_FINISH = 4 -} tdefl_flush; - -// tdefl's compression state structure. -typedef struct -{ - tdefl_put_buf_func_ptr m_pPut_buf_func; - void *m_pPut_buf_user; - mz_uint m_flags, m_max_probes[2]; - int m_greedy_parsing; - mz_uint m_adler32, m_lookahead_pos, m_lookahead_size, m_dict_size; - mz_uint8 *m_pLZ_code_buf, *m_pLZ_flags, *m_pOutput_buf, *m_pOutput_buf_end; - mz_uint m_num_flags_left, m_total_lz_bytes, m_lz_code_buf_dict_pos, m_bits_in, m_bit_buffer; - mz_uint m_saved_match_dist, m_saved_match_len, m_saved_lit, m_output_flush_ofs, m_output_flush_remaining, m_finished, m_block_index, m_wants_to_finish; - tdefl_status m_prev_return_status; - const void *m_pIn_buf; - void *m_pOut_buf; - size_t *m_pIn_buf_size, *m_pOut_buf_size; - tdefl_flush m_flush; - const mz_uint8 *m_pSrc; - size_t m_src_buf_left, m_out_buf_ofs; - mz_uint8 m_dict[TDEFL_LZ_DICT_SIZE + TDEFL_MAX_MATCH_LEN - 1]; - mz_uint16 m_huff_count[TDEFL_MAX_HUFF_TABLES][TDEFL_MAX_HUFF_SYMBOLS]; - mz_uint16 m_huff_codes[TDEFL_MAX_HUFF_TABLES][TDEFL_MAX_HUFF_SYMBOLS]; - mz_uint8 m_huff_code_sizes[TDEFL_MAX_HUFF_TABLES][TDEFL_MAX_HUFF_SYMBOLS]; - mz_uint8 m_lz_code_buf[TDEFL_LZ_CODE_BUF_SIZE]; - mz_uint16 m_next[TDEFL_LZ_DICT_SIZE]; - mz_uint16 m_hash[TDEFL_LZ_HASH_SIZE]; - mz_uint8 m_output_buf[TDEFL_OUT_BUF_SIZE]; -} tdefl_compressor; - -// Initializes the compressor. -// There is no corresponding deinit() function because the tdefl API's do not dynamically allocate memory. -// pBut_buf_func: If NULL, output data will be supplied to the specified callback. In this case, the user should call the tdefl_compress_buffer() API for compression. -// If pBut_buf_func is NULL the user should always call the tdefl_compress() API. -// flags: See the above enums (TDEFL_HUFFMAN_ONLY, TDEFL_WRITE_ZLIB_HEADER, etc.) -tdefl_status tdefl_init(tdefl_compressor *d, tdefl_put_buf_func_ptr pPut_buf_func, void *pPut_buf_user, int flags); - -// Compresses a block of data, consuming as much of the specified input buffer as possible, and writing as much compressed data to the specified output buffer as possible. -tdefl_status tdefl_compress(tdefl_compressor *d, const void *pIn_buf, size_t *pIn_buf_size, void *pOut_buf, size_t *pOut_buf_size, tdefl_flush flush); - -// tdefl_compress_buffer() is only usable when the tdefl_init() is called with a non-NULL tdefl_put_buf_func_ptr. -// tdefl_compress_buffer() always consumes the entire input buffer. -tdefl_status tdefl_compress_buffer(tdefl_compressor *d, const void *pIn_buf, size_t in_buf_size, tdefl_flush flush); - -tdefl_status tdefl_get_prev_return_status(tdefl_compressor *d); -mz_uint32 tdefl_get_adler32(tdefl_compressor *d); - -// Can't use tdefl_create_comp_flags_from_zip_params if MINIZ_NO_ZLIB_APIS isn't defined, because it uses some of its macros. -#ifndef MINIZ_NO_ZLIB_APIS -// Create tdefl_compress() flags given zlib-style compression parameters. -// level may range from [0,10] (where 10 is absolute max compression, but may be much slower on some files) -// window_bits may be -15 (raw deflate) or 15 (zlib) -// strategy may be either MZ_DEFAULT_STRATEGY, MZ_FILTERED, MZ_HUFFMAN_ONLY, MZ_RLE, or MZ_FIXED -mz_uint tdefl_create_comp_flags_from_zip_params(int level, int window_bits, int strategy); -#endif // #ifndef MINIZ_NO_ZLIB_APIS - -#ifdef __cplusplus -} -#endif - -#endif // MINIZ_HEADER_INCLUDED - -// ------------------- End of Header: Implementation follows. (If you only want the header, define MINIZ_HEADER_FILE_ONLY.) - -#ifndef MINIZ_HEADER_FILE_ONLY - -typedef unsigned char mz_validate_uint16[sizeof(mz_uint16)==2 ? 1 : -1]; -typedef unsigned char mz_validate_uint32[sizeof(mz_uint32)==4 ? 1 : -1]; -typedef unsigned char mz_validate_uint64[sizeof(mz_uint64)==8 ? 1 : -1]; - -#include -#include - -#define MZ_ASSERT(x) assert(x) - -#ifdef MINIZ_NO_MALLOC - #define MZ_MALLOC(x) NULL - #define MZ_FREE(x) (void)x, ((void)0) - #define MZ_REALLOC(p, x) NULL -#else - #define MZ_MALLOC(x) malloc(x) - #define MZ_FREE(x) free(x) - #define MZ_REALLOC(p, x) realloc(p, x) -#endif - -#define MZ_MAX(a,b) (((a)>(b))?(a):(b)) -#define MZ_MIN(a,b) (((a)<(b))?(a):(b)) -#define MZ_CLEAR_OBJ(obj) memset(&(obj), 0, sizeof(obj)) - -#if MINIZ_USE_UNALIGNED_LOADS_AND_STORES && MINIZ_LITTLE_ENDIAN - #define MZ_READ_LE16(p) *((const mz_uint16 *)(p)) - #define MZ_READ_LE32(p) *((const mz_uint32 *)(p)) -#else - #define MZ_READ_LE16(p) ((mz_uint32)(((const mz_uint8 *)(p))[0]) | ((mz_uint32)(((const mz_uint8 *)(p))[1]) << 8U)) - #define MZ_READ_LE32(p) ((mz_uint32)(((const mz_uint8 *)(p))[0]) | ((mz_uint32)(((const mz_uint8 *)(p))[1]) << 8U) | ((mz_uint32)(((const mz_uint8 *)(p))[2]) << 16U) | ((mz_uint32)(((const mz_uint8 *)(p))[3]) << 24U)) -#endif - -#ifdef _MSC_VER - #define MZ_FORCEINLINE __forceinline -#elif defined(__GNUC__) - #define MZ_FORCEINLINE __attribute__((__always_inline__)) -#else - #define MZ_FORCEINLINE inline -#endif - -#ifdef __cplusplus - extern "C" { -#endif - -// ------------------- zlib-style API's - -mz_ulong mz_adler32(mz_ulong adler, const unsigned char *ptr, size_t buf_len) -{ - mz_uint32 i, s1 = (mz_uint32)(adler & 0xffff), s2 = (mz_uint32)(adler >> 16); size_t block_len = buf_len % 5552; - if (!ptr) return MZ_ADLER32_INIT; - while (buf_len) { - for (i = 0; i + 7 < block_len; i += 8, ptr += 8) { - s1 += ptr[0], s2 += s1; s1 += ptr[1], s2 += s1; s1 += ptr[2], s2 += s1; s1 += ptr[3], s2 += s1; - s1 += ptr[4], s2 += s1; s1 += ptr[5], s2 += s1; s1 += ptr[6], s2 += s1; s1 += ptr[7], s2 += s1; - } - for ( ; i < block_len; ++i) s1 += *ptr++, s2 += s1; - s1 %= 65521U, s2 %= 65521U; buf_len -= block_len; block_len = 5552; - } - return (s2 << 16) + s1; -} - -// Karl Malbrain's compact CRC-32. See "A compact CCITT crc16 and crc32 C implementation that balances processor cache usage against speed": http://www.geocities.com/malbrain/ -mz_ulong mz_crc32(mz_ulong crc, const mz_uint8 *ptr, size_t buf_len) -{ - static const mz_uint32 s_crc32[16] = { 0, 0x1db71064, 0x3b6e20c8, 0x26d930ac, 0x76dc4190, 0x6b6b51f4, 0x4db26158, 0x5005713c, - 0xedb88320, 0xf00f9344, 0xd6d6a3e8, 0xcb61b38c, 0x9b64c2b0, 0x86d3d2d4, 0xa00ae278, 0xbdbdf21c }; - mz_uint32 crcu32 = (mz_uint32)crc; - if (!ptr) return MZ_CRC32_INIT; - crcu32 = ~crcu32; while (buf_len--) { mz_uint8 b = *ptr++; crcu32 = (crcu32 >> 4) ^ s_crc32[(crcu32 & 0xF) ^ (b & 0xF)]; crcu32 = (crcu32 >> 4) ^ s_crc32[(crcu32 & 0xF) ^ (b >> 4)]; } - return ~crcu32; -} - -#ifndef MINIZ_NO_ZLIB_APIS - -static void *def_alloc_func(void *opaque, size_t items, size_t size) { (void)opaque, (void)items, (void)size; return MZ_MALLOC(items * size); } -static void def_free_func(void *opaque, void *address) { (void)opaque, (void)address; MZ_FREE(address); } -static void *def_realloc_func(void *opaque, void *address, size_t items, size_t size) { (void)opaque, (void)address, (void)items, (void)size; return MZ_REALLOC(address, items * size); } - -const char *mz_version(void) -{ - return MZ_VERSION; -} - -int mz_deflateInit(mz_streamp pStream, int level) -{ - return mz_deflateInit2(pStream, level, MZ_DEFLATED, MZ_DEFAULT_WINDOW_BITS, 9, MZ_DEFAULT_STRATEGY); -} - -int mz_deflateInit2(mz_streamp pStream, int level, int method, int window_bits, int mem_level, int strategy) -{ - tdefl_compressor *pComp; - mz_uint comp_flags = TDEFL_COMPUTE_ADLER32 | tdefl_create_comp_flags_from_zip_params(level, window_bits, strategy); - - if (!pStream) return MZ_STREAM_ERROR; - if ((method != MZ_DEFLATED) || ((mem_level < 1) || (mem_level > 9)) || ((window_bits != MZ_DEFAULT_WINDOW_BITS) && (-window_bits != MZ_DEFAULT_WINDOW_BITS))) return MZ_PARAM_ERROR; - - pStream->data_type = 0; - pStream->adler = MZ_ADLER32_INIT; - pStream->msg = NULL; - pStream->reserved = 0; - pStream->total_in = 0; - pStream->total_out = 0; - if (!pStream->zalloc) pStream->zalloc = def_alloc_func; - if (!pStream->zfree) pStream->zfree = def_free_func; - - pComp = (tdefl_compressor *)pStream->zalloc(pStream->opaque, 1, sizeof(tdefl_compressor)); - if (!pComp) - return MZ_MEM_ERROR; - - pStream->state = (struct mz_internal_state *)pComp; - - if (tdefl_init(pComp, NULL, NULL, comp_flags) != TDEFL_STATUS_OKAY) - { - mz_deflateEnd(pStream); - return MZ_PARAM_ERROR; - } - - return MZ_OK; -} - -int mz_deflateReset(mz_streamp pStream) -{ - if ((!pStream) || (!pStream->state) || (!pStream->zalloc) || (!pStream->zfree)) return MZ_STREAM_ERROR; - pStream->total_in = pStream->total_out = 0; - tdefl_init((tdefl_compressor*)pStream->state, NULL, NULL, ((tdefl_compressor*)pStream->state)->m_flags); - return MZ_OK; -} - -int mz_deflate(mz_streamp pStream, int flush) -{ - size_t in_bytes, out_bytes; - mz_ulong orig_total_in, orig_total_out; - int mz_status = MZ_OK; - - if ((!pStream) || (!pStream->state) || (flush < 0) || (flush > MZ_FINISH) || (!pStream->next_out)) return MZ_STREAM_ERROR; - if (!pStream->avail_out) return MZ_BUF_ERROR; - - if (flush == MZ_PARTIAL_FLUSH) flush = MZ_SYNC_FLUSH; - - if (((tdefl_compressor*)pStream->state)->m_prev_return_status == TDEFL_STATUS_DONE) - return (flush == MZ_FINISH) ? MZ_STREAM_END : MZ_BUF_ERROR; - - orig_total_in = pStream->total_in; orig_total_out = pStream->total_out; - for ( ; ; ) - { - tdefl_status defl_status; - in_bytes = pStream->avail_in; out_bytes = pStream->avail_out; - - defl_status = tdefl_compress((tdefl_compressor*)pStream->state, pStream->next_in, &in_bytes, pStream->next_out, &out_bytes, (tdefl_flush)flush); - pStream->next_in += (mz_uint)in_bytes; pStream->avail_in -= (mz_uint)in_bytes; - pStream->total_in += (mz_uint)in_bytes; pStream->adler = tdefl_get_adler32((tdefl_compressor*)pStream->state); - - pStream->next_out += (mz_uint)out_bytes; pStream->avail_out -= (mz_uint)out_bytes; - pStream->total_out += (mz_uint)out_bytes; - - if (defl_status < 0) - { - mz_status = MZ_STREAM_ERROR; - break; - } - else if (defl_status == TDEFL_STATUS_DONE) - { - mz_status = MZ_STREAM_END; - break; - } - else if (!pStream->avail_out) - break; - else if ((!pStream->avail_in) && (flush != MZ_FINISH)) - { - if ((flush) || (pStream->total_in != orig_total_in) || (pStream->total_out != orig_total_out)) - break; - return MZ_BUF_ERROR; // Can't make forward progress without some input. - } - } - return mz_status; -} - -int mz_deflateEnd(mz_streamp pStream) -{ - if (!pStream) return MZ_STREAM_ERROR; - if (pStream->state) - { - pStream->zfree(pStream->opaque, pStream->state); - pStream->state = NULL; - } - return MZ_OK; -} - -mz_ulong mz_deflateBound(mz_streamp pStream, mz_ulong source_len) -{ - (void)pStream; - // This is really over conservative. (And lame, but it's actually pretty tricky to compute a true upper bound given the way tdefl's blocking works.) - return MZ_MAX(128 + (source_len * 110) / 100, 128 + source_len + ((source_len / (31 * 1024)) + 1) * 5); -} - -int mz_compress2(unsigned char *pDest, mz_ulong *pDest_len, const unsigned char *pSource, mz_ulong source_len, int level) -{ - int status; - mz_stream stream; - memset(&stream, 0, sizeof(stream)); - - // In case mz_ulong is 64-bits (argh I hate longs). - if ((source_len | *pDest_len) > 0xFFFFFFFFU) return MZ_PARAM_ERROR; - - stream.next_in = pSource; - stream.avail_in = (mz_uint32)source_len; - stream.next_out = pDest; - stream.avail_out = (mz_uint32)*pDest_len; - - status = mz_deflateInit(&stream, level); - if (status != MZ_OK) return status; - - status = mz_deflate(&stream, MZ_FINISH); - if (status != MZ_STREAM_END) - { - mz_deflateEnd(&stream); - return (status == MZ_OK) ? MZ_BUF_ERROR : status; - } - - *pDest_len = stream.total_out; - return mz_deflateEnd(&stream); -} - -int mz_compress(unsigned char *pDest, mz_ulong *pDest_len, const unsigned char *pSource, mz_ulong source_len) -{ - return mz_compress2(pDest, pDest_len, pSource, source_len, MZ_DEFAULT_COMPRESSION); -} - -mz_ulong mz_compressBound(mz_ulong source_len) -{ - return mz_deflateBound(NULL, source_len); -} - -typedef struct -{ - tinfl_decompressor m_decomp; - mz_uint m_dict_ofs, m_dict_avail, m_first_call, m_has_flushed; int m_window_bits; - mz_uint8 m_dict[TINFL_LZ_DICT_SIZE]; - tinfl_status m_last_status; -} inflate_state; - -int mz_inflateInit2(mz_streamp pStream, int window_bits) -{ - inflate_state *pDecomp; - if (!pStream) return MZ_STREAM_ERROR; - if ((window_bits != MZ_DEFAULT_WINDOW_BITS) && (-window_bits != MZ_DEFAULT_WINDOW_BITS)) return MZ_PARAM_ERROR; - - pStream->data_type = 0; - pStream->adler = 0; - pStream->msg = NULL; - pStream->total_in = 0; - pStream->total_out = 0; - pStream->reserved = 0; - if (!pStream->zalloc) pStream->zalloc = def_alloc_func; - if (!pStream->zfree) pStream->zfree = def_free_func; - - pDecomp = (inflate_state*)pStream->zalloc(pStream->opaque, 1, sizeof(inflate_state)); - if (!pDecomp) return MZ_MEM_ERROR; - - pStream->state = (struct mz_internal_state *)pDecomp; - - tinfl_init(&pDecomp->m_decomp); - pDecomp->m_dict_ofs = 0; - pDecomp->m_dict_avail = 0; - pDecomp->m_last_status = TINFL_STATUS_NEEDS_MORE_INPUT; - pDecomp->m_first_call = 1; - pDecomp->m_has_flushed = 0; - pDecomp->m_window_bits = window_bits; - - return MZ_OK; -} - -int mz_inflateInit(mz_streamp pStream) -{ - return mz_inflateInit2(pStream, MZ_DEFAULT_WINDOW_BITS); -} - -int mz_inflate(mz_streamp pStream, int flush) -{ - inflate_state* pState; - mz_uint n, first_call, decomp_flags = TINFL_FLAG_COMPUTE_ADLER32; - size_t in_bytes, out_bytes, orig_avail_in; - tinfl_status status; - - if ((!pStream) || (!pStream->state)) return MZ_STREAM_ERROR; - if (flush == MZ_PARTIAL_FLUSH) flush = MZ_SYNC_FLUSH; - if ((flush) && (flush != MZ_SYNC_FLUSH) && (flush != MZ_FINISH)) return MZ_STREAM_ERROR; - - pState = (inflate_state*)pStream->state; - if (pState->m_window_bits > 0) decomp_flags |= TINFL_FLAG_PARSE_ZLIB_HEADER; - orig_avail_in = pStream->avail_in; - - first_call = pState->m_first_call; pState->m_first_call = 0; - if (pState->m_last_status < 0) return MZ_DATA_ERROR; - - if (pState->m_has_flushed && (flush != MZ_FINISH)) return MZ_STREAM_ERROR; - pState->m_has_flushed |= (flush == MZ_FINISH); - - if ((flush == MZ_FINISH) && (first_call)) - { - // MZ_FINISH on the first call implies that the input and output buffers are large enough to hold the entire compressed/decompressed file. - decomp_flags |= TINFL_FLAG_USING_NON_WRAPPING_OUTPUT_BUF; - in_bytes = pStream->avail_in; out_bytes = pStream->avail_out; - status = tinfl_decompress(&pState->m_decomp, pStream->next_in, &in_bytes, pStream->next_out, pStream->next_out, &out_bytes, decomp_flags); - pState->m_last_status = status; - pStream->next_in += (mz_uint)in_bytes; pStream->avail_in -= (mz_uint)in_bytes; pStream->total_in += (mz_uint)in_bytes; - pStream->adler = tinfl_get_adler32(&pState->m_decomp); - pStream->next_out += (mz_uint)out_bytes; pStream->avail_out -= (mz_uint)out_bytes; pStream->total_out += (mz_uint)out_bytes; - - if (status < 0) - return MZ_DATA_ERROR; - else if (status != TINFL_STATUS_DONE) - { - pState->m_last_status = TINFL_STATUS_FAILED; - return MZ_BUF_ERROR; - } - return MZ_STREAM_END; - } - // flush != MZ_FINISH then we must assume there's more input. - if (flush != MZ_FINISH) decomp_flags |= TINFL_FLAG_HAS_MORE_INPUT; - - if (pState->m_dict_avail) - { - n = MZ_MIN(pState->m_dict_avail, pStream->avail_out); - memcpy(pStream->next_out, pState->m_dict + pState->m_dict_ofs, n); - pStream->next_out += n; pStream->avail_out -= n; pStream->total_out += n; - pState->m_dict_avail -= n; pState->m_dict_ofs = (pState->m_dict_ofs + n) & (TINFL_LZ_DICT_SIZE - 1); - return ((pState->m_last_status == TINFL_STATUS_DONE) && (!pState->m_dict_avail)) ? MZ_STREAM_END : MZ_OK; - } - - for ( ; ; ) - { - in_bytes = pStream->avail_in; - out_bytes = TINFL_LZ_DICT_SIZE - pState->m_dict_ofs; - - status = tinfl_decompress(&pState->m_decomp, pStream->next_in, &in_bytes, pState->m_dict, pState->m_dict + pState->m_dict_ofs, &out_bytes, decomp_flags); - pState->m_last_status = status; - - pStream->next_in += (mz_uint)in_bytes; pStream->avail_in -= (mz_uint)in_bytes; - pStream->total_in += (mz_uint)in_bytes; pStream->adler = tinfl_get_adler32(&pState->m_decomp); - - pState->m_dict_avail = (mz_uint)out_bytes; - - n = MZ_MIN(pState->m_dict_avail, pStream->avail_out); - memcpy(pStream->next_out, pState->m_dict + pState->m_dict_ofs, n); - pStream->next_out += n; pStream->avail_out -= n; pStream->total_out += n; - pState->m_dict_avail -= n; pState->m_dict_ofs = (pState->m_dict_ofs + n) & (TINFL_LZ_DICT_SIZE - 1); - - if (status < 0) - return MZ_DATA_ERROR; // Stream is corrupted (there could be some uncompressed data left in the output dictionary - oh well). - else if ((status == TINFL_STATUS_NEEDS_MORE_INPUT) && (!orig_avail_in)) - return MZ_BUF_ERROR; // Signal caller that we can't make forward progress without supplying more input or by setting flush to MZ_FINISH. - else if (flush == MZ_FINISH) - { - // The output buffer MUST be large to hold the remaining uncompressed data when flush==MZ_FINISH. - if (status == TINFL_STATUS_DONE) - return pState->m_dict_avail ? MZ_BUF_ERROR : MZ_STREAM_END; - // status here must be TINFL_STATUS_HAS_MORE_OUTPUT, which means there's at least 1 more byte on the way. If there's no more room left in the output buffer then something is wrong. - else if (!pStream->avail_out) - return MZ_BUF_ERROR; - } - else if ((status == TINFL_STATUS_DONE) || (!pStream->avail_in) || (!pStream->avail_out) || (pState->m_dict_avail)) - break; - } - - return ((status == TINFL_STATUS_DONE) && (!pState->m_dict_avail)) ? MZ_STREAM_END : MZ_OK; -} - -int mz_inflateEnd(mz_streamp pStream) -{ - if (!pStream) - return MZ_STREAM_ERROR; - if (pStream->state) - { - pStream->zfree(pStream->opaque, pStream->state); - pStream->state = NULL; - } - return MZ_OK; -} - -int mz_uncompress(unsigned char *pDest, mz_ulong *pDest_len, const unsigned char *pSource, mz_ulong source_len) -{ - mz_stream stream; - int status; - memset(&stream, 0, sizeof(stream)); - - // In case mz_ulong is 64-bits (argh I hate longs). - if ((source_len | *pDest_len) > 0xFFFFFFFFU) return MZ_PARAM_ERROR; - - stream.next_in = pSource; - stream.avail_in = (mz_uint32)source_len; - stream.next_out = pDest; - stream.avail_out = (mz_uint32)*pDest_len; - - status = mz_inflateInit(&stream); - if (status != MZ_OK) - return status; - - status = mz_inflate(&stream, MZ_FINISH); - if (status != MZ_STREAM_END) - { - mz_inflateEnd(&stream); - return ((status == MZ_BUF_ERROR) && (!stream.avail_in)) ? MZ_DATA_ERROR : status; - } - *pDest_len = stream.total_out; - - return mz_inflateEnd(&stream); -} - -const char *mz_error(int err) -{ - static struct { int m_err; const char *m_pDesc; } s_error_descs[] = - { - { MZ_OK, "" }, { MZ_STREAM_END, "stream end" }, { MZ_NEED_DICT, "need dictionary" }, { MZ_ERRNO, "file error" }, { MZ_STREAM_ERROR, "stream error" }, - { MZ_DATA_ERROR, "data error" }, { MZ_MEM_ERROR, "out of memory" }, { MZ_BUF_ERROR, "buf error" }, { MZ_VERSION_ERROR, "version error" }, { MZ_PARAM_ERROR, "parameter error" } - }; - mz_uint i; for (i = 0; i < sizeof(s_error_descs) / sizeof(s_error_descs[0]); ++i) if (s_error_descs[i].m_err == err) return s_error_descs[i].m_pDesc; - return NULL; -} - -#endif //MINIZ_NO_ZLIB_APIS - -// ------------------- Low-level Decompression (completely independent from all compression API's) - -#define TINFL_MEMCPY(d, s, l) memcpy(d, s, l) -#define TINFL_MEMSET(p, c, l) memset(p, c, l) - -#define TINFL_CR_BEGIN switch(r->m_state) { case 0: -#define TINFL_CR_RETURN(state_index, result) do { status = result; r->m_state = state_index; goto common_exit; case state_index:; } MZ_MACRO_END -#define TINFL_CR_RETURN_FOREVER(state_index, result) do { for ( ; ; ) { TINFL_CR_RETURN(state_index, result); } } MZ_MACRO_END -#define TINFL_CR_FINISH } - -// TODO: If the caller has indicated that there's no more input, and we attempt to read beyond the input buf, then something is wrong with the input because the inflator never -// reads ahead more than it needs to. Currently TINFL_GET_BYTE() pads the end of the stream with 0's in this scenario. -#define TINFL_GET_BYTE(state_index, c) do { \ - if (pIn_buf_cur >= pIn_buf_end) { \ - for ( ; ; ) { \ - if (decomp_flags & TINFL_FLAG_HAS_MORE_INPUT) { \ - TINFL_CR_RETURN(state_index, TINFL_STATUS_NEEDS_MORE_INPUT); \ - if (pIn_buf_cur < pIn_buf_end) { \ - c = *pIn_buf_cur++; \ - break; \ - } \ - } else { \ - c = 0; \ - break; \ - } \ - } \ - } else c = *pIn_buf_cur++; } MZ_MACRO_END - -#define TINFL_NEED_BITS(state_index, n) do { mz_uint c; TINFL_GET_BYTE(state_index, c); bit_buf |= (((tinfl_bit_buf_t)c) << num_bits); num_bits += 8; } while (num_bits < (mz_uint)(n)) -#define TINFL_SKIP_BITS(state_index, n) do { if (num_bits < (mz_uint)(n)) { TINFL_NEED_BITS(state_index, n); } bit_buf >>= (n); num_bits -= (n); } MZ_MACRO_END -#define TINFL_GET_BITS(state_index, b, n) do { if (num_bits < (mz_uint)(n)) { TINFL_NEED_BITS(state_index, n); } b = bit_buf & ((1 << (n)) - 1); bit_buf >>= (n); num_bits -= (n); } MZ_MACRO_END - -// TINFL_HUFF_BITBUF_FILL() is only used rarely, when the number of bytes remaining in the input buffer falls below 2. -// It reads just enough bytes from the input stream that are needed to decode the next Huffman code (and absolutely no more). It works by trying to fully decode a -// Huffman code by using whatever bits are currently present in the bit buffer. If this fails, it reads another byte, and tries again until it succeeds or until the -// bit buffer contains >=15 bits (deflate's max. Huffman code size). -#define TINFL_HUFF_BITBUF_FILL(state_index, pHuff) \ - do { \ - temp = (pHuff)->m_look_up[bit_buf & (TINFL_FAST_LOOKUP_SIZE - 1)]; \ - if (temp >= 0) { \ - code_len = temp >> 9; \ - if ((code_len) && (num_bits >= code_len)) \ - break; \ - } else if (num_bits > TINFL_FAST_LOOKUP_BITS) { \ - code_len = TINFL_FAST_LOOKUP_BITS; \ - do { \ - temp = (pHuff)->m_tree[~temp + ((bit_buf >> code_len++) & 1)]; \ - } while ((temp < 0) && (num_bits >= (code_len + 1))); if (temp >= 0) break; \ - } TINFL_GET_BYTE(state_index, c); bit_buf |= (((tinfl_bit_buf_t)c) << num_bits); num_bits += 8; \ - } while (num_bits < 15); - -// TINFL_HUFF_DECODE() decodes the next Huffman coded symbol. It's more complex than you would initially expect because the zlib API expects the decompressor to never read -// beyond the final byte of the deflate stream. (In other words, when this macro wants to read another byte from the input, it REALLY needs another byte in order to fully -// decode the next Huffman code.) Handling this properly is particularly important on raw deflate (non-zlib) streams, which aren't followed by a byte aligned adler-32. -// The slow path is only executed at the very end of the input buffer. -#define TINFL_HUFF_DECODE(state_index, sym, pHuff) do { \ - int temp; mz_uint code_len, c; \ - if (num_bits < 15) { \ - if ((pIn_buf_end - pIn_buf_cur) < 2) { \ - TINFL_HUFF_BITBUF_FILL(state_index, pHuff); \ - } else { \ - bit_buf |= (((tinfl_bit_buf_t)pIn_buf_cur[0]) << num_bits) | (((tinfl_bit_buf_t)pIn_buf_cur[1]) << (num_bits + 8)); pIn_buf_cur += 2; num_bits += 16; \ - } \ - } \ - if ((temp = (pHuff)->m_look_up[bit_buf & (TINFL_FAST_LOOKUP_SIZE - 1)]) >= 0) \ - code_len = temp >> 9, temp &= 511; \ - else { \ - code_len = TINFL_FAST_LOOKUP_BITS; do { temp = (pHuff)->m_tree[~temp + ((bit_buf >> code_len++) & 1)]; } while (temp < 0); \ - } sym = temp; bit_buf >>= code_len; num_bits -= code_len; } MZ_MACRO_END - -tinfl_status tinfl_decompress(tinfl_decompressor *r, const mz_uint8 *pIn_buf_next, size_t *pIn_buf_size, mz_uint8 *pOut_buf_start, mz_uint8 *pOut_buf_next, size_t *pOut_buf_size, const mz_uint32 decomp_flags) -{ - static const int s_length_base[31] = { 3,4,5,6,7,8,9,10,11,13, 15,17,19,23,27,31,35,43,51,59, 67,83,99,115,131,163,195,227,258,0,0 }; - static const int s_length_extra[31]= { 0,0,0,0,0,0,0,0,1,1,1,1,2,2,2,2,3,3,3,3,4,4,4,4,5,5,5,5,0,0,0 }; - static const int s_dist_base[32] = { 1,2,3,4,5,7,9,13,17,25,33,49,65,97,129,193, 257,385,513,769,1025,1537,2049,3073,4097,6145,8193,12289,16385,24577,0,0}; - static const int s_dist_extra[32] = { 0,0,0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,8,8,9,9,10,10,11,11,12,12,13,13}; - static const mz_uint8 s_length_dezigzag[19] = { 16,17,18,0,8,7,9,6,10,5,11,4,12,3,13,2,14,1,15 }; - static const int s_min_table_sizes[3] = { 257, 1, 4 }; - - tinfl_status status = TINFL_STATUS_FAILED; mz_uint32 num_bits, dist, counter, num_extra; tinfl_bit_buf_t bit_buf; - const mz_uint8 *pIn_buf_cur = pIn_buf_next, *const pIn_buf_end = pIn_buf_next + *pIn_buf_size; - mz_uint8 *pOut_buf_cur = pOut_buf_next, *const pOut_buf_end = pOut_buf_next + *pOut_buf_size; - size_t out_buf_size_mask = (decomp_flags & TINFL_FLAG_USING_NON_WRAPPING_OUTPUT_BUF) ? (size_t)-1 : ((pOut_buf_next - pOut_buf_start) + *pOut_buf_size) - 1, dist_from_out_buf_start; - - // Ensure the output buffer's size is a power of 2, unless the output buffer is large enough to hold the entire output file (in which case it doesn't matter). - if (((out_buf_size_mask + 1) & out_buf_size_mask) || (pOut_buf_next < pOut_buf_start)) { *pIn_buf_size = *pOut_buf_size = 0; return TINFL_STATUS_BAD_PARAM; } - - num_bits = r->m_num_bits; bit_buf = r->m_bit_buf; dist = r->m_dist; counter = r->m_counter; num_extra = r->m_num_extra; dist_from_out_buf_start = r->m_dist_from_out_buf_start; - TINFL_CR_BEGIN - - bit_buf = num_bits = dist = counter = num_extra = r->m_zhdr0 = r->m_zhdr1 = 0; r->m_z_adler32 = r->m_check_adler32 = 1; - if (decomp_flags & TINFL_FLAG_PARSE_ZLIB_HEADER) - { - TINFL_GET_BYTE(1, r->m_zhdr0); TINFL_GET_BYTE(2, r->m_zhdr1); - counter = (((r->m_zhdr0 * 256 + r->m_zhdr1) % 31 != 0) || (r->m_zhdr1 & 32) || ((r->m_zhdr0 & 15) != 8)); - if (!(decomp_flags & TINFL_FLAG_USING_NON_WRAPPING_OUTPUT_BUF)) counter |= (((1U << (8U + (r->m_zhdr0 >> 4))) > 32768U) || ((out_buf_size_mask + 1) < (size_t)(1U << (8U + (r->m_zhdr0 >> 4))))); - if (counter) { TINFL_CR_RETURN_FOREVER(36, TINFL_STATUS_FAILED); } - } - - do - { - TINFL_GET_BITS(3, r->m_final, 3); r->m_type = r->m_final >> 1; - if (r->m_type == 0) - { - TINFL_SKIP_BITS(5, num_bits & 7); - for (counter = 0; counter < 4; ++counter) { if (num_bits) TINFL_GET_BITS(6, r->m_raw_header[counter], 8); else TINFL_GET_BYTE(7, r->m_raw_header[counter]); } - if ((counter = (r->m_raw_header[0] | (r->m_raw_header[1] << 8))) != (mz_uint)(0xFFFF ^ (r->m_raw_header[2] | (r->m_raw_header[3] << 8)))) { TINFL_CR_RETURN_FOREVER(39, TINFL_STATUS_FAILED); } - while ((counter) && (num_bits)) - { - TINFL_GET_BITS(51, dist, 8); - while (pOut_buf_cur >= pOut_buf_end) { TINFL_CR_RETURN(52, TINFL_STATUS_HAS_MORE_OUTPUT); } - *pOut_buf_cur++ = (mz_uint8)dist; - counter--; - } - while (counter) - { - size_t n; while (pOut_buf_cur >= pOut_buf_end) { TINFL_CR_RETURN(9, TINFL_STATUS_HAS_MORE_OUTPUT); } - while (pIn_buf_cur >= pIn_buf_end) - { - if (decomp_flags & TINFL_FLAG_HAS_MORE_INPUT) - { - TINFL_CR_RETURN(38, TINFL_STATUS_NEEDS_MORE_INPUT); - } - else - { - TINFL_CR_RETURN_FOREVER(40, TINFL_STATUS_FAILED); - } - } - n = MZ_MIN(MZ_MIN((size_t)(pOut_buf_end - pOut_buf_cur), (size_t)(pIn_buf_end - pIn_buf_cur)), counter); - TINFL_MEMCPY(pOut_buf_cur, pIn_buf_cur, n); pIn_buf_cur += n; pOut_buf_cur += n; counter -= (mz_uint)n; - } - } - else if (r->m_type == 3) - { - TINFL_CR_RETURN_FOREVER(10, TINFL_STATUS_FAILED); - } - else - { - if (r->m_type == 1) - { - mz_uint8 *p = r->m_tables[0].m_code_size; mz_uint i; - r->m_table_sizes[0] = 288; r->m_table_sizes[1] = 32; TINFL_MEMSET(r->m_tables[1].m_code_size, 5, 32); - for ( i = 0; i <= 143; ++i) *p++ = 8; for ( ; i <= 255; ++i) *p++ = 9; for ( ; i <= 279; ++i) *p++ = 7; for ( ; i <= 287; ++i) *p++ = 8; - } - else - { - for (counter = 0; counter < 3; counter++) { TINFL_GET_BITS(11, r->m_table_sizes[counter], "\05\05\04"[counter]); r->m_table_sizes[counter] += s_min_table_sizes[counter]; } - MZ_CLEAR_OBJ(r->m_tables[2].m_code_size); for (counter = 0; counter < r->m_table_sizes[2]; counter++) { mz_uint s; TINFL_GET_BITS(14, s, 3); r->m_tables[2].m_code_size[s_length_dezigzag[counter]] = (mz_uint8)s; } - r->m_table_sizes[2] = 19; - } - for ( ; (int)r->m_type >= 0; r->m_type--) - { - int tree_next, tree_cur; tinfl_huff_table *pTable; - mz_uint i, j, used_syms, total, sym_index, next_code[17], total_syms[16]; pTable = &r->m_tables[r->m_type]; MZ_CLEAR_OBJ(total_syms); MZ_CLEAR_OBJ(pTable->m_look_up); MZ_CLEAR_OBJ(pTable->m_tree); - for (i = 0; i < r->m_table_sizes[r->m_type]; ++i) total_syms[pTable->m_code_size[i]]++; - used_syms = 0, total = 0; next_code[0] = next_code[1] = 0; - for (i = 1; i <= 15; ++i) { used_syms += total_syms[i]; next_code[i + 1] = (total = ((total + total_syms[i]) << 1)); } - if ((65536 != total) && (used_syms > 1)) - { - TINFL_CR_RETURN_FOREVER(35, TINFL_STATUS_FAILED); - } - for (tree_next = -1, sym_index = 0; sym_index < r->m_table_sizes[r->m_type]; ++sym_index) - { - mz_uint rev_code = 0, l, cur_code, code_size = pTable->m_code_size[sym_index]; if (!code_size) continue; - cur_code = next_code[code_size]++; for (l = code_size; l > 0; l--, cur_code >>= 1) rev_code = (rev_code << 1) | (cur_code & 1); - if (code_size <= TINFL_FAST_LOOKUP_BITS) { mz_int16 k = (mz_int16)((code_size << 9) | sym_index); while (rev_code < TINFL_FAST_LOOKUP_SIZE) { pTable->m_look_up[rev_code] = k; rev_code += (1 << code_size); } continue; } - if (0 == (tree_cur = pTable->m_look_up[rev_code & (TINFL_FAST_LOOKUP_SIZE - 1)])) { pTable->m_look_up[rev_code & (TINFL_FAST_LOOKUP_SIZE - 1)] = (mz_int16)tree_next; tree_cur = tree_next; tree_next -= 2; } - rev_code >>= (TINFL_FAST_LOOKUP_BITS - 1); - for (j = code_size; j > (TINFL_FAST_LOOKUP_BITS + 1); j--) - { - tree_cur -= ((rev_code >>= 1) & 1); - if (!pTable->m_tree[-tree_cur - 1]) { pTable->m_tree[-tree_cur - 1] = (mz_int16)tree_next; tree_cur = tree_next; tree_next -= 2; } else tree_cur = pTable->m_tree[-tree_cur - 1]; - } - tree_cur -= ((rev_code >>= 1) & 1); pTable->m_tree[-tree_cur - 1] = (mz_int16)sym_index; - } - if (r->m_type == 2) - { - for (counter = 0; counter < (r->m_table_sizes[0] + r->m_table_sizes[1]); ) - { - mz_uint s; TINFL_HUFF_DECODE(16, dist, &r->m_tables[2]); if (dist < 16) { r->m_len_codes[counter++] = (mz_uint8)dist; continue; } - if ((dist == 16) && (!counter)) - { - TINFL_CR_RETURN_FOREVER(17, TINFL_STATUS_FAILED); - } - num_extra = "\02\03\07"[dist - 16]; TINFL_GET_BITS(18, s, num_extra); s += "\03\03\013"[dist - 16]; - TINFL_MEMSET(r->m_len_codes + counter, (dist == 16) ? r->m_len_codes[counter - 1] : 0, s); counter += s; - } - if ((r->m_table_sizes[0] + r->m_table_sizes[1]) != counter) - { - TINFL_CR_RETURN_FOREVER(21, TINFL_STATUS_FAILED); - } - TINFL_MEMCPY(r->m_tables[0].m_code_size, r->m_len_codes, r->m_table_sizes[0]); TINFL_MEMCPY(r->m_tables[1].m_code_size, r->m_len_codes + r->m_table_sizes[0], r->m_table_sizes[1]); - } - } - for ( ; ; ) - { - mz_uint8 *pSrc; - for ( ; ; ) - { - if (((pIn_buf_end - pIn_buf_cur) < 4) || ((pOut_buf_end - pOut_buf_cur) < 2)) - { - TINFL_HUFF_DECODE(23, counter, &r->m_tables[0]); - if (counter >= 256) - break; - while (pOut_buf_cur >= pOut_buf_end) { TINFL_CR_RETURN(24, TINFL_STATUS_HAS_MORE_OUTPUT); } - *pOut_buf_cur++ = (mz_uint8)counter; - } - else - { - int sym2; mz_uint code_len; -#if TINFL_USE_64BIT_BITBUF - if (num_bits < 30) { bit_buf |= (((tinfl_bit_buf_t)MZ_READ_LE32(pIn_buf_cur)) << num_bits); pIn_buf_cur += 4; num_bits += 32; } -#else - if (num_bits < 15) { bit_buf |= (((tinfl_bit_buf_t)MZ_READ_LE16(pIn_buf_cur)) << num_bits); pIn_buf_cur += 2; num_bits += 16; } -#endif - if ((sym2 = r->m_tables[0].m_look_up[bit_buf & (TINFL_FAST_LOOKUP_SIZE - 1)]) >= 0) - code_len = sym2 >> 9; - else - { - code_len = TINFL_FAST_LOOKUP_BITS; do { sym2 = r->m_tables[0].m_tree[~sym2 + ((bit_buf >> code_len++) & 1)]; } while (sym2 < 0); - } - counter = sym2; bit_buf >>= code_len; num_bits -= code_len; - if (counter & 256) - break; - -#if !TINFL_USE_64BIT_BITBUF - if (num_bits < 15) { bit_buf |= (((tinfl_bit_buf_t)MZ_READ_LE16(pIn_buf_cur)) << num_bits); pIn_buf_cur += 2; num_bits += 16; } -#endif - if ((sym2 = r->m_tables[0].m_look_up[bit_buf & (TINFL_FAST_LOOKUP_SIZE - 1)]) >= 0) - code_len = sym2 >> 9; - else - { - code_len = TINFL_FAST_LOOKUP_BITS; do { sym2 = r->m_tables[0].m_tree[~sym2 + ((bit_buf >> code_len++) & 1)]; } while (sym2 < 0); - } - bit_buf >>= code_len; num_bits -= code_len; - - pOut_buf_cur[0] = (mz_uint8)counter; - if (sym2 & 256) - { - pOut_buf_cur++; - counter = sym2; - break; - } - pOut_buf_cur[1] = (mz_uint8)sym2; - pOut_buf_cur += 2; - } - } - if ((counter &= 511) == 256) break; - - num_extra = s_length_extra[counter - 257]; counter = s_length_base[counter - 257]; - if (num_extra) { mz_uint extra_bits; TINFL_GET_BITS(25, extra_bits, num_extra); counter += extra_bits; } - - TINFL_HUFF_DECODE(26, dist, &r->m_tables[1]); - num_extra = s_dist_extra[dist]; dist = s_dist_base[dist]; - if (num_extra) { mz_uint extra_bits; TINFL_GET_BITS(27, extra_bits, num_extra); dist += extra_bits; } - - dist_from_out_buf_start = pOut_buf_cur - pOut_buf_start; - if ((dist > dist_from_out_buf_start) && (decomp_flags & TINFL_FLAG_USING_NON_WRAPPING_OUTPUT_BUF)) - { - TINFL_CR_RETURN_FOREVER(37, TINFL_STATUS_FAILED); - } - - pSrc = pOut_buf_start + ((dist_from_out_buf_start - dist) & out_buf_size_mask); - - if ((MZ_MAX(pOut_buf_cur, pSrc) + counter) > pOut_buf_end) - { - while (counter--) - { - while (pOut_buf_cur >= pOut_buf_end) { TINFL_CR_RETURN(53, TINFL_STATUS_HAS_MORE_OUTPUT); } - *pOut_buf_cur++ = pOut_buf_start[(dist_from_out_buf_start++ - dist) & out_buf_size_mask]; - } - continue; - } -#if MINIZ_USE_UNALIGNED_LOADS_AND_STORES - else if ((counter >= 9) && (counter <= dist)) - { - const mz_uint8 *pSrc_end = pSrc + (counter & ~7); - do - { - ((mz_uint32 *)pOut_buf_cur)[0] = ((const mz_uint32 *)pSrc)[0]; - ((mz_uint32 *)pOut_buf_cur)[1] = ((const mz_uint32 *)pSrc)[1]; - pOut_buf_cur += 8; - } while ((pSrc += 8) < pSrc_end); - if ((counter &= 7) < 3) - { - if (counter) - { - pOut_buf_cur[0] = pSrc[0]; - if (counter > 1) - pOut_buf_cur[1] = pSrc[1]; - pOut_buf_cur += counter; - } - continue; - } - } -#endif - do - { - pOut_buf_cur[0] = pSrc[0]; - pOut_buf_cur[1] = pSrc[1]; - pOut_buf_cur[2] = pSrc[2]; - pOut_buf_cur += 3; pSrc += 3; - } while ((int)(counter -= 3) > 2); - if ((int)counter > 0) - { - pOut_buf_cur[0] = pSrc[0]; - if ((int)counter > 1) - pOut_buf_cur[1] = pSrc[1]; - pOut_buf_cur += counter; - } - } - } - } while (!(r->m_final & 1)); - if (decomp_flags & TINFL_FLAG_PARSE_ZLIB_HEADER) - { - TINFL_SKIP_BITS(32, num_bits & 7); for (counter = 0; counter < 4; ++counter) { mz_uint s; if (num_bits) TINFL_GET_BITS(41, s, 8); else TINFL_GET_BYTE(42, s); r->m_z_adler32 = (r->m_z_adler32 << 8) | s; } - } - TINFL_CR_RETURN_FOREVER(34, TINFL_STATUS_DONE); - TINFL_CR_FINISH - -common_exit: - r->m_num_bits = num_bits; r->m_bit_buf = bit_buf; r->m_dist = dist; r->m_counter = counter; r->m_num_extra = num_extra; r->m_dist_from_out_buf_start = dist_from_out_buf_start; - *pIn_buf_size = pIn_buf_cur - pIn_buf_next; *pOut_buf_size = pOut_buf_cur - pOut_buf_next; - if ((decomp_flags & (TINFL_FLAG_PARSE_ZLIB_HEADER | TINFL_FLAG_COMPUTE_ADLER32)) && (status >= 0)) - { - const mz_uint8 *ptr = pOut_buf_next; size_t buf_len = *pOut_buf_size; - mz_uint32 i, s1 = r->m_check_adler32 & 0xffff, s2 = r->m_check_adler32 >> 16; size_t block_len = buf_len % 5552; - while (buf_len) - { - for (i = 0; i + 7 < block_len; i += 8, ptr += 8) - { - s1 += ptr[0], s2 += s1; s1 += ptr[1], s2 += s1; s1 += ptr[2], s2 += s1; s1 += ptr[3], s2 += s1; - s1 += ptr[4], s2 += s1; s1 += ptr[5], s2 += s1; s1 += ptr[6], s2 += s1; s1 += ptr[7], s2 += s1; - } - for ( ; i < block_len; ++i) s1 += *ptr++, s2 += s1; - s1 %= 65521U, s2 %= 65521U; buf_len -= block_len; block_len = 5552; - } - r->m_check_adler32 = (s2 << 16) + s1; if ((status == TINFL_STATUS_DONE) && (decomp_flags & TINFL_FLAG_PARSE_ZLIB_HEADER) && (r->m_check_adler32 != r->m_z_adler32)) status = TINFL_STATUS_ADLER32_MISMATCH; - } - return status; -} - -// Higher level helper functions. -void *tinfl_decompress_mem_to_heap(const void *pSrc_buf, size_t src_buf_len, size_t *pOut_len, int flags) -{ - tinfl_decompressor decomp; void *pBuf = NULL, *pNew_buf; size_t src_buf_ofs = 0, out_buf_capacity = 0; - *pOut_len = 0; - tinfl_init(&decomp); - for ( ; ; ) - { - size_t src_buf_size = src_buf_len - src_buf_ofs, dst_buf_size = out_buf_capacity - *pOut_len, new_out_buf_capacity; - tinfl_status status = tinfl_decompress(&decomp, (const mz_uint8*)pSrc_buf + src_buf_ofs, &src_buf_size, (mz_uint8*)pBuf, pBuf ? (mz_uint8*)pBuf + *pOut_len : NULL, &dst_buf_size, - (flags & ~TINFL_FLAG_HAS_MORE_INPUT) | TINFL_FLAG_USING_NON_WRAPPING_OUTPUT_BUF); - if ((status < 0) || (status == TINFL_STATUS_NEEDS_MORE_INPUT)) - { - MZ_FREE(pBuf); *pOut_len = 0; return NULL; - } - src_buf_ofs += src_buf_size; - *pOut_len += dst_buf_size; - if (status == TINFL_STATUS_DONE) break; - new_out_buf_capacity = out_buf_capacity * 2; if (new_out_buf_capacity < 128) new_out_buf_capacity = 128; - pNew_buf = MZ_REALLOC(pBuf, new_out_buf_capacity); - if (!pNew_buf) - { - MZ_FREE(pBuf); *pOut_len = 0; return NULL; - } - pBuf = pNew_buf; out_buf_capacity = new_out_buf_capacity; - } - return pBuf; -} - -size_t tinfl_decompress_mem_to_mem(void *pOut_buf, size_t out_buf_len, const void *pSrc_buf, size_t src_buf_len, int flags) -{ - tinfl_decompressor decomp; tinfl_status status; tinfl_init(&decomp); - status = tinfl_decompress(&decomp, (const mz_uint8*)pSrc_buf, &src_buf_len, (mz_uint8*)pOut_buf, (mz_uint8*)pOut_buf, &out_buf_len, (flags & ~TINFL_FLAG_HAS_MORE_INPUT) | TINFL_FLAG_USING_NON_WRAPPING_OUTPUT_BUF); - return (status != TINFL_STATUS_DONE) ? TINFL_DECOMPRESS_MEM_TO_MEM_FAILED : out_buf_len; -} - -int tinfl_decompress_mem_to_callback(const void *pIn_buf, size_t *pIn_buf_size, tinfl_put_buf_func_ptr pPut_buf_func, void *pPut_buf_user, int flags) -{ - int result = 0; - tinfl_decompressor decomp; - mz_uint8 *pDict = (mz_uint8*)MZ_MALLOC(TINFL_LZ_DICT_SIZE); size_t in_buf_ofs = 0, dict_ofs = 0; - if (!pDict) - return TINFL_STATUS_FAILED; - tinfl_init(&decomp); - for ( ; ; ) - { - size_t in_buf_size = *pIn_buf_size - in_buf_ofs, dst_buf_size = TINFL_LZ_DICT_SIZE - dict_ofs; - tinfl_status status = tinfl_decompress(&decomp, (const mz_uint8*)pIn_buf + in_buf_ofs, &in_buf_size, pDict, pDict + dict_ofs, &dst_buf_size, - (flags & ~(TINFL_FLAG_HAS_MORE_INPUT | TINFL_FLAG_USING_NON_WRAPPING_OUTPUT_BUF))); - in_buf_ofs += in_buf_size; - if ((dst_buf_size) && (!(*pPut_buf_func)(pDict + dict_ofs, (int)dst_buf_size, pPut_buf_user))) - break; - if (status != TINFL_STATUS_HAS_MORE_OUTPUT) - { - result = (status == TINFL_STATUS_DONE); - break; - } - dict_ofs = (dict_ofs + dst_buf_size) & (TINFL_LZ_DICT_SIZE - 1); - } - MZ_FREE(pDict); - *pIn_buf_size = in_buf_ofs; - return result; -} - -// ------------------- Low-level Compression (independent from all decompression API's) - -// Purposely making these tables static for faster init and thread safety. -static const mz_uint16 s_tdefl_len_sym[256] = { - 257,258,259,260,261,262,263,264,265,265,266,266,267,267,268,268,269,269,269,269,270,270,270,270,271,271,271,271,272,272,272,272, - 273,273,273,273,273,273,273,273,274,274,274,274,274,274,274,274,275,275,275,275,275,275,275,275,276,276,276,276,276,276,276,276, - 277,277,277,277,277,277,277,277,277,277,277,277,277,277,277,277,278,278,278,278,278,278,278,278,278,278,278,278,278,278,278,278, - 279,279,279,279,279,279,279,279,279,279,279,279,279,279,279,279,280,280,280,280,280,280,280,280,280,280,280,280,280,280,280,280, - 281,281,281,281,281,281,281,281,281,281,281,281,281,281,281,281,281,281,281,281,281,281,281,281,281,281,281,281,281,281,281,281, - 282,282,282,282,282,282,282,282,282,282,282,282,282,282,282,282,282,282,282,282,282,282,282,282,282,282,282,282,282,282,282,282, - 283,283,283,283,283,283,283,283,283,283,283,283,283,283,283,283,283,283,283,283,283,283,283,283,283,283,283,283,283,283,283,283, - 284,284,284,284,284,284,284,284,284,284,284,284,284,284,284,284,284,284,284,284,284,284,284,284,284,284,284,284,284,284,284,285 }; - -static const mz_uint8 s_tdefl_len_extra[256] = { - 0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3, - 4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4, - 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5, - 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,0 }; - -static const mz_uint8 s_tdefl_small_dist_sym[512] = { - 0,1,2,3,4,4,5,5,6,6,6,6,7,7,7,7,8,8,8,8,8,8,8,8,9,9,9,9,9,9,9,9,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,11,11,11,11,11,11, - 11,11,11,11,11,11,11,11,11,11,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,13, - 13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,14,14,14,14,14,14,14,14,14,14,14,14, - 14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14, - 14,14,14,14,14,14,14,14,14,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15, - 15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,16,16,16,16,16,16,16,16,16,16,16,16,16, - 16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16, - 16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16, - 16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,17,17,17,17,17,17,17,17,17,17,17,17,17,17, - 17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17, - 17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17, - 17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17 }; - -static const mz_uint8 s_tdefl_small_dist_extra[512] = { - 0,0,0,0,1,1,1,1,2,2,2,2,2,2,2,2,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,5,5,5,5,5,5,5,5, - 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6, - 6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6, - 6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7, - 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7, - 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7, - 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7, - 7,7,7,7,7,7,7,7 }; - -static const mz_uint8 s_tdefl_large_dist_sym[128] = { - 0,0,18,19,20,20,21,21,22,22,22,22,23,23,23,23,24,24,24,24,24,24,24,24,25,25,25,25,25,25,25,25,26,26,26,26,26,26,26,26,26,26,26,26, - 26,26,26,26,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28, - 28,28,28,28,28,28,28,28,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29 }; - -static const mz_uint8 s_tdefl_large_dist_extra[128] = { - 0,0,8,8,9,9,9,9,10,10,10,10,10,10,10,10,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12, - 12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13, - 13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13 }; - -// Radix sorts tdefl_sym_freq[] array by 16-bit key m_key. Returns ptr to sorted values. -typedef struct { mz_uint16 m_key, m_sym_index; } tdefl_sym_freq; -static tdefl_sym_freq* tdefl_radix_sort_syms(mz_uint num_syms, tdefl_sym_freq* pSyms0, tdefl_sym_freq* pSyms1) -{ - mz_uint32 total_passes = 2, pass_shift, pass, i, hist[256 * 2]; tdefl_sym_freq* pCur_syms = pSyms0, *pNew_syms = pSyms1; MZ_CLEAR_OBJ(hist); - for (i = 0; i < num_syms; i++) { mz_uint freq = pSyms0[i].m_key; hist[freq & 0xFF]++; hist[256 + ((freq >> 8) & 0xFF)]++; } - while ((total_passes > 1) && (num_syms == hist[(total_passes - 1) * 256])) total_passes--; - for (pass_shift = 0, pass = 0; pass < total_passes; pass++, pass_shift += 8) - { - const mz_uint32* pHist = &hist[pass << 8]; - mz_uint offsets[256], cur_ofs = 0; - for (i = 0; i < 256; i++) { offsets[i] = cur_ofs; cur_ofs += pHist[i]; } - for (i = 0; i < num_syms; i++) pNew_syms[offsets[(pCur_syms[i].m_key >> pass_shift) & 0xFF]++] = pCur_syms[i]; - { tdefl_sym_freq* t = pCur_syms; pCur_syms = pNew_syms; pNew_syms = t; } - } - return pCur_syms; -} - -// tdefl_calculate_minimum_redundancy() originally written by: Alistair Moffat, alistair@cs.mu.oz.au, Jyrki Katajainen, jyrki@diku.dk, November 1996. -static void tdefl_calculate_minimum_redundancy(tdefl_sym_freq *A, int n) -{ - int root, leaf, next, avbl, used, dpth; - if (n==0) return; else if (n==1) { A[0].m_key = 1; return; } - A[0].m_key += A[1].m_key; root = 0; leaf = 2; - for (next=1; next < n-1; next++) - { - if (leaf>=n || A[root].m_key=n || (root=0; next--) A[next].m_key = A[A[next].m_key].m_key+1; - avbl = 1; used = dpth = 0; root = n-2; next = n-1; - while (avbl>0) - { - while (root>=0 && (int)A[root].m_key==dpth) { used++; root--; } - while (avbl>used) { A[next--].m_key = (mz_uint16)(dpth); avbl--; } - avbl = 2*used; dpth++; used = 0; - } -} - -// Limits canonical Huffman code table's max code size. -enum { TDEFL_MAX_SUPPORTED_HUFF_CODESIZE = 32 }; -static void tdefl_huffman_enforce_max_code_size(int *pNum_codes, int code_list_len, int max_code_size) -{ - int i; mz_uint32 total = 0; if (code_list_len <= 1) return; - for (i = max_code_size + 1; i <= TDEFL_MAX_SUPPORTED_HUFF_CODESIZE; i++) pNum_codes[max_code_size] += pNum_codes[i]; - for (i = max_code_size; i > 0; i--) total += (((mz_uint32)pNum_codes[i]) << (max_code_size - i)); - while (total != (1UL << max_code_size)) - { - pNum_codes[max_code_size]--; - for (i = max_code_size - 1; i > 0; i--) if (pNum_codes[i]) { pNum_codes[i]--; pNum_codes[i + 1] += 2; break; } - total--; - } -} - -static void tdefl_optimize_huffman_table(tdefl_compressor *d, int table_num, int table_len, int code_size_limit, int static_table) -{ - int i, j, l, num_codes[1 + TDEFL_MAX_SUPPORTED_HUFF_CODESIZE]; mz_uint next_code[TDEFL_MAX_SUPPORTED_HUFF_CODESIZE + 1]; MZ_CLEAR_OBJ(num_codes); - if (static_table) - { - for (i = 0; i < table_len; i++) num_codes[d->m_huff_code_sizes[table_num][i]]++; - } - else - { - tdefl_sym_freq syms0[TDEFL_MAX_HUFF_SYMBOLS], syms1[TDEFL_MAX_HUFF_SYMBOLS], *pSyms; - int num_used_syms = 0; - const mz_uint16 *pSym_count = &d->m_huff_count[table_num][0]; - for (i = 0; i < table_len; i++) if (pSym_count[i]) { syms0[num_used_syms].m_key = (mz_uint16)pSym_count[i]; syms0[num_used_syms++].m_sym_index = (mz_uint16)i; } - - pSyms = tdefl_radix_sort_syms(num_used_syms, syms0, syms1); tdefl_calculate_minimum_redundancy(pSyms, num_used_syms); - - for (i = 0; i < num_used_syms; i++) num_codes[pSyms[i].m_key]++; - - tdefl_huffman_enforce_max_code_size(num_codes, num_used_syms, code_size_limit); - - MZ_CLEAR_OBJ(d->m_huff_code_sizes[table_num]); MZ_CLEAR_OBJ(d->m_huff_codes[table_num]); - for (i = 1, j = num_used_syms; i <= code_size_limit; i++) - for (l = num_codes[i]; l > 0; l--) d->m_huff_code_sizes[table_num][pSyms[--j].m_sym_index] = (mz_uint8)(i); - } - - next_code[1] = 0; for (j = 0, i = 2; i <= code_size_limit; i++) next_code[i] = j = ((j + num_codes[i - 1]) << 1); - - for (i = 0; i < table_len; i++) - { - mz_uint rev_code = 0, code, code_size; if ((code_size = d->m_huff_code_sizes[table_num][i]) == 0) continue; - code = next_code[code_size]++; for (l = code_size; l > 0; l--, code >>= 1) rev_code = (rev_code << 1) | (code & 1); - d->m_huff_codes[table_num][i] = (mz_uint16)rev_code; - } -} - -#define TDEFL_PUT_BITS(b, l) do { \ - mz_uint bits = b; mz_uint len = l; MZ_ASSERT(bits <= ((1U << len) - 1U)); \ - d->m_bit_buffer |= (bits << d->m_bits_in); d->m_bits_in += len; \ - while (d->m_bits_in >= 8) { \ - if (d->m_pOutput_buf < d->m_pOutput_buf_end) \ - *d->m_pOutput_buf++ = (mz_uint8)(d->m_bit_buffer); \ - d->m_bit_buffer >>= 8; \ - d->m_bits_in -= 8; \ - } \ -} MZ_MACRO_END - -#define TDEFL_RLE_PREV_CODE_SIZE() { if (rle_repeat_count) { \ - if (rle_repeat_count < 3) { \ - d->m_huff_count[2][prev_code_size] = (mz_uint16)(d->m_huff_count[2][prev_code_size] + rle_repeat_count); \ - while (rle_repeat_count--) packed_code_sizes[num_packed_code_sizes++] = prev_code_size; \ - } else { \ - d->m_huff_count[2][16] = (mz_uint16)(d->m_huff_count[2][16] + 1); packed_code_sizes[num_packed_code_sizes++] = 16; packed_code_sizes[num_packed_code_sizes++] = (mz_uint8)(rle_repeat_count - 3); \ -} rle_repeat_count = 0; } } - -#define TDEFL_RLE_ZERO_CODE_SIZE() { if (rle_z_count) { \ - if (rle_z_count < 3) { \ - d->m_huff_count[2][0] = (mz_uint16)(d->m_huff_count[2][0] + rle_z_count); while (rle_z_count--) packed_code_sizes[num_packed_code_sizes++] = 0; \ - } else if (rle_z_count <= 10) { \ - d->m_huff_count[2][17] = (mz_uint16)(d->m_huff_count[2][17] + 1); packed_code_sizes[num_packed_code_sizes++] = 17; packed_code_sizes[num_packed_code_sizes++] = (mz_uint8)(rle_z_count - 3); \ - } else { \ - d->m_huff_count[2][18] = (mz_uint16)(d->m_huff_count[2][18] + 1); packed_code_sizes[num_packed_code_sizes++] = 18; packed_code_sizes[num_packed_code_sizes++] = (mz_uint8)(rle_z_count - 11); \ -} rle_z_count = 0; } } - -static mz_uint8 s_tdefl_packed_code_size_syms_swizzle[] = { 16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15 }; - -static void tdefl_start_dynamic_block(tdefl_compressor *d) -{ - int num_lit_codes, num_dist_codes, num_bit_lengths; mz_uint i, total_code_sizes_to_pack, num_packed_code_sizes, rle_z_count, rle_repeat_count, packed_code_sizes_index; - mz_uint8 code_sizes_to_pack[TDEFL_MAX_HUFF_SYMBOLS_0 + TDEFL_MAX_HUFF_SYMBOLS_1], packed_code_sizes[TDEFL_MAX_HUFF_SYMBOLS_0 + TDEFL_MAX_HUFF_SYMBOLS_1], prev_code_size = 0xFF; - - d->m_huff_count[0][256] = 1; - - tdefl_optimize_huffman_table(d, 0, TDEFL_MAX_HUFF_SYMBOLS_0, 15, MZ_FALSE); - tdefl_optimize_huffman_table(d, 1, TDEFL_MAX_HUFF_SYMBOLS_1, 15, MZ_FALSE); - - for (num_lit_codes = 286; num_lit_codes > 257; num_lit_codes--) if (d->m_huff_code_sizes[0][num_lit_codes - 1]) break; - for (num_dist_codes = 30; num_dist_codes > 1; num_dist_codes--) if (d->m_huff_code_sizes[1][num_dist_codes - 1]) break; - - memcpy(code_sizes_to_pack, &d->m_huff_code_sizes[0][0], num_lit_codes); - memcpy(code_sizes_to_pack + num_lit_codes, &d->m_huff_code_sizes[1][0], num_dist_codes); - total_code_sizes_to_pack = num_lit_codes + num_dist_codes; num_packed_code_sizes = 0; rle_z_count = 0; rle_repeat_count = 0; - - memset(&d->m_huff_count[2][0], 0, sizeof(d->m_huff_count[2][0]) * TDEFL_MAX_HUFF_SYMBOLS_2); - for (i = 0; i < total_code_sizes_to_pack; i++) - { - mz_uint8 code_size = code_sizes_to_pack[i]; - if (!code_size) - { - TDEFL_RLE_PREV_CODE_SIZE(); - if (++rle_z_count == 138) { TDEFL_RLE_ZERO_CODE_SIZE(); } - } - else - { - TDEFL_RLE_ZERO_CODE_SIZE(); - if (code_size != prev_code_size) - { - TDEFL_RLE_PREV_CODE_SIZE(); - d->m_huff_count[2][code_size] = (mz_uint16)(d->m_huff_count[2][code_size] + 1); packed_code_sizes[num_packed_code_sizes++] = code_size; - } - else if (++rle_repeat_count == 6) - { - TDEFL_RLE_PREV_CODE_SIZE(); - } - } - prev_code_size = code_size; - } - if (rle_repeat_count) { TDEFL_RLE_PREV_CODE_SIZE(); } else { TDEFL_RLE_ZERO_CODE_SIZE(); } - - tdefl_optimize_huffman_table(d, 2, TDEFL_MAX_HUFF_SYMBOLS_2, 7, MZ_FALSE); - - TDEFL_PUT_BITS(2, 2); - - TDEFL_PUT_BITS(num_lit_codes - 257, 5); - TDEFL_PUT_BITS(num_dist_codes - 1, 5); - - for (num_bit_lengths = 18; num_bit_lengths >= 0; num_bit_lengths--) if (d->m_huff_code_sizes[2][s_tdefl_packed_code_size_syms_swizzle[num_bit_lengths]]) break; - num_bit_lengths = MZ_MAX(4, (num_bit_lengths + 1)); TDEFL_PUT_BITS(num_bit_lengths - 4, 4); - for (i = 0; (int)i < num_bit_lengths; i++) TDEFL_PUT_BITS(d->m_huff_code_sizes[2][s_tdefl_packed_code_size_syms_swizzle[i]], 3); - - for (packed_code_sizes_index = 0; packed_code_sizes_index < num_packed_code_sizes; ) - { - mz_uint code = packed_code_sizes[packed_code_sizes_index++]; MZ_ASSERT(code < TDEFL_MAX_HUFF_SYMBOLS_2); - TDEFL_PUT_BITS(d->m_huff_codes[2][code], d->m_huff_code_sizes[2][code]); - if (code >= 16) TDEFL_PUT_BITS(packed_code_sizes[packed_code_sizes_index++], "\02\03\07"[code - 16]); - } -} - -static void tdefl_start_static_block(tdefl_compressor *d) -{ - mz_uint i; - mz_uint8 *p = &d->m_huff_code_sizes[0][0]; - - for (i = 0; i <= 143; ++i) *p++ = 8; - for ( ; i <= 255; ++i) *p++ = 9; - for ( ; i <= 279; ++i) *p++ = 7; - for ( ; i <= 287; ++i) *p++ = 8; - - memset(d->m_huff_code_sizes[1], 5, 32); - - tdefl_optimize_huffman_table(d, 0, 288, 15, MZ_TRUE); - tdefl_optimize_huffman_table(d, 1, 32, 15, MZ_TRUE); - - TDEFL_PUT_BITS(1, 2); -} - -static const mz_uint mz_bitmasks[17] = { 0x0000, 0x0001, 0x0003, 0x0007, 0x000F, 0x001F, 0x003F, 0x007F, 0x00FF, 0x01FF, 0x03FF, 0x07FF, 0x0FFF, 0x1FFF, 0x3FFF, 0x7FFF, 0xFFFF }; - -#if MINIZ_USE_UNALIGNED_LOADS_AND_STORES && MINIZ_LITTLE_ENDIAN && MINIZ_HAS_64BIT_REGISTERS -static mz_bool tdefl_compress_lz_codes(tdefl_compressor *d) -{ - mz_uint flags; - mz_uint8 *pLZ_codes; - mz_uint8 *pOutput_buf = d->m_pOutput_buf; - mz_uint8 *pLZ_code_buf_end = d->m_pLZ_code_buf; - mz_uint64 bit_buffer = d->m_bit_buffer; - mz_uint bits_in = d->m_bits_in; - -#define TDEFL_PUT_BITS_FAST(b, l) { bit_buffer |= (((mz_uint64)(b)) << bits_in); bits_in += (l); } - - flags = 1; - for (pLZ_codes = d->m_lz_code_buf; pLZ_codes < pLZ_code_buf_end; flags >>= 1) - { - if (flags == 1) - flags = *pLZ_codes++ | 0x100; - - if (flags & 1) - { - mz_uint s0, s1, n0, n1, sym, num_extra_bits; - mz_uint match_len = pLZ_codes[0], match_dist = *(const mz_uint16 *)(pLZ_codes + 1); pLZ_codes += 3; - - MZ_ASSERT(d->m_huff_code_sizes[0][s_tdefl_len_sym[match_len]]); - TDEFL_PUT_BITS_FAST(d->m_huff_codes[0][s_tdefl_len_sym[match_len]], d->m_huff_code_sizes[0][s_tdefl_len_sym[match_len]]); - TDEFL_PUT_BITS_FAST(match_len & mz_bitmasks[s_tdefl_len_extra[match_len]], s_tdefl_len_extra[match_len]); - - // This sequence coaxes MSVC into using cmov's vs. jmp's. - s0 = s_tdefl_small_dist_sym[match_dist & 511]; - n0 = s_tdefl_small_dist_extra[match_dist & 511]; - s1 = s_tdefl_large_dist_sym[match_dist >> 8]; - n1 = s_tdefl_large_dist_extra[match_dist >> 8]; - sym = (match_dist < 512) ? s0 : s1; - num_extra_bits = (match_dist < 512) ? n0 : n1; - - MZ_ASSERT(d->m_huff_code_sizes[1][sym]); - TDEFL_PUT_BITS_FAST(d->m_huff_codes[1][sym], d->m_huff_code_sizes[1][sym]); - TDEFL_PUT_BITS_FAST(match_dist & mz_bitmasks[num_extra_bits], num_extra_bits); - } - else - { - mz_uint lit = *pLZ_codes++; - MZ_ASSERT(d->m_huff_code_sizes[0][lit]); - TDEFL_PUT_BITS_FAST(d->m_huff_codes[0][lit], d->m_huff_code_sizes[0][lit]); - - if (((flags & 2) == 0) && (pLZ_codes < pLZ_code_buf_end)) - { - flags >>= 1; - lit = *pLZ_codes++; - MZ_ASSERT(d->m_huff_code_sizes[0][lit]); - TDEFL_PUT_BITS_FAST(d->m_huff_codes[0][lit], d->m_huff_code_sizes[0][lit]); - - if (((flags & 2) == 0) && (pLZ_codes < pLZ_code_buf_end)) - { - flags >>= 1; - lit = *pLZ_codes++; - MZ_ASSERT(d->m_huff_code_sizes[0][lit]); - TDEFL_PUT_BITS_FAST(d->m_huff_codes[0][lit], d->m_huff_code_sizes[0][lit]); - } - } - } - - if (pOutput_buf >= d->m_pOutput_buf_end) - return MZ_FALSE; - - *(mz_uint64*)pOutput_buf = bit_buffer; - pOutput_buf += (bits_in >> 3); - bit_buffer >>= (bits_in & ~7); - bits_in &= 7; - } - -#undef TDEFL_PUT_BITS_FAST - - d->m_pOutput_buf = pOutput_buf; - d->m_bits_in = 0; - d->m_bit_buffer = 0; - - while (bits_in) - { - mz_uint32 n = MZ_MIN(bits_in, 16); - TDEFL_PUT_BITS((mz_uint)bit_buffer & mz_bitmasks[n], n); - bit_buffer >>= n; - bits_in -= n; - } - - TDEFL_PUT_BITS(d->m_huff_codes[0][256], d->m_huff_code_sizes[0][256]); - - return (d->m_pOutput_buf < d->m_pOutput_buf_end); -} -#else -static mz_bool tdefl_compress_lz_codes(tdefl_compressor *d) -{ - mz_uint flags; - mz_uint8 *pLZ_codes; - - flags = 1; - for (pLZ_codes = d->m_lz_code_buf; pLZ_codes < d->m_pLZ_code_buf; flags >>= 1) - { - if (flags == 1) - flags = *pLZ_codes++ | 0x100; - if (flags & 1) - { - mz_uint sym, num_extra_bits; - mz_uint match_len = pLZ_codes[0], match_dist = (pLZ_codes[1] | (pLZ_codes[2] << 8)); pLZ_codes += 3; - - MZ_ASSERT(d->m_huff_code_sizes[0][s_tdefl_len_sym[match_len]]); - TDEFL_PUT_BITS(d->m_huff_codes[0][s_tdefl_len_sym[match_len]], d->m_huff_code_sizes[0][s_tdefl_len_sym[match_len]]); - TDEFL_PUT_BITS(match_len & mz_bitmasks[s_tdefl_len_extra[match_len]], s_tdefl_len_extra[match_len]); - - if (match_dist < 512) - { - sym = s_tdefl_small_dist_sym[match_dist]; num_extra_bits = s_tdefl_small_dist_extra[match_dist]; - } - else - { - sym = s_tdefl_large_dist_sym[match_dist >> 8]; num_extra_bits = s_tdefl_large_dist_extra[match_dist >> 8]; - } - MZ_ASSERT(d->m_huff_code_sizes[1][sym]); - TDEFL_PUT_BITS(d->m_huff_codes[1][sym], d->m_huff_code_sizes[1][sym]); - TDEFL_PUT_BITS(match_dist & mz_bitmasks[num_extra_bits], num_extra_bits); - } - else - { - mz_uint lit = *pLZ_codes++; - MZ_ASSERT(d->m_huff_code_sizes[0][lit]); - TDEFL_PUT_BITS(d->m_huff_codes[0][lit], d->m_huff_code_sizes[0][lit]); - } - } - - TDEFL_PUT_BITS(d->m_huff_codes[0][256], d->m_huff_code_sizes[0][256]); - - return (d->m_pOutput_buf < d->m_pOutput_buf_end); -} -#endif // MINIZ_USE_UNALIGNED_LOADS_AND_STORES && MINIZ_LITTLE_ENDIAN && MINIZ_HAS_64BIT_REGISTERS - -static mz_bool tdefl_compress_block(tdefl_compressor *d, mz_bool static_block) -{ - if (static_block) - tdefl_start_static_block(d); - else - tdefl_start_dynamic_block(d); - return tdefl_compress_lz_codes(d); -} - -static int tdefl_flush_block(tdefl_compressor *d, int flush) -{ - mz_uint saved_bit_buf, saved_bits_in; - mz_uint8 *pSaved_output_buf; - mz_bool comp_block_succeeded = MZ_FALSE; - int n, use_raw_block = ((d->m_flags & TDEFL_FORCE_ALL_RAW_BLOCKS) != 0) && (d->m_lookahead_pos - d->m_lz_code_buf_dict_pos) <= d->m_dict_size; - mz_uint8 *pOutput_buf_start = ((d->m_pPut_buf_func == NULL) && ((*d->m_pOut_buf_size - d->m_out_buf_ofs) >= TDEFL_OUT_BUF_SIZE)) ? ((mz_uint8 *)d->m_pOut_buf + d->m_out_buf_ofs) : d->m_output_buf; - - d->m_pOutput_buf = pOutput_buf_start; - d->m_pOutput_buf_end = d->m_pOutput_buf + TDEFL_OUT_BUF_SIZE - 16; - - MZ_ASSERT(!d->m_output_flush_remaining); - d->m_output_flush_ofs = 0; - d->m_output_flush_remaining = 0; - - *d->m_pLZ_flags = (mz_uint8)(*d->m_pLZ_flags >> d->m_num_flags_left); - d->m_pLZ_code_buf -= (d->m_num_flags_left == 8); - - if ((d->m_flags & TDEFL_WRITE_ZLIB_HEADER) && (!d->m_block_index)) - { - TDEFL_PUT_BITS(0x78, 8); TDEFL_PUT_BITS(0x01, 8); - } - - TDEFL_PUT_BITS(flush == TDEFL_FINISH, 1); - - pSaved_output_buf = d->m_pOutput_buf; saved_bit_buf = d->m_bit_buffer; saved_bits_in = d->m_bits_in; - - if (!use_raw_block) - comp_block_succeeded = tdefl_compress_block(d, (d->m_flags & TDEFL_FORCE_ALL_STATIC_BLOCKS) || (d->m_total_lz_bytes < 48)); - - // If the block gets expanded, forget the current contents of the output buffer and send a raw block instead. - if ( ((use_raw_block) || ((d->m_total_lz_bytes) && ((d->m_pOutput_buf - pSaved_output_buf + 1U) >= d->m_total_lz_bytes))) && - ((d->m_lookahead_pos - d->m_lz_code_buf_dict_pos) <= d->m_dict_size) ) - { - mz_uint i; d->m_pOutput_buf = pSaved_output_buf; d->m_bit_buffer = saved_bit_buf, d->m_bits_in = saved_bits_in; - TDEFL_PUT_BITS(0, 2); - if (d->m_bits_in) { TDEFL_PUT_BITS(0, 8 - d->m_bits_in); } - for (i = 2; i; --i, d->m_total_lz_bytes ^= 0xFFFF) - { - TDEFL_PUT_BITS(d->m_total_lz_bytes & 0xFFFF, 16); - } - for (i = 0; i < d->m_total_lz_bytes; ++i) - { - TDEFL_PUT_BITS(d->m_dict[(d->m_lz_code_buf_dict_pos + i) & TDEFL_LZ_DICT_SIZE_MASK], 8); - } - } - // Check for the extremely unlikely (if not impossible) case of the compressed block not fitting into the output buffer when using dynamic codes. - else if (!comp_block_succeeded) - { - d->m_pOutput_buf = pSaved_output_buf; d->m_bit_buffer = saved_bit_buf, d->m_bits_in = saved_bits_in; - tdefl_compress_block(d, MZ_TRUE); - } - - if (flush) - { - if (flush == TDEFL_FINISH) - { - if (d->m_bits_in) { TDEFL_PUT_BITS(0, 8 - d->m_bits_in); } - if (d->m_flags & TDEFL_WRITE_ZLIB_HEADER) { mz_uint i, a = d->m_adler32; for (i = 0; i < 4; i++) { TDEFL_PUT_BITS((a >> 24) & 0xFF, 8); a <<= 8; } } - } - else - { - mz_uint i, z = 0; TDEFL_PUT_BITS(0, 3); if (d->m_bits_in) { TDEFL_PUT_BITS(0, 8 - d->m_bits_in); } for (i = 2; i; --i, z ^= 0xFFFF) { TDEFL_PUT_BITS(z & 0xFFFF, 16); } - } - } - - MZ_ASSERT(d->m_pOutput_buf < d->m_pOutput_buf_end); - - memset(&d->m_huff_count[0][0], 0, sizeof(d->m_huff_count[0][0]) * TDEFL_MAX_HUFF_SYMBOLS_0); - memset(&d->m_huff_count[1][0], 0, sizeof(d->m_huff_count[1][0]) * TDEFL_MAX_HUFF_SYMBOLS_1); - - d->m_pLZ_code_buf = d->m_lz_code_buf + 1; d->m_pLZ_flags = d->m_lz_code_buf; d->m_num_flags_left = 8; d->m_lz_code_buf_dict_pos += d->m_total_lz_bytes; d->m_total_lz_bytes = 0; d->m_block_index++; - - if ((n = (int)(d->m_pOutput_buf - pOutput_buf_start)) != 0) - { - if (d->m_pPut_buf_func) - { - *d->m_pIn_buf_size = d->m_pSrc - (const mz_uint8 *)d->m_pIn_buf; - if (!(*d->m_pPut_buf_func)(d->m_output_buf, n, d->m_pPut_buf_user)) - return (d->m_prev_return_status = TDEFL_STATUS_PUT_BUF_FAILED); - } - else if (pOutput_buf_start == d->m_output_buf) - { - int bytes_to_copy = (int)MZ_MIN((size_t)n, (size_t)(*d->m_pOut_buf_size - d->m_out_buf_ofs)); - memcpy((mz_uint8 *)d->m_pOut_buf + d->m_out_buf_ofs, d->m_output_buf, bytes_to_copy); - d->m_out_buf_ofs += bytes_to_copy; - if ((n -= bytes_to_copy) != 0) - { - d->m_output_flush_ofs = bytes_to_copy; - d->m_output_flush_remaining = n; - } - } - else - { - d->m_out_buf_ofs += n; - } - } - - return d->m_output_flush_remaining; -} - -#if MINIZ_USE_UNALIGNED_LOADS_AND_STORES -#define TDEFL_READ_UNALIGNED_WORD(p) *(const mz_uint16*)(p) -static MZ_FORCEINLINE void tdefl_find_match(tdefl_compressor *d, mz_uint lookahead_pos, mz_uint max_dist, mz_uint max_match_len, mz_uint *pMatch_dist, mz_uint *pMatch_len) -{ - mz_uint dist, pos = lookahead_pos & TDEFL_LZ_DICT_SIZE_MASK, match_len = *pMatch_len, probe_pos = pos, next_probe_pos, probe_len; - mz_uint num_probes_left = d->m_max_probes[match_len >= 32]; - const mz_uint16 *s = (const mz_uint16*)(d->m_dict + pos), *p, *q; - mz_uint16 c01 = TDEFL_READ_UNALIGNED_WORD(&d->m_dict[pos + match_len - 1]), s01 = TDEFL_READ_UNALIGNED_WORD(s); - MZ_ASSERT(max_match_len <= TDEFL_MAX_MATCH_LEN); if (max_match_len <= match_len) return; - for ( ; ; ) - { - for ( ; ; ) - { - if (--num_probes_left == 0) return; - #define TDEFL_PROBE \ - next_probe_pos = d->m_next[probe_pos]; \ - if ((!next_probe_pos) || ((dist = (mz_uint16)(lookahead_pos - next_probe_pos)) > max_dist)) return; \ - probe_pos = next_probe_pos & TDEFL_LZ_DICT_SIZE_MASK; \ - if (TDEFL_READ_UNALIGNED_WORD(&d->m_dict[probe_pos + match_len - 1]) == c01) break; - TDEFL_PROBE; TDEFL_PROBE; TDEFL_PROBE; - } - if (!dist) break; q = (const mz_uint16*)(d->m_dict + probe_pos); if (TDEFL_READ_UNALIGNED_WORD(q) != s01) continue; p = s; probe_len = 32; - do { } while ( (TDEFL_READ_UNALIGNED_WORD(++p) == TDEFL_READ_UNALIGNED_WORD(++q)) && (TDEFL_READ_UNALIGNED_WORD(++p) == TDEFL_READ_UNALIGNED_WORD(++q)) && - (TDEFL_READ_UNALIGNED_WORD(++p) == TDEFL_READ_UNALIGNED_WORD(++q)) && (TDEFL_READ_UNALIGNED_WORD(++p) == TDEFL_READ_UNALIGNED_WORD(++q)) && (--probe_len > 0) ); - if (!probe_len) - { - *pMatch_dist = dist; *pMatch_len = MZ_MIN(max_match_len, TDEFL_MAX_MATCH_LEN); break; - } - else if ((probe_len = ((mz_uint)(p - s) * 2) + (mz_uint)(*(const mz_uint8*)p == *(const mz_uint8*)q)) > match_len) - { - *pMatch_dist = dist; if ((*pMatch_len = match_len = MZ_MIN(max_match_len, probe_len)) == max_match_len) break; - c01 = TDEFL_READ_UNALIGNED_WORD(&d->m_dict[pos + match_len - 1]); - } - } -} -#else -static MZ_FORCEINLINE void tdefl_find_match(tdefl_compressor *d, mz_uint lookahead_pos, mz_uint max_dist, mz_uint max_match_len, mz_uint *pMatch_dist, mz_uint *pMatch_len) -{ - mz_uint dist, pos = lookahead_pos & TDEFL_LZ_DICT_SIZE_MASK, match_len = *pMatch_len, probe_pos = pos, next_probe_pos, probe_len; - mz_uint num_probes_left = d->m_max_probes[match_len >= 32]; - const mz_uint8 *s = d->m_dict + pos, *p, *q; - mz_uint8 c0 = d->m_dict[pos + match_len], c1 = d->m_dict[pos + match_len - 1]; - MZ_ASSERT(max_match_len <= TDEFL_MAX_MATCH_LEN); if (max_match_len <= match_len) return; - for ( ; ; ) - { - for ( ; ; ) - { - if (--num_probes_left == 0) return; - #define TDEFL_PROBE \ - next_probe_pos = d->m_next[probe_pos]; \ - if ((!next_probe_pos) || ((dist = (mz_uint16)(lookahead_pos - next_probe_pos)) > max_dist)) return; \ - probe_pos = next_probe_pos & TDEFL_LZ_DICT_SIZE_MASK; \ - if ((d->m_dict[probe_pos + match_len] == c0) && (d->m_dict[probe_pos + match_len - 1] == c1)) break; - TDEFL_PROBE; TDEFL_PROBE; TDEFL_PROBE; - } - if (!dist) break; p = s; q = d->m_dict + probe_pos; for (probe_len = 0; probe_len < max_match_len; probe_len++) if (*p++ != *q++) break; - if (probe_len > match_len) - { - *pMatch_dist = dist; if ((*pMatch_len = match_len = probe_len) == max_match_len) return; - c0 = d->m_dict[pos + match_len]; c1 = d->m_dict[pos + match_len - 1]; - } - } -} -#endif // #if MINIZ_USE_UNALIGNED_LOADS_AND_STORES - -#if MINIZ_USE_UNALIGNED_LOADS_AND_STORES && MINIZ_LITTLE_ENDIAN -static mz_bool tdefl_compress_fast(tdefl_compressor *d) -{ - // Faster, minimally featured LZRW1-style match+parse loop with better register utilization. Intended for applications where raw throughput is valued more highly than ratio. - mz_uint lookahead_pos = d->m_lookahead_pos, lookahead_size = d->m_lookahead_size, dict_size = d->m_dict_size, total_lz_bytes = d->m_total_lz_bytes, num_flags_left = d->m_num_flags_left; - mz_uint8 *pLZ_code_buf = d->m_pLZ_code_buf, *pLZ_flags = d->m_pLZ_flags; - mz_uint cur_pos = lookahead_pos & TDEFL_LZ_DICT_SIZE_MASK; - - while ((d->m_src_buf_left) || ((d->m_flush) && (lookahead_size))) - { - const mz_uint TDEFL_COMP_FAST_LOOKAHEAD_SIZE = 4096; - mz_uint dst_pos = (lookahead_pos + lookahead_size) & TDEFL_LZ_DICT_SIZE_MASK; - mz_uint num_bytes_to_process = (mz_uint)MZ_MIN(d->m_src_buf_left, TDEFL_COMP_FAST_LOOKAHEAD_SIZE - lookahead_size); - d->m_src_buf_left -= num_bytes_to_process; - lookahead_size += num_bytes_to_process; - - while (num_bytes_to_process) - { - mz_uint32 n = MZ_MIN(TDEFL_LZ_DICT_SIZE - dst_pos, num_bytes_to_process); - memcpy(d->m_dict + dst_pos, d->m_pSrc, n); - if (dst_pos < (TDEFL_MAX_MATCH_LEN - 1)) - memcpy(d->m_dict + TDEFL_LZ_DICT_SIZE + dst_pos, d->m_pSrc, MZ_MIN(n, (TDEFL_MAX_MATCH_LEN - 1) - dst_pos)); - d->m_pSrc += n; - dst_pos = (dst_pos + n) & TDEFL_LZ_DICT_SIZE_MASK; - num_bytes_to_process -= n; - } - - dict_size = MZ_MIN(TDEFL_LZ_DICT_SIZE - lookahead_size, dict_size); - if ((!d->m_flush) && (lookahead_size < TDEFL_COMP_FAST_LOOKAHEAD_SIZE)) break; - - while (lookahead_size >= 4) - { - mz_uint cur_match_dist, cur_match_len = 1; - mz_uint8 *pCur_dict = d->m_dict + cur_pos; - mz_uint first_trigram = (*(const mz_uint32 *)pCur_dict) & 0xFFFFFF; - mz_uint hash = (first_trigram ^ (first_trigram >> (24 - (TDEFL_LZ_HASH_BITS - 8)))) & TDEFL_LEVEL1_HASH_SIZE_MASK; - mz_uint probe_pos = d->m_hash[hash]; - d->m_hash[hash] = (mz_uint16)lookahead_pos; - - if (((cur_match_dist = (mz_uint16)(lookahead_pos - probe_pos)) <= dict_size) && ((*(const mz_uint32 *)(d->m_dict + (probe_pos &= TDEFL_LZ_DICT_SIZE_MASK)) & 0xFFFFFF) == first_trigram)) - { - const mz_uint16 *p = (const mz_uint16 *)pCur_dict; - const mz_uint16 *q = (const mz_uint16 *)(d->m_dict + probe_pos); - mz_uint32 probe_len = 32; - do { } while ( (TDEFL_READ_UNALIGNED_WORD(++p) == TDEFL_READ_UNALIGNED_WORD(++q)) && (TDEFL_READ_UNALIGNED_WORD(++p) == TDEFL_READ_UNALIGNED_WORD(++q)) && - (TDEFL_READ_UNALIGNED_WORD(++p) == TDEFL_READ_UNALIGNED_WORD(++q)) && (TDEFL_READ_UNALIGNED_WORD(++p) == TDEFL_READ_UNALIGNED_WORD(++q)) && (--probe_len > 0) ); - cur_match_len = ((mz_uint)(p - (const mz_uint16 *)pCur_dict) * 2) + (mz_uint)(*(const mz_uint8 *)p == *(const mz_uint8 *)q); - if (!probe_len) - cur_match_len = cur_match_dist ? TDEFL_MAX_MATCH_LEN : 0; - - if ((cur_match_len < TDEFL_MIN_MATCH_LEN) || ((cur_match_len == TDEFL_MIN_MATCH_LEN) && (cur_match_dist >= 8U*1024U))) - { - cur_match_len = 1; - *pLZ_code_buf++ = (mz_uint8)first_trigram; - *pLZ_flags = (mz_uint8)(*pLZ_flags >> 1); - d->m_huff_count[0][(mz_uint8)first_trigram]++; - } - else - { - mz_uint32 s0, s1; - cur_match_len = MZ_MIN(cur_match_len, lookahead_size); - - MZ_ASSERT((cur_match_len >= TDEFL_MIN_MATCH_LEN) && (cur_match_dist >= 1) && (cur_match_dist <= TDEFL_LZ_DICT_SIZE)); - - cur_match_dist--; - - pLZ_code_buf[0] = (mz_uint8)(cur_match_len - TDEFL_MIN_MATCH_LEN); - *(mz_uint16 *)(&pLZ_code_buf[1]) = (mz_uint16)cur_match_dist; - pLZ_code_buf += 3; - *pLZ_flags = (mz_uint8)((*pLZ_flags >> 1) | 0x80); - - s0 = s_tdefl_small_dist_sym[cur_match_dist & 511]; - s1 = s_tdefl_large_dist_sym[cur_match_dist >> 8]; - d->m_huff_count[1][(cur_match_dist < 512) ? s0 : s1]++; - - d->m_huff_count[0][s_tdefl_len_sym[cur_match_len - TDEFL_MIN_MATCH_LEN]]++; - } - } - else - { - *pLZ_code_buf++ = (mz_uint8)first_trigram; - *pLZ_flags = (mz_uint8)(*pLZ_flags >> 1); - d->m_huff_count[0][(mz_uint8)first_trigram]++; - } - - if (--num_flags_left == 0) { num_flags_left = 8; pLZ_flags = pLZ_code_buf++; } - - total_lz_bytes += cur_match_len; - lookahead_pos += cur_match_len; - dict_size = MZ_MIN(dict_size + cur_match_len, TDEFL_LZ_DICT_SIZE); - cur_pos = (cur_pos + cur_match_len) & TDEFL_LZ_DICT_SIZE_MASK; - MZ_ASSERT(lookahead_size >= cur_match_len); - lookahead_size -= cur_match_len; - - if (pLZ_code_buf > &d->m_lz_code_buf[TDEFL_LZ_CODE_BUF_SIZE - 8]) - { - int n; - d->m_lookahead_pos = lookahead_pos; d->m_lookahead_size = lookahead_size; d->m_dict_size = dict_size; - d->m_total_lz_bytes = total_lz_bytes; d->m_pLZ_code_buf = pLZ_code_buf; d->m_pLZ_flags = pLZ_flags; d->m_num_flags_left = num_flags_left; - if ((n = tdefl_flush_block(d, 0)) != 0) - return (n < 0) ? MZ_FALSE : MZ_TRUE; - total_lz_bytes = d->m_total_lz_bytes; pLZ_code_buf = d->m_pLZ_code_buf; pLZ_flags = d->m_pLZ_flags; num_flags_left = d->m_num_flags_left; - } - } - - while (lookahead_size) - { - mz_uint8 lit = d->m_dict[cur_pos]; - - total_lz_bytes++; - *pLZ_code_buf++ = lit; - *pLZ_flags = (mz_uint8)(*pLZ_flags >> 1); - if (--num_flags_left == 0) { num_flags_left = 8; pLZ_flags = pLZ_code_buf++; } - - d->m_huff_count[0][lit]++; - - lookahead_pos++; - dict_size = MZ_MIN(dict_size + 1, TDEFL_LZ_DICT_SIZE); - cur_pos = (cur_pos + 1) & TDEFL_LZ_DICT_SIZE_MASK; - lookahead_size--; - - if (pLZ_code_buf > &d->m_lz_code_buf[TDEFL_LZ_CODE_BUF_SIZE - 8]) - { - int n; - d->m_lookahead_pos = lookahead_pos; d->m_lookahead_size = lookahead_size; d->m_dict_size = dict_size; - d->m_total_lz_bytes = total_lz_bytes; d->m_pLZ_code_buf = pLZ_code_buf; d->m_pLZ_flags = pLZ_flags; d->m_num_flags_left = num_flags_left; - if ((n = tdefl_flush_block(d, 0)) != 0) - return (n < 0) ? MZ_FALSE : MZ_TRUE; - total_lz_bytes = d->m_total_lz_bytes; pLZ_code_buf = d->m_pLZ_code_buf; pLZ_flags = d->m_pLZ_flags; num_flags_left = d->m_num_flags_left; - } - } - } - - d->m_lookahead_pos = lookahead_pos; d->m_lookahead_size = lookahead_size; d->m_dict_size = dict_size; - d->m_total_lz_bytes = total_lz_bytes; d->m_pLZ_code_buf = pLZ_code_buf; d->m_pLZ_flags = pLZ_flags; d->m_num_flags_left = num_flags_left; - return MZ_TRUE; -} -#endif // MINIZ_USE_UNALIGNED_LOADS_AND_STORES && MINIZ_LITTLE_ENDIAN - -static MZ_FORCEINLINE void tdefl_record_literal(tdefl_compressor *d, mz_uint8 lit) -{ - d->m_total_lz_bytes++; - *d->m_pLZ_code_buf++ = lit; - *d->m_pLZ_flags = (mz_uint8)(*d->m_pLZ_flags >> 1); if (--d->m_num_flags_left == 0) { d->m_num_flags_left = 8; d->m_pLZ_flags = d->m_pLZ_code_buf++; } - d->m_huff_count[0][lit]++; -} - -static MZ_FORCEINLINE void tdefl_record_match(tdefl_compressor *d, mz_uint match_len, mz_uint match_dist) -{ - mz_uint32 s0, s1; - - MZ_ASSERT((match_len >= TDEFL_MIN_MATCH_LEN) && (match_dist >= 1) && (match_dist <= TDEFL_LZ_DICT_SIZE)); - - d->m_total_lz_bytes += match_len; - - d->m_pLZ_code_buf[0] = (mz_uint8)(match_len - TDEFL_MIN_MATCH_LEN); - - match_dist -= 1; - d->m_pLZ_code_buf[1] = (mz_uint8)(match_dist & 0xFF); - d->m_pLZ_code_buf[2] = (mz_uint8)(match_dist >> 8); d->m_pLZ_code_buf += 3; - - *d->m_pLZ_flags = (mz_uint8)((*d->m_pLZ_flags >> 1) | 0x80); if (--d->m_num_flags_left == 0) { d->m_num_flags_left = 8; d->m_pLZ_flags = d->m_pLZ_code_buf++; } - - s0 = s_tdefl_small_dist_sym[match_dist & 511]; s1 = s_tdefl_large_dist_sym[(match_dist >> 8) & 127]; - d->m_huff_count[1][(match_dist < 512) ? s0 : s1]++; - - if (match_len >= TDEFL_MIN_MATCH_LEN) d->m_huff_count[0][s_tdefl_len_sym[match_len - TDEFL_MIN_MATCH_LEN]]++; -} - -static mz_bool tdefl_compress_normal(tdefl_compressor *d) -{ - const mz_uint8 *pSrc = d->m_pSrc; size_t src_buf_left = d->m_src_buf_left; - tdefl_flush flush = d->m_flush; - - while ((src_buf_left) || ((flush) && (d->m_lookahead_size))) - { - mz_uint len_to_move, cur_match_dist, cur_match_len, cur_pos; - // Update dictionary and hash chains. Keeps the lookahead size equal to TDEFL_MAX_MATCH_LEN. - if ((d->m_lookahead_size + d->m_dict_size) >= (TDEFL_MIN_MATCH_LEN - 1)) - { - mz_uint dst_pos = (d->m_lookahead_pos + d->m_lookahead_size) & TDEFL_LZ_DICT_SIZE_MASK, ins_pos = d->m_lookahead_pos + d->m_lookahead_size - 2; - mz_uint hash = (d->m_dict[ins_pos & TDEFL_LZ_DICT_SIZE_MASK] << TDEFL_LZ_HASH_SHIFT) ^ d->m_dict[(ins_pos + 1) & TDEFL_LZ_DICT_SIZE_MASK]; - mz_uint num_bytes_to_process = (mz_uint)MZ_MIN(src_buf_left, TDEFL_MAX_MATCH_LEN - d->m_lookahead_size); - const mz_uint8 *pSrc_end = pSrc + num_bytes_to_process; - src_buf_left -= num_bytes_to_process; - d->m_lookahead_size += num_bytes_to_process; - while (pSrc != pSrc_end) - { - mz_uint8 c = *pSrc++; d->m_dict[dst_pos] = c; if (dst_pos < (TDEFL_MAX_MATCH_LEN - 1)) d->m_dict[TDEFL_LZ_DICT_SIZE + dst_pos] = c; - hash = ((hash << TDEFL_LZ_HASH_SHIFT) ^ c) & (TDEFL_LZ_HASH_SIZE - 1); - d->m_next[ins_pos & TDEFL_LZ_DICT_SIZE_MASK] = d->m_hash[hash]; d->m_hash[hash] = (mz_uint16)(ins_pos); - dst_pos = (dst_pos + 1) & TDEFL_LZ_DICT_SIZE_MASK; ins_pos++; - } - } - else - { - while ((src_buf_left) && (d->m_lookahead_size < TDEFL_MAX_MATCH_LEN)) - { - mz_uint8 c = *pSrc++; - mz_uint dst_pos = (d->m_lookahead_pos + d->m_lookahead_size) & TDEFL_LZ_DICT_SIZE_MASK; - src_buf_left--; - d->m_dict[dst_pos] = c; - if (dst_pos < (TDEFL_MAX_MATCH_LEN - 1)) - d->m_dict[TDEFL_LZ_DICT_SIZE + dst_pos] = c; - if ((++d->m_lookahead_size + d->m_dict_size) >= TDEFL_MIN_MATCH_LEN) - { - mz_uint ins_pos = d->m_lookahead_pos + (d->m_lookahead_size - 1) - 2; - mz_uint hash = ((d->m_dict[ins_pos & TDEFL_LZ_DICT_SIZE_MASK] << (TDEFL_LZ_HASH_SHIFT * 2)) ^ (d->m_dict[(ins_pos + 1) & TDEFL_LZ_DICT_SIZE_MASK] << TDEFL_LZ_HASH_SHIFT) ^ c) & (TDEFL_LZ_HASH_SIZE - 1); - d->m_next[ins_pos & TDEFL_LZ_DICT_SIZE_MASK] = d->m_hash[hash]; d->m_hash[hash] = (mz_uint16)(ins_pos); - } - } - } - d->m_dict_size = MZ_MIN(TDEFL_LZ_DICT_SIZE - d->m_lookahead_size, d->m_dict_size); - if ((!flush) && (d->m_lookahead_size < TDEFL_MAX_MATCH_LEN)) - break; - - // Simple lazy/greedy parsing state machine. - len_to_move = 1; cur_match_dist = 0; cur_match_len = d->m_saved_match_len ? d->m_saved_match_len : (TDEFL_MIN_MATCH_LEN - 1); cur_pos = d->m_lookahead_pos & TDEFL_LZ_DICT_SIZE_MASK; - if (d->m_flags & (TDEFL_RLE_MATCHES | TDEFL_FORCE_ALL_RAW_BLOCKS)) - { - if ((d->m_dict_size) && (!(d->m_flags & TDEFL_FORCE_ALL_RAW_BLOCKS))) - { - mz_uint8 c = d->m_dict[(cur_pos - 1) & TDEFL_LZ_DICT_SIZE_MASK]; - cur_match_len = 0; while (cur_match_len < d->m_lookahead_size) { if (d->m_dict[cur_pos + cur_match_len] != c) break; cur_match_len++; } - if (cur_match_len < TDEFL_MIN_MATCH_LEN) cur_match_len = 0; else cur_match_dist = 1; - } - } - else - { - tdefl_find_match(d, d->m_lookahead_pos, d->m_dict_size, d->m_lookahead_size, &cur_match_dist, &cur_match_len); - } - if (((cur_match_len == TDEFL_MIN_MATCH_LEN) && (cur_match_dist >= 8U*1024U)) || (cur_pos == cur_match_dist) || ((d->m_flags & TDEFL_FILTER_MATCHES) && (cur_match_len <= 5))) - { - cur_match_dist = cur_match_len = 0; - } - if (d->m_saved_match_len) - { - if (cur_match_len > d->m_saved_match_len) - { - tdefl_record_literal(d, (mz_uint8)d->m_saved_lit); - if (cur_match_len >= 128) - { - tdefl_record_match(d, cur_match_len, cur_match_dist); - d->m_saved_match_len = 0; len_to_move = cur_match_len; - } - else - { - d->m_saved_lit = d->m_dict[cur_pos]; d->m_saved_match_dist = cur_match_dist; d->m_saved_match_len = cur_match_len; - } - } - else - { - tdefl_record_match(d, d->m_saved_match_len, d->m_saved_match_dist); - len_to_move = d->m_saved_match_len - 1; d->m_saved_match_len = 0; - } - } - else if (!cur_match_dist) - tdefl_record_literal(d, d->m_dict[MZ_MIN(cur_pos, sizeof(d->m_dict) - 1)]); - else if ((d->m_greedy_parsing) || (d->m_flags & TDEFL_RLE_MATCHES) || (cur_match_len >= 128)) - { - tdefl_record_match(d, cur_match_len, cur_match_dist); - len_to_move = cur_match_len; - } - else - { - d->m_saved_lit = d->m_dict[MZ_MIN(cur_pos, sizeof(d->m_dict) - 1)]; d->m_saved_match_dist = cur_match_dist; d->m_saved_match_len = cur_match_len; - } - // Move the lookahead forward by len_to_move bytes. - d->m_lookahead_pos += len_to_move; - MZ_ASSERT(d->m_lookahead_size >= len_to_move); - d->m_lookahead_size -= len_to_move; - d->m_dict_size = MZ_MIN(d->m_dict_size + len_to_move, TDEFL_LZ_DICT_SIZE); - // Check if it's time to flush the current LZ codes to the internal output buffer. - if ( (d->m_pLZ_code_buf > &d->m_lz_code_buf[TDEFL_LZ_CODE_BUF_SIZE - 8]) || - ( (d->m_total_lz_bytes > 31*1024) && (((((mz_uint)(d->m_pLZ_code_buf - d->m_lz_code_buf) * 115) >> 7) >= d->m_total_lz_bytes) || (d->m_flags & TDEFL_FORCE_ALL_RAW_BLOCKS))) ) - { - int n; - d->m_pSrc = pSrc; d->m_src_buf_left = src_buf_left; - if ((n = tdefl_flush_block(d, 0)) != 0) - return (n < 0) ? MZ_FALSE : MZ_TRUE; - } - } - - d->m_pSrc = pSrc; d->m_src_buf_left = src_buf_left; - return MZ_TRUE; -} - -static tdefl_status tdefl_flush_output_buffer(tdefl_compressor *d) -{ - if (d->m_pIn_buf_size) - { - *d->m_pIn_buf_size = d->m_pSrc - (const mz_uint8 *)d->m_pIn_buf; - } - - if (d->m_pOut_buf_size) - { - size_t n = MZ_MIN(*d->m_pOut_buf_size - d->m_out_buf_ofs, d->m_output_flush_remaining); - memcpy((mz_uint8 *)d->m_pOut_buf + d->m_out_buf_ofs, d->m_output_buf + d->m_output_flush_ofs, n); - d->m_output_flush_ofs += (mz_uint)n; - d->m_output_flush_remaining -= (mz_uint)n; - d->m_out_buf_ofs += n; - - *d->m_pOut_buf_size = d->m_out_buf_ofs; - } - - return (d->m_finished && !d->m_output_flush_remaining) ? TDEFL_STATUS_DONE : TDEFL_STATUS_OKAY; -} - -tdefl_status tdefl_compress(tdefl_compressor *d, const void *pIn_buf, size_t *pIn_buf_size, void *pOut_buf, size_t *pOut_buf_size, tdefl_flush flush) -{ - if (!d) - { - if (pIn_buf_size) *pIn_buf_size = 0; - if (pOut_buf_size) *pOut_buf_size = 0; - return TDEFL_STATUS_BAD_PARAM; - } - - d->m_pIn_buf = pIn_buf; d->m_pIn_buf_size = pIn_buf_size; - d->m_pOut_buf = pOut_buf; d->m_pOut_buf_size = pOut_buf_size; - d->m_pSrc = (const mz_uint8 *)(pIn_buf); d->m_src_buf_left = pIn_buf_size ? *pIn_buf_size : 0; - d->m_out_buf_ofs = 0; - d->m_flush = flush; - - if ( ((d->m_pPut_buf_func != NULL) == ((pOut_buf != NULL) || (pOut_buf_size != NULL))) || (d->m_prev_return_status != TDEFL_STATUS_OKAY) || - (d->m_wants_to_finish && (flush != TDEFL_FINISH)) || (pIn_buf_size && *pIn_buf_size && !pIn_buf) || (pOut_buf_size && *pOut_buf_size && !pOut_buf) ) - { - if (pIn_buf_size) *pIn_buf_size = 0; - if (pOut_buf_size) *pOut_buf_size = 0; - return (d->m_prev_return_status = TDEFL_STATUS_BAD_PARAM); - } - d->m_wants_to_finish |= (flush == TDEFL_FINISH); - - if ((d->m_output_flush_remaining) || (d->m_finished)) - return (d->m_prev_return_status = tdefl_flush_output_buffer(d)); - -#if MINIZ_USE_UNALIGNED_LOADS_AND_STORES && MINIZ_LITTLE_ENDIAN - if (((d->m_flags & TDEFL_MAX_PROBES_MASK) == 1) && - ((d->m_flags & TDEFL_GREEDY_PARSING_FLAG) != 0) && - ((d->m_flags & (TDEFL_FILTER_MATCHES | TDEFL_FORCE_ALL_RAW_BLOCKS | TDEFL_RLE_MATCHES)) == 0)) - { - if (!tdefl_compress_fast(d)) - return d->m_prev_return_status; - } - else -#endif // #if MINIZ_USE_UNALIGNED_LOADS_AND_STORES && MINIZ_LITTLE_ENDIAN - { - if (!tdefl_compress_normal(d)) - return d->m_prev_return_status; - } - - if ((d->m_flags & (TDEFL_WRITE_ZLIB_HEADER | TDEFL_COMPUTE_ADLER32)) && (pIn_buf)) - d->m_adler32 = (mz_uint32)mz_adler32(d->m_adler32, (const mz_uint8 *)pIn_buf, d->m_pSrc - (const mz_uint8 *)pIn_buf); - - if ((flush) && (!d->m_lookahead_size) && (!d->m_src_buf_left) && (!d->m_output_flush_remaining)) - { - if (tdefl_flush_block(d, flush) < 0) - return d->m_prev_return_status; - d->m_finished = (flush == TDEFL_FINISH); - if (flush == TDEFL_FULL_FLUSH) { MZ_CLEAR_OBJ(d->m_hash); MZ_CLEAR_OBJ(d->m_next); d->m_dict_size = 0; } - } - - return (d->m_prev_return_status = tdefl_flush_output_buffer(d)); -} - -tdefl_status tdefl_compress_buffer(tdefl_compressor *d, const void *pIn_buf, size_t in_buf_size, tdefl_flush flush) -{ - MZ_ASSERT(d->m_pPut_buf_func); return tdefl_compress(d, pIn_buf, &in_buf_size, NULL, NULL, flush); -} - -tdefl_status tdefl_init(tdefl_compressor *d, tdefl_put_buf_func_ptr pPut_buf_func, void *pPut_buf_user, int flags) -{ - d->m_pPut_buf_func = pPut_buf_func; d->m_pPut_buf_user = pPut_buf_user; - d->m_flags = (mz_uint)(flags); d->m_max_probes[0] = 1 + ((flags & 0xFFF) + 2) / 3; d->m_greedy_parsing = (flags & TDEFL_GREEDY_PARSING_FLAG) != 0; - d->m_max_probes[1] = 1 + (((flags & 0xFFF) >> 2) + 2) / 3; - if (!(flags & TDEFL_NONDETERMINISTIC_PARSING_FLAG)) MZ_CLEAR_OBJ(d->m_hash); - d->m_lookahead_pos = d->m_lookahead_size = d->m_dict_size = d->m_total_lz_bytes = d->m_lz_code_buf_dict_pos = d->m_bits_in = 0; - d->m_output_flush_ofs = d->m_output_flush_remaining = d->m_finished = d->m_block_index = d->m_bit_buffer = d->m_wants_to_finish = 0; - d->m_pLZ_code_buf = d->m_lz_code_buf + 1; d->m_pLZ_flags = d->m_lz_code_buf; d->m_num_flags_left = 8; - d->m_pOutput_buf = d->m_output_buf; d->m_pOutput_buf_end = d->m_output_buf; d->m_prev_return_status = TDEFL_STATUS_OKAY; - d->m_saved_match_dist = d->m_saved_match_len = d->m_saved_lit = 0; d->m_adler32 = 1; - d->m_pIn_buf = NULL; d->m_pOut_buf = NULL; - d->m_pIn_buf_size = NULL; d->m_pOut_buf_size = NULL; - d->m_flush = TDEFL_NO_FLUSH; d->m_pSrc = NULL; d->m_src_buf_left = 0; d->m_out_buf_ofs = 0; - memset(&d->m_huff_count[0][0], 0, sizeof(d->m_huff_count[0][0]) * TDEFL_MAX_HUFF_SYMBOLS_0); - memset(&d->m_huff_count[1][0], 0, sizeof(d->m_huff_count[1][0]) * TDEFL_MAX_HUFF_SYMBOLS_1); - return TDEFL_STATUS_OKAY; -} - -tdefl_status tdefl_get_prev_return_status(tdefl_compressor *d) -{ - return d->m_prev_return_status; -} - -mz_uint32 tdefl_get_adler32(tdefl_compressor *d) -{ - return d->m_adler32; -} - -mz_bool tdefl_compress_mem_to_output(const void *pBuf, size_t buf_len, tdefl_put_buf_func_ptr pPut_buf_func, void *pPut_buf_user, int flags) -{ - tdefl_compressor *pComp; mz_bool succeeded; if (((buf_len) && (!pBuf)) || (!pPut_buf_func)) return MZ_FALSE; - pComp = (tdefl_compressor*)MZ_MALLOC(sizeof(tdefl_compressor)); if (!pComp) return MZ_FALSE; - succeeded = (tdefl_init(pComp, pPut_buf_func, pPut_buf_user, flags) == TDEFL_STATUS_OKAY); - succeeded = succeeded && (tdefl_compress_buffer(pComp, pBuf, buf_len, TDEFL_FINISH) == TDEFL_STATUS_DONE); - MZ_FREE(pComp); return succeeded; -} - -typedef struct -{ - size_t m_size, m_capacity; - mz_uint8 *m_pBuf; - mz_bool m_expandable; -} tdefl_output_buffer; - -static mz_bool tdefl_output_buffer_putter(const void *pBuf, int len, void *pUser) -{ - tdefl_output_buffer *p = (tdefl_output_buffer *)pUser; - size_t new_size = p->m_size + len; - if (new_size > p->m_capacity) - { - size_t new_capacity = p->m_capacity; mz_uint8 *pNew_buf; if (!p->m_expandable) return MZ_FALSE; - do { new_capacity = MZ_MAX(128U, new_capacity << 1U); } while (new_size > new_capacity); - pNew_buf = (mz_uint8*)MZ_REALLOC(p->m_pBuf, new_capacity); if (!pNew_buf) return MZ_FALSE; - p->m_pBuf = pNew_buf; p->m_capacity = new_capacity; - } - memcpy((mz_uint8*)p->m_pBuf + p->m_size, pBuf, len); p->m_size = new_size; - return MZ_TRUE; -} - -void *tdefl_compress_mem_to_heap(const void *pSrc_buf, size_t src_buf_len, size_t *pOut_len, int flags) -{ - tdefl_output_buffer out_buf; MZ_CLEAR_OBJ(out_buf); - if (!pOut_len) return MZ_FALSE; else *pOut_len = 0; - out_buf.m_expandable = MZ_TRUE; - if (!tdefl_compress_mem_to_output(pSrc_buf, src_buf_len, tdefl_output_buffer_putter, &out_buf, flags)) return NULL; - *pOut_len = out_buf.m_size; return out_buf.m_pBuf; -} - -size_t tdefl_compress_mem_to_mem(void *pOut_buf, size_t out_buf_len, const void *pSrc_buf, size_t src_buf_len, int flags) -{ - tdefl_output_buffer out_buf; MZ_CLEAR_OBJ(out_buf); - if (!pOut_buf) return 0; - out_buf.m_pBuf = (mz_uint8*)pOut_buf; out_buf.m_capacity = out_buf_len; - if (!tdefl_compress_mem_to_output(pSrc_buf, src_buf_len, tdefl_output_buffer_putter, &out_buf, flags)) return 0; - return out_buf.m_size; -} - -#ifndef MINIZ_NO_ZLIB_APIS -static const mz_uint s_tdefl_num_probes[11] = { 0, 1, 6, 32, 16, 32, 128, 256, 512, 768, 1500 }; - -// level may actually range from [0,10] (10 is a "hidden" max level, where we want a bit more compression and it's fine if throughput to fall off a cliff on some files). -mz_uint tdefl_create_comp_flags_from_zip_params(int level, int window_bits, int strategy) -{ - mz_uint comp_flags = s_tdefl_num_probes[(level >= 0) ? MZ_MIN(10, level) : MZ_DEFAULT_LEVEL] | ((level <= 3) ? TDEFL_GREEDY_PARSING_FLAG : 0); - if (window_bits > 0) comp_flags |= TDEFL_WRITE_ZLIB_HEADER; - - if (!level) comp_flags |= TDEFL_FORCE_ALL_RAW_BLOCKS; - else if (strategy == MZ_FILTERED) comp_flags |= TDEFL_FILTER_MATCHES; - else if (strategy == MZ_HUFFMAN_ONLY) comp_flags &= ~TDEFL_MAX_PROBES_MASK; - else if (strategy == MZ_FIXED) comp_flags |= TDEFL_FORCE_ALL_STATIC_BLOCKS; - else if (strategy == MZ_RLE) comp_flags |= TDEFL_RLE_MATCHES; - - return comp_flags; -} -#endif //MINIZ_NO_ZLIB_APIS - -#ifdef _MSC_VER -#pragma warning (push) -#pragma warning (disable:4204) // nonstandard extension used : non-constant aggregate initializer (also supported by GNU C and C99, so no big deal) -#endif - -// Simple PNG writer function by Alex Evans, 2011. Released into the public domain: https://gist.github.com/908299, more context at -// http://altdevblogaday.org/2011/04/06/a-smaller-jpg-encoder/. -void *tdefl_write_image_to_png_file_in_memory(const void *pImage, int w, int h, int num_chans, size_t *pLen_out) -{ - tdefl_compressor *pComp = (tdefl_compressor *)MZ_MALLOC(sizeof(tdefl_compressor)); tdefl_output_buffer out_buf; int i, bpl = w * num_chans, y, z; mz_uint32 c; *pLen_out = 0; - if (!pComp) return NULL; - MZ_CLEAR_OBJ(out_buf); out_buf.m_expandable = MZ_TRUE; out_buf.m_capacity = 57+MZ_MAX(64, (1+bpl)*h); if (NULL == (out_buf.m_pBuf = (mz_uint8*)MZ_MALLOC(out_buf.m_capacity))) { MZ_FREE(pComp); return NULL; } - // write dummy header - for (z = 41; z; --z) tdefl_output_buffer_putter(&z, 1, &out_buf); - // compress image data - tdefl_init(pComp, tdefl_output_buffer_putter, &out_buf, TDEFL_DEFAULT_MAX_PROBES | TDEFL_WRITE_ZLIB_HEADER); - for (y = 0; y < h; ++y) { tdefl_compress_buffer(pComp, &z, 1, TDEFL_NO_FLUSH); tdefl_compress_buffer(pComp, (mz_uint8*)pImage + y * bpl, bpl, TDEFL_NO_FLUSH); } - if (tdefl_compress_buffer(pComp, NULL, 0, TDEFL_FINISH) != TDEFL_STATUS_DONE) { MZ_FREE(pComp); MZ_FREE(out_buf.m_pBuf); return NULL; } - // write real header - *pLen_out = out_buf.m_size-41; - { - mz_uint8 pnghdr[41]={0x89,0x50,0x4e,0x47,0x0d,0x0a,0x1a,0x0a,0x00,0x00,0x00,0x0d,0x49,0x48,0x44,0x52, - 0,0,(mz_uint8)(w>>8),(mz_uint8)w,0,0,(mz_uint8)(h>>8),(mz_uint8)h,8,"\0\0\04\02\06"[num_chans],0,0,0,0,0,0,0, - (mz_uint8)(*pLen_out>>24),(mz_uint8)(*pLen_out>>16),(mz_uint8)(*pLen_out>>8),(mz_uint8)*pLen_out,0x49,0x44,0x41,0x54}; - c=(mz_uint32)mz_crc32(MZ_CRC32_INIT,pnghdr+12,17); for (i=0; i<4; ++i, c<<=8) ((mz_uint8*)(pnghdr+29))[i]=(mz_uint8)(c>>24); - memcpy(out_buf.m_pBuf, pnghdr, 41); - } - // write footer (IDAT CRC-32, followed by IEND chunk) - if (!tdefl_output_buffer_putter("\0\0\0\0\0\0\0\0\x49\x45\x4e\x44\xae\x42\x60\x82", 16, &out_buf)) { *pLen_out = 0; MZ_FREE(pComp); MZ_FREE(out_buf.m_pBuf); return NULL; } - c = (mz_uint32)mz_crc32(MZ_CRC32_INIT,out_buf.m_pBuf+41-4, *pLen_out+4); for (i=0; i<4; ++i, c<<=8) (out_buf.m_pBuf+out_buf.m_size-16)[i] = (mz_uint8)(c >> 24); - // compute final size of file, grab compressed data buffer and return - *pLen_out += 57; MZ_FREE(pComp); return out_buf.m_pBuf; -} - -#ifdef _MSC_VER -#pragma warning (pop) -#endif - -// ------------------- .ZIP archive reading - -#ifndef MINIZ_NO_ARCHIVE_APIS - -#ifdef MINIZ_NO_STDIO - #define MZ_FILE void * -#else - #include - #include - #if defined(_MSC_VER) || defined(__MINGW64__) - #include - #define MZ_FILE FILE - #define MZ_FOPEN fopen - #define MZ_FCLOSE fclose - #define MZ_FREAD fread - #define MZ_FWRITE fwrite - #define MZ_FTELL64 _ftelli64 - #define MZ_FSEEK64 _fseeki64 - #define MZ_FILE_STAT_STRUCT _stat - #define MZ_FILE_STAT _stat - #define MZ_FFLUSH fflush - #define MZ_FREOPEN freopen - #define MZ_DELETE_FILE remove - #elif defined(__MINGW32__) - #include - #define MZ_FILE FILE - #define MZ_FOPEN fopen - #define MZ_FCLOSE fclose - #define MZ_FREAD fread - #define MZ_FWRITE fwrite - #define MZ_FTELL64 ftello64 - #define MZ_FSEEK64 fseeko64 - #define MZ_FILE_STAT_STRUCT _stat - #define MZ_FILE_STAT _stat - #define MZ_FFLUSH fflush - #define MZ_FREOPEN freopen - #define MZ_DELETE_FILE remove - #else - #include - #define MZ_FILE FILE - #define MZ_FOPEN fopen - #define MZ_FCLOSE fclose - #define MZ_FREAD fread - #define MZ_FWRITE fwrite - #define MZ_FTELL64 ftello - #define MZ_FSEEK64 fseeko - #define MZ_FILE_STAT_STRUCT stat - #define MZ_FILE_STAT stat - #define MZ_FFLUSH fflush - #define MZ_FREOPEN freopen - #define MZ_DELETE_FILE remove - #endif // #ifdef _MSC_VER -#endif // #ifdef MINIZ_NO_STDIO - -#define MZ_TOLOWER(c) ((((c) >= 'A') && ((c) <= 'Z')) ? ((c) - 'A' + 'a') : (c)) - -// Various ZIP archive enums. To completely avoid cross platform compiler alignment and platform endian issues, miniz.c doesn't use structs for any of this stuff. -enum -{ - // ZIP archive identifiers and record sizes - MZ_ZIP_END_OF_CENTRAL_DIR_HEADER_SIG = 0x06054b50, MZ_ZIP_CENTRAL_DIR_HEADER_SIG = 0x02014b50, MZ_ZIP_LOCAL_DIR_HEADER_SIG = 0x04034b50, - MZ_ZIP_LOCAL_DIR_HEADER_SIZE = 30, MZ_ZIP_CENTRAL_DIR_HEADER_SIZE = 46, MZ_ZIP_END_OF_CENTRAL_DIR_HEADER_SIZE = 22, - // Central directory header record offsets - MZ_ZIP_CDH_SIG_OFS = 0, MZ_ZIP_CDH_VERSION_MADE_BY_OFS = 4, MZ_ZIP_CDH_VERSION_NEEDED_OFS = 6, MZ_ZIP_CDH_BIT_FLAG_OFS = 8, - MZ_ZIP_CDH_METHOD_OFS = 10, MZ_ZIP_CDH_FILE_TIME_OFS = 12, MZ_ZIP_CDH_FILE_DATE_OFS = 14, MZ_ZIP_CDH_CRC32_OFS = 16, - MZ_ZIP_CDH_COMPRESSED_SIZE_OFS = 20, MZ_ZIP_CDH_DECOMPRESSED_SIZE_OFS = 24, MZ_ZIP_CDH_FILENAME_LEN_OFS = 28, MZ_ZIP_CDH_EXTRA_LEN_OFS = 30, - MZ_ZIP_CDH_COMMENT_LEN_OFS = 32, MZ_ZIP_CDH_DISK_START_OFS = 34, MZ_ZIP_CDH_INTERNAL_ATTR_OFS = 36, MZ_ZIP_CDH_EXTERNAL_ATTR_OFS = 38, MZ_ZIP_CDH_LOCAL_HEADER_OFS = 42, - // Local directory header offsets - MZ_ZIP_LDH_SIG_OFS = 0, MZ_ZIP_LDH_VERSION_NEEDED_OFS = 4, MZ_ZIP_LDH_BIT_FLAG_OFS = 6, MZ_ZIP_LDH_METHOD_OFS = 8, MZ_ZIP_LDH_FILE_TIME_OFS = 10, - MZ_ZIP_LDH_FILE_DATE_OFS = 12, MZ_ZIP_LDH_CRC32_OFS = 14, MZ_ZIP_LDH_COMPRESSED_SIZE_OFS = 18, MZ_ZIP_LDH_DECOMPRESSED_SIZE_OFS = 22, - MZ_ZIP_LDH_FILENAME_LEN_OFS = 26, MZ_ZIP_LDH_EXTRA_LEN_OFS = 28, - // End of central directory offsets - MZ_ZIP_ECDH_SIG_OFS = 0, MZ_ZIP_ECDH_NUM_THIS_DISK_OFS = 4, MZ_ZIP_ECDH_NUM_DISK_CDIR_OFS = 6, MZ_ZIP_ECDH_CDIR_NUM_ENTRIES_ON_DISK_OFS = 8, - MZ_ZIP_ECDH_CDIR_TOTAL_ENTRIES_OFS = 10, MZ_ZIP_ECDH_CDIR_SIZE_OFS = 12, MZ_ZIP_ECDH_CDIR_OFS_OFS = 16, MZ_ZIP_ECDH_COMMENT_SIZE_OFS = 20, -}; - -typedef struct -{ - void *m_p; - size_t m_size, m_capacity; - mz_uint m_element_size; -} mz_zip_array; - -struct mz_zip_internal_state_tag -{ - mz_zip_array m_central_dir; - mz_zip_array m_central_dir_offsets; - mz_zip_array m_sorted_central_dir_offsets; - MZ_FILE *m_pFile; - void *m_pMem; - size_t m_mem_size; - size_t m_mem_capacity; -}; - -#define MZ_ZIP_ARRAY_SET_ELEMENT_SIZE(array_ptr, element_size) (array_ptr)->m_element_size = element_size -#define MZ_ZIP_ARRAY_ELEMENT(array_ptr, element_type, index) ((element_type *)((array_ptr)->m_p))[index] - -static MZ_FORCEINLINE void mz_zip_array_clear(mz_zip_archive *pZip, mz_zip_array *pArray) -{ - pZip->m_pFree(pZip->m_pAlloc_opaque, pArray->m_p); - memset(pArray, 0, sizeof(mz_zip_array)); -} - -static mz_bool mz_zip_array_ensure_capacity(mz_zip_archive *pZip, mz_zip_array *pArray, size_t min_new_capacity, mz_uint growing) -{ - void *pNew_p; size_t new_capacity = min_new_capacity; MZ_ASSERT(pArray->m_element_size); if (pArray->m_capacity >= min_new_capacity) return MZ_TRUE; - if (growing) { new_capacity = MZ_MAX(1, pArray->m_capacity); while (new_capacity < min_new_capacity) new_capacity *= 2; } - if (NULL == (pNew_p = pZip->m_pRealloc(pZip->m_pAlloc_opaque, pArray->m_p, pArray->m_element_size, new_capacity))) return MZ_FALSE; - pArray->m_p = pNew_p; pArray->m_capacity = new_capacity; - return MZ_TRUE; -} - -static MZ_FORCEINLINE mz_bool mz_zip_array_reserve(mz_zip_archive *pZip, mz_zip_array *pArray, size_t new_capacity, mz_uint growing) -{ - if (new_capacity > pArray->m_capacity) { if (!mz_zip_array_ensure_capacity(pZip, pArray, new_capacity, growing)) return MZ_FALSE; } - return MZ_TRUE; -} - -static MZ_FORCEINLINE mz_bool mz_zip_array_resize(mz_zip_archive *pZip, mz_zip_array *pArray, size_t new_size, mz_uint growing) -{ - if (new_size > pArray->m_capacity) { if (!mz_zip_array_ensure_capacity(pZip, pArray, new_size, growing)) return MZ_FALSE; } - pArray->m_size = new_size; - return MZ_TRUE; -} - -static MZ_FORCEINLINE mz_bool mz_zip_array_ensure_room(mz_zip_archive *pZip, mz_zip_array *pArray, size_t n) -{ - return mz_zip_array_reserve(pZip, pArray, pArray->m_size + n, MZ_TRUE); -} - -static MZ_FORCEINLINE mz_bool mz_zip_array_push_back(mz_zip_archive *pZip, mz_zip_array *pArray, const void *pElements, size_t n) -{ - size_t orig_size = pArray->m_size; if (!mz_zip_array_resize(pZip, pArray, orig_size + n, MZ_TRUE)) return MZ_FALSE; - memcpy((mz_uint8*)pArray->m_p + orig_size * pArray->m_element_size, pElements, n * pArray->m_element_size); - return MZ_TRUE; -} - -#ifndef MINIZ_NO_TIME -static time_t mz_zip_dos_to_time_t(int dos_time, int dos_date) -{ - struct tm tm; - memset(&tm, 0, sizeof(tm)); tm.tm_isdst = -1; - tm.tm_year = ((dos_date >> 9) & 127) + 1980 - 1900; tm.tm_mon = ((dos_date >> 5) & 15) - 1; tm.tm_mday = dos_date & 31; - tm.tm_hour = (dos_time >> 11) & 31; tm.tm_min = (dos_time >> 5) & 63; tm.tm_sec = (dos_time << 1) & 62; - return mktime(&tm); -} - -static void mz_zip_time_to_dos_time(time_t time, mz_uint16 *pDOS_time, mz_uint16 *pDOS_date) -{ - struct tm *tm = localtime(&time); - *pDOS_time = (mz_uint16)(((tm->tm_hour) << 11) + ((tm->tm_min) << 5) + ((tm->tm_sec) >> 1)); - *pDOS_date = (mz_uint16)(((tm->tm_year + 1900 - 1980) << 9) + ((tm->tm_mon + 1) << 5) + tm->tm_mday); -} -#endif - -#ifndef MINIZ_NO_STDIO -static mz_bool mz_zip_get_file_modified_time(const char *pFilename, mz_uint16 *pDOS_time, mz_uint16 *pDOS_date) -{ -#ifdef MINIZ_NO_TIME - (void)pFilename; *pDOS_date = *pDOS_time = 0; -#else - struct MZ_FILE_STAT_STRUCT file_stat; if (MZ_FILE_STAT(pFilename, &file_stat) != 0) return MZ_FALSE; - mz_zip_time_to_dos_time(file_stat.st_mtime, pDOS_time, pDOS_date); -#endif // #ifdef MINIZ_NO_TIME - return MZ_TRUE; -} - -static mz_bool mz_zip_set_file_times(const char *pFilename, time_t access_time, time_t modified_time) -{ -#ifndef MINIZ_NO_TIME - struct utimbuf t; t.actime = access_time; t.modtime = modified_time; - return !utime(pFilename, &t); -#else - pFilename, access_time, modified_time; - return MZ_TRUE; -#endif // #ifndef MINIZ_NO_TIME -} -#endif - -static mz_bool mz_zip_reader_init_internal(mz_zip_archive *pZip, mz_uint32 flags) -{ - (void)flags; - if ((!pZip) || (pZip->m_pState) || (pZip->m_zip_mode != MZ_ZIP_MODE_INVALID)) - return MZ_FALSE; - - if (!pZip->m_pAlloc) pZip->m_pAlloc = def_alloc_func; - if (!pZip->m_pFree) pZip->m_pFree = def_free_func; - if (!pZip->m_pRealloc) pZip->m_pRealloc = def_realloc_func; - - pZip->m_zip_mode = MZ_ZIP_MODE_READING; - pZip->m_archive_size = 0; - pZip->m_central_directory_file_ofs = 0; - pZip->m_total_files = 0; - - if (NULL == (pZip->m_pState = (mz_zip_internal_state *)pZip->m_pAlloc(pZip->m_pAlloc_opaque, 1, sizeof(mz_zip_internal_state)))) - return MZ_FALSE; - memset(pZip->m_pState, 0, sizeof(mz_zip_internal_state)); - MZ_ZIP_ARRAY_SET_ELEMENT_SIZE(&pZip->m_pState->m_central_dir, sizeof(mz_uint8)); - MZ_ZIP_ARRAY_SET_ELEMENT_SIZE(&pZip->m_pState->m_central_dir_offsets, sizeof(mz_uint32)); - MZ_ZIP_ARRAY_SET_ELEMENT_SIZE(&pZip->m_pState->m_sorted_central_dir_offsets, sizeof(mz_uint32)); - return MZ_TRUE; -} - -static MZ_FORCEINLINE mz_bool mz_zip_reader_filename_less(const mz_zip_array *pCentral_dir_array, const mz_zip_array *pCentral_dir_offsets, mz_uint l_index, mz_uint r_index) -{ - const mz_uint8 *pL = &MZ_ZIP_ARRAY_ELEMENT(pCentral_dir_array, mz_uint8, MZ_ZIP_ARRAY_ELEMENT(pCentral_dir_offsets, mz_uint32, l_index)), *pE; - const mz_uint8 *pR = &MZ_ZIP_ARRAY_ELEMENT(pCentral_dir_array, mz_uint8, MZ_ZIP_ARRAY_ELEMENT(pCentral_dir_offsets, mz_uint32, r_index)); - mz_uint l_len = MZ_READ_LE16(pL + MZ_ZIP_CDH_FILENAME_LEN_OFS), r_len = MZ_READ_LE16(pR + MZ_ZIP_CDH_FILENAME_LEN_OFS); - mz_uint8 l = 0, r = 0; - pL += MZ_ZIP_CENTRAL_DIR_HEADER_SIZE; pR += MZ_ZIP_CENTRAL_DIR_HEADER_SIZE; - pE = pL + MZ_MIN(l_len, r_len); - while (pL < pE) - { - if ((l = MZ_TOLOWER(*pL)) != (r = MZ_TOLOWER(*pR))) - break; - pL++; pR++; - } - return (pL == pE) ? (l_len < r_len) : (l < r); -} - -#define MZ_SWAP_UINT32(a, b) do { mz_uint32 t = a; a = b; b = t; } MZ_MACRO_END - -// Heap sort of lowercased filenames, used to help accelerate plain central directory searches by mz_zip_reader_locate_file(). (Could also use qsort(), but it could allocate memory.) -static void mz_zip_reader_sort_central_dir_offsets_by_filename(mz_zip_archive *pZip) -{ - mz_zip_internal_state *pState = pZip->m_pState; - const mz_zip_array *pCentral_dir_offsets = &pState->m_central_dir_offsets; - const mz_zip_array *pCentral_dir = &pState->m_central_dir; - mz_uint32 *pIndices = &MZ_ZIP_ARRAY_ELEMENT(&pState->m_sorted_central_dir_offsets, mz_uint32, 0); - const int size = pZip->m_total_files; - int start = (size - 2) >> 1, end; - while (start >= 0) - { - int child, root = start; - for ( ; ; ) - { - if ((child = (root << 1) + 1) >= size) - break; - child += (((child + 1) < size) && (mz_zip_reader_filename_less(pCentral_dir, pCentral_dir_offsets, pIndices[child], pIndices[child + 1]))); - if (!mz_zip_reader_filename_less(pCentral_dir, pCentral_dir_offsets, pIndices[root], pIndices[child])) - break; - MZ_SWAP_UINT32(pIndices[root], pIndices[child]); root = child; - } - start--; - } - - end = size - 1; - while (end > 0) - { - int child, root = 0; - MZ_SWAP_UINT32(pIndices[end], pIndices[0]); - for ( ; ; ) - { - if ((child = (root << 1) + 1) >= end) - break; - child += (((child + 1) < end) && mz_zip_reader_filename_less(pCentral_dir, pCentral_dir_offsets, pIndices[child], pIndices[child + 1])); - if (!mz_zip_reader_filename_less(pCentral_dir, pCentral_dir_offsets, pIndices[root], pIndices[child])) - break; - MZ_SWAP_UINT32(pIndices[root], pIndices[child]); root = child; - } - end--; - } -} - -static mz_bool mz_zip_reader_read_central_dir(mz_zip_archive *pZip, mz_uint32 flags) -{ - mz_uint cdir_size, num_this_disk, cdir_disk_index; - mz_uint64 cdir_ofs; - mz_int64 cur_file_ofs; - const mz_uint8 *p; - mz_uint32 buf_u32[4096 / sizeof(mz_uint32)]; mz_uint8 *pBuf = (mz_uint8 *)buf_u32; - // Basic sanity checks - reject files which are too small, and check the first 4 bytes of the file to make sure a local header is there. - if (pZip->m_archive_size < MZ_ZIP_END_OF_CENTRAL_DIR_HEADER_SIZE) - return MZ_FALSE; - // Find the end of central directory record by scanning the file from the end towards the beginning. - cur_file_ofs = MZ_MAX((mz_int64)pZip->m_archive_size - (mz_int64)sizeof(buf_u32), 0); - for ( ; ; ) - { - int i, n = (int)MZ_MIN(sizeof(buf_u32), pZip->m_archive_size - cur_file_ofs); - if (pZip->m_pRead(pZip->m_pIO_opaque, cur_file_ofs, pBuf, n) != (mz_uint)n) - return MZ_FALSE; - for (i = n - 4; i >= 0; --i) - if (MZ_READ_LE32(pBuf + i) == MZ_ZIP_END_OF_CENTRAL_DIR_HEADER_SIG) - break; - if (i >= 0) - { - cur_file_ofs += i; - break; - } - if ((!cur_file_ofs) || ((pZip->m_archive_size - cur_file_ofs) >= (0xFFFF + MZ_ZIP_END_OF_CENTRAL_DIR_HEADER_SIZE))) - return MZ_FALSE; - cur_file_ofs = MZ_MAX(cur_file_ofs - (sizeof(buf_u32) - 3), 0); - } - // Read and verify the end of central directory record. - if (pZip->m_pRead(pZip->m_pIO_opaque, cur_file_ofs, pBuf, MZ_ZIP_END_OF_CENTRAL_DIR_HEADER_SIZE) != MZ_ZIP_END_OF_CENTRAL_DIR_HEADER_SIZE) - return MZ_FALSE; - if ((MZ_READ_LE32(pBuf + MZ_ZIP_ECDH_SIG_OFS) != MZ_ZIP_END_OF_CENTRAL_DIR_HEADER_SIG) || - ((pZip->m_total_files = MZ_READ_LE16(pBuf + MZ_ZIP_ECDH_CDIR_TOTAL_ENTRIES_OFS)) != MZ_READ_LE16(pBuf + MZ_ZIP_ECDH_CDIR_NUM_ENTRIES_ON_DISK_OFS))) - return MZ_FALSE; - - num_this_disk = MZ_READ_LE16(pBuf + MZ_ZIP_ECDH_NUM_THIS_DISK_OFS); - cdir_disk_index = MZ_READ_LE16(pBuf + MZ_ZIP_ECDH_NUM_DISK_CDIR_OFS); - if (((num_this_disk | cdir_disk_index) != 0) && ((num_this_disk != 1) || (cdir_disk_index != 1))) - return MZ_FALSE; - - if ((cdir_size = MZ_READ_LE32(pBuf + MZ_ZIP_ECDH_CDIR_SIZE_OFS)) < pZip->m_total_files * MZ_ZIP_CENTRAL_DIR_HEADER_SIZE) - return MZ_FALSE; - - cdir_ofs = MZ_READ_LE32(pBuf + MZ_ZIP_ECDH_CDIR_OFS_OFS); - if ((cdir_ofs + (mz_uint64)cdir_size) > pZip->m_archive_size) - return MZ_FALSE; - - pZip->m_central_directory_file_ofs = cdir_ofs; - - if (pZip->m_total_files) - { - mz_uint i, n; - // Read the entire central directory into a heap block, and allocate another heap block to hold the unsorted central dir file record offsets, and another to hold the sorted indices. - if ((!mz_zip_array_resize(pZip, &pZip->m_pState->m_central_dir, cdir_size, MZ_FALSE)) || - (!mz_zip_array_resize(pZip, &pZip->m_pState->m_central_dir_offsets, pZip->m_total_files, MZ_FALSE)) || - (!mz_zip_array_resize(pZip, &pZip->m_pState->m_sorted_central_dir_offsets, pZip->m_total_files, MZ_FALSE))) - return MZ_FALSE; - if (pZip->m_pRead(pZip->m_pIO_opaque, cdir_ofs, pZip->m_pState->m_central_dir.m_p, cdir_size) != cdir_size) - return MZ_FALSE; - - // Now create an index into the central directory file records, do some basic sanity checking on each record, and check for zip64 entries (which are not yet supported). - p = (const mz_uint8 *)pZip->m_pState->m_central_dir.m_p; - for (n = cdir_size, i = 0; i < pZip->m_total_files; ++i) - { - mz_uint total_header_size, comp_size, decomp_size, disk_index; - if ((n < MZ_ZIP_CENTRAL_DIR_HEADER_SIZE) || (MZ_READ_LE32(p) != MZ_ZIP_CENTRAL_DIR_HEADER_SIG)) - return MZ_FALSE; - MZ_ZIP_ARRAY_ELEMENT(&pZip->m_pState->m_central_dir_offsets, mz_uint32, i) = (mz_uint32)(p - (const mz_uint8 *)pZip->m_pState->m_central_dir.m_p); - MZ_ZIP_ARRAY_ELEMENT(&pZip->m_pState->m_sorted_central_dir_offsets, mz_uint32, i) = i; - comp_size = MZ_READ_LE32(p + MZ_ZIP_CDH_COMPRESSED_SIZE_OFS); - decomp_size = MZ_READ_LE32(p + MZ_ZIP_CDH_DECOMPRESSED_SIZE_OFS); - if (((!MZ_READ_LE32(p + MZ_ZIP_CDH_METHOD_OFS)) && (decomp_size != comp_size)) || (decomp_size && !comp_size) || (decomp_size == 0xFFFFFFFF) || (comp_size == 0xFFFFFFFF)) - return MZ_FALSE; - disk_index = MZ_READ_LE16(p + MZ_ZIP_CDH_DISK_START_OFS); - if ((disk_index != num_this_disk) && (disk_index != 1)) - return MZ_FALSE; - if (((mz_uint64)MZ_READ_LE32(p + MZ_ZIP_CDH_LOCAL_HEADER_OFS) + MZ_ZIP_LOCAL_DIR_HEADER_SIZE + comp_size) > pZip->m_archive_size) - return MZ_FALSE; - if ((total_header_size = MZ_ZIP_CENTRAL_DIR_HEADER_SIZE + MZ_READ_LE16(p + MZ_ZIP_CDH_FILENAME_LEN_OFS) + MZ_READ_LE16(p + MZ_ZIP_CDH_EXTRA_LEN_OFS) + MZ_READ_LE16(p + MZ_ZIP_CDH_COMMENT_LEN_OFS)) > n) - return MZ_FALSE; - n -= total_header_size; p += total_header_size; - } - } - - if ((flags & MZ_ZIP_FLAG_DO_NOT_SORT_CENTRAL_DIRECTORY) == 0) - mz_zip_reader_sort_central_dir_offsets_by_filename(pZip); - - return MZ_TRUE; -} - -mz_bool mz_zip_reader_init(mz_zip_archive *pZip, mz_uint64 size, mz_uint32 flags) -{ - if ((!pZip) || (!pZip->m_pRead)) - return MZ_FALSE; - if (!mz_zip_reader_init_internal(pZip, flags)) - return MZ_FALSE; - pZip->m_archive_size = size; - if (!mz_zip_reader_read_central_dir(pZip, flags)) - { - mz_zip_reader_end(pZip); - return MZ_FALSE; - } - return MZ_TRUE; -} - -static size_t mz_zip_mem_read_func(void *pOpaque, mz_uint64 file_ofs, void *pBuf, size_t n) -{ - mz_zip_archive *pZip = (mz_zip_archive *)pOpaque; - size_t s = (file_ofs >= pZip->m_archive_size) ? 0 : (size_t)MZ_MIN(pZip->m_archive_size - file_ofs, n); - memcpy(pBuf, (const mz_uint8 *)pZip->m_pState->m_pMem + file_ofs, s); - return s; -} - -mz_bool mz_zip_reader_init_mem(mz_zip_archive *pZip, const void *pMem, size_t size, mz_uint32 flags) -{ - if (!mz_zip_reader_init_internal(pZip, flags)) - return MZ_FALSE; - pZip->m_archive_size = size; - pZip->m_pRead = mz_zip_mem_read_func; - pZip->m_pIO_opaque = pZip; - pZip->m_pState->m_pMem = (void *)pMem; - pZip->m_pState->m_mem_size = size; - if (!mz_zip_reader_read_central_dir(pZip, flags)) - { - mz_zip_reader_end(pZip); - return MZ_FALSE; - } - return MZ_TRUE; -} - -#ifndef MINIZ_NO_STDIO -static size_t mz_zip_file_read_func(void *pOpaque, mz_uint64 file_ofs, void *pBuf, size_t n) -{ - mz_zip_archive *pZip = (mz_zip_archive *)pOpaque; - mz_int64 cur_ofs = MZ_FTELL64(pZip->m_pState->m_pFile); - if (((mz_int64)file_ofs < 0) || (((cur_ofs != (mz_int64)file_ofs)) && (MZ_FSEEK64(pZip->m_pState->m_pFile, (mz_int64)file_ofs, SEEK_SET)))) - return 0; - return MZ_FREAD(pBuf, 1, n, pZip->m_pState->m_pFile); -} - -mz_bool mz_zip_reader_init_file(mz_zip_archive *pZip, const char *pFilename, mz_uint32 flags) -{ - mz_uint64 file_size; - MZ_FILE *pFile = MZ_FOPEN(pFilename, "rb"); - if (!pFile) - return MZ_FALSE; - if (MZ_FSEEK64(pFile, 0, SEEK_END)) - return MZ_FALSE; - file_size = MZ_FTELL64(pFile); - if (!mz_zip_reader_init_internal(pZip, flags)) - { - MZ_FCLOSE(pFile); - return MZ_FALSE; - } - pZip->m_pRead = mz_zip_file_read_func; - pZip->m_pIO_opaque = pZip; - pZip->m_pState->m_pFile = pFile; - pZip->m_archive_size = file_size; - if (!mz_zip_reader_read_central_dir(pZip, flags)) - { - mz_zip_reader_end(pZip); - return MZ_FALSE; - } - return MZ_TRUE; -} -#endif // #ifndef MINIZ_NO_STDIO - -mz_uint mz_zip_reader_get_num_files(mz_zip_archive *pZip) -{ - return pZip ? pZip->m_total_files : 0; -} - -static MZ_FORCEINLINE const mz_uint8 *mz_zip_reader_get_cdh(mz_zip_archive *pZip, mz_uint file_index) -{ - if ((!pZip) || (!pZip->m_pState) || (file_index >= pZip->m_total_files) || (pZip->m_zip_mode != MZ_ZIP_MODE_READING)) - return NULL; - return &MZ_ZIP_ARRAY_ELEMENT(&pZip->m_pState->m_central_dir, mz_uint8, MZ_ZIP_ARRAY_ELEMENT(&pZip->m_pState->m_central_dir_offsets, mz_uint32, file_index)); -} - -mz_bool mz_zip_reader_is_file_encrypted(mz_zip_archive *pZip, mz_uint file_index) -{ - mz_uint m_bit_flag; - const mz_uint8 *p = mz_zip_reader_get_cdh(pZip, file_index); - if (!p) - return MZ_FALSE; - m_bit_flag = MZ_READ_LE16(p + MZ_ZIP_CDH_BIT_FLAG_OFS); - return (m_bit_flag & 1); -} - -mz_bool mz_zip_reader_is_file_a_directory(mz_zip_archive *pZip, mz_uint file_index) -{ - mz_uint filename_len, internal_attr, external_attr; - const mz_uint8 *p = mz_zip_reader_get_cdh(pZip, file_index); - if (!p) - return MZ_FALSE; - - internal_attr = MZ_READ_LE16(p + MZ_ZIP_CDH_INTERNAL_ATTR_OFS); - external_attr = MZ_READ_LE32(p + MZ_ZIP_CDH_EXTERNAL_ATTR_OFS); - if ((!internal_attr) && ((external_attr & 0x10) != 0)) - return MZ_TRUE; - - filename_len = MZ_READ_LE16(p + MZ_ZIP_CDH_FILENAME_LEN_OFS); - if (filename_len) - { - if (*(p + MZ_ZIP_CENTRAL_DIR_HEADER_SIZE + filename_len - 1) == '/') - return MZ_TRUE; - } - - return MZ_FALSE; -} - -mz_bool mz_zip_reader_file_stat(mz_zip_archive *pZip, mz_uint file_index, mz_zip_archive_file_stat *pStat) -{ - mz_uint n; - const mz_uint8 *p = mz_zip_reader_get_cdh(pZip, file_index); - if ((!p) || (!pStat)) - return MZ_FALSE; - - // Unpack the central directory record. - pStat->m_file_index = file_index; - pStat->m_central_dir_ofs = MZ_ZIP_ARRAY_ELEMENT(&pZip->m_pState->m_central_dir_offsets, mz_uint32, file_index); - pStat->m_version_made_by = MZ_READ_LE16(p + MZ_ZIP_CDH_VERSION_MADE_BY_OFS); - pStat->m_version_needed = MZ_READ_LE16(p + MZ_ZIP_CDH_VERSION_NEEDED_OFS); - pStat->m_bit_flag = MZ_READ_LE16(p + MZ_ZIP_CDH_BIT_FLAG_OFS); - pStat->m_method = MZ_READ_LE16(p + MZ_ZIP_CDH_METHOD_OFS); -#ifndef MINIZ_NO_TIME - pStat->m_time = mz_zip_dos_to_time_t(MZ_READ_LE16(p + MZ_ZIP_CDH_FILE_TIME_OFS), MZ_READ_LE16(p + MZ_ZIP_CDH_FILE_DATE_OFS)); -#endif - pStat->m_crc32 = MZ_READ_LE32(p + MZ_ZIP_CDH_CRC32_OFS); - pStat->m_comp_size = MZ_READ_LE32(p + MZ_ZIP_CDH_COMPRESSED_SIZE_OFS); - pStat->m_uncomp_size = MZ_READ_LE32(p + MZ_ZIP_CDH_DECOMPRESSED_SIZE_OFS); - pStat->m_internal_attr = MZ_READ_LE16(p + MZ_ZIP_CDH_INTERNAL_ATTR_OFS); - pStat->m_external_attr = MZ_READ_LE32(p + MZ_ZIP_CDH_EXTERNAL_ATTR_OFS); - pStat->m_local_header_ofs = MZ_READ_LE32(p + MZ_ZIP_CDH_LOCAL_HEADER_OFS); - - // Copy as much of the filename and comment as possible. - n = MZ_READ_LE16(p + MZ_ZIP_CDH_FILENAME_LEN_OFS); n = MZ_MIN(n, MZ_ZIP_MAX_ARCHIVE_FILENAME_SIZE - 1); - memcpy(pStat->m_filename, p + MZ_ZIP_CENTRAL_DIR_HEADER_SIZE, n); pStat->m_filename[n] = '\0'; - - n = MZ_READ_LE16(p + MZ_ZIP_CDH_COMMENT_LEN_OFS); n = MZ_MIN(n, MZ_ZIP_MAX_ARCHIVE_FILE_COMMENT_SIZE - 1); - pStat->m_comment_size = n; - memcpy(pStat->m_comment, p + MZ_ZIP_CENTRAL_DIR_HEADER_SIZE + MZ_READ_LE16(p + MZ_ZIP_CDH_FILENAME_LEN_OFS) + MZ_READ_LE16(p + MZ_ZIP_CDH_EXTRA_LEN_OFS), n); pStat->m_comment[n] = '\0'; - - return MZ_TRUE; -} - -mz_uint mz_zip_reader_get_filename(mz_zip_archive *pZip, mz_uint file_index, char *pFilename, mz_uint filename_buf_size) -{ - mz_uint n; - const mz_uint8 *p = mz_zip_reader_get_cdh(pZip, file_index); - if (!p) { if (filename_buf_size) pFilename[0] = '\0'; return 0; } - n = MZ_READ_LE16(p + MZ_ZIP_CDH_FILENAME_LEN_OFS); - if (filename_buf_size) - { - n = MZ_MIN(n, filename_buf_size - 1); - memcpy(pFilename, p + MZ_ZIP_CENTRAL_DIR_HEADER_SIZE, n); - pFilename[n] = '\0'; - } - return n + 1; -} - -static MZ_FORCEINLINE mz_bool mz_zip_reader_string_equal(const char *pA, const char *pB, mz_uint len, mz_uint flags) -{ - mz_uint i; - if (flags & MZ_ZIP_FLAG_CASE_SENSITIVE) - return 0 == memcmp(pA, pB, len); - for (i = 0; i < len; ++i) - if (MZ_TOLOWER(pA[i]) != MZ_TOLOWER(pB[i])) - return MZ_FALSE; - return MZ_TRUE; -} - -static MZ_FORCEINLINE int mz_zip_reader_filename_compare(const mz_zip_array *pCentral_dir_array, const mz_zip_array *pCentral_dir_offsets, mz_uint l_index, const char *pR, mz_uint r_len) -{ - const mz_uint8 *pL = &MZ_ZIP_ARRAY_ELEMENT(pCentral_dir_array, mz_uint8, MZ_ZIP_ARRAY_ELEMENT(pCentral_dir_offsets, mz_uint32, l_index)), *pE; - mz_uint l_len = MZ_READ_LE16(pL + MZ_ZIP_CDH_FILENAME_LEN_OFS); - mz_uint8 l = 0, r = 0; - pL += MZ_ZIP_CENTRAL_DIR_HEADER_SIZE; - pE = pL + MZ_MIN(l_len, r_len); - while (pL < pE) - { - if ((l = MZ_TOLOWER(*pL)) != (r = MZ_TOLOWER(*pR))) - break; - pL++; pR++; - } - return (pL == pE) ? (int)(l_len - r_len) : (l - r); -} - -static int mz_zip_reader_locate_file_binary_search(mz_zip_archive *pZip, const char *pFilename) -{ - mz_zip_internal_state *pState = pZip->m_pState; - const mz_zip_array *pCentral_dir_offsets = &pState->m_central_dir_offsets; - const mz_zip_array *pCentral_dir = &pState->m_central_dir; - mz_uint32 *pIndices = &MZ_ZIP_ARRAY_ELEMENT(&pState->m_sorted_central_dir_offsets, mz_uint32, 0); - const int size = pZip->m_total_files; - const mz_uint filename_len = (mz_uint)strlen(pFilename); - int l = 0, h = size - 1; - while (l <= h) - { - int m = (l + h) >> 1, file_index = pIndices[m], comp = mz_zip_reader_filename_compare(pCentral_dir, pCentral_dir_offsets, file_index, pFilename, filename_len); - if (!comp) - return file_index; - else if (comp < 0) - l = m + 1; - else - h = m - 1; - } - return -1; -} - -int mz_zip_reader_locate_file(mz_zip_archive *pZip, const char *pName, const char *pComment, mz_uint flags) -{ - mz_uint file_index; size_t name_len, comment_len; - if ((!pZip) || (!pZip->m_pState) || (!pName) || (pZip->m_zip_mode != MZ_ZIP_MODE_READING)) - return -1; - if (((flags & (MZ_ZIP_FLAG_IGNORE_PATH | MZ_ZIP_FLAG_CASE_SENSITIVE)) == 0) && (!pComment) && (pZip->m_pState->m_sorted_central_dir_offsets.m_p)) - return mz_zip_reader_locate_file_binary_search(pZip, pName); - name_len = strlen(pName); if (name_len > 0xFFFF) return -1; - comment_len = pComment ? strlen(pComment) : 0; if (comment_len > 0xFFFF) return -1; - for (file_index = 0; file_index < pZip->m_total_files; file_index++) - { - const mz_uint8 *pHeader = &MZ_ZIP_ARRAY_ELEMENT(&pZip->m_pState->m_central_dir, mz_uint8, MZ_ZIP_ARRAY_ELEMENT(&pZip->m_pState->m_central_dir_offsets, mz_uint32, file_index)); - mz_uint filename_len = MZ_READ_LE16(pHeader + MZ_ZIP_CDH_FILENAME_LEN_OFS); - const char *pFilename = (const char *)pHeader + MZ_ZIP_CENTRAL_DIR_HEADER_SIZE; - if (filename_len < name_len) - continue; - if (comment_len) - { - mz_uint file_extra_len = MZ_READ_LE16(pHeader + MZ_ZIP_CDH_EXTRA_LEN_OFS), file_comment_len = MZ_READ_LE16(pHeader + MZ_ZIP_CDH_COMMENT_LEN_OFS); - const char *pFile_comment = pFilename + filename_len + file_extra_len; - if ((file_comment_len != comment_len) || (!mz_zip_reader_string_equal(pComment, pFile_comment, file_comment_len, flags))) - continue; - } - if ((flags & MZ_ZIP_FLAG_IGNORE_PATH) && (filename_len)) - { - int ofs = filename_len - 1; - do - { - if ((pFilename[ofs] == '/') || (pFilename[ofs] == '\\') || (pFilename[ofs] == ':')) - break; - } while (--ofs >= 0); - ofs++; - pFilename += ofs; filename_len -= ofs; - } - if ((filename_len == name_len) && (mz_zip_reader_string_equal(pName, pFilename, filename_len, flags))) - return file_index; - } - return -1; -} - -mz_bool mz_zip_reader_extract_to_mem_no_alloc(mz_zip_archive *pZip, mz_uint file_index, void *pBuf, size_t buf_size, mz_uint flags, void *pUser_read_buf, size_t user_read_buf_size) -{ - int status = TINFL_STATUS_DONE; - mz_uint64 needed_size, cur_file_ofs, comp_remaining, out_buf_ofs = 0, read_buf_size, read_buf_ofs = 0, read_buf_avail; - mz_zip_archive_file_stat file_stat; - void *pRead_buf; - mz_uint32 local_header_u32[(MZ_ZIP_LOCAL_DIR_HEADER_SIZE + sizeof(mz_uint32) - 1) / sizeof(mz_uint32)]; mz_uint8 *pLocal_header = (mz_uint8 *)local_header_u32; - tinfl_decompressor inflator; - - if ((buf_size) && (!pBuf)) - return MZ_FALSE; - - if (!mz_zip_reader_file_stat(pZip, file_index, &file_stat)) - return MZ_FALSE; - - if (!file_stat.m_comp_size) - return MZ_TRUE; - - // Encryption and patch files are not supported. - if (file_stat.m_bit_flag & (1 | 32)) - return MZ_FALSE; - - // This function only supports stored and deflate. - if ((!(flags & MZ_ZIP_FLAG_COMPRESSED_DATA)) && (file_stat.m_method != 0) && (file_stat.m_method != MZ_DEFLATED)) - return MZ_FALSE; - - // Ensure supplied output buffer is large enough. - needed_size = (flags & MZ_ZIP_FLAG_COMPRESSED_DATA) ? file_stat.m_comp_size : file_stat.m_uncomp_size; - if (buf_size < needed_size) - return MZ_FALSE; - - // Read and parse the local directory entry. - cur_file_ofs = file_stat.m_local_header_ofs; - if (pZip->m_pRead(pZip->m_pIO_opaque, cur_file_ofs, pLocal_header, MZ_ZIP_LOCAL_DIR_HEADER_SIZE) != MZ_ZIP_LOCAL_DIR_HEADER_SIZE) - return MZ_FALSE; - if (MZ_READ_LE32(pLocal_header) != MZ_ZIP_LOCAL_DIR_HEADER_SIG) - return MZ_FALSE; - - cur_file_ofs += MZ_ZIP_LOCAL_DIR_HEADER_SIZE + MZ_READ_LE16(pLocal_header + MZ_ZIP_LDH_FILENAME_LEN_OFS) + MZ_READ_LE16(pLocal_header + MZ_ZIP_LDH_EXTRA_LEN_OFS); - if ((cur_file_ofs + file_stat.m_comp_size) > pZip->m_archive_size) - return MZ_FALSE; - - if ((flags & MZ_ZIP_FLAG_COMPRESSED_DATA) || (!file_stat.m_method)) - { - // The file is stored or the caller has requested the compressed data. - if (pZip->m_pRead(pZip->m_pIO_opaque, cur_file_ofs, pBuf, (size_t)needed_size) != needed_size) - return MZ_FALSE; - return ((flags & MZ_ZIP_FLAG_COMPRESSED_DATA) != 0) || (mz_crc32(MZ_CRC32_INIT, (const mz_uint8 *)pBuf, (size_t)file_stat.m_uncomp_size) == file_stat.m_crc32); - } - - // Decompress the file either directly from memory or from a file input buffer. - tinfl_init(&inflator); - - if (pZip->m_pState->m_pMem) - { - // Read directly from the archive in memory. - pRead_buf = (mz_uint8 *)pZip->m_pState->m_pMem + cur_file_ofs; - read_buf_size = read_buf_avail = file_stat.m_comp_size; - comp_remaining = 0; - } - else if (pUser_read_buf) - { - // Use a user provided read buffer. - if (!user_read_buf_size) - return MZ_FALSE; - pRead_buf = (mz_uint8 *)pUser_read_buf; - read_buf_size = user_read_buf_size; - read_buf_avail = 0; - comp_remaining = file_stat.m_uncomp_size; - } - else - { - // Temporarily allocate a read buffer. - read_buf_size = MZ_MIN(file_stat.m_comp_size, MZ_ZIP_MAX_IO_BUF_SIZE); -#ifdef _MSC_VER - if (((0, sizeof(size_t) == sizeof(mz_uint32))) && (read_buf_size > 0x7FFFFFFF)) -#else - if (((sizeof(size_t) == sizeof(mz_uint32))) && (read_buf_size > 0x7FFFFFFF)) -#endif - return MZ_FALSE; - if (NULL == (pRead_buf = pZip->m_pAlloc(pZip->m_pAlloc_opaque, 1, (size_t)read_buf_size))) - return MZ_FALSE; - read_buf_avail = 0; - comp_remaining = file_stat.m_comp_size; - } - - do - { - size_t in_buf_size, out_buf_size = (size_t)(file_stat.m_uncomp_size - out_buf_ofs); - if ((!read_buf_avail) && (!pZip->m_pState->m_pMem)) - { - read_buf_avail = MZ_MIN(read_buf_size, comp_remaining); - if (pZip->m_pRead(pZip->m_pIO_opaque, cur_file_ofs, pRead_buf, (size_t)read_buf_avail) != read_buf_avail) - { - status = TINFL_STATUS_FAILED; - break; - } - cur_file_ofs += read_buf_avail; - comp_remaining -= read_buf_avail; - read_buf_ofs = 0; - } - in_buf_size = (size_t)read_buf_avail; - status = tinfl_decompress(&inflator, (mz_uint8 *)pRead_buf + read_buf_ofs, &in_buf_size, (mz_uint8 *)pBuf, (mz_uint8 *)pBuf + out_buf_ofs, &out_buf_size, TINFL_FLAG_USING_NON_WRAPPING_OUTPUT_BUF | (comp_remaining ? TINFL_FLAG_HAS_MORE_INPUT : 0)); - read_buf_avail -= in_buf_size; - read_buf_ofs += in_buf_size; - out_buf_ofs += out_buf_size; - } while (status == TINFL_STATUS_NEEDS_MORE_INPUT); - - if (status == TINFL_STATUS_DONE) - { - // Make sure the entire file was decompressed, and check its CRC. - if ((out_buf_ofs != file_stat.m_uncomp_size) || (mz_crc32(MZ_CRC32_INIT, (const mz_uint8 *)pBuf, (size_t)file_stat.m_uncomp_size) != file_stat.m_crc32)) - status = TINFL_STATUS_FAILED; - } - - if ((!pZip->m_pState->m_pMem) && (!pUser_read_buf)) - pZip->m_pFree(pZip->m_pAlloc_opaque, pRead_buf); - - return status == TINFL_STATUS_DONE; -} - -mz_bool mz_zip_reader_extract_file_to_mem_no_alloc(mz_zip_archive *pZip, const char *pFilename, void *pBuf, size_t buf_size, mz_uint flags, void *pUser_read_buf, size_t user_read_buf_size) -{ - int file_index = mz_zip_reader_locate_file(pZip, pFilename, NULL, flags); - if (file_index < 0) - return MZ_FALSE; - return mz_zip_reader_extract_to_mem_no_alloc(pZip, file_index, pBuf, buf_size, flags, pUser_read_buf, user_read_buf_size); -} - -mz_bool mz_zip_reader_extract_to_mem(mz_zip_archive *pZip, mz_uint file_index, void *pBuf, size_t buf_size, mz_uint flags) -{ - return mz_zip_reader_extract_to_mem_no_alloc(pZip, file_index, pBuf, buf_size, flags, NULL, 0); -} - -mz_bool mz_zip_reader_extract_file_to_mem(mz_zip_archive *pZip, const char *pFilename, void *pBuf, size_t buf_size, mz_uint flags) -{ - return mz_zip_reader_extract_file_to_mem_no_alloc(pZip, pFilename, pBuf, buf_size, flags, NULL, 0); -} - -void *mz_zip_reader_extract_to_heap(mz_zip_archive *pZip, mz_uint file_index, size_t *pSize, mz_uint flags) -{ - mz_uint64 comp_size, uncomp_size, alloc_size; - const mz_uint8 *p = mz_zip_reader_get_cdh(pZip, file_index); - void *pBuf; - - if (pSize) - *pSize = 0; - if (!p) - return NULL; - - comp_size = MZ_READ_LE32(p + MZ_ZIP_CDH_COMPRESSED_SIZE_OFS); - uncomp_size = MZ_READ_LE32(p + MZ_ZIP_CDH_DECOMPRESSED_SIZE_OFS); - - alloc_size = (flags & MZ_ZIP_FLAG_COMPRESSED_DATA) ? comp_size : uncomp_size; -#ifdef _MSC_VER - if (((0, sizeof(size_t) == sizeof(mz_uint32))) && (alloc_size > 0x7FFFFFFF)) -#else - if (((sizeof(size_t) == sizeof(mz_uint32))) && (alloc_size > 0x7FFFFFFF)) -#endif - return NULL; - if (NULL == (pBuf = pZip->m_pAlloc(pZip->m_pAlloc_opaque, 1, (size_t)alloc_size))) - return NULL; - - if (!mz_zip_reader_extract_to_mem(pZip, file_index, pBuf, (size_t)alloc_size, flags)) - { - pZip->m_pFree(pZip->m_pAlloc_opaque, pBuf); - return NULL; - } - - if (pSize) *pSize = (size_t)alloc_size; - return pBuf; -} - -void *mz_zip_reader_extract_file_to_heap(mz_zip_archive *pZip, const char *pFilename, size_t *pSize, mz_uint flags) -{ - int file_index = mz_zip_reader_locate_file(pZip, pFilename, NULL, flags); - if (file_index < 0) - { - if (pSize) *pSize = 0; - return MZ_FALSE; - } - return mz_zip_reader_extract_to_heap(pZip, file_index, pSize, flags); -} - -mz_bool mz_zip_reader_extract_to_callback(mz_zip_archive *pZip, mz_uint file_index, mz_file_write_func pCallback, void *pOpaque, mz_uint flags) -{ - int status = TINFL_STATUS_DONE; mz_uint file_crc32 = MZ_CRC32_INIT; - mz_uint64 read_buf_size, read_buf_ofs = 0, read_buf_avail, comp_remaining, out_buf_ofs = 0, cur_file_ofs; - mz_zip_archive_file_stat file_stat; - void *pRead_buf = NULL; void *pWrite_buf = NULL; - mz_uint32 local_header_u32[(MZ_ZIP_LOCAL_DIR_HEADER_SIZE + sizeof(mz_uint32) - 1) / sizeof(mz_uint32)]; mz_uint8 *pLocal_header = (mz_uint8 *)local_header_u32; - - if (!mz_zip_reader_file_stat(pZip, file_index, &file_stat)) - return MZ_FALSE; - - if (!file_stat.m_comp_size) - return MZ_TRUE; - - // Encryption and patch files are not supported. - if (file_stat.m_bit_flag & (1 | 32)) - return MZ_FALSE; - - // This function only supports stored and deflate. - if ((!(flags & MZ_ZIP_FLAG_COMPRESSED_DATA)) && (file_stat.m_method != 0) && (file_stat.m_method != MZ_DEFLATED)) - return MZ_FALSE; - - // Read and parse the local directory entry. - cur_file_ofs = file_stat.m_local_header_ofs; - if (pZip->m_pRead(pZip->m_pIO_opaque, cur_file_ofs, pLocal_header, MZ_ZIP_LOCAL_DIR_HEADER_SIZE) != MZ_ZIP_LOCAL_DIR_HEADER_SIZE) - return MZ_FALSE; - if (MZ_READ_LE32(pLocal_header) != MZ_ZIP_LOCAL_DIR_HEADER_SIG) - return MZ_FALSE; - - cur_file_ofs += MZ_ZIP_LOCAL_DIR_HEADER_SIZE + MZ_READ_LE16(pLocal_header + MZ_ZIP_LDH_FILENAME_LEN_OFS) + MZ_READ_LE16(pLocal_header + MZ_ZIP_LDH_EXTRA_LEN_OFS); - if ((cur_file_ofs + file_stat.m_comp_size) > pZip->m_archive_size) - return MZ_FALSE; - - // Decompress the file either directly from memory or from a file input buffer. - if (pZip->m_pState->m_pMem) - { - pRead_buf = (mz_uint8 *)pZip->m_pState->m_pMem + cur_file_ofs; - read_buf_size = read_buf_avail = file_stat.m_comp_size; - comp_remaining = 0; - } - else - { - read_buf_size = MZ_MIN(file_stat.m_comp_size, MZ_ZIP_MAX_IO_BUF_SIZE); - if (NULL == (pRead_buf = pZip->m_pAlloc(pZip->m_pAlloc_opaque, 1, (size_t)read_buf_size))) - return MZ_FALSE; - read_buf_avail = 0; - comp_remaining = file_stat.m_comp_size; - } - - if ((flags & MZ_ZIP_FLAG_COMPRESSED_DATA) || (!file_stat.m_method)) - { - // The file is stored or the caller has requested the compressed data. - if (pZip->m_pState->m_pMem) - { -#ifdef _MSC_VER - if (((0, sizeof(size_t) == sizeof(mz_uint32))) && (file_stat.m_comp_size > 0xFFFFFFFF)) -#else - if (((sizeof(size_t) == sizeof(mz_uint32))) && (file_stat.m_comp_size > 0xFFFFFFFF)) -#endif - return MZ_FALSE; - if (pCallback(pOpaque, out_buf_ofs, pRead_buf, (size_t)file_stat.m_comp_size) != file_stat.m_comp_size) - status = TINFL_STATUS_FAILED; - else if (!(flags & MZ_ZIP_FLAG_COMPRESSED_DATA)) - file_crc32 = (mz_uint32)mz_crc32(file_crc32, (const mz_uint8 *)pRead_buf, (size_t)file_stat.m_comp_size); - cur_file_ofs += file_stat.m_comp_size; - out_buf_ofs += file_stat.m_comp_size; - comp_remaining = 0; - } - else - { - while (comp_remaining) - { - read_buf_avail = MZ_MIN(read_buf_size, comp_remaining); - if (pZip->m_pRead(pZip->m_pIO_opaque, cur_file_ofs, pRead_buf, (size_t)read_buf_avail) != read_buf_avail) - { - status = TINFL_STATUS_FAILED; - break; - } - - if (!(flags & MZ_ZIP_FLAG_COMPRESSED_DATA)) - file_crc32 = (mz_uint32)mz_crc32(file_crc32, (const mz_uint8 *)pRead_buf, (size_t)read_buf_avail); - - if (pCallback(pOpaque, out_buf_ofs, pRead_buf, (size_t)read_buf_avail) != read_buf_avail) - { - status = TINFL_STATUS_FAILED; - break; - } - cur_file_ofs += read_buf_avail; - out_buf_ofs += read_buf_avail; - comp_remaining -= read_buf_avail; - } - } - } - else - { - tinfl_decompressor inflator; - tinfl_init(&inflator); - - if (NULL == (pWrite_buf = pZip->m_pAlloc(pZip->m_pAlloc_opaque, 1, TINFL_LZ_DICT_SIZE))) - status = TINFL_STATUS_FAILED; - else - { - do - { - mz_uint8 *pWrite_buf_cur = (mz_uint8 *)pWrite_buf + (out_buf_ofs & (TINFL_LZ_DICT_SIZE - 1)); - size_t in_buf_size, out_buf_size = TINFL_LZ_DICT_SIZE - (out_buf_ofs & (TINFL_LZ_DICT_SIZE - 1)); - if ((!read_buf_avail) && (!pZip->m_pState->m_pMem)) - { - read_buf_avail = MZ_MIN(read_buf_size, comp_remaining); - if (pZip->m_pRead(pZip->m_pIO_opaque, cur_file_ofs, pRead_buf, (size_t)read_buf_avail) != read_buf_avail) - { - status = TINFL_STATUS_FAILED; - break; - } - cur_file_ofs += read_buf_avail; - comp_remaining -= read_buf_avail; - read_buf_ofs = 0; - } - - in_buf_size = (size_t)read_buf_avail; - status = tinfl_decompress(&inflator, (const mz_uint8 *)pRead_buf + read_buf_ofs, &in_buf_size, (mz_uint8 *)pWrite_buf, pWrite_buf_cur, &out_buf_size, comp_remaining ? TINFL_FLAG_HAS_MORE_INPUT : 0); - read_buf_avail -= in_buf_size; - read_buf_ofs += in_buf_size; - - if (out_buf_size) - { - if (pCallback(pOpaque, out_buf_ofs, pWrite_buf_cur, out_buf_size) != out_buf_size) - { - status = TINFL_STATUS_FAILED; - break; - } - file_crc32 = (mz_uint32)mz_crc32(file_crc32, pWrite_buf_cur, out_buf_size); - if ((out_buf_ofs += out_buf_size) > file_stat.m_uncomp_size) - { - status = TINFL_STATUS_FAILED; - break; - } - } - } while ((status == TINFL_STATUS_NEEDS_MORE_INPUT) || (status == TINFL_STATUS_HAS_MORE_OUTPUT)); - } - } - - if ((status == TINFL_STATUS_DONE) && (!(flags & MZ_ZIP_FLAG_COMPRESSED_DATA))) - { - // Make sure the entire file was decompressed, and check its CRC. - if ((out_buf_ofs != file_stat.m_uncomp_size) || (file_crc32 != file_stat.m_crc32)) - status = TINFL_STATUS_FAILED; - } - - if (!pZip->m_pState->m_pMem) - pZip->m_pFree(pZip->m_pAlloc_opaque, pRead_buf); - if (pWrite_buf) - pZip->m_pFree(pZip->m_pAlloc_opaque, pWrite_buf); - - return status == TINFL_STATUS_DONE; -} - -mz_bool mz_zip_reader_extract_file_to_callback(mz_zip_archive *pZip, const char *pFilename, mz_file_write_func pCallback, void *pOpaque, mz_uint flags) -{ - int file_index = mz_zip_reader_locate_file(pZip, pFilename, NULL, flags); - if (file_index < 0) - return MZ_FALSE; - return mz_zip_reader_extract_to_callback(pZip, file_index, pCallback, pOpaque, flags); -} - -#ifndef MINIZ_NO_STDIO -static size_t mz_zip_file_write_callback(void *pOpaque, mz_uint64 ofs, const void *pBuf, size_t n) -{ - (void)ofs; return MZ_FWRITE(pBuf, 1, n, (MZ_FILE*)pOpaque); -} - -mz_bool mz_zip_reader_extract_to_file(mz_zip_archive *pZip, mz_uint file_index, const char *pDst_filename, mz_uint flags) -{ - mz_bool status; - mz_zip_archive_file_stat file_stat; - MZ_FILE *pFile; - if (!mz_zip_reader_file_stat(pZip, file_index, &file_stat)) - return MZ_FALSE; - pFile = MZ_FOPEN(pDst_filename, "wb"); - if (!pFile) - return MZ_FALSE; - status = mz_zip_reader_extract_to_callback(pZip, file_index, mz_zip_file_write_callback, pFile, flags); - if (MZ_FCLOSE(pFile) == EOF) - return MZ_FALSE; -#ifndef MINIZ_NO_TIME - if (status) - mz_zip_set_file_times(pDst_filename, file_stat.m_time, file_stat.m_time); -#endif - return status; -} -#endif // #ifndef MINIZ_NO_STDIO - -mz_bool mz_zip_reader_end(mz_zip_archive *pZip) -{ - if ((!pZip) || (!pZip->m_pState) || (!pZip->m_pAlloc) || (!pZip->m_pFree) || (pZip->m_zip_mode != MZ_ZIP_MODE_READING)) - return MZ_FALSE; - - if (pZip->m_pState) - { - mz_zip_internal_state *pState = pZip->m_pState; pZip->m_pState = NULL; - mz_zip_array_clear(pZip, &pState->m_central_dir); - mz_zip_array_clear(pZip, &pState->m_central_dir_offsets); - mz_zip_array_clear(pZip, &pState->m_sorted_central_dir_offsets); - -#ifndef MINIZ_NO_STDIO - if (pState->m_pFile) - { - MZ_FCLOSE(pState->m_pFile); - pState->m_pFile = NULL; - } -#endif // #ifndef MINIZ_NO_STDIO - - pZip->m_pFree(pZip->m_pAlloc_opaque, pState); - } - pZip->m_zip_mode = MZ_ZIP_MODE_INVALID; - - return MZ_TRUE; -} - -#ifndef MINIZ_NO_STDIO -mz_bool mz_zip_reader_extract_file_to_file(mz_zip_archive *pZip, const char *pArchive_filename, const char *pDst_filename, mz_uint flags) -{ - int file_index = mz_zip_reader_locate_file(pZip, pArchive_filename, NULL, flags); - if (file_index < 0) - return MZ_FALSE; - return mz_zip_reader_extract_to_file(pZip, file_index, pDst_filename, flags); -} -#endif - -// ------------------- .ZIP archive writing - -#ifndef MINIZ_NO_ARCHIVE_WRITING_APIS - -static void mz_write_le16(mz_uint8 *p, mz_uint16 v) { p[0] = (mz_uint8)v; p[1] = (mz_uint8)(v >> 8); } -static void mz_write_le32(mz_uint8 *p, mz_uint32 v) { p[0] = (mz_uint8)v; p[1] = (mz_uint8)(v >> 8); p[2] = (mz_uint8)(v >> 16); p[3] = (mz_uint8)(v >> 24); } -#define MZ_WRITE_LE16(p, v) mz_write_le16((mz_uint8 *)(p), (mz_uint16)(v)) -#define MZ_WRITE_LE32(p, v) mz_write_le32((mz_uint8 *)(p), (mz_uint32)(v)) - -mz_bool mz_zip_writer_init(mz_zip_archive *pZip, mz_uint64 existing_size) -{ - if ((!pZip) || (pZip->m_pState) || (!pZip->m_pWrite) || (pZip->m_zip_mode != MZ_ZIP_MODE_INVALID)) - return MZ_FALSE; - - if (pZip->m_file_offset_alignment) - { - // Ensure user specified file offset alignment is a power of 2. - if (pZip->m_file_offset_alignment & (pZip->m_file_offset_alignment - 1)) - return MZ_FALSE; - } - - if (!pZip->m_pAlloc) pZip->m_pAlloc = def_alloc_func; - if (!pZip->m_pFree) pZip->m_pFree = def_free_func; - if (!pZip->m_pRealloc) pZip->m_pRealloc = def_realloc_func; - - pZip->m_zip_mode = MZ_ZIP_MODE_WRITING; - pZip->m_archive_size = existing_size; - pZip->m_central_directory_file_ofs = 0; - pZip->m_total_files = 0; - - if (NULL == (pZip->m_pState = (mz_zip_internal_state *)pZip->m_pAlloc(pZip->m_pAlloc_opaque, 1, sizeof(mz_zip_internal_state)))) - return MZ_FALSE; - memset(pZip->m_pState, 0, sizeof(mz_zip_internal_state)); - MZ_ZIP_ARRAY_SET_ELEMENT_SIZE(&pZip->m_pState->m_central_dir, sizeof(mz_uint8)); - MZ_ZIP_ARRAY_SET_ELEMENT_SIZE(&pZip->m_pState->m_central_dir_offsets, sizeof(mz_uint32)); - MZ_ZIP_ARRAY_SET_ELEMENT_SIZE(&pZip->m_pState->m_sorted_central_dir_offsets, sizeof(mz_uint32)); - return MZ_TRUE; -} - -static size_t mz_zip_heap_write_func(void *pOpaque, mz_uint64 file_ofs, const void *pBuf, size_t n) -{ - mz_zip_archive *pZip = (mz_zip_archive *)pOpaque; - mz_zip_internal_state *pState = pZip->m_pState; - mz_uint64 new_size = MZ_MAX(file_ofs + n, pState->m_mem_size); -#ifdef _MSC_VER - if ((!n) || ((0, sizeof(size_t) == sizeof(mz_uint32)) && (new_size > 0x7FFFFFFF))) -#else - if ((!n) || ((sizeof(size_t) == sizeof(mz_uint32)) && (new_size > 0x7FFFFFFF))) -#endif - return 0; - if (new_size > pState->m_mem_capacity) - { - void *pNew_block; - size_t new_capacity = MZ_MAX(64, pState->m_mem_capacity); while (new_capacity < new_size) new_capacity *= 2; - if (NULL == (pNew_block = pZip->m_pRealloc(pZip->m_pAlloc_opaque, pState->m_pMem, 1, new_capacity))) - return 0; - pState->m_pMem = pNew_block; pState->m_mem_capacity = new_capacity; - } - memcpy((mz_uint8 *)pState->m_pMem + file_ofs, pBuf, n); - pState->m_mem_size = (size_t)new_size; - return n; -} - -mz_bool mz_zip_writer_init_heap(mz_zip_archive *pZip, size_t size_to_reserve_at_beginning, size_t initial_allocation_size) -{ - pZip->m_pWrite = mz_zip_heap_write_func; - pZip->m_pIO_opaque = pZip; - if (!mz_zip_writer_init(pZip, size_to_reserve_at_beginning)) - return MZ_FALSE; - if (0 != (initial_allocation_size = MZ_MAX(initial_allocation_size, size_to_reserve_at_beginning))) - { - if (NULL == (pZip->m_pState->m_pMem = pZip->m_pAlloc(pZip->m_pAlloc_opaque, 1, initial_allocation_size))) - { - mz_zip_writer_end(pZip); - return MZ_FALSE; - } - pZip->m_pState->m_mem_capacity = initial_allocation_size; - } - return MZ_TRUE; -} - -#ifndef MINIZ_NO_STDIO -static size_t mz_zip_file_write_func(void *pOpaque, mz_uint64 file_ofs, const void *pBuf, size_t n) -{ - mz_zip_archive *pZip = (mz_zip_archive *)pOpaque; - mz_int64 cur_ofs = MZ_FTELL64(pZip->m_pState->m_pFile); - if (((mz_int64)file_ofs < 0) || (((cur_ofs != (mz_int64)file_ofs)) && (MZ_FSEEK64(pZip->m_pState->m_pFile, (mz_int64)file_ofs, SEEK_SET)))) - return 0; - return MZ_FWRITE(pBuf, 1, n, pZip->m_pState->m_pFile); -} - -mz_bool mz_zip_writer_init_file(mz_zip_archive *pZip, const char *pFilename, mz_uint64 size_to_reserve_at_beginning) -{ - MZ_FILE *pFile; - pZip->m_pWrite = mz_zip_file_write_func; - pZip->m_pIO_opaque = pZip; - if (!mz_zip_writer_init(pZip, size_to_reserve_at_beginning)) - return MZ_FALSE; - if (NULL == (pFile = MZ_FOPEN(pFilename, "wb"))) - { - mz_zip_writer_end(pZip); - return MZ_FALSE; - } - pZip->m_pState->m_pFile = pFile; - if (size_to_reserve_at_beginning) - { - mz_uint64 cur_ofs = 0; char buf[4096]; MZ_CLEAR_OBJ(buf); - do - { - size_t n = (size_t)MZ_MIN(sizeof(buf), size_to_reserve_at_beginning); - if (pZip->m_pWrite(pZip->m_pIO_opaque, cur_ofs, buf, n) != n) - { - mz_zip_writer_end(pZip); - return MZ_FALSE; - } - cur_ofs += n; size_to_reserve_at_beginning -= n; - } while (size_to_reserve_at_beginning); - } - return MZ_TRUE; -} -#endif // #ifndef MINIZ_NO_STDIO - -mz_bool mz_zip_writer_init_from_reader(mz_zip_archive *pZip, const char *pFilename) -{ - mz_zip_internal_state *pState; - if ((!pZip) || (!pZip->m_pState) || (pZip->m_zip_mode != MZ_ZIP_MODE_READING)) - return MZ_FALSE; - // No sense in trying to write to an archive that's already at the support max size - if ((pZip->m_total_files == 0xFFFF) || ((pZip->m_archive_size + MZ_ZIP_CENTRAL_DIR_HEADER_SIZE + MZ_ZIP_LOCAL_DIR_HEADER_SIZE) > 0xFFFFFFFF)) - return MZ_FALSE; - - pState = pZip->m_pState; - - if (pState->m_pFile) - { -#ifdef MINIZ_NO_STDIO - pFilename; return MZ_FALSE; -#else - // Archive is being read from stdio - try to reopen as writable. - if (pZip->m_pIO_opaque != pZip) - return MZ_FALSE; - if (!pFilename) - return MZ_FALSE; - pZip->m_pWrite = mz_zip_file_write_func; - if (NULL == (pState->m_pFile = MZ_FREOPEN(pFilename, "r+b", pState->m_pFile))) - { - // The mz_zip_archive is now in a bogus state because pState->m_pFile is NULL, so just close it. - mz_zip_reader_end(pZip); - return MZ_FALSE; - } -#endif // #ifdef MINIZ_NO_STDIO - } - else if (pState->m_pMem) - { - // Archive lives in a memory block. Assume it's from the heap that we can resize using the realloc callback. - if (pZip->m_pIO_opaque != pZip) - return MZ_FALSE; - pState->m_mem_capacity = pState->m_mem_size; - pZip->m_pWrite = mz_zip_heap_write_func; - } - // Archive is being read via a user provided read function - make sure the user has specified a write function too. - else if (!pZip->m_pWrite) - return MZ_FALSE; - - // Start writing new files at the archive's current central directory location. - pZip->m_archive_size = pZip->m_central_directory_file_ofs; - pZip->m_zip_mode = MZ_ZIP_MODE_WRITING; - pZip->m_central_directory_file_ofs = 0; - - return MZ_TRUE; -} - -mz_bool mz_zip_writer_add_mem(mz_zip_archive *pZip, const char *pArchive_name, const void *pBuf, size_t buf_size, mz_uint level_and_flags) -{ - return mz_zip_writer_add_mem_ex(pZip, pArchive_name, pBuf, buf_size, NULL, 0, level_and_flags, 0, 0); -} - -typedef struct -{ - mz_zip_archive *m_pZip; - mz_uint64 m_cur_archive_file_ofs; - mz_uint64 m_comp_size; -} mz_zip_writer_add_state; - -static mz_bool mz_zip_writer_add_put_buf_callback(const void* pBuf, int len, void *pUser) -{ - mz_zip_writer_add_state *pState = (mz_zip_writer_add_state *)pUser; - if ((int)pState->m_pZip->m_pWrite(pState->m_pZip->m_pIO_opaque, pState->m_cur_archive_file_ofs, pBuf, len) != len) - return MZ_FALSE; - pState->m_cur_archive_file_ofs += len; - pState->m_comp_size += len; - return MZ_TRUE; -} - -static mz_bool mz_zip_writer_create_local_dir_header(mz_zip_archive *pZip, mz_uint8 *pDst, mz_uint16 filename_size, mz_uint16 extra_size, mz_uint64 uncomp_size, mz_uint64 comp_size, mz_uint32 uncomp_crc32, mz_uint16 method, mz_uint16 bit_flags, mz_uint16 dos_time, mz_uint16 dos_date) -{ - (void)pZip; - memset(pDst, 0, MZ_ZIP_LOCAL_DIR_HEADER_SIZE); - MZ_WRITE_LE32(pDst + MZ_ZIP_LDH_SIG_OFS, MZ_ZIP_LOCAL_DIR_HEADER_SIG); - MZ_WRITE_LE16(pDst + MZ_ZIP_LDH_VERSION_NEEDED_OFS, method ? 20 : 0); - MZ_WRITE_LE16(pDst + MZ_ZIP_LDH_BIT_FLAG_OFS, bit_flags); - MZ_WRITE_LE16(pDst + MZ_ZIP_LDH_METHOD_OFS, method); - MZ_WRITE_LE16(pDst + MZ_ZIP_LDH_FILE_TIME_OFS, dos_time); - MZ_WRITE_LE16(pDst + MZ_ZIP_LDH_FILE_DATE_OFS, dos_date); - MZ_WRITE_LE32(pDst + MZ_ZIP_LDH_CRC32_OFS, uncomp_crc32); - MZ_WRITE_LE32(pDst + MZ_ZIP_LDH_COMPRESSED_SIZE_OFS, comp_size); - MZ_WRITE_LE32(pDst + MZ_ZIP_LDH_DECOMPRESSED_SIZE_OFS, uncomp_size); - MZ_WRITE_LE16(pDst + MZ_ZIP_LDH_FILENAME_LEN_OFS, filename_size); - MZ_WRITE_LE16(pDst + MZ_ZIP_LDH_EXTRA_LEN_OFS, extra_size); - return MZ_TRUE; -} - -static mz_bool mz_zip_writer_create_central_dir_header(mz_zip_archive *pZip, mz_uint8 *pDst, mz_uint16 filename_size, mz_uint16 extra_size, mz_uint16 comment_size, mz_uint64 uncomp_size, mz_uint64 comp_size, mz_uint32 uncomp_crc32, mz_uint16 method, mz_uint16 bit_flags, mz_uint16 dos_time, mz_uint16 dos_date, mz_uint64 local_header_ofs, mz_uint32 ext_attributes) -{ - (void)pZip; - memset(pDst, 0, MZ_ZIP_CENTRAL_DIR_HEADER_SIZE); - MZ_WRITE_LE32(pDst + MZ_ZIP_CDH_SIG_OFS, MZ_ZIP_CENTRAL_DIR_HEADER_SIG); - MZ_WRITE_LE16(pDst + MZ_ZIP_CDH_VERSION_NEEDED_OFS, method ? 20 : 0); - MZ_WRITE_LE16(pDst + MZ_ZIP_CDH_BIT_FLAG_OFS, bit_flags); - MZ_WRITE_LE16(pDst + MZ_ZIP_CDH_METHOD_OFS, method); - MZ_WRITE_LE16(pDst + MZ_ZIP_CDH_FILE_TIME_OFS, dos_time); - MZ_WRITE_LE16(pDst + MZ_ZIP_CDH_FILE_DATE_OFS, dos_date); - MZ_WRITE_LE32(pDst + MZ_ZIP_CDH_CRC32_OFS, uncomp_crc32); - MZ_WRITE_LE32(pDst + MZ_ZIP_CDH_COMPRESSED_SIZE_OFS, comp_size); - MZ_WRITE_LE32(pDst + MZ_ZIP_CDH_DECOMPRESSED_SIZE_OFS, uncomp_size); - MZ_WRITE_LE16(pDst + MZ_ZIP_CDH_FILENAME_LEN_OFS, filename_size); - MZ_WRITE_LE16(pDst + MZ_ZIP_CDH_EXTRA_LEN_OFS, extra_size); - MZ_WRITE_LE16(pDst + MZ_ZIP_CDH_COMMENT_LEN_OFS, comment_size); - MZ_WRITE_LE32(pDst + MZ_ZIP_CDH_EXTERNAL_ATTR_OFS, ext_attributes); - MZ_WRITE_LE32(pDst + MZ_ZIP_CDH_LOCAL_HEADER_OFS, local_header_ofs); - return MZ_TRUE; -} - -static mz_bool mz_zip_writer_add_to_central_dir(mz_zip_archive *pZip, const char *pFilename, mz_uint16 filename_size, const void *pExtra, mz_uint16 extra_size, const void *pComment, mz_uint16 comment_size, mz_uint64 uncomp_size, mz_uint64 comp_size, mz_uint32 uncomp_crc32, mz_uint16 method, mz_uint16 bit_flags, mz_uint16 dos_time, mz_uint16 dos_date, mz_uint64 local_header_ofs, mz_uint32 ext_attributes) -{ - mz_zip_internal_state *pState = pZip->m_pState; - mz_uint32 central_dir_ofs = (mz_uint32)pState->m_central_dir.m_size; - size_t orig_central_dir_size = pState->m_central_dir.m_size; - mz_uint8 central_dir_header[MZ_ZIP_CENTRAL_DIR_HEADER_SIZE]; - - // No zip64 support yet - if ((local_header_ofs > 0xFFFFFFFF) || (((mz_uint64)pState->m_central_dir.m_size + MZ_ZIP_CENTRAL_DIR_HEADER_SIZE + filename_size + extra_size + comment_size) > 0xFFFFFFFF)) - return MZ_FALSE; - - if (!mz_zip_writer_create_central_dir_header(pZip, central_dir_header, filename_size, extra_size, comment_size, uncomp_size, comp_size, uncomp_crc32, method, bit_flags, dos_time, dos_date, local_header_ofs, ext_attributes)) - return MZ_FALSE; - - if ((!mz_zip_array_push_back(pZip, &pState->m_central_dir, central_dir_header, MZ_ZIP_CENTRAL_DIR_HEADER_SIZE)) || - (!mz_zip_array_push_back(pZip, &pState->m_central_dir, pFilename, filename_size)) || - (!mz_zip_array_push_back(pZip, &pState->m_central_dir, pExtra, extra_size)) || - (!mz_zip_array_push_back(pZip, &pState->m_central_dir, pComment, comment_size)) || - (!mz_zip_array_push_back(pZip, &pState->m_central_dir_offsets, ¢ral_dir_ofs, 1))) - { - // Try to push the central directory array back into its original state. - mz_zip_array_resize(pZip, &pState->m_central_dir, orig_central_dir_size, MZ_FALSE); - return MZ_FALSE; - } - - return MZ_TRUE; -} - -static mz_bool mz_zip_writer_validate_archive_name(const char *pArchive_name) -{ - // Basic ZIP archive filename validity checks: Valid filenames cannot start with a forward slash, cannot contain a drive letter, and cannot use DOS-style backward slashes. - if (*pArchive_name == '/') - return MZ_FALSE; - while (*pArchive_name) - { - if ((*pArchive_name == '\\') || (*pArchive_name == ':')) - return MZ_FALSE; - pArchive_name++; - } - return MZ_TRUE; -} - -static mz_uint mz_zip_writer_compute_padding_needed_for_file_alignment(mz_zip_archive *pZip) -{ - mz_uint32 n; - if (!pZip->m_file_offset_alignment) - return 0; - n = (mz_uint32)(pZip->m_archive_size & (pZip->m_file_offset_alignment - 1)); - return (pZip->m_file_offset_alignment - n) & (pZip->m_file_offset_alignment - 1); -} - -static mz_bool mz_zip_writer_write_zeros(mz_zip_archive *pZip, mz_uint64 cur_file_ofs, mz_uint32 n) -{ - char buf[4096]; - memset(buf, 0, MZ_MIN(sizeof(buf), n)); - while (n) - { - mz_uint32 s = MZ_MIN(sizeof(buf), n); - if (pZip->m_pWrite(pZip->m_pIO_opaque, cur_file_ofs, buf, s) != s) - return MZ_FALSE; - cur_file_ofs += s; n -= s; - } - return MZ_TRUE; -} - -mz_bool mz_zip_writer_add_mem_ex(mz_zip_archive *pZip, const char *pArchive_name, const void *pBuf, size_t buf_size, const void *pComment, mz_uint16 comment_size, mz_uint level_and_flags, mz_uint64 uncomp_size, mz_uint32 uncomp_crc32) -{ - mz_uint16 method = 0, dos_time = 0, dos_date = 0; - mz_uint level, ext_attributes = 0, num_alignment_padding_bytes; - mz_uint64 local_dir_header_ofs = pZip->m_archive_size, cur_archive_file_ofs = pZip->m_archive_size, comp_size = 0; - size_t archive_name_size; - mz_uint8 local_dir_header[MZ_ZIP_LOCAL_DIR_HEADER_SIZE]; - tdefl_compressor *pComp = NULL; - mz_bool store_data_uncompressed; - mz_zip_internal_state *pState; - - if ((int)level_and_flags < 0) - level_and_flags = MZ_DEFAULT_LEVEL; - level = level_and_flags & 0xF; - store_data_uncompressed = ((!level) || (level_and_flags & MZ_ZIP_FLAG_COMPRESSED_DATA)); - - if ((!pZip) || (!pZip->m_pState) || (pZip->m_zip_mode != MZ_ZIP_MODE_WRITING) || ((buf_size) && (!pBuf)) || (!pArchive_name) || ((comment_size) && (!pComment)) || (pZip->m_total_files == 0xFFFF) || (level > MZ_UBER_COMPRESSION)) - return MZ_FALSE; - - pState = pZip->m_pState; - - if ((!(level_and_flags & MZ_ZIP_FLAG_COMPRESSED_DATA)) && (uncomp_size)) - return MZ_FALSE; - // No zip64 support yet - if ((buf_size > 0xFFFFFFFF) || (uncomp_size > 0xFFFFFFFF)) - return MZ_FALSE; - if (!mz_zip_writer_validate_archive_name(pArchive_name)) - return MZ_FALSE; - -#ifndef MINIZ_NO_TIME - { - time_t cur_time; time(&cur_time); - mz_zip_time_to_dos_time(cur_time, &dos_time, &dos_date); - } -#endif // #ifndef MINIZ_NO_TIME - - archive_name_size = strlen(pArchive_name); - if (archive_name_size > 0xFFFF) - return MZ_FALSE; - - num_alignment_padding_bytes = mz_zip_writer_compute_padding_needed_for_file_alignment(pZip); - - // no zip64 support yet - if ((pZip->m_total_files == 0xFFFF) || ((pZip->m_archive_size + num_alignment_padding_bytes + MZ_ZIP_LOCAL_DIR_HEADER_SIZE + MZ_ZIP_CENTRAL_DIR_HEADER_SIZE + comment_size + archive_name_size) > 0xFFFFFFFF)) - return MZ_FALSE; - - if ((archive_name_size) && (pArchive_name[archive_name_size - 1] == '/')) - { - // Set DOS Subdirectory attribute bit. - ext_attributes |= 0x10; - // Subdirectories cannot contain data. - if ((buf_size) || (uncomp_size)) - return MZ_FALSE; - } - - // Try to do any allocations before writing to the archive, so if an allocation fails the file remains unmodified. (A good idea if we're doing an in-place modification.) - if ((!mz_zip_array_ensure_room(pZip, &pState->m_central_dir, MZ_ZIP_CENTRAL_DIR_HEADER_SIZE + archive_name_size + comment_size)) || (!mz_zip_array_ensure_room(pZip, &pState->m_central_dir_offsets, 1))) - return MZ_FALSE; - - if ((!store_data_uncompressed) && (buf_size)) - { - if (NULL == (pComp = (tdefl_compressor *)pZip->m_pAlloc(pZip->m_pAlloc_opaque, 1, sizeof(tdefl_compressor)))) - return MZ_FALSE; - } - - if (!mz_zip_writer_write_zeros(pZip, cur_archive_file_ofs, num_alignment_padding_bytes + sizeof(local_dir_header))) - { - pZip->m_pFree(pZip->m_pAlloc_opaque, pComp); - return MZ_FALSE; - } - local_dir_header_ofs += num_alignment_padding_bytes; - if (pZip->m_file_offset_alignment) { MZ_ASSERT((local_dir_header_ofs & (pZip->m_file_offset_alignment - 1)) == 0); } - cur_archive_file_ofs += num_alignment_padding_bytes + sizeof(local_dir_header); - - MZ_CLEAR_OBJ(local_dir_header); - if (pZip->m_pWrite(pZip->m_pIO_opaque, cur_archive_file_ofs, pArchive_name, archive_name_size) != archive_name_size) - { - pZip->m_pFree(pZip->m_pAlloc_opaque, pComp); - return MZ_FALSE; - } - cur_archive_file_ofs += archive_name_size; - - if (!(level_and_flags & MZ_ZIP_FLAG_COMPRESSED_DATA)) - { - uncomp_crc32 = (mz_uint32)mz_crc32(MZ_CRC32_INIT, (const mz_uint8*)pBuf, buf_size); - uncomp_size = buf_size; - if (uncomp_size <= 3) - { - level = 0; - store_data_uncompressed = MZ_TRUE; - } - } - - if (store_data_uncompressed) - { - if (pZip->m_pWrite(pZip->m_pIO_opaque, cur_archive_file_ofs, pBuf, buf_size) != buf_size) - { - pZip->m_pFree(pZip->m_pAlloc_opaque, pComp); - return MZ_FALSE; - } - - cur_archive_file_ofs += buf_size; - comp_size = buf_size; - - if (level_and_flags & MZ_ZIP_FLAG_COMPRESSED_DATA) - method = MZ_DEFLATED; - } - else if (buf_size) - { - mz_zip_writer_add_state state; - - state.m_pZip = pZip; - state.m_cur_archive_file_ofs = cur_archive_file_ofs; - state.m_comp_size = 0; - - if ((tdefl_init(pComp, mz_zip_writer_add_put_buf_callback, &state, tdefl_create_comp_flags_from_zip_params(level, -15, MZ_DEFAULT_STRATEGY)) != TDEFL_STATUS_OKAY) || - (tdefl_compress_buffer(pComp, pBuf, buf_size, TDEFL_FINISH) != TDEFL_STATUS_DONE)) - { - pZip->m_pFree(pZip->m_pAlloc_opaque, pComp); - return MZ_FALSE; - } - - comp_size = state.m_comp_size; - cur_archive_file_ofs = state.m_cur_archive_file_ofs; - - method = MZ_DEFLATED; - } - - pZip->m_pFree(pZip->m_pAlloc_opaque, pComp); - pComp = NULL; - - // no zip64 support yet - if ((comp_size > 0xFFFFFFFF) || (cur_archive_file_ofs > 0xFFFFFFFF)) - return MZ_FALSE; - - if (!mz_zip_writer_create_local_dir_header(pZip, local_dir_header, (mz_uint16)archive_name_size, 0, uncomp_size, comp_size, uncomp_crc32, method, 0, dos_time, dos_date)) - return MZ_FALSE; - - if (pZip->m_pWrite(pZip->m_pIO_opaque, local_dir_header_ofs, local_dir_header, sizeof(local_dir_header)) != sizeof(local_dir_header)) - return MZ_FALSE; - - if (!mz_zip_writer_add_to_central_dir(pZip, pArchive_name, (mz_uint16)archive_name_size, NULL, 0, pComment, comment_size, uncomp_size, comp_size, uncomp_crc32, method, 0, dos_time, dos_date, local_dir_header_ofs, ext_attributes)) - return MZ_FALSE; - - pZip->m_total_files++; - pZip->m_archive_size = cur_archive_file_ofs; - - return MZ_TRUE; -} - -#ifndef MINIZ_NO_STDIO -mz_bool mz_zip_writer_add_file(mz_zip_archive *pZip, const char *pArchive_name, const char *pSrc_filename, const void *pComment, mz_uint16 comment_size, mz_uint level_and_flags) -{ - mz_uint uncomp_crc32 = MZ_CRC32_INIT, level, num_alignment_padding_bytes; - mz_uint16 method = 0, dos_time = 0, dos_date = 0, ext_attributes = 0; - mz_uint64 local_dir_header_ofs = pZip->m_archive_size, cur_archive_file_ofs = pZip->m_archive_size, uncomp_size = 0, comp_size = 0; - size_t archive_name_size; - mz_uint8 local_dir_header[MZ_ZIP_LOCAL_DIR_HEADER_SIZE]; - MZ_FILE *pSrc_file = NULL; - - if ((int)level_and_flags < 0) - level_and_flags = MZ_DEFAULT_LEVEL; - level = level_and_flags & 0xF; - - if ((!pZip) || (!pZip->m_pState) || (pZip->m_zip_mode != MZ_ZIP_MODE_WRITING) || (!pArchive_name) || ((comment_size) && (!pComment)) || (level > MZ_UBER_COMPRESSION)) - return MZ_FALSE; - if (level_and_flags & MZ_ZIP_FLAG_COMPRESSED_DATA) - return MZ_FALSE; - if (!mz_zip_writer_validate_archive_name(pArchive_name)) - return MZ_FALSE; - - archive_name_size = strlen(pArchive_name); - if (archive_name_size > 0xFFFF) - return MZ_FALSE; - - num_alignment_padding_bytes = mz_zip_writer_compute_padding_needed_for_file_alignment(pZip); - - // no zip64 support yet - if ((pZip->m_total_files == 0xFFFF) || ((pZip->m_archive_size + num_alignment_padding_bytes + MZ_ZIP_LOCAL_DIR_HEADER_SIZE + MZ_ZIP_CENTRAL_DIR_HEADER_SIZE + comment_size + archive_name_size) > 0xFFFFFFFF)) - return MZ_FALSE; - - if (!mz_zip_get_file_modified_time(pSrc_filename, &dos_time, &dos_date)) - return MZ_FALSE; - - pSrc_file = MZ_FOPEN(pSrc_filename, "rb"); - if (!pSrc_file) - return MZ_FALSE; - MZ_FSEEK64(pSrc_file, 0, SEEK_END); - uncomp_size = MZ_FTELL64(pSrc_file); - MZ_FSEEK64(pSrc_file, 0, SEEK_SET); - - if (uncomp_size > 0xFFFFFFFF) - { - // No zip64 support yet - MZ_FCLOSE(pSrc_file); - return MZ_FALSE; - } - if (uncomp_size <= 3) - level = 0; - - if (!mz_zip_writer_write_zeros(pZip, cur_archive_file_ofs, num_alignment_padding_bytes + sizeof(local_dir_header))) - return MZ_FALSE; - local_dir_header_ofs += num_alignment_padding_bytes; - if (pZip->m_file_offset_alignment) { MZ_ASSERT((local_dir_header_ofs & (pZip->m_file_offset_alignment - 1)) == 0); } - cur_archive_file_ofs += num_alignment_padding_bytes + sizeof(local_dir_header); - - MZ_CLEAR_OBJ(local_dir_header); - if (pZip->m_pWrite(pZip->m_pIO_opaque, cur_archive_file_ofs, pArchive_name, archive_name_size) != archive_name_size) - { - MZ_FCLOSE(pSrc_file); - return MZ_FALSE; - } - cur_archive_file_ofs += archive_name_size; - - if (uncomp_size) - { - mz_uint64 uncomp_remaining = uncomp_size; - void *pRead_buf = pZip->m_pAlloc(pZip->m_pAlloc_opaque, 1, MZ_ZIP_MAX_IO_BUF_SIZE); - if (!pRead_buf) - { - MZ_FCLOSE(pSrc_file); - return MZ_FALSE; - } - - if (!level) - { - while (uncomp_remaining) - { - mz_uint n = (mz_uint)MZ_MIN(MZ_ZIP_MAX_IO_BUF_SIZE, uncomp_remaining); - if ((MZ_FREAD(pRead_buf, 1, n, pSrc_file) != n) || (pZip->m_pWrite(pZip->m_pIO_opaque, cur_archive_file_ofs, pRead_buf, n) != n)) - { - pZip->m_pFree(pZip->m_pAlloc_opaque, pRead_buf); - MZ_FCLOSE(pSrc_file); - return MZ_FALSE; - } - uncomp_crc32 = (mz_uint32)mz_crc32(uncomp_crc32, (const mz_uint8 *)pRead_buf, n); - uncomp_remaining -= n; - cur_archive_file_ofs += n; - } - comp_size = uncomp_size; - } - else - { - mz_bool result = MZ_FALSE; - mz_zip_writer_add_state state; - tdefl_compressor *pComp = (tdefl_compressor *)pZip->m_pAlloc(pZip->m_pAlloc_opaque, 1, sizeof(tdefl_compressor)); - if (!pComp) - { - pZip->m_pFree(pZip->m_pAlloc_opaque, pRead_buf); - MZ_FCLOSE(pSrc_file); - return MZ_FALSE; - } - - state.m_pZip = pZip; - state.m_cur_archive_file_ofs = cur_archive_file_ofs; - state.m_comp_size = 0; - - if (tdefl_init(pComp, mz_zip_writer_add_put_buf_callback, &state, tdefl_create_comp_flags_from_zip_params(level, -15, MZ_DEFAULT_STRATEGY)) != TDEFL_STATUS_OKAY) - { - pZip->m_pFree(pZip->m_pAlloc_opaque, pComp); - pZip->m_pFree(pZip->m_pAlloc_opaque, pRead_buf); - MZ_FCLOSE(pSrc_file); - return MZ_FALSE; - } - - for ( ; ; ) - { - size_t in_buf_size = (mz_uint32)MZ_MIN(uncomp_remaining, MZ_ZIP_MAX_IO_BUF_SIZE); - tdefl_status status; - - if (MZ_FREAD(pRead_buf, 1, in_buf_size, pSrc_file) != in_buf_size) - break; - - uncomp_crc32 = (mz_uint32)mz_crc32(uncomp_crc32, (const mz_uint8 *)pRead_buf, in_buf_size); - uncomp_remaining -= in_buf_size; - - status = tdefl_compress_buffer(pComp, pRead_buf, in_buf_size, uncomp_remaining ? TDEFL_NO_FLUSH : TDEFL_FINISH); - if (status == TDEFL_STATUS_DONE) - { - result = MZ_TRUE; - break; - } - else if (status != TDEFL_STATUS_OKAY) - break; - } - - pZip->m_pFree(pZip->m_pAlloc_opaque, pComp); - - if (!result) - { - pZip->m_pFree(pZip->m_pAlloc_opaque, pRead_buf); - MZ_FCLOSE(pSrc_file); - return MZ_FALSE; - } - - comp_size = state.m_comp_size; - cur_archive_file_ofs = state.m_cur_archive_file_ofs; - - method = MZ_DEFLATED; - } - - pZip->m_pFree(pZip->m_pAlloc_opaque, pRead_buf); - } - - MZ_FCLOSE(pSrc_file); pSrc_file = NULL; - - // no zip64 support yet - if ((comp_size > 0xFFFFFFFF) || (cur_archive_file_ofs > 0xFFFFFFFF)) - return MZ_FALSE; - - if (!mz_zip_writer_create_local_dir_header(pZip, local_dir_header, (mz_uint16)archive_name_size, 0, uncomp_size, comp_size, uncomp_crc32, method, 0, dos_time, dos_date)) - return MZ_FALSE; - - if (pZip->m_pWrite(pZip->m_pIO_opaque, local_dir_header_ofs, local_dir_header, sizeof(local_dir_header)) != sizeof(local_dir_header)) - return MZ_FALSE; - - if (!mz_zip_writer_add_to_central_dir(pZip, pArchive_name, (mz_uint16)archive_name_size, NULL, 0, pComment, comment_size, uncomp_size, comp_size, uncomp_crc32, method, 0, dos_time, dos_date, local_dir_header_ofs, ext_attributes)) - return MZ_FALSE; - - pZip->m_total_files++; - pZip->m_archive_size = cur_archive_file_ofs; - - return MZ_TRUE; -} -#endif // #ifndef MINIZ_NO_STDIO - -mz_bool mz_zip_writer_add_from_zip_reader(mz_zip_archive *pZip, mz_zip_archive *pSource_zip, mz_uint file_index) -{ - mz_uint n, bit_flags, num_alignment_padding_bytes; - mz_uint64 comp_bytes_remaining, local_dir_header_ofs; - mz_uint64 cur_src_file_ofs, cur_dst_file_ofs; - mz_uint32 local_header_u32[(MZ_ZIP_LOCAL_DIR_HEADER_SIZE + sizeof(mz_uint32) - 1) / sizeof(mz_uint32)]; mz_uint8 *pLocal_header = (mz_uint8 *)local_header_u32; - mz_uint8 central_header[MZ_ZIP_CENTRAL_DIR_HEADER_SIZE]; - size_t orig_central_dir_size; - mz_zip_internal_state *pState; - void *pBuf; const mz_uint8 *pSrc_central_header; - - if ((!pZip) || (!pZip->m_pState) || (pZip->m_zip_mode != MZ_ZIP_MODE_WRITING)) - return MZ_FALSE; - if (NULL == (pSrc_central_header = mz_zip_reader_get_cdh(pSource_zip, file_index))) - return MZ_FALSE; - pState = pZip->m_pState; - - num_alignment_padding_bytes = mz_zip_writer_compute_padding_needed_for_file_alignment(pZip); - - // no zip64 support yet - if ((pZip->m_total_files == 0xFFFF) || ((pZip->m_archive_size + num_alignment_padding_bytes + MZ_ZIP_LOCAL_DIR_HEADER_SIZE + MZ_ZIP_CENTRAL_DIR_HEADER_SIZE) > 0xFFFFFFFF)) - return MZ_FALSE; - - cur_src_file_ofs = MZ_READ_LE32(pSrc_central_header + MZ_ZIP_CDH_LOCAL_HEADER_OFS); - cur_dst_file_ofs = pZip->m_archive_size; - - if (pSource_zip->m_pRead(pSource_zip->m_pIO_opaque, cur_src_file_ofs, pLocal_header, MZ_ZIP_LOCAL_DIR_HEADER_SIZE) != MZ_ZIP_LOCAL_DIR_HEADER_SIZE) - return MZ_FALSE; - if (MZ_READ_LE32(pLocal_header) != MZ_ZIP_LOCAL_DIR_HEADER_SIG) - return MZ_FALSE; - cur_src_file_ofs += MZ_ZIP_LOCAL_DIR_HEADER_SIZE; - - if (!mz_zip_writer_write_zeros(pZip, cur_dst_file_ofs, num_alignment_padding_bytes)) - return MZ_FALSE; - cur_dst_file_ofs += num_alignment_padding_bytes; - local_dir_header_ofs = cur_dst_file_ofs; - if (pZip->m_file_offset_alignment) { MZ_ASSERT((local_dir_header_ofs & (pZip->m_file_offset_alignment - 1)) == 0); } - - if (pZip->m_pWrite(pZip->m_pIO_opaque, cur_dst_file_ofs, pLocal_header, MZ_ZIP_LOCAL_DIR_HEADER_SIZE) != MZ_ZIP_LOCAL_DIR_HEADER_SIZE) - return MZ_FALSE; - cur_dst_file_ofs += MZ_ZIP_LOCAL_DIR_HEADER_SIZE; - - n = MZ_READ_LE16(pLocal_header + MZ_ZIP_LDH_FILENAME_LEN_OFS) + MZ_READ_LE16(pLocal_header + MZ_ZIP_LDH_EXTRA_LEN_OFS); - comp_bytes_remaining = n + MZ_READ_LE32(pSrc_central_header + MZ_ZIP_CDH_COMPRESSED_SIZE_OFS); - - if (NULL == (pBuf = pZip->m_pAlloc(pZip->m_pAlloc_opaque, 1, (size_t)MZ_MAX(sizeof(mz_uint32) * 4, MZ_MIN(MZ_ZIP_MAX_IO_BUF_SIZE, comp_bytes_remaining))))) - return MZ_FALSE; - - while (comp_bytes_remaining) - { - n = (mz_uint)MZ_MIN(MZ_ZIP_MAX_IO_BUF_SIZE, comp_bytes_remaining); - if (pSource_zip->m_pRead(pSource_zip->m_pIO_opaque, cur_src_file_ofs, pBuf, n) != n) - { - pZip->m_pFree(pZip->m_pAlloc_opaque, pBuf); - return MZ_FALSE; - } - cur_src_file_ofs += n; - - if (pZip->m_pWrite(pZip->m_pIO_opaque, cur_dst_file_ofs, pBuf, n) != n) - { - pZip->m_pFree(pZip->m_pAlloc_opaque, pBuf); - return MZ_FALSE; - } - cur_dst_file_ofs += n; - - comp_bytes_remaining -= n; - } - - bit_flags = MZ_READ_LE16(pLocal_header + MZ_ZIP_LDH_BIT_FLAG_OFS); - if (bit_flags & 8) - { - // Copy data descriptor - if (pSource_zip->m_pRead(pSource_zip->m_pIO_opaque, cur_src_file_ofs, pBuf, sizeof(mz_uint32) * 4) != sizeof(mz_uint32) * 4) - { - pZip->m_pFree(pZip->m_pAlloc_opaque, pBuf); - return MZ_FALSE; - } - - n = sizeof(mz_uint32) * ((MZ_READ_LE32(pBuf) == 0x08074b50) ? 4 : 3); - if (pZip->m_pWrite(pZip->m_pIO_opaque, cur_dst_file_ofs, pBuf, n) != n) - { - pZip->m_pFree(pZip->m_pAlloc_opaque, pBuf); - return MZ_FALSE; - } - - cur_src_file_ofs += n; - cur_dst_file_ofs += n; - } - pZip->m_pFree(pZip->m_pAlloc_opaque, pBuf); - - // no zip64 support yet - if (cur_dst_file_ofs > 0xFFFFFFFF) - return MZ_FALSE; - - orig_central_dir_size = pState->m_central_dir.m_size; - - memcpy(central_header, pSrc_central_header, MZ_ZIP_CENTRAL_DIR_HEADER_SIZE); - MZ_WRITE_LE32(central_header + MZ_ZIP_CDH_LOCAL_HEADER_OFS, local_dir_header_ofs); - if (!mz_zip_array_push_back(pZip, &pState->m_central_dir, central_header, MZ_ZIP_CENTRAL_DIR_HEADER_SIZE)) - return MZ_FALSE; - - n = MZ_READ_LE16(pSrc_central_header + MZ_ZIP_CDH_FILENAME_LEN_OFS) + MZ_READ_LE16(pSrc_central_header + MZ_ZIP_CDH_EXTRA_LEN_OFS) + MZ_READ_LE16(pSrc_central_header + MZ_ZIP_CDH_COMMENT_LEN_OFS); - if (!mz_zip_array_push_back(pZip, &pState->m_central_dir, pSrc_central_header + MZ_ZIP_CENTRAL_DIR_HEADER_SIZE, n)) - { - mz_zip_array_resize(pZip, &pState->m_central_dir, orig_central_dir_size, MZ_FALSE); - return MZ_FALSE; - } - - if (pState->m_central_dir.m_size > 0xFFFFFFFF) - return MZ_FALSE; - n = (mz_uint32)pState->m_central_dir.m_size; - if (!mz_zip_array_push_back(pZip, &pState->m_central_dir_offsets, &n, 1)) - { - mz_zip_array_resize(pZip, &pState->m_central_dir, orig_central_dir_size, MZ_FALSE); - return MZ_FALSE; - } - - pZip->m_total_files++; - pZip->m_archive_size = cur_dst_file_ofs; - - return MZ_TRUE; -} - -mz_bool mz_zip_writer_finalize_archive(mz_zip_archive *pZip) -{ - mz_zip_internal_state *pState; - mz_uint64 central_dir_ofs, central_dir_size; - mz_uint8 hdr[MZ_ZIP_END_OF_CENTRAL_DIR_HEADER_SIZE]; - - if ((!pZip) || (!pZip->m_pState) || (pZip->m_zip_mode != MZ_ZIP_MODE_WRITING)) - return MZ_FALSE; - - pState = pZip->m_pState; - - // no zip64 support yet - if ((pZip->m_total_files > 0xFFFF) || ((pZip->m_archive_size + pState->m_central_dir.m_size + MZ_ZIP_END_OF_CENTRAL_DIR_HEADER_SIZE) > 0xFFFFFFFF)) - return MZ_FALSE; - - central_dir_ofs = 0; - central_dir_size = 0; - if (pZip->m_total_files) - { - // Write central directory - central_dir_ofs = pZip->m_archive_size; - central_dir_size = pState->m_central_dir.m_size; - pZip->m_central_directory_file_ofs = central_dir_ofs; - if (pZip->m_pWrite(pZip->m_pIO_opaque, central_dir_ofs, pState->m_central_dir.m_p, (size_t)central_dir_size) != central_dir_size) - return MZ_FALSE; - pZip->m_archive_size += central_dir_size; - } - - // Write end of central directory record - MZ_CLEAR_OBJ(hdr); - MZ_WRITE_LE32(hdr + MZ_ZIP_ECDH_SIG_OFS, MZ_ZIP_END_OF_CENTRAL_DIR_HEADER_SIG); - MZ_WRITE_LE16(hdr + MZ_ZIP_ECDH_CDIR_NUM_ENTRIES_ON_DISK_OFS, pZip->m_total_files); - MZ_WRITE_LE16(hdr + MZ_ZIP_ECDH_CDIR_TOTAL_ENTRIES_OFS, pZip->m_total_files); - MZ_WRITE_LE32(hdr + MZ_ZIP_ECDH_CDIR_SIZE_OFS, central_dir_size); - MZ_WRITE_LE32(hdr + MZ_ZIP_ECDH_CDIR_OFS_OFS, central_dir_ofs); - - if (pZip->m_pWrite(pZip->m_pIO_opaque, pZip->m_archive_size, hdr, sizeof(hdr)) != sizeof(hdr)) - return MZ_FALSE; -#ifndef MINIZ_NO_STDIO - if ((pState->m_pFile) && (MZ_FFLUSH(pState->m_pFile) == EOF)) - return MZ_FALSE; -#endif // #ifndef MINIZ_NO_STDIO - - pZip->m_archive_size += sizeof(hdr); - - pZip->m_zip_mode = MZ_ZIP_MODE_WRITING_HAS_BEEN_FINALIZED; - return MZ_TRUE; -} - -mz_bool mz_zip_writer_finalize_heap_archive(mz_zip_archive *pZip, void **pBuf, size_t *pSize) -{ - if ((!pZip) || (!pZip->m_pState) || (!pBuf) || (!pSize)) - return MZ_FALSE; - if (pZip->m_pWrite != mz_zip_heap_write_func) - return MZ_FALSE; - if (!mz_zip_writer_finalize_archive(pZip)) - return MZ_FALSE; - - *pBuf = pZip->m_pState->m_pMem; - *pSize = pZip->m_pState->m_mem_size; - pZip->m_pState->m_pMem = NULL; - pZip->m_pState->m_mem_size = pZip->m_pState->m_mem_capacity = 0; - return MZ_TRUE; -} - -mz_bool mz_zip_writer_end(mz_zip_archive *pZip) -{ - mz_zip_internal_state *pState; - mz_bool status = MZ_TRUE; - if ((!pZip) || (!pZip->m_pState) || (!pZip->m_pAlloc) || (!pZip->m_pFree) || ((pZip->m_zip_mode != MZ_ZIP_MODE_WRITING) && (pZip->m_zip_mode != MZ_ZIP_MODE_WRITING_HAS_BEEN_FINALIZED))) - return MZ_FALSE; - - pState = pZip->m_pState; - pZip->m_pState = NULL; - mz_zip_array_clear(pZip, &pState->m_central_dir); - mz_zip_array_clear(pZip, &pState->m_central_dir_offsets); - mz_zip_array_clear(pZip, &pState->m_sorted_central_dir_offsets); - -#ifndef MINIZ_NO_STDIO - if (pState->m_pFile) - { - MZ_FCLOSE(pState->m_pFile); - pState->m_pFile = NULL; - } -#endif // #ifndef MINIZ_NO_STDIO - - if ((pZip->m_pWrite == mz_zip_heap_write_func) && (pState->m_pMem)) - { - pZip->m_pFree(pZip->m_pAlloc_opaque, pState->m_pMem); - pState->m_pMem = NULL; - } - - pZip->m_pFree(pZip->m_pAlloc_opaque, pState); - pZip->m_zip_mode = MZ_ZIP_MODE_INVALID; - return status; -} - -#ifndef MINIZ_NO_STDIO -mz_bool mz_zip_add_mem_to_archive_file_in_place(const char *pZip_filename, const char *pArchive_name, const void *pBuf, size_t buf_size, const void *pComment, mz_uint16 comment_size, mz_uint level_and_flags) -{ - mz_bool status, created_new_archive = MZ_FALSE; - mz_zip_archive zip_archive; - struct MZ_FILE_STAT_STRUCT file_stat; - MZ_CLEAR_OBJ(zip_archive); - if ((int)level_and_flags < 0) - level_and_flags = MZ_DEFAULT_LEVEL; - if ((!pZip_filename) || (!pArchive_name) || ((buf_size) && (!pBuf)) || ((comment_size) && (!pComment)) || ((level_and_flags & 0xF) > MZ_UBER_COMPRESSION)) - return MZ_FALSE; - if (!mz_zip_writer_validate_archive_name(pArchive_name)) - return MZ_FALSE; - if (MZ_FILE_STAT(pZip_filename, &file_stat) != 0) - { - // Create a new archive. - if (!mz_zip_writer_init_file(&zip_archive, pZip_filename, 0)) - return MZ_FALSE; - created_new_archive = MZ_TRUE; - } - else - { - // Append to an existing archive. - if (!mz_zip_reader_init_file(&zip_archive, pZip_filename, level_and_flags | MZ_ZIP_FLAG_DO_NOT_SORT_CENTRAL_DIRECTORY)) - return MZ_FALSE; - if (!mz_zip_writer_init_from_reader(&zip_archive, pZip_filename)) - { - mz_zip_reader_end(&zip_archive); - return MZ_FALSE; - } - } - status = mz_zip_writer_add_mem_ex(&zip_archive, pArchive_name, pBuf, buf_size, pComment, comment_size, level_and_flags, 0, 0); - // Always finalize, even if adding failed for some reason, so we have a valid central directory. (This may not always succeed, but we can try.) - if (!mz_zip_writer_finalize_archive(&zip_archive)) - status = MZ_FALSE; - if (!mz_zip_writer_end(&zip_archive)) - status = MZ_FALSE; - if ((!status) && (created_new_archive)) - { - // It's a new archive and something went wrong, so just delete it. - int ignoredStatus = MZ_DELETE_FILE(pZip_filename); - (void)ignoredStatus; - } - return status; -} - -void *mz_zip_extract_archive_file_to_heap(const char *pZip_filename, const char *pArchive_name, size_t *pSize, mz_uint flags) -{ - int file_index; - mz_zip_archive zip_archive; - void *p = NULL; - - if (pSize) - *pSize = 0; - - if ((!pZip_filename) || (!pArchive_name)) - return NULL; - - MZ_CLEAR_OBJ(zip_archive); - if (!mz_zip_reader_init_file(&zip_archive, pZip_filename, flags | MZ_ZIP_FLAG_DO_NOT_SORT_CENTRAL_DIRECTORY)) - return NULL; - - if ((file_index = mz_zip_reader_locate_file(&zip_archive, pArchive_name, NULL, flags)) >= 0) - p = mz_zip_reader_extract_to_heap(&zip_archive, file_index, pSize, flags); - - mz_zip_reader_end(&zip_archive); - return p; -} - -#endif // #ifndef MINIZ_NO_STDIO - -#endif // #ifndef MINIZ_NO_ARCHIVE_WRITING_APIS - -#endif // #ifndef MINIZ_NO_ARCHIVE_APIS - -#ifdef __cplusplus -} -#endif - -#endif // MINIZ_HEADER_FILE_ONLY - -/* - This is free and unencumbered software released into the public domain. - - Anyone is free to copy, modify, publish, use, compile, sell, or - distribute this software, either in source code form or as a compiled - binary, for any purpose, commercial or non-commercial, and by any - means. - - In jurisdictions that recognize copyright laws, the author or authors - of this software dedicate any and all copyright interest in the - software to the public domain. We make this dedication for the benefit - of the public at large and to the detriment of our heirs and - successors. We intend this dedication to be an overt act of - relinquishment in perpetuity of all present and future rights to this - software under copyright law. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR - OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - OTHER DEALINGS IN THE SOFTWARE. - - For more information, please refer to -*/ diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h index 01741c2b..543906eb 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h @@ -26,7 +26,6 @@ freely, subject to the following restrictions: #include "PolyVoxCore/BaseVolume.h" #include "PolyVoxCore/Block.h" -#include "PolyVoxCore/BlockCompressor.h" #include "PolyVoxCore/Pager.h" #include "PolyVoxCore/Region.h" #include "PolyVoxCore/Vector.h" diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl index f3914a64..5e3ab738 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl @@ -23,8 +23,6 @@ freely, subject to the following restrictions: #include "PolyVoxCore/Impl/ErrorHandling.h" -#include "PolyVoxCore/MinizBlockCompressor.h" // For creating a default compressor when none is provided. - #include namespace PolyVox diff --git a/library/PolyVoxCore/include/PolyVoxCore/MinizBlockCompressor.h b/library/PolyVoxCore/include/PolyVoxCore/MinizBlockCompressor.h deleted file mode 100644 index f701101c..00000000 --- a/library/PolyVoxCore/include/PolyVoxCore/MinizBlockCompressor.h +++ /dev/null @@ -1,65 +0,0 @@ -/******************************************************************************* -Copyright (c) 2005-2013 David Williams and Matthew Williams - -This software is provided 'as-is', without any express or implied -warranty. In no event will the authors be held liable for any damages -arising from the use of this software. - -Permission is granted to anyone to use this software for any purpose, -including commercial applications, and to alter it and redistribute it -freely, subject to the following restrictions: - - 1. The origin of this software must not be misrepresented; you must not - claim that you wrote the original software. If you use this software - in a product, an acknowledgment in the product documentation would be - appreciated but is not required. - - 2. Altered source versions must be plainly marked as such, and must not be - misrepresented as being the original software. - - 3. This notice may not be removed or altered from any source - distribution. -*******************************************************************************/ - -#ifndef __PolyVox_MinizBlockCompressor_H__ -#define __PolyVox_MinizBlockCompressor_H__ - -#include "PolyVoxCore/BlockCompressor.h" - -#include "PolyVoxCore/Impl/MinizWrapper.h" - -namespace PolyVox -{ - /** - * Provides an interface for performing paging of data. - */ - template - class MinizBlockCompressor : public BlockCompressor - { - public: - MinizBlockCompressor(int iCompressionLevel = 6); // Miniz defines MZ_DEFAULT_LEVEL = 6 so we use the same here - ~MinizBlockCompressor(); - - void compressData(UncompressedBlock* pSrcBlock, CompressedBlock* pDstBlock); - void decompress(CompressedBlock* pSrcBlock, UncompressedBlock* pDstBlock); - - public: - uint32_t getExpectedCompressedSize(uint32_t uUncompressedInputSize); - uint32_t getMaxCompressedSize(uint32_t uUncompressedInputSize); - uint32_t compressWithMiniz(const void* pSrcData, size_t uSrcLength, void* pDstData, size_t uDstLength); - uint32_t decompressWithMiniz(const void* pSrcData, size_t uSrcLength, void* pDstData, size_t uDstLength); - - unsigned int m_uCompressionFlags; - - // Data gets compressed into this, then we check how big the result - // is and copy the required number of bytes to the destination block. - std::vector m_vecTempBuffer; - - // tdefl_compressor contains all the state needed by the low-level compressor so it's a pretty big struct (~300k). - tdefl_compressor* m_pDeflator; - }; -} - -#include "PolyVoxCore/MinizBlockCompressor.inl" - -#endif //__PolyVox_MinizBlockCompressor_H__ diff --git a/library/PolyVoxCore/include/PolyVoxCore/MinizBlockCompressor.inl b/library/PolyVoxCore/include/PolyVoxCore/MinizBlockCompressor.inl deleted file mode 100644 index a4e41e72..00000000 --- a/library/PolyVoxCore/include/PolyVoxCore/MinizBlockCompressor.inl +++ /dev/null @@ -1,210 +0,0 @@ -/******************************************************************************* -Copyright (c) 2005-2013 David Williams and Matthew Williams - -This software is provided 'as-is', without any express or implied -warranty. In no event will the authors be held liable for any damages -arising from the use of this software. - -Permission is granted to anyone to use this software for any purpose, -including commercial applications, and to alter it and redistribute it -freely, subject to the following restrictions: - - 1. The origin of this software must not be misrepresented; you must not - claim that you wrote the original software. If you use this software - in a product, an acknowledgment in the product documentation would be - appreciated but is not required. - - 2. Altered source versions must be plainly marked as such, and must not be - misrepresented as being the original software. - - 3. This notice may not be removed or altered from any source - distribution. -*******************************************************************************/ - -#include - -namespace PolyVox -{ - /** - * You can specify a compression level when constructing this compressor. This controls the tradeoff between speed and compression - * rate. Levels 0-9 are the standard zlib-style levels, 10 is best possible compression (not zlib compatible, and may be very slow). - * \param iCompressionLevel The desired compression level. - */ - template - MinizBlockCompressor::MinizBlockCompressor(int iCompressionLevel) - :m_pDeflator(0) - { - // Create and store the deflator. - m_pDeflator = new tdefl_compressor; - - // The number of dictionary probes to use at each compression level (0-10). 0=implies fastest/minimal possible probing. - // The discontinuity is unsettling but may be explained by the 'iCompressionLevel <= 3' check later? - static const mz_uint s_tdefl_num_probes[11] = { 0, 1, 6, 32, 16, 32, 128, 256, 512, 768, 1500 }; - - // Create tdefl() compatible flags (we have to compose the low-level flags ourselves, or use tdefl_create_comp_flags_from_zip_params() but that means MINIZ_NO_ZLIB_APIS can't be defined). - m_uCompressionFlags = TDEFL_WRITE_ZLIB_HEADER | s_tdefl_num_probes[(std::min)(10, iCompressionLevel)] | ((iCompressionLevel <= 3) ? TDEFL_GREEDY_PARSING_FLAG : 0); - if (!iCompressionLevel) - { - m_uCompressionFlags |= TDEFL_FORCE_ALL_RAW_BLOCKS; - } - } - - template - MinizBlockCompressor::~MinizBlockCompressor() - { - // Delete the deflator - delete m_pDeflator; - } - - template - void MinizBlockCompressor::compressData(UncompressedBlock* pSrcBlock, CompressedBlock* pDstBlock) - { - // The uncompressed data will be read straight out of the block - void* pSrcData = reinterpret_cast(pSrcBlock->getData()); - size_t uSrcLength = pSrcBlock->getDataSizeInBytes(); - - // This compressor is expected to be used many times to compress a large number of blocks, but they are all - // expected to have the same size. Therefore the resize() function below should only perform allocation once. - uint32_t expectedCompressedSize = getExpectedCompressedSize(uSrcLength); - if(m_vecTempBuffer.size() != expectedCompressedSize) - { - POLYVOX_LOG_INFO("Resizing temp buffer to " << expectedCompressedSize << "bytes. " - << "This should only happen the first time the MinizBlockCompressor is used"); - m_vecTempBuffer.resize(expectedCompressedSize); - } - - // The compressed data will be written into this temporary buffer. - uint8_t* pDstData = &(m_vecTempBuffer[0]); - size_t uDstLength = m_vecTempBuffer.size(); - - try - { - // Perform the compression - uint32_t uCompressedLength = compressWithMiniz(pSrcData, uSrcLength, pDstData, uDstLength); - - // Copy the resulting compressed data into the compressed block - pDstBlock->setData(pDstData, uCompressedLength); - } - catch(std::exception&) - { - // It is possible for the compression to fail. A common cause for this would be if the destination - // buffer is not big enough. So now we try again using a buffer that is definitely big enough. - // Note that ideally we will choose our earlier buffer size so that this almost never happens. - POLYVOX_LOG_WARNING("The compressor failed to compress the block, probabaly due to the buffer being too small."); - POLYVOX_LOG_WARNING("The compression will be tried again with a larger buffer."); - - std::vector vecExtraBigBuffer; - vecExtraBigBuffer.resize(getMaxCompressedSize(uSrcLength)); - - uint8_t* pDstData = &(vecExtraBigBuffer[0]); - size_t uDstLength = vecExtraBigBuffer.size(); - - try - { - // Perform the compression - uint32_t uCompressedLength = compressWithMiniz(pSrcData, uSrcLength, pDstData, uDstLength); - - // Copy the resulting compressed data into the compressed block - pDstBlock->setData(pDstData, uCompressedLength); - } - catch(std::exception&) - { - // At this point it didn't work even with a bigger buffer. - // Not much more we can do so just rethrow the exception. - POLYVOX_THROW(std::runtime_error, "Failed to compress block data"); - } - } - } - - template - void MinizBlockCompressor::decompress(CompressedBlock* pSrcBlock, UncompressedBlock* pDstBlock) - { - // Get raw pointers so that the data can be decompressed directly into the destination block. - const void* pSrcData = reinterpret_cast(pSrcBlock->getData()); - void* pDstData = reinterpret_cast(pDstBlock->getData()); - size_t uSrcLength = pSrcBlock->getDataSizeInBytes(); - size_t uDstLength = pDstBlock->getDataSizeInBytes(); - - // Perform the decompression - uint32_t uUncompressedLength = decompressWithMiniz(pSrcData, uSrcLength, pDstData, uDstLength); - - // We know we should have received exactly one block's worth of data. If not then something went wrong. - POLYVOX_THROW_IF(uUncompressedLength != pDstBlock->getDataSizeInBytes(), std::runtime_error, "Decompressed data does not have the expected length"); - } - - template - uint32_t MinizBlockCompressor::getExpectedCompressedSize(uint32_t uUncompressedInputSize) - { - // We expect this block compressor will be used for smoothly changing volume data such as density fields and so - // the compression rate might not be great. The value beloew is basically a guess based on previous experience. - uint32_t uExpectedCompressionRate = 4; - return uUncompressedInputSize / uExpectedCompressionRate; - } - - template - uint32_t MinizBlockCompressor::getMaxCompressedSize(uint32_t uUncompressedInputSize) - { - // The contents of this function are copied from miniz's 'mz_deflateBound()' - // (which we can't use because it is part of the zlib-style higher level API). - unsigned long source_len = uUncompressedInputSize; - - // This is really over conservative. (And lame, but it's actually pretty tricky to compute a true upper bound given the way tdefl's blocking works.) - return (std::max)(128 + (source_len * 110) / 100, 128 + source_len + ((source_len / (31 * 1024)) + 1) * 5); - } - - template - uint32_t MinizBlockCompressor::compressWithMiniz(const void* pSrcData, size_t uSrcLength, void* pDstData, size_t uDstLength) - { - // It seems we have to reinitialise the deflator for each fresh dataset (it's probably intended for streaming, which we're not doing here) - tdefl_status status = tdefl_init(m_pDeflator, NULL, NULL, m_uCompressionFlags); - if (status != TDEFL_STATUS_OKAY) - { - std::stringstream ss; - ss << "tdefl_init() failed with return code '" << status << "'"; - POLYVOX_THROW(std::runtime_error, ss.str()); - } - - // Compress as much of the input as possible (or all of it) to the output buffer. - status = tdefl_compress(m_pDeflator, pSrcData, &uSrcLength, pDstData, &uDstLength, TDEFL_FINISH); - - //Check whether the compression was successful. - if (status != TDEFL_STATUS_DONE) - { - std::stringstream ss; - ss << "tdefl_compress() failed with return code '" << status << "'"; - POLYVOX_THROW(std::runtime_error, ss.str()); - } - - // The compression modifies 'ulDstLength' to hold the new length. - return uDstLength; - } - - template - uint32_t MinizBlockCompressor::decompressWithMiniz(const void* pSrcData, size_t uSrcLength, void* pDstData, size_t uDstLength) - { - // I don't know exactly why this limitation exists but it's an implementation detail of miniz. It shouldn't matter for our purposes - // as our destination is a Block and those are always a power of two. If you need to use this code for other purposes then you'll - // probably have to scale up your dst buffer to the nearest appropriate size. Alternatively you can use the mz_uncompress function, - // but that means enabling parts of the miniz API which are #defined out at the top of this file. - POLYVOX_THROW_IF(isPowerOf2(uDstLength) == false, std::invalid_argument, "Miniz decompressor requires the destination buffer to have a size which is a power of two."); - - // Create and initialise the decompressor (I believe this is much small than the compressor). - tinfl_decompressor inflator; - tinfl_init(&inflator); - - // Do the decompression. In some scenarios 'tinfl_decompress' would be called multiple times with the same dest buffer but - // different locations within it. In our scenario it's only called once so the start and the location are the same (both pDstData). - tinfl_status status = tinfl_decompress(&inflator, (const mz_uint8 *)pSrcData, &uSrcLength, (mz_uint8 *)pDstData, (mz_uint8 *)pDstData, &uDstLength, TINFL_FLAG_PARSE_ZLIB_HEADER); - - //Check whether the decompression was successful. - if (status != TINFL_STATUS_DONE) - { - std::stringstream ss; - ss << "tinfl_decompress() failed with return code '" << status << "'"; - POLYVOX_THROW(std::runtime_error, ss.str()); - } - - // The decompression modifies 'ulDstLength' to hold the new length. - return uDstLength; - } -} \ No newline at end of file diff --git a/library/PolyVoxCore/include/PolyVoxCore/RLEBlockCompressor.h b/library/PolyVoxCore/include/PolyVoxCore/RLEBlockCompressor.h deleted file mode 100644 index cb4fc38d..00000000 --- a/library/PolyVoxCore/include/PolyVoxCore/RLEBlockCompressor.h +++ /dev/null @@ -1,59 +0,0 @@ -/******************************************************************************* -Copyright (c) 2005-2013 David Williams and Matthew Williams - -This software is provided 'as-is', without any express or implied -warranty. In no event will the authors be held liable for any damages -arising from the use of this software. - -Permission is granted to anyone to use this software for any purpose, -including commercial applications, and to alter it and redistribute it -freely, subject to the following restrictions: - - 1. The origin of this software must not be misrepresented; you must not - claim that you wrote the original software. If you use this software - in a product, an acknowledgment in the product documentation would be - appreciated but is not required. - - 2. Altered source versions must be plainly marked as such, and must not be - misrepresented as being the original software. - - 3. This notice may not be removed or altered from any source - distribution. -*******************************************************************************/ - -#ifndef __PolyVox_RLEBlockCompressor_H__ -#define __PolyVox_RLEBlockCompressor_H__ - -#include "PolyVoxCore/BlockCompressor.h" - -namespace PolyVox -{ - template - class Run - { - public: - typedef uint16_t LengthType; - VoxelType value; - LengthType length; - }; - - /** - * Provides an interface for performing paging of data. - */ - template - class RLEBlockCompressor : public BlockCompressor - { - - - public: - RLEBlockCompressor(); - ~RLEBlockCompressor(); - - void compressData(UncompressedBlock* pSrcBlock, CompressedBlock* pDstBlock); - void decompress(CompressedBlock* pSrcBlock, UncompressedBlock* pDstBlock); - }; -} - -#include "PolyVoxCore/RLEBlockCompressor.inl" - -#endif //__PolyVox_RLEBlockCompressor_H__ diff --git a/library/PolyVoxCore/include/PolyVoxCore/RLEBlockCompressor.inl b/library/PolyVoxCore/include/PolyVoxCore/RLEBlockCompressor.inl deleted file mode 100644 index a8408059..00000000 --- a/library/PolyVoxCore/include/PolyVoxCore/RLEBlockCompressor.inl +++ /dev/null @@ -1,167 +0,0 @@ -/******************************************************************************* -Copyright (c) 2005-2013 David Williams and Matthew Williams - -This software is provided 'as-is', without any express or implied -warranty. In no event will the authors be held liable for any damages -arising from the use of this software. - -Permission is granted to anyone to use this software for any purpose, -including commercial applications, and to alter it and redistribute it -freely, subject to the following restrictions: - - 1. The origin of this software must not be misrepresented; you must not - claim that you wrote the original software. If you use this software - in a product, an acknowledgment in the product documentation would be - appreciated but is not required. - - 2. Altered source versions must be plainly marked as such, and must not be - misrepresented as being the original software. - - 3. This notice may not be removed or altered from any source - distribution. -*******************************************************************************/ - -#include - -namespace PolyVox -{ - template - RLEBlockCompressor::RLEBlockCompressor() - { - } - - template - RLEBlockCompressor::~RLEBlockCompressor() - { - } - - template - void RLEBlockCompressor::compressData(UncompressedBlock* pSrcBlock, CompressedBlock* pDstBlock) - { - void* pSrcData = reinterpret_cast(pSrcBlock->getData()); - uint32_t uSrcLength = pSrcBlock->getDataSizeInBytes(); - - if(uSrcLength % sizeof(VoxelType) != 0) - { - POLYVOX_THROW(std::length_error, "Source length must be a integer multiple of the ValueType size"); - } - - // Lengths provided are in bytes, so convert them to be in terms of our types. - uSrcLength /= sizeof(VoxelType); - //uDstLength /= sizeof(Run); - - // Get data pointers in the appropriate type - const VoxelType* pSrcDataAsType = reinterpret_cast(pSrcData); - //Run* pDstDataAsRun = reinterpret_cast(pDstData); - std::vector< Run > vecDstDataAsRuns; - - // Pointers to just past the end of the data - const VoxelType* pSrcDataEnd = pSrcDataAsType + uSrcLength; - //Run* pDstDataEnd = pDstDataAsRun + uDstLength; - - //Counter for the output length - //uint32_t uDstLengthInBytes = 0; - - // Read the first element of the source and set up the first run based on it. - /*pDstDataAsRun->value = *pSrcDataAsType; - pSrcDataAsType++; - pDstDataAsRun->length = 1; - uDstLengthInBytes += sizeof(Run);*/ - Run firstRun; - firstRun.value = *pSrcDataAsType; - firstRun.length = 1; - vecDstDataAsRuns.push_back(firstRun); - - pSrcDataAsType++; - - //Now process all remaining elements of the source. - while(pSrcDataAsType < pSrcDataEnd) - { - // If the value is the same as the current run (and we have not - // reached the maximum run length) then extend the current run. - typename std::vector< Run< VoxelType> >::iterator currentRun = (vecDstDataAsRuns.end() - 1); - if((*pSrcDataAsType == currentRun->value) && (currentRun->length < (std::numeric_limits::LengthType>::max)())) - { - currentRun->length++; - } - // Otherwise we need to start a new Run. - else - { - /*pDstDataAsRun++; - - // Check if we have enough space in the destination buffer. - if(pDstDataAsRun >= pDstDataEnd) - { - POLYVOX_THROW(std::runtime_error, "Insufficient space in destination buffer."); - }*/ - - // Create the new run. - /*pDstDataAsRun->value = *pSrcDataAsType; - pDstDataAsRun->length = 1; - uDstLengthInBytes += sizeof(Run);*/ - - Run newRun; - newRun.value = *pSrcDataAsType; - newRun.length = 1; - vecDstDataAsRuns.push_back(newRun); - } - - pSrcDataAsType++; - } - - //Now copy the data into the compressed block. - - pDstBlock->setData(reinterpret_cast(&(vecDstDataAsRuns[0])), vecDstDataAsRuns.size() * sizeof(Run)); - } - - template - void RLEBlockCompressor::decompress(CompressedBlock* pSrcBlock, UncompressedBlock* pDstBlock) - { - const void* pSrcData = reinterpret_cast(pSrcBlock->getData()); - uint32_t uSrcLength = pSrcBlock->getDataSizeInBytes(); - - void* pDstData = pDstBlock->getData(); - uint32_t uDstLength = pDstBlock->getDataSizeInBytes(); - - if(uSrcLength % sizeof(Run) != 0) - { - POLYVOX_THROW(std::length_error, "Source length must be a integer multiple of the Run size"); - } - - // Lengths provided are in bytes, so convert them to be in terms of our types. - uSrcLength /= sizeof(Run); - uDstLength /= sizeof(VoxelType); - - // Get data pointers in the appropriate type - const Run* pSrcDataAsRun = reinterpret_cast*>(pSrcData); - VoxelType* pDstDataAsType = reinterpret_cast(pDstData); - - // Pointers to just past the end of the data - const Run* pSrcDataEnd = pSrcDataAsRun + uSrcLength; - VoxelType* pDstDataEnd = pDstDataAsType + uDstLength; - - //Counter for the output length - uint32_t uDstLengthInBytes = 0; - - while(pSrcDataAsRun < pSrcDataEnd) - { - // Check if we have enough space in the destination buffer. - if(pDstDataAsType + pSrcDataAsRun->length > pDstDataEnd) - { - POLYVOX_THROW(std::runtime_error, "Insufficient space in destination buffer."); - } - else - { - - // Write the run into the destination - std::fill(pDstDataAsType, pDstDataAsType + pSrcDataAsRun->length, pSrcDataAsRun->value); - pDstDataAsType += pSrcDataAsRun->length; - - uDstLengthInBytes += pSrcDataAsRun->length * sizeof(VoxelType); - } - pSrcDataAsRun++; - } - - POLYVOX_ASSERT(uDstLengthInBytes == pDstBlock->getDataSizeInBytes(), "Uncompressed data does not have the correct length."); - } -} \ No newline at end of file diff --git a/library/PolyVoxCore/source/Impl/MinizWrapper.cpp b/library/PolyVoxCore/source/Impl/MinizWrapper.cpp deleted file mode 100644 index 6dc6656f..00000000 --- a/library/PolyVoxCore/source/Impl/MinizWrapper.cpp +++ /dev/null @@ -1,29 +0,0 @@ -/******************************************************************************* -Copyright (c) 2005-2013 David Williams - -This software is provided 'as-is', without any express or implied -warranty. In no event will the authors be held liable for any damages -arising from the use of this software. - -Permission is granted to anyone to use this software for any purpose, -including commercial applications, and to alter it and redistribute it -freely, subject to the following restrictions: - - 1. The origin of this software must not be misrepresented; you must not - claim that you wrote the original software. If you use this software - in a product, an acknowledgment in the product documentation would be - appreciated but is not required. - - 2. Altered source versions must be plainly marked as such, and must not be - misrepresented as being the original software. - - 3. This notice may not be removed or altered from any source - distribution. -*******************************************************************************/ - -// Please see this header file for comment about the purpose of MinizWrapper.h/cpp -#include "PolyVoxCore/Impl/MinizWrapper.h" - -// Get the full definitions, not just the declarations. -#undef MINIZ_HEADER_FILE_ONLY -#include "PolyVoxCore/Impl/miniz.c" \ No newline at end of file diff --git a/tests/TestSurfaceExtractor.cpp b/tests/TestSurfaceExtractor.cpp index 71e74d70..fc7f92ba 100644 --- a/tests/TestSurfaceExtractor.cpp +++ b/tests/TestSurfaceExtractor.cpp @@ -25,7 +25,6 @@ freely, subject to the following restrictions: #include "PolyVoxCore/Density.h" #include "PolyVoxCore/FilePager.h" -#include "PolyVoxCore/MinizBlockCompressor.h" #include "PolyVoxCore/MaterialDensityPair.h" #include "PolyVoxCore/RawVolume.h" #include "PolyVoxCore/SimpleVolume.h" @@ -102,8 +101,7 @@ template VolumeType* createAndFillVolume(void) { const int32_t uVolumeSideLength = 64; - - MinizBlockCompressor* compressor = new MinizBlockCompressor(); + FilePager* pager = new FilePager("./"); //Create empty volume @@ -140,8 +138,7 @@ float randomFloat(float a, float b) template VolumeType* createAndFillVolumeWithNoise(int32_t iVolumeSideLength, float minValue, float maxValue) -{ - MinizBlockCompressor* compressor = new MinizBlockCompressor(); +{ FilePager* pager = new FilePager("./"); //Create empty volume diff --git a/tests/testvolume.cpp b/tests/testvolume.cpp index c3090f2c..3d737c0d 100644 --- a/tests/testvolume.cpp +++ b/tests/testvolume.cpp @@ -25,7 +25,6 @@ freely, subject to the following restrictions: #include "PolyVoxCore/FilePager.h" #include "PolyVoxCore/LargeVolume.h" -#include "PolyVoxCore/MinizBlockCompressor.h" #include "PolyVoxCore/RawVolume.h" #include "PolyVoxCore/SimpleVolume.h" @@ -271,8 +270,6 @@ TestVolume::TestVolume() { Region region(-57, -31, 12, 64, 96, 131); // Deliberatly awkward size - //m_pCompressor = new RLECompressor; - m_pBlockCompressor = new MinizBlockCompressor; m_pFilePager = new FilePager("./"); //Create the volumes From 5ff54c041fcf754bf563df21f0e58819648fe53c Mon Sep 17 00:00:00 2001 From: David Williams Date: Sat, 13 Sep 2014 23:14:07 +0200 Subject: [PATCH 06/42] Renamed variable. --- .../include/PolyVoxCore/LargeVolume.h | 4 +- .../include/PolyVoxCore/LargeVolume.inl | 41 ++++++------------- 2 files changed, 14 insertions(+), 31 deletions(-) diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h index 543906eb..f4a0d67e 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h @@ -302,8 +302,6 @@ namespace PolyVox typedef std::map*, BlockPositionCompare> UncompressedBlockMap; - uint32_t calculateBlockMemoryUsage(void) const; - void ensureUncompressedBlockMapHasFreeSpace(void) const; void initialise(); @@ -321,7 +319,7 @@ namespace PolyVox void eraseBlock(typename UncompressedBlockMap::iterator itUncompressedBlock) const; // The block data - mutable UncompressedBlockMap m_pUncompressedBlockCache; + mutable UncompressedBlockMap m_pBlocks; mutable uint32_t m_uTimestamper; mutable Vector3DInt32 m_v3dLastAccessedBlockPos; diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl index 5e3ab738..a7bd07a7 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl @@ -391,9 +391,9 @@ namespace PolyVox //Replaced the for loop here as the call to //eraseBlock was invalidating the iterator. - while(m_pUncompressedBlockCache.size() > 0) + while(m_pBlocks.size() > 0) { - eraseBlock(m_pUncompressedBlockCache.begin()); + eraseBlock(m_pBlocks.begin()); } } @@ -422,8 +422,8 @@ namespace PolyVox for(int32_t z = v3dStart.getZ(); z <= v3dEnd.getZ(); z++) { Vector3DInt32 pos(x,y,z); - typename UncompressedBlockMap::iterator itBlock = m_pUncompressedBlockCache.find(pos); - if (itBlock == m_pUncompressedBlockCache.end()) + typename UncompressedBlockMap::iterator itBlock = m_pBlocks.find(pos); + if (itBlock == m_pBlocks.end()) { // not loaded, not unloading continue; @@ -475,7 +475,7 @@ namespace PolyVox //setMaxNumberOfUncompressedBlocks(m_uMaxNumberOfUncompressedBlocks); //Clear the previous data - m_pUncompressedBlockCache.clear(); + m_pBlocks.clear(); //Other properties we might find useful later this->m_uLongestSideLength = (std::max)((std::max)(this->getWidth(),this->getHeight()),this->getDepth()); @@ -514,7 +514,7 @@ namespace PolyVox delete itUncompressedBlock->second; // We can now remove the block data from memory. - m_pUncompressedBlockCache.erase(itUncompressedBlock); + m_pBlocks.erase(itUncompressedBlock); } template @@ -533,9 +533,9 @@ namespace PolyVox UncompressedBlock* pUncompressedBlock = 0; - typename UncompressedBlockMap::iterator itUncompressedBlock = m_pUncompressedBlockCache.find(v3dBlockPos); + typename UncompressedBlockMap::iterator itUncompressedBlock = m_pBlocks.find(v3dBlockPos); // check whether the block is already loaded - if(itUncompressedBlock != m_pUncompressedBlockCache.end()) + if(itUncompressedBlock != m_pBlocks.end()) { pUncompressedBlock = itUncompressedBlock->second; } @@ -565,7 +565,7 @@ namespace PolyVox m_pPager->pageIn(reg, pUncompressedBlock); // Add our new block to the map. - m_pUncompressedBlockCache.insert(std::make_pair(v3dBlockPos, pUncompressedBlock)); + m_pBlocks.insert(std::make_pair(v3dBlockPos, pUncompressedBlock)); } pUncompressedBlock->m_uBlockLastAccessed = ++m_uTimestamper; @@ -585,7 +585,7 @@ namespace PolyVox //Memory used by the blocks typename UncompressedBlockMap::iterator i; - for (i = m_pUncompressedBlockCache.begin(); i != m_pUncompressedBlockCache.end(); i++) + for (i = m_pBlocks.begin(); i != m_pBlocks.end(); i++) { //Inaccurate - account for rest of loaded block. uSizeInBytes += i->second->calculateSizeInBytes(); @@ -598,31 +598,16 @@ namespace PolyVox return uSizeInBytes; } - template - uint32_t LargeVolume::calculateBlockMemoryUsage(void) const - { - uint32_t uMemoryUsage = 0; - - typename CompressedBlockMap::iterator i; - for(i = m_pBlocks.begin(); i != m_pBlocks.end(); i++) - { - //Inaccurate - account for rest of loaded block. - uMemoryUsage += i->second->calculateSizeInBytes(); - } - - return uMemoryUsage; - } - template void LargeVolume::ensureUncompressedBlockMapHasFreeSpace(void) const { - while(m_pUncompressedBlockCache.size() > m_uMaxNumberOfUncompressedBlocks) + while(m_pBlocks.size() > m_uMaxNumberOfUncompressedBlocks) { // Find the least recently used block. The uncompressed block cache should be // much smaller than the total number of blocks, so hopefully this isn't too slow. typename UncompressedBlockMap::iterator i; - typename UncompressedBlockMap::iterator itUnloadBlock = m_pUncompressedBlockCache.begin(); - for(i = m_pUncompressedBlockCache.begin(); i != m_pUncompressedBlockCache.end(); i++) + typename UncompressedBlockMap::iterator itUnloadBlock = m_pBlocks.begin(); + for(i = m_pBlocks.begin(); i != m_pBlocks.end(); i++) { if(i->second->m_uBlockLastAccessed < itUnloadBlock->second->m_uBlockLastAccessed) { From 3cf8f38793bfc972bcd7dd14fb03e7e0a68c1882 Mon Sep 17 00:00:00 2001 From: David Williams Date: Sat, 13 Sep 2014 23:38:11 +0200 Subject: [PATCH 07/42] Merged Block and UncompressedBlock classes. --- library/PolyVoxCore/CMakeLists.txt | 1 - .../PolyVoxCore/include/PolyVoxCore/Block.h | 66 ------------------- .../include/PolyVoxCore/FilePager.h | 1 + .../include/PolyVoxCore/LargeVolume.h | 2 +- .../include/PolyVoxCore/UncompressedBlock.h | 12 +++- .../include/PolyVoxCore/UncompressedBlock.inl | 4 +- 6 files changed, 15 insertions(+), 71 deletions(-) delete mode 100644 library/PolyVoxCore/include/PolyVoxCore/Block.h diff --git a/library/PolyVoxCore/CMakeLists.txt b/library/PolyVoxCore/CMakeLists.txt index c8b13071..486cdf97 100644 --- a/library/PolyVoxCore/CMakeLists.txt +++ b/library/PolyVoxCore/CMakeLists.txt @@ -38,7 +38,6 @@ SET(CORE_INC_FILES include/PolyVoxCore/BaseVolume.h include/PolyVoxCore/BaseVolume.inl include/PolyVoxCore/BaseVolumeSampler.inl - include/PolyVoxCore/Block.h include/PolyVoxCore/CubicSurfaceExtractor.h include/PolyVoxCore/CubicSurfaceExtractor.inl include/PolyVoxCore/DefaultIsQuadNeeded.h diff --git a/library/PolyVoxCore/include/PolyVoxCore/Block.h b/library/PolyVoxCore/include/PolyVoxCore/Block.h deleted file mode 100644 index e2bb16cb..00000000 --- a/library/PolyVoxCore/include/PolyVoxCore/Block.h +++ /dev/null @@ -1,66 +0,0 @@ -/******************************************************************************* -Copyright (c) 2005-2013 David Williams and Matthew Williams - -This software is provided 'as-is', without any express or implied -warranty. In no event will the authors be held liable for any damages -arising from the use of this software. - -Permission is granted to anyone to use this software for any purpose, -including commercial applications, and to alter it and redistribute it -freely, subject to the following restrictions: - - 1. The origin of this software must not be misrepresented; you must not - claim that you wrote the original software. If you use this software - in a product, an acknowledgment in the product documentation would be - appreciated but is not required. - - 2. Altered source versions must be plainly marked as such, and must not be - misrepresented as being the original software. - - 3. This notice may not be removed or altered from any source - distribution. -*******************************************************************************/ - -#ifndef __PolyVox_Block_H__ -#define __PolyVox_Block_H__ - -#include "PolyVoxCore/Impl/TypeDef.h" - -#include "PolyVoxCore/PolyVoxForwardDeclarations.h" -#include "PolyVoxCore/Vector.h" - -#include -#include - -namespace PolyVox -{ - template - class Block - { - friend class LargeVolume; - - public: - Block() - :m_uBlockLastAccessed(0) - ,m_bDataModified(true) - { - } - - protected: - // This is updated by the LargeVolume and used to discard the least recently used blocks. - uint32_t m_uBlockLastAccessed; - - // This is so we can tell whether a uncompressed block has to be recompressed and whether - // a compressed block has to be paged back to disk, or whether they can just be discarded. - bool m_bDataModified; - - private: - /// Private copy constructor to prevent accisdental copying - Block(const Block& /*rhs*/) {}; - - /// Private assignment operator to prevent accisdental copying - Block& operator=(const Block& /*rhs*/) {}; - }; -} - -#endif diff --git a/library/PolyVoxCore/include/PolyVoxCore/FilePager.h b/library/PolyVoxCore/include/PolyVoxCore/FilePager.h index 5e0a6081..8e3e36ed 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/FilePager.h +++ b/library/PolyVoxCore/include/PolyVoxCore/FilePager.h @@ -34,6 +34,7 @@ freely, subject to the following restrictions: #include #include #include +#include namespace PolyVox { diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h index f4a0d67e..57e90120 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h @@ -25,9 +25,9 @@ freely, subject to the following restrictions: #define __PolyVox_LargeVolume_H__ #include "PolyVoxCore/BaseVolume.h" -#include "PolyVoxCore/Block.h" #include "PolyVoxCore/Pager.h" #include "PolyVoxCore/Region.h" +#include "PolyVoxCore/UncompressedBlock.h" #include "PolyVoxCore/Vector.h" #include diff --git a/library/PolyVoxCore/include/PolyVoxCore/UncompressedBlock.h b/library/PolyVoxCore/include/PolyVoxCore/UncompressedBlock.h index 54a1aacb..a0ed3c75 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/UncompressedBlock.h +++ b/library/PolyVoxCore/include/PolyVoxCore/UncompressedBlock.h @@ -24,12 +24,13 @@ freely, subject to the following restrictions: #ifndef __PolyVox_UncompressedBlock_H__ #define __PolyVox_UncompressedBlock_H__ -#include "PolyVoxCore/Block.h" +#include "PolyVoxCore/PolyVoxForwardDeclarations.h" +#include "PolyVoxCore/Vector.h" namespace PolyVox { template - class UncompressedBlock : public Block + class UncompressedBlock { friend class LargeVolume; @@ -53,6 +54,13 @@ namespace PolyVox /// Private assignment operator to prevent accisdental copying UncompressedBlock& operator=(const UncompressedBlock& /*rhs*/) {}; + // This is updated by the LargeVolume and used to discard the least recently used blocks. + uint32_t m_uBlockLastAccessed; + + // This is so we can tell whether a uncompressed block has to be recompressed and whether + // a compressed block has to be paged back to disk, or whether they can just be discarded. + bool m_bDataModified; + // Made this private for consistancy with CompressedBlock. // Users shouldn't really need this for UncompressedBlock anyway. uint32_t calculateSizeInBytes(void); diff --git a/library/PolyVoxCore/include/PolyVoxCore/UncompressedBlock.inl b/library/PolyVoxCore/include/PolyVoxCore/UncompressedBlock.inl index 2239b267..d5252d2c 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/UncompressedBlock.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/UncompressedBlock.inl @@ -27,7 +27,9 @@ namespace PolyVox { template UncompressedBlock::UncompressedBlock(uint16_t uSideLength) - :m_tData(0) + :m_uBlockLastAccessed(0) + ,m_bDataModified(true) + ,m_tData(0) ,m_uSideLength(0) ,m_uSideLengthPower(0) { From 47ace554ccfdf870d38d1af75fc01a912e220c7b Mon Sep 17 00:00:00 2001 From: David Williams Date: Sun, 14 Sep 2014 11:47:17 +0200 Subject: [PATCH 08/42] Making use of shared_ptr to track blocks. --- examples/Paging/main.cpp | 4 ++-- .../include/PolyVoxCore/FilePager.h | 4 ++-- .../include/PolyVoxCore/LargeVolume.h | 6 +++--- .../include/PolyVoxCore/LargeVolume.inl | 19 ++++++++++--------- .../PolyVoxCore/LargeVolumeSampler.inl | 2 +- .../PolyVoxCore/include/PolyVoxCore/Pager.h | 6 ++++-- 6 files changed, 22 insertions(+), 19 deletions(-) diff --git a/examples/Paging/main.cpp b/examples/Paging/main.cpp index a01779fb..04d20067 100644 --- a/examples/Paging/main.cpp +++ b/examples/Paging/main.cpp @@ -90,7 +90,7 @@ public: /// Destructor virtual ~PerlinNoisePager() {}; - virtual void pageIn(const PolyVox::Region& region, UncompressedBlock* pBlockData) + virtual void pageIn(const PolyVox::Region& region, std::shared_ptr< UncompressedBlock > pBlockData) { // FIXME - this isn't a great example... it's a shame we have to hard clode the block size and also create/destroy // a compressor each time. These could at least be moved outside somewhere if we can't fix it in a better way... @@ -143,7 +143,7 @@ public: //delete compressor; } - virtual void pageOut(const PolyVox::Region& region, UncompressedBlock* /*pBlockData*/) + virtual void pageOut(const PolyVox::Region& region, std::shared_ptr< UncompressedBlock > /*pBlockData*/) { std::cout << "warning unloading region: " << region.getLowerCorner() << " -> " << region.getUpperCorner() << std::endl; } diff --git a/library/PolyVoxCore/include/PolyVoxCore/FilePager.h b/library/PolyVoxCore/include/PolyVoxCore/FilePager.h index 8e3e36ed..8da3adcf 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/FilePager.h +++ b/library/PolyVoxCore/include/PolyVoxCore/FilePager.h @@ -69,7 +69,7 @@ namespace PolyVox m_vecCreatedFiles.clear(); } - virtual void pageIn(const Region& region, UncompressedBlock* pBlockData) + virtual void pageIn(const Region& region, std::shared_ptr< UncompressedBlock > pBlockData) { POLYVOX_ASSERT(pBlockData, "Attempting to page in NULL block"); //POLYVOX_ASSERT(pBlockData->hasUncompressedData() == false, "Block should not have uncompressed data"); @@ -113,7 +113,7 @@ namespace PolyVox } } - virtual void pageOut(const Region& region, UncompressedBlock* pBlockData) + virtual void pageOut(const Region& region, std::shared_ptr< UncompressedBlock > pBlockData) { POLYVOX_ASSERT(pBlockData, "Attempting to page out NULL block"); //POLYVOX_ASSERT(pBlockData->hasUncompressedData() == false, "Block should not have uncompressed data"); diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h index 57e90120..ca4f60d6 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h @@ -300,7 +300,7 @@ namespace PolyVox } }; - typedef std::map*, BlockPositionCompare> UncompressedBlockMap; + typedef std::map >, BlockPositionCompare> UncompressedBlockMap; void ensureUncompressedBlockMapHasFreeSpace(void) const; @@ -314,7 +314,7 @@ namespace PolyVox VoxelType getVoxelImpl(int32_t uXPos, int32_t uYPos, int32_t uZPos, WrapModeType, VoxelType tBorder) const; VoxelType getVoxelImpl(int32_t uXPos, int32_t uYPos, int32_t uZPos, WrapModeType, VoxelType tBorder) const; - UncompressedBlock* getUncompressedBlock(int32_t uBlockX, int32_t uBlockY, int32_t uBlockZ) const; + std::shared_ptr< UncompressedBlock > getUncompressedBlock(int32_t uBlockX, int32_t uBlockY, int32_t uBlockZ) const; void eraseBlock(typename UncompressedBlockMap::iterator itUncompressedBlock) const; @@ -323,7 +323,7 @@ namespace PolyVox mutable uint32_t m_uTimestamper; mutable Vector3DInt32 m_v3dLastAccessedBlockPos; - mutable UncompressedBlock* m_pLastAccessedBlock; + mutable std::shared_ptr< UncompressedBlock > m_pLastAccessedBlock; uint32_t m_uMaxNumberOfUncompressedBlocks; // The size of the volume diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl index a7bd07a7..4cbeb632 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl @@ -186,7 +186,7 @@ namespace PolyVox const uint16_t yOffset = static_cast(uYPos - (blockY << m_uBlockSideLengthPower)); const uint16_t zOffset = static_cast(uZPos - (blockZ << m_uBlockSideLengthPower)); - const UncompressedBlock* pUncompressedBlock = getUncompressedBlock(blockX, blockY, blockZ); + auto pUncompressedBlock = getUncompressedBlock(blockX, blockY, blockZ); return pUncompressedBlock->getVoxel(xOffset, yOffset, zOffset); } @@ -262,7 +262,7 @@ namespace PolyVox const uint16_t yOffset = static_cast(uYPos - (blockY << m_uBlockSideLengthPower)); const uint16_t zOffset = static_cast(uZPos - (blockZ << m_uBlockSideLengthPower)); - UncompressedBlock* pUncompressedBlock = getUncompressedBlock(blockX, blockY, blockZ); + auto pUncompressedBlock = getUncompressedBlock(blockX, blockY, blockZ); pUncompressedBlock->setVoxelAt(xOffset, yOffset, zOffset, tValue); } @@ -299,7 +299,7 @@ namespace PolyVox const uint16_t yOffset = static_cast(uYPos - (blockY << m_uBlockSideLengthPower)); const uint16_t zOffset = static_cast(uZPos - (blockZ << m_uBlockSideLengthPower)); - UncompressedBlock* pUncompressedBlock = getUncompressedBlock(blockX, blockY, blockZ); + auto pUncompressedBlock = getUncompressedBlock(blockX, blockY, blockZ); pUncompressedBlock->setVoxelAt(xOffset, yOffset, zOffset, tValue); @@ -486,7 +486,7 @@ namespace PolyVox template void LargeVolume::eraseBlock(typename UncompressedBlockMap::iterator itUncompressedBlock) const { - UncompressedBlock* pUncompressedBlock = itUncompressedBlock->second; + std::shared_ptr< UncompressedBlock > pUncompressedBlock = itUncompressedBlock->second; // This should never happen as blocks are deleted based on being least recently used. // I the case that we are flushing we delete all blocks, but the flush function will @@ -511,14 +511,14 @@ namespace PolyVox pUncompressedBlock->m_bDataModified = false; } - delete itUncompressedBlock->second; + //delete itUncompressedBlock->second; // We can now remove the block data from memory. m_pBlocks.erase(itUncompressedBlock); } template - UncompressedBlock* LargeVolume::getUncompressedBlock(int32_t uBlockX, int32_t uBlockY, int32_t uBlockZ) const + std::shared_ptr< UncompressedBlock > LargeVolume::getUncompressedBlock(int32_t uBlockX, int32_t uBlockY, int32_t uBlockZ) const { Vector3DInt32 v3dBlockPos(uBlockX, uBlockY, uBlockZ); @@ -531,7 +531,7 @@ namespace PolyVox return m_pLastAccessedBlock; } - UncompressedBlock* pUncompressedBlock = 0; + std::shared_ptr< UncompressedBlock > pUncompressedBlock = nullptr; typename UncompressedBlockMap::iterator itUncompressedBlock = m_pBlocks.find(v3dBlockPos); // check whether the block is already loaded @@ -546,7 +546,8 @@ namespace PolyVox ensureUncompressedBlockMapHasFreeSpace(); // We can now create a new block. - pUncompressedBlock = new UncompressedBlock(m_uBlockSideLength); + //pUncompressedBlock = new UncompressedBlock(m_uBlockSideLength); + pUncompressedBlock = std::make_shared< UncompressedBlock >(m_uBlockSideLength); // An uncompressed bock is always backed by a compressed one, and this is created by getCompressedBlock() if it doesn't // already exist. If it does already exist and has data then we bring this across into the ucompressed version. @@ -677,7 +678,7 @@ namespace PolyVox const uint16_t yOffset = static_cast(uYPos - (blockY << m_uBlockSideLengthPower)); const uint16_t zOffset = static_cast(uZPos - (blockZ << m_uBlockSideLengthPower)); - UncompressedBlock* pUncompressedBlock = getUncompressedBlock(blockX, blockY, blockZ); + auto pUncompressedBlock = getUncompressedBlock(blockX, blockY, blockZ); return pUncompressedBlock->getVoxel(xOffset, yOffset, zOffset); } } diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolumeSampler.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolumeSampler.inl index c5e4e12c..df07bb77 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolumeSampler.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolumeSampler.inl @@ -119,7 +119,7 @@ namespace PolyVox uYPosInBlock * this->mVolume->m_uBlockSideLength + uZPosInBlock * this->mVolume->m_uBlockSideLength * this->mVolume->m_uBlockSideLength; - UncompressedBlock* pUncompressedCurrentBlock = this->mVolume->getUncompressedBlock(uXBlock, uYBlock, uZBlock); + auto pUncompressedCurrentBlock = this->mVolume->getUncompressedBlock(uXBlock, uYBlock, uZBlock); mCurrentVoxel = pUncompressedCurrentBlock->m_tData + uVoxelIndexInBlock; } diff --git a/library/PolyVoxCore/include/PolyVoxCore/Pager.h b/library/PolyVoxCore/include/PolyVoxCore/Pager.h index c11d8afd..0970c943 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/Pager.h +++ b/library/PolyVoxCore/include/PolyVoxCore/Pager.h @@ -27,6 +27,8 @@ freely, subject to the following restrictions: #include "PolyVoxCore/UncompressedBlock.h" #include "PolyVoxCore/Impl/TypeDef.h" +#include + namespace PolyVox { /** @@ -41,8 +43,8 @@ namespace PolyVox /// Destructor virtual ~Pager() {}; - virtual void pageIn(const Region& region, UncompressedBlock* pBlockData) = 0; - virtual void pageOut(const Region& region, UncompressedBlock* pBlockData) = 0; + virtual void pageIn(const Region& region, std::shared_ptr< UncompressedBlock > pBlockData) = 0; + virtual void pageOut(const Region& region, std::shared_ptr< UncompressedBlock > pBlockData) = 0; }; } From 704eeaf94829842b91e5a37ea5c2d56fd7b65250 Mon Sep 17 00:00:00 2001 From: David Williams Date: Tue, 16 Sep 2014 17:02:53 +0200 Subject: [PATCH 09/42] Work on supporting no pager being attached. --- .../include/PolyVoxCore/LargeVolume.h | 7 ---- .../include/PolyVoxCore/LargeVolume.inl | 32 ++++++++++++------- 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h index ca4f60d6..1f080523 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h @@ -334,13 +334,6 @@ namespace PolyVox uint8_t m_uBlockSideLengthPower; Pager* m_pPager; - - // Compressed data for an empty block (sometimes needed for initialisation). - //CompressedBlock* m_pCompressedEmptyBlock; - - // Whether we created the compressor or whether it was provided - // by the user. This controls whether we delete it on destruction. - //bool m_bIsOurCompressor; }; } diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl index 4cbeb632..44116cd5 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl @@ -24,6 +24,7 @@ freely, subject to the following restrictions: #include "PolyVoxCore/Impl/ErrorHandling.h" #include +#include namespace PolyVox { @@ -46,9 +47,20 @@ namespace PolyVox :BaseVolume(regValid) { m_uBlockSideLength = uBlockSideLength; - m_pPager = pPager; + if (m_pPager) + { + // If a pager is available then we can set a sensible limit on our memory usage. + m_uMaxNumberOfUncompressedBlocks = 256; + } + else + { + // If there is no pager provided then we set the block limit to the maximum + // value to ensure the system never attempts to page blocks out of memory. + m_uMaxNumberOfUncompressedBlocks = (std::numeric_limits::max)(); + } + initialise(); } @@ -215,6 +227,8 @@ namespace PolyVox template void LargeVolume::setMaxNumberOfUncompressedBlocks(uint32_t uMaxNumberOfUncompressedBlocks) { + POLYVOX_THROW_IF(!m_pPager, invalid_operation, "You cannot limit the memory usage of the volume because it was created without a pager attached."); + //clearBlockCache(); /*if (m_pBlocks.size() > uMaxNumberOfBlocksInMemory) @@ -386,6 +400,8 @@ namespace PolyVox template void LargeVolume::flushAll() { + POLYVOX_THROW_IF(!m_pPager, invalid_operation, "You cannot flush data out of the volume because it was created without a pager attached."); + // Flushing will remove the most accessed block, so invalidate the pointer. m_pLastAccessedBlock = 0; @@ -403,6 +419,8 @@ namespace PolyVox template void LargeVolume::flush(Region regFlush) { + POLYVOX_THROW_IF(!m_pPager, invalid_operation, "You cannot flush data out of the volume because it was created without a pager attached."); + Vector3DInt32 v3dStart; for(int i = 0; i < 3; i++) { @@ -486,6 +504,8 @@ namespace PolyVox template void LargeVolume::eraseBlock(typename UncompressedBlockMap::iterator itUncompressedBlock) const { + POLYVOX_ASSERT(m_pPager, "A block should never be erased if there is no pager attached to handle it"); + std::shared_ptr< UncompressedBlock > pUncompressedBlock = itUncompressedBlock->second; // This should never happen as blocks are deleted based on being least recently used. @@ -549,16 +569,6 @@ namespace PolyVox //pUncompressedBlock = new UncompressedBlock(m_uBlockSideLength); pUncompressedBlock = std::make_shared< UncompressedBlock >(m_uBlockSideLength); - // An uncompressed bock is always backed by a compressed one, and this is created by getCompressedBlock() if it doesn't - // already exist. If it does already exist and has data then we bring this across into the ucompressed version. - /*if(getCompressedBlock(uBlockX, uBlockY, uBlockZ)->getData() != 0) - { - // FIXME - multiple getCompressedBlock() calls (including the one above) - CompressedBlock* pBlock = getCompressedBlock(uBlockX, uBlockY, ); - - m_pBlockCompressor->decompress(pBlock, pUncompressedBlock); - }*/ - // Pass the block to the Pager to give it a chance to initialise it with any data Vector3DInt32 v3dLower(uBlockX << m_uBlockSideLengthPower, uBlockY << m_uBlockSideLengthPower, uBlockZ << m_uBlockSideLengthPower); Vector3DInt32 v3dUpper = v3dLower + Vector3DInt32(m_uBlockSideLength - 1, m_uBlockSideLength - 1, m_uBlockSideLength - 1); From 2b47c959a56e03bd2be702c38b23072b1bcd9e90 Mon Sep 17 00:00:00 2001 From: David Williams Date: Tue, 16 Sep 2014 21:39:55 +0200 Subject: [PATCH 10/42] Work on getUncompressedVoxel(). --- .../include/PolyVoxCore/LargeVolume.h | 2 - .../include/PolyVoxCore/LargeVolume.inl | 67 +++++++++---------- 2 files changed, 31 insertions(+), 38 deletions(-) diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h index 1f080523..17954659 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h @@ -302,8 +302,6 @@ namespace PolyVox typedef std::map >, BlockPositionCompare> UncompressedBlockMap; - void ensureUncompressedBlockMapHasFreeSpace(void) const; - void initialise(); // A trick to implement specialization of template member functions in template classes. See http://stackoverflow.com/a/4951057 diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl index 44116cd5..47068b36 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl @@ -551,32 +551,49 @@ namespace PolyVox return m_pLastAccessedBlock; } + // Try to find the required block in our block list. std::shared_ptr< UncompressedBlock > pUncompressedBlock = nullptr; - typename UncompressedBlockMap::iterator itUncompressedBlock = m_pBlocks.find(v3dBlockPos); - // check whether the block is already loaded + + // Check whether the block was found. if(itUncompressedBlock != m_pBlocks.end()) { + // The block was found so we can use it. pUncompressedBlock = itUncompressedBlock->second; } else { - // At this point we know that the uncompresed block did not exist in the cache. We will - // create it and add it to the cache, which means we need to make sure there is space. - ensureUncompressedBlockMapHasFreeSpace(); - - // We can now create a new block. - //pUncompressedBlock = new UncompressedBlock(m_uBlockSideLength); + // The blcok was not found so we will create a new one. pUncompressedBlock = std::make_shared< UncompressedBlock >(m_uBlockSideLength); // Pass the block to the Pager to give it a chance to initialise it with any data - Vector3DInt32 v3dLower(uBlockX << m_uBlockSideLengthPower, uBlockY << m_uBlockSideLengthPower, uBlockZ << m_uBlockSideLengthPower); - Vector3DInt32 v3dUpper = v3dLower + Vector3DInt32(m_uBlockSideLength - 1, m_uBlockSideLength - 1, m_uBlockSideLength - 1); - Region reg(v3dLower, v3dUpper); - m_pPager->pageIn(reg, pUncompressedBlock); - + if (m_pPager) + { + Vector3DInt32 v3dLower(uBlockX << m_uBlockSideLengthPower, uBlockY << m_uBlockSideLengthPower, uBlockZ << m_uBlockSideLengthPower); + Vector3DInt32 v3dUpper = v3dLower + Vector3DInt32(m_uBlockSideLength - 1, m_uBlockSideLength - 1, m_uBlockSideLength - 1); + Region reg(v3dLower, v3dUpper); + m_pPager->pageIn(reg, pUncompressedBlock); + } + + while (m_pBlocks.size() + 1 > m_uMaxNumberOfUncompressedBlocks) // +1 ready for new block we will add next. + { + // Find the least recently used block. Hopefully this isn't too slow. + typename UncompressedBlockMap::iterator i; + typename UncompressedBlockMap::iterator itUnloadBlock = m_pBlocks.begin(); + for (i = m_pBlocks.begin(); i != m_pBlocks.end(); i++) + { + if (i->second->m_uBlockLastAccessed < itUnloadBlock->second->m_uBlockLastAccessed) + { + itUnloadBlock = i; + } + } + + // Erase the least recently used block + eraseBlock(itUnloadBlock); + } + // Add our new block to the map. - m_pBlocks.insert(std::make_pair(v3dBlockPos, pUncompressedBlock)); + m_pBlocks.insert(std::make_pair(v3dBlockPos, pUncompressedBlock)); } pUncompressedBlock->m_uBlockLastAccessed = ++m_uTimestamper; @@ -609,28 +626,6 @@ namespace PolyVox return uSizeInBytes; } - template - void LargeVolume::ensureUncompressedBlockMapHasFreeSpace(void) const - { - while(m_pBlocks.size() > m_uMaxNumberOfUncompressedBlocks) - { - // Find the least recently used block. The uncompressed block cache should be - // much smaller than the total number of blocks, so hopefully this isn't too slow. - typename UncompressedBlockMap::iterator i; - typename UncompressedBlockMap::iterator itUnloadBlock = m_pBlocks.begin(); - for(i = m_pBlocks.begin(); i != m_pBlocks.end(); i++) - { - if(i->second->m_uBlockLastAccessed < itUnloadBlock->second->m_uBlockLastAccessed) - { - itUnloadBlock = i; - } - } - - // Erase the least recently used block - eraseBlock(itUnloadBlock); - } - } - template template VoxelType LargeVolume::getVoxelImpl(int32_t uXPos, int32_t uYPos, int32_t uZPos, WrapModeType, VoxelType tBorder) const From 5b59bc2d8c07fe0455eafced9d86462daa91d4c3 Mon Sep 17 00:00:00 2001 From: David Williams Date: Wed, 17 Sep 2014 13:35:43 +0200 Subject: [PATCH 11/42] Moved paging code into block constructor/destructor. --- examples/Paging/main.cpp | 4 +-- .../include/PolyVoxCore/FilePager.h | 4 +-- .../include/PolyVoxCore/LargeVolume.inl | 27 ++-------------- .../PolyVoxCore/include/PolyVoxCore/Pager.h | 4 +-- .../include/PolyVoxCore/UncompressedBlock.h | 6 ++-- .../include/PolyVoxCore/UncompressedBlock.inl | 31 +++++++++++++++++-- 6 files changed, 41 insertions(+), 35 deletions(-) diff --git a/examples/Paging/main.cpp b/examples/Paging/main.cpp index 04d20067..a01779fb 100644 --- a/examples/Paging/main.cpp +++ b/examples/Paging/main.cpp @@ -90,7 +90,7 @@ public: /// Destructor virtual ~PerlinNoisePager() {}; - virtual void pageIn(const PolyVox::Region& region, std::shared_ptr< UncompressedBlock > pBlockData) + virtual void pageIn(const PolyVox::Region& region, UncompressedBlock* pBlockData) { // FIXME - this isn't a great example... it's a shame we have to hard clode the block size and also create/destroy // a compressor each time. These could at least be moved outside somewhere if we can't fix it in a better way... @@ -143,7 +143,7 @@ public: //delete compressor; } - virtual void pageOut(const PolyVox::Region& region, std::shared_ptr< UncompressedBlock > /*pBlockData*/) + virtual void pageOut(const PolyVox::Region& region, UncompressedBlock* /*pBlockData*/) { std::cout << "warning unloading region: " << region.getLowerCorner() << " -> " << region.getUpperCorner() << std::endl; } diff --git a/library/PolyVoxCore/include/PolyVoxCore/FilePager.h b/library/PolyVoxCore/include/PolyVoxCore/FilePager.h index 8da3adcf..8e3e36ed 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/FilePager.h +++ b/library/PolyVoxCore/include/PolyVoxCore/FilePager.h @@ -69,7 +69,7 @@ namespace PolyVox m_vecCreatedFiles.clear(); } - virtual void pageIn(const Region& region, std::shared_ptr< UncompressedBlock > pBlockData) + virtual void pageIn(const Region& region, UncompressedBlock* pBlockData) { POLYVOX_ASSERT(pBlockData, "Attempting to page in NULL block"); //POLYVOX_ASSERT(pBlockData->hasUncompressedData() == false, "Block should not have uncompressed data"); @@ -113,7 +113,7 @@ namespace PolyVox } } - virtual void pageOut(const Region& region, std::shared_ptr< UncompressedBlock > pBlockData) + virtual void pageOut(const Region& region, UncompressedBlock* pBlockData) { POLYVOX_ASSERT(pBlockData, "Attempting to page out NULL block"); //POLYVOX_ASSERT(pBlockData->hasUncompressedData() == false, "Block should not have uncompressed data"); diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl index 47068b36..218bff10 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl @@ -515,21 +515,7 @@ namespace PolyVox // Before deleting the block we may need to recompress its data. We // only do this if the data has been modified since it was decompressed. - if(pUncompressedBlock->m_bDataModified) - { - // Get the compressed block which we will copy the data back in to. - Vector3DInt32 v3dBlockPos = itUncompressedBlock->first; - - // From the coordinates of the block we deduce the coordinates of the contained voxels. - Vector3DInt32 v3dLower(v3dBlockPos.getX() << m_uBlockSideLengthPower, v3dBlockPos.getY() << m_uBlockSideLengthPower, v3dBlockPos.getZ() << m_uBlockSideLengthPower); - Vector3DInt32 v3dUpper = v3dLower + Vector3DInt32(m_uBlockSideLength - 1, m_uBlockSideLength - 1, m_uBlockSideLength - 1); - - // Page the data out - m_pPager->pageOut(Region(v3dLower, v3dUpper), itUncompressedBlock->second); - - // The compressed data has been updated, so the uncompressed data is no longer modified with respect to it. - pUncompressedBlock->m_bDataModified = false; - } + //delete itUncompressedBlock->second; @@ -564,16 +550,7 @@ namespace PolyVox else { // The blcok was not found so we will create a new one. - pUncompressedBlock = std::make_shared< UncompressedBlock >(m_uBlockSideLength); - - // Pass the block to the Pager to give it a chance to initialise it with any data - if (m_pPager) - { - Vector3DInt32 v3dLower(uBlockX << m_uBlockSideLengthPower, uBlockY << m_uBlockSideLengthPower, uBlockZ << m_uBlockSideLengthPower); - Vector3DInt32 v3dUpper = v3dLower + Vector3DInt32(m_uBlockSideLength - 1, m_uBlockSideLength - 1, m_uBlockSideLength - 1); - Region reg(v3dLower, v3dUpper); - m_pPager->pageIn(reg, pUncompressedBlock); - } + pUncompressedBlock = std::make_shared< UncompressedBlock >(v3dBlockPos, m_uBlockSideLength, m_pPager); while (m_pBlocks.size() + 1 > m_uMaxNumberOfUncompressedBlocks) // +1 ready for new block we will add next. { diff --git a/library/PolyVoxCore/include/PolyVoxCore/Pager.h b/library/PolyVoxCore/include/PolyVoxCore/Pager.h index 0970c943..6b670302 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/Pager.h +++ b/library/PolyVoxCore/include/PolyVoxCore/Pager.h @@ -43,8 +43,8 @@ namespace PolyVox /// Destructor virtual ~Pager() {}; - virtual void pageIn(const Region& region, std::shared_ptr< UncompressedBlock > pBlockData) = 0; - virtual void pageOut(const Region& region, std::shared_ptr< UncompressedBlock > pBlockData) = 0; + virtual void pageIn(const Region& region, UncompressedBlock* pBlockData) = 0; + virtual void pageOut(const Region& region, UncompressedBlock* pBlockData) = 0; }; } diff --git a/library/PolyVoxCore/include/PolyVoxCore/UncompressedBlock.h b/library/PolyVoxCore/include/PolyVoxCore/UncompressedBlock.h index a0ed3c75..c8479322 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/UncompressedBlock.h +++ b/library/PolyVoxCore/include/PolyVoxCore/UncompressedBlock.h @@ -35,7 +35,7 @@ namespace PolyVox friend class LargeVolume; public: - UncompressedBlock(uint16_t uSideLength); + UncompressedBlock(Vector3DInt32 v3dPosition, uint16_t uSideLength, Pager* pPager = nullptr); ~UncompressedBlock(); VoxelType* getData(void) const; @@ -67,7 +67,9 @@ namespace PolyVox VoxelType* m_tData; uint16_t m_uSideLength; - uint8_t m_uSideLengthPower; + uint8_t m_uSideLengthPower; + Pager* m_pPager; + Vector3DInt32 m_v3dBlockSpacePosition; }; } diff --git a/library/PolyVoxCore/include/PolyVoxCore/UncompressedBlock.inl b/library/PolyVoxCore/include/PolyVoxCore/UncompressedBlock.inl index d5252d2c..401ff40e 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/UncompressedBlock.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/UncompressedBlock.inl @@ -26,12 +26,14 @@ freely, subject to the following restrictions: namespace PolyVox { template - UncompressedBlock::UncompressedBlock(uint16_t uSideLength) + UncompressedBlock::UncompressedBlock(Vector3DInt32 v3dPosition, uint16_t uSideLength, Pager* pPager) :m_uBlockLastAccessed(0) ,m_bDataModified(true) ,m_tData(0) ,m_uSideLength(0) ,m_uSideLengthPower(0) + ,m_pPager(pPager) + ,m_v3dBlockSpacePosition(v3dPosition) { // Compute the side length m_uSideLength = uSideLength; @@ -39,12 +41,37 @@ namespace PolyVox // Allocate the data const uint32_t uNoOfVoxels = m_uSideLength * m_uSideLength * m_uSideLength; - m_tData = new VoxelType[uNoOfVoxels]; + m_tData = new VoxelType[uNoOfVoxels]; + + // Pass the block to the Pager to give it a chance to initialise it with any data + if (m_pPager) + { + // From the coordinates of the block we deduce the coordinates of the contained voxels. + Vector3DInt32 v3dLower = m_v3dBlockSpacePosition * static_cast(m_uSideLength); + Vector3DInt32 v3dUpper = v3dLower + Vector3DInt32(m_uSideLength - 1, m_uSideLength - 1, m_uSideLength - 1); + Region reg(v3dLower, v3dUpper); + + // Page the data in + m_pPager->pageIn(reg, this); + } + + // We'll use this later to decide if data needs to be paged out again. + m_bDataModified = false; } template UncompressedBlock::~UncompressedBlock() { + if (m_pPager && m_bDataModified) + { + // From the coordinates of the block we deduce the coordinates of the contained voxels. + Vector3DInt32 v3dLower = m_v3dBlockSpacePosition * static_cast(m_uSideLength); + Vector3DInt32 v3dUpper = v3dLower + Vector3DInt32(m_uSideLength - 1, m_uSideLength - 1, m_uSideLength - 1); + + // Page the data out + m_pPager->pageOut(Region(v3dLower, v3dUpper), this); + } + delete[] m_tData; m_tData = 0; } From 37f1ddaed21fe7a48193cda832ac2917eb60aa4e Mon Sep 17 00:00:00 2001 From: David Williams Date: Wed, 17 Sep 2014 17:10:13 +0200 Subject: [PATCH 12/42] Work on block storage. --- .../include/PolyVoxCore/LargeVolume.h | 8 +- .../include/PolyVoxCore/LargeVolume.inl | 77 ++++++++++++------- 2 files changed, 53 insertions(+), 32 deletions(-) diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h index 17954659..df966f6f 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h @@ -300,7 +300,8 @@ namespace PolyVox } }; - typedef std::map >, BlockPositionCompare> UncompressedBlockMap; + typedef std::map >, BlockPositionCompare> SharedPtrBlockMap; + typedef std::map >, BlockPositionCompare> WeakPtrBlockMap; void initialise(); @@ -314,10 +315,11 @@ namespace PolyVox std::shared_ptr< UncompressedBlock > getUncompressedBlock(int32_t uBlockX, int32_t uBlockY, int32_t uBlockZ) const; - void eraseBlock(typename UncompressedBlockMap::iterator itUncompressedBlock) const; + void eraseBlock(typename SharedPtrBlockMap::iterator itUncompressedBlock) const; // The block data - mutable UncompressedBlockMap m_pBlocks; + mutable WeakPtrBlockMap m_pAllBlocks; + mutable SharedPtrBlockMap m_pRecentlyUsedBlocks; mutable uint32_t m_uTimestamper; mutable Vector3DInt32 m_v3dLastAccessedBlockPos; diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl index 218bff10..0c7335d7 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl @@ -231,7 +231,7 @@ namespace PolyVox //clearBlockCache(); - /*if (m_pBlocks.size() > uMaxNumberOfBlocksInMemory) + /*if (m_pRecentlyUsedBlocks.size() > uMaxNumberOfBlocksInMemory) { flushAll(); }*/ @@ -369,9 +369,9 @@ namespace PolyVox for(int32_t z = v3dStart.getZ(); z <= v3dEnd.getZ(); z++) { Vector3DInt32 pos(x,y,z); - /*typename CompressedBlockMap::iterator itBlock = m_pBlocks.find(pos); + /*typename CompressedBlockMap::iterator itBlock = m_pRecentlyUsedBlocks.find(pos); - if(itBlock != m_pBlocks.end()) + if(itBlock != m_pRecentlyUsedBlocks.end()) { // If the block is already loaded then we don't load it again. This means it does not get uncompressed, // whereas if we were to call getUncompressedBlock() regardless then it would also get uncompressed. @@ -407,9 +407,9 @@ namespace PolyVox //Replaced the for loop here as the call to //eraseBlock was invalidating the iterator. - while(m_pBlocks.size() > 0) + while (m_pRecentlyUsedBlocks.size() > 0) { - eraseBlock(m_pBlocks.begin()); + eraseBlock(m_pRecentlyUsedBlocks.begin()); } } @@ -440,8 +440,8 @@ namespace PolyVox for(int32_t z = v3dStart.getZ(); z <= v3dEnd.getZ(); z++) { Vector3DInt32 pos(x,y,z); - typename UncompressedBlockMap::iterator itBlock = m_pBlocks.find(pos); - if (itBlock == m_pBlocks.end()) + typename SharedPtrBlockMap::iterator itBlock = m_pRecentlyUsedBlocks.find(pos); + if (itBlock == m_pRecentlyUsedBlocks.end()) { // not loaded, not unloading continue; @@ -493,7 +493,7 @@ namespace PolyVox //setMaxNumberOfUncompressedBlocks(m_uMaxNumberOfUncompressedBlocks); //Clear the previous data - m_pBlocks.clear(); + m_pRecentlyUsedBlocks.clear(); //Other properties we might find useful later this->m_uLongestSideLength = (std::max)((std::max)(this->getWidth(),this->getHeight()),this->getDepth()); @@ -502,7 +502,7 @@ namespace PolyVox } template - void LargeVolume::eraseBlock(typename UncompressedBlockMap::iterator itUncompressedBlock) const + void LargeVolume::eraseBlock(typename SharedPtrBlockMap::iterator itUncompressedBlock) const { POLYVOX_ASSERT(m_pPager, "A block should never be erased if there is no pager attached to handle it"); @@ -520,7 +520,7 @@ namespace PolyVox //delete itUncompressedBlock->second; // We can now remove the block data from memory. - m_pBlocks.erase(itUncompressedBlock); + m_pRecentlyUsedBlocks.erase(itUncompressedBlock); } template @@ -539,38 +539,57 @@ namespace PolyVox // Try to find the required block in our block list. std::shared_ptr< UncompressedBlock > pUncompressedBlock = nullptr; - typename UncompressedBlockMap::iterator itUncompressedBlock = m_pBlocks.find(v3dBlockPos); + typename SharedPtrBlockMap::iterator itUncompressedBlock = m_pRecentlyUsedBlocks.find(v3dBlockPos); // Check whether the block was found. - if(itUncompressedBlock != m_pBlocks.end()) + if (itUncompressedBlock != m_pRecentlyUsedBlocks.end()) { // The block was found so we can use it. pUncompressedBlock = itUncompressedBlock->second; } else { - // The blcok was not found so we will create a new one. - pUncompressedBlock = std::make_shared< UncompressedBlock >(v3dBlockPos, m_uBlockSideLength, m_pPager); - - while (m_pBlocks.size() + 1 > m_uMaxNumberOfUncompressedBlocks) // +1 ready for new block we will add next. + // There's some (slim) chance that it exists in the list of all blocks, because a sampler may be holding on to it. + typename WeakPtrBlockMap::iterator itWeakUncompressedBlock = m_pAllBlocks.find(v3dBlockPos); + if (itWeakUncompressedBlock != m_pAllBlocks.end()) { - // Find the least recently used block. Hopefully this isn't too slow. - typename UncompressedBlockMap::iterator i; - typename UncompressedBlockMap::iterator itUnloadBlock = m_pBlocks.begin(); - for (i = m_pBlocks.begin(); i != m_pBlocks.end(); i++) + if (!itWeakUncompressedBlock->second.expired()) { - if (i->second->m_uBlockLastAccessed < itUnloadBlock->second->m_uBlockLastAccessed) + // The block was found so we can use it. + pUncompressedBlock = std::shared_ptr< UncompressedBlock >(itWeakUncompressedBlock->second); + m_pRecentlyUsedBlocks.insert(std::make_pair(v3dBlockPos, pUncompressedBlock)); + } + else + { + m_pAllBlocks.erase(itWeakUncompressedBlock); + } + } + else + { + // The block was not found so we will create a new one. + pUncompressedBlock = std::make_shared< UncompressedBlock >(v3dBlockPos, m_uBlockSideLength, m_pPager); + + while (m_pRecentlyUsedBlocks.size() + 1 > m_uMaxNumberOfUncompressedBlocks) // +1 ready for new block we will add next. + { + // Find the least recently used block. Hopefully this isn't too slow. + typename SharedPtrBlockMap::iterator i; + typename SharedPtrBlockMap::iterator itUnloadBlock = m_pRecentlyUsedBlocks.begin(); + for (i = m_pRecentlyUsedBlocks.begin(); i != m_pRecentlyUsedBlocks.end(); i++) { - itUnloadBlock = i; + if (i->second->m_uBlockLastAccessed < itUnloadBlock->second->m_uBlockLastAccessed) + { + itUnloadBlock = i; + } } + + // Erase the least recently used block + eraseBlock(itUnloadBlock); } - // Erase the least recently used block - eraseBlock(itUnloadBlock); + // Add our new block to the map. + m_pAllBlocks.insert(std::make_pair(v3dBlockPos, pUncompressedBlock)); + m_pRecentlyUsedBlocks.insert(std::make_pair(v3dBlockPos, pUncompressedBlock)); } - - // Add our new block to the map. - m_pBlocks.insert(std::make_pair(v3dBlockPos, pUncompressedBlock)); } pUncompressedBlock->m_uBlockLastAccessed = ++m_uTimestamper; @@ -589,8 +608,8 @@ namespace PolyVox uint32_t uSizeInBytes = sizeof(LargeVolume); //Memory used by the blocks - typename UncompressedBlockMap::iterator i; - for (i = m_pBlocks.begin(); i != m_pBlocks.end(); i++) + typename SharedPtrBlockMap::iterator i; + for (i = m_pRecentlyUsedBlocks.begin(); i != m_pRecentlyUsedBlocks.end(); i++) { //Inaccurate - account for rest of loaded block. uSizeInBytes += i->second->calculateSizeInBytes(); From 98526d38d35ccbac2dbb241c84074e50ddd14664 Mon Sep 17 00:00:00 2001 From: David Williams Date: Wed, 17 Sep 2014 21:04:47 +0200 Subject: [PATCH 13/42] Crash fix. --- .../include/PolyVoxCore/LargeVolume.inl | 42 ++++++++++--------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl index 0c7335d7..86fb8244 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl @@ -547,7 +547,8 @@ namespace PolyVox // The block was found so we can use it. pUncompressedBlock = itUncompressedBlock->second; } - else + + if (!pUncompressedBlock) { // There's some (slim) chance that it exists in the list of all blocks, because a sampler may be holding on to it. typename WeakPtrBlockMap::iterator itWeakUncompressedBlock = m_pAllBlocks.find(v3dBlockPos); @@ -564,32 +565,33 @@ namespace PolyVox m_pAllBlocks.erase(itWeakUncompressedBlock); } } - else + } + + if (!pUncompressedBlock) + { + // The block was not found so we will create a new one. + pUncompressedBlock = std::make_shared< UncompressedBlock >(v3dBlockPos, m_uBlockSideLength, m_pPager); + + while (m_pRecentlyUsedBlocks.size() + 1 > m_uMaxNumberOfUncompressedBlocks) // +1 ready for new block we will add next. { - // The block was not found so we will create a new one. - pUncompressedBlock = std::make_shared< UncompressedBlock >(v3dBlockPos, m_uBlockSideLength, m_pPager); - - while (m_pRecentlyUsedBlocks.size() + 1 > m_uMaxNumberOfUncompressedBlocks) // +1 ready for new block we will add next. + // Find the least recently used block. Hopefully this isn't too slow. + typename SharedPtrBlockMap::iterator i; + typename SharedPtrBlockMap::iterator itUnloadBlock = m_pRecentlyUsedBlocks.begin(); + for (i = m_pRecentlyUsedBlocks.begin(); i != m_pRecentlyUsedBlocks.end(); i++) { - // Find the least recently used block. Hopefully this isn't too slow. - typename SharedPtrBlockMap::iterator i; - typename SharedPtrBlockMap::iterator itUnloadBlock = m_pRecentlyUsedBlocks.begin(); - for (i = m_pRecentlyUsedBlocks.begin(); i != m_pRecentlyUsedBlocks.end(); i++) + if (i->second->m_uBlockLastAccessed < itUnloadBlock->second->m_uBlockLastAccessed) { - if (i->second->m_uBlockLastAccessed < itUnloadBlock->second->m_uBlockLastAccessed) - { - itUnloadBlock = i; - } + itUnloadBlock = i; } - - // Erase the least recently used block - eraseBlock(itUnloadBlock); } - // Add our new block to the map. - m_pAllBlocks.insert(std::make_pair(v3dBlockPos, pUncompressedBlock)); - m_pRecentlyUsedBlocks.insert(std::make_pair(v3dBlockPos, pUncompressedBlock)); + // Erase the least recently used block + eraseBlock(itUnloadBlock); } + + // Add our new block to the map. + m_pAllBlocks.insert(std::make_pair(v3dBlockPos, pUncompressedBlock)); + m_pRecentlyUsedBlocks.insert(std::make_pair(v3dBlockPos, pUncompressedBlock)); } pUncompressedBlock->m_uBlockLastAccessed = ++m_uTimestamper; From daa93aab199cd8c1510b81f18568f0ba67c21898 Mon Sep 17 00:00:00 2001 From: David Williams Date: Wed, 17 Sep 2014 21:27:52 +0200 Subject: [PATCH 14/42] Tidying and commenting. --- .../include/PolyVoxCore/LargeVolume.inl | 24 ++++++++++++------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl index 86fb8244..38f05adf 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl @@ -537,36 +537,42 @@ namespace PolyVox return m_pLastAccessedBlock; } - // Try to find the required block in our block list. + // The block was not the same as last time, but we can now hope it is in the set of most recently used blocks. std::shared_ptr< UncompressedBlock > pUncompressedBlock = nullptr; typename SharedPtrBlockMap::iterator itUncompressedBlock = m_pRecentlyUsedBlocks.find(v3dBlockPos); // Check whether the block was found. - if (itUncompressedBlock != m_pRecentlyUsedBlocks.end()) + if ((itUncompressedBlock) != m_pRecentlyUsedBlocks.end()) { // The block was found so we can use it. pUncompressedBlock = itUncompressedBlock->second; + POLYVOX_ASSERT(pUncompressedBlock, "Recent block list shold never contain a null pointer."); } if (!pUncompressedBlock) { - // There's some (slim) chance that it exists in the list of all blocks, because a sampler may be holding on to it. + // Although it's not in our recently use blocks, there's some (slim) chance that it + // exists in the list of all loaded blocks, because a sampler may be holding on to it. typename WeakPtrBlockMap::iterator itWeakUncompressedBlock = m_pAllBlocks.find(v3dBlockPos); if (itWeakUncompressedBlock != m_pAllBlocks.end()) { - if (!itWeakUncompressedBlock->second.expired()) + // We've found an entry in the 'all blocks' list, but it can be null. This happens if a sampler was the + // last thing to be keeping it alive and then the sampler let it go. In this case we remove it from the + // list, and it will get added again soon when we pag e it in and fil it with valid data. + if (itWeakUncompressedBlock->second.expired()) { - // The block was found so we can use it. - pUncompressedBlock = std::shared_ptr< UncompressedBlock >(itWeakUncompressedBlock->second); - m_pRecentlyUsedBlocks.insert(std::make_pair(v3dBlockPos, pUncompressedBlock)); + m_pAllBlocks.erase(itWeakUncompressedBlock); } else { - m_pAllBlocks.erase(itWeakUncompressedBlock); + // The block is valid. We know it's not in the recently used list (we checked earlier) so it should be added. + pUncompressedBlock = std::shared_ptr< UncompressedBlock >(itWeakUncompressedBlock->second); + m_pRecentlyUsedBlocks.insert(std::make_pair(v3dBlockPos, pUncompressedBlock)); } } } + // If we still haven't found the block then it's time to create a new one and page it in from disk. if (!pUncompressedBlock) { // The block was not found so we will create a new one. @@ -589,7 +595,7 @@ namespace PolyVox eraseBlock(itUnloadBlock); } - // Add our new block to the map. + // Add our new block to the maps. m_pAllBlocks.insert(std::make_pair(v3dBlockPos, pUncompressedBlock)); m_pRecentlyUsedBlocks.insert(std::make_pair(v3dBlockPos, pUncompressedBlock)); } From 177eb16becfd3c902ef98de946f94744fc7a0578 Mon Sep 17 00:00:00 2001 From: David Williams Date: Wed, 17 Sep 2014 21:40:27 +0200 Subject: [PATCH 15/42] Tidying and commenting. --- .../include/PolyVoxCore/LargeVolume.h | 2 - .../include/PolyVoxCore/LargeVolume.inl | 40 +++++-------------- 2 files changed, 11 insertions(+), 31 deletions(-) diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h index df966f6f..f5b22a07 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h @@ -315,8 +315,6 @@ namespace PolyVox std::shared_ptr< UncompressedBlock > getUncompressedBlock(int32_t uBlockX, int32_t uBlockY, int32_t uBlockZ) const; - void eraseBlock(typename SharedPtrBlockMap::iterator itUncompressedBlock) const; - // The block data mutable WeakPtrBlockMap m_pAllBlocks; mutable SharedPtrBlockMap m_pRecentlyUsedBlocks; diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl index 38f05adf..bb164054 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl @@ -407,10 +407,10 @@ namespace PolyVox //Replaced the for loop here as the call to //eraseBlock was invalidating the iterator. - while (m_pRecentlyUsedBlocks.size() > 0) + /*while (m_pRecentlyUsedBlocks.size() > 0) { eraseBlock(m_pRecentlyUsedBlocks.begin()); - } + }*/ } //////////////////////////////////////////////////////////////////////////////// @@ -419,7 +419,7 @@ namespace PolyVox template void LargeVolume::flush(Region regFlush) { - POLYVOX_THROW_IF(!m_pPager, invalid_operation, "You cannot flush data out of the volume because it was created without a pager attached."); + /*POLYVOX_THROW_IF(!m_pPager, invalid_operation, "You cannot flush data out of the volume because it was created without a pager attached."); Vector3DInt32 v3dStart; for(int i = 0; i < 3; i++) @@ -454,7 +454,7 @@ namespace PolyVox } } // for z } // for y - } // for x + } // for x*/ } //////////////////////////////////////////////////////////////////////////////// @@ -501,28 +501,6 @@ namespace PolyVox this->m_fDiagonalLength = sqrtf(static_cast(this->getWidth() * this->getWidth() + this->getHeight() * this->getHeight() + this->getDepth() * this->getDepth())); } - template - void LargeVolume::eraseBlock(typename SharedPtrBlockMap::iterator itUncompressedBlock) const - { - POLYVOX_ASSERT(m_pPager, "A block should never be erased if there is no pager attached to handle it"); - - std::shared_ptr< UncompressedBlock > pUncompressedBlock = itUncompressedBlock->second; - - // This should never happen as blocks are deleted based on being least recently used. - // I the case that we are flushing we delete all blocks, but the flush function will - // reset the 'm_pLastAccessedBlock' anyway to prevent it being accidentally reused. - //POLYVOX_ASSERT(pUncompressedBlock != m_pLastAccessedBlock, "Attempted to delete last accessed block."); - - // Before deleting the block we may need to recompress its data. We - // only do this if the data has been modified since it was decompressed. - - - //delete itUncompressedBlock->second; - - // We can now remove the block data from memory. - m_pRecentlyUsedBlocks.erase(itUncompressedBlock); - } - template std::shared_ptr< UncompressedBlock > LargeVolume::getUncompressedBlock(int32_t uBlockX, int32_t uBlockY, int32_t uBlockZ) const { @@ -578,12 +556,16 @@ namespace PolyVox // The block was not found so we will create a new one. pUncompressedBlock = std::make_shared< UncompressedBlock >(v3dBlockPos, m_uBlockSideLength, m_pPager); + // As we are loading a new block we should try to ensure we don't go over our target memory usage. while (m_pRecentlyUsedBlocks.size() + 1 > m_uMaxNumberOfUncompressedBlocks) // +1 ready for new block we will add next. { + // This should never hit, because it should not have been possible for + // the user to limit the number of blocks if they did not provide a pager. + POLYVOX_ASSERT(m_pPager, "A valid pager is required to limit number of blocks"); + // Find the least recently used block. Hopefully this isn't too slow. - typename SharedPtrBlockMap::iterator i; typename SharedPtrBlockMap::iterator itUnloadBlock = m_pRecentlyUsedBlocks.begin(); - for (i = m_pRecentlyUsedBlocks.begin(); i != m_pRecentlyUsedBlocks.end(); i++) + for (typename SharedPtrBlockMap::iterator i = m_pRecentlyUsedBlocks.begin(); i != m_pRecentlyUsedBlocks.end(); i++) { if (i->second->m_uBlockLastAccessed < itUnloadBlock->second->m_uBlockLastAccessed) { @@ -592,7 +574,7 @@ namespace PolyVox } // Erase the least recently used block - eraseBlock(itUnloadBlock); + m_pRecentlyUsedBlocks.erase(itUnloadBlock); } // Add our new block to the maps. From 916fe310ad7da78a37eff3244e12e81f28dadfe8 Mon Sep 17 00:00:00 2001 From: David Williams Date: Wed, 17 Sep 2014 23:31:23 +0200 Subject: [PATCH 16/42] Added function to purge null blocks. --- .../include/PolyVoxCore/LargeVolume.h | 2 ++ .../include/PolyVoxCore/LargeVolume.inl | 21 +++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h index f5b22a07..2bd177ea 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h @@ -315,6 +315,8 @@ namespace PolyVox std::shared_ptr< UncompressedBlock > getUncompressedBlock(int32_t uBlockX, int32_t uBlockY, int32_t uBlockZ) const; + void purgeNullPtrsFromAllBlocks(void) const; + // The block data mutable WeakPtrBlockMap m_pAllBlocks; mutable SharedPtrBlockMap m_pRecentlyUsedBlocks; diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl index bb164054..0247f190 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl @@ -557,6 +557,7 @@ namespace PolyVox pUncompressedBlock = std::make_shared< UncompressedBlock >(v3dBlockPos, m_uBlockSideLength, m_pPager); // As we are loading a new block we should try to ensure we don't go over our target memory usage. + bool erasedBlock = false; while (m_pRecentlyUsedBlocks.size() + 1 > m_uMaxNumberOfUncompressedBlocks) // +1 ready for new block we will add next. { // This should never hit, because it should not have been possible for @@ -575,6 +576,14 @@ namespace PolyVox // Erase the least recently used block m_pRecentlyUsedBlocks.erase(itUnloadBlock); + erasedBlock = true; + } + + // If we've deleted any blocks from the recently used list then this + // seems like a good place to purge the 'all blocks' list as well. + if (erasedBlock) + { + purgeNullPtrsFromAllBlocks(); } // Add our new block to the maps. @@ -612,6 +621,18 @@ namespace PolyVox return uSizeInBytes; } + template + void LargeVolume::purgeNullPtrsFromAllBlocks(void) const + { + for (auto blockIter = m_pAllBlocks.begin(); blockIter != m_pAllBlocks.end(); blockIter++) + { + if (blockIter->second.expired()) + { + blockIter = m_pAllBlocks.erase(blockIter); + } + } + } + template template VoxelType LargeVolume::getVoxelImpl(int32_t uXPos, int32_t uYPos, int32_t uZPos, WrapModeType, VoxelType tBorder) const From 277b6c7b9bb736444a161f942631244159294533 Mon Sep 17 00:00:00 2001 From: David Williams Date: Wed, 17 Sep 2014 23:53:14 +0200 Subject: [PATCH 17/42] Tidying up. --- .../include/PolyVoxCore/LargeVolume.inl | 25 +++++++++++-------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl index 0247f190..5be837b7 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl @@ -400,17 +400,19 @@ namespace PolyVox template void LargeVolume::flushAll() { - POLYVOX_THROW_IF(!m_pPager, invalid_operation, "You cannot flush data out of the volume because it was created without a pager attached."); + POLYVOX_LOG_WARNING_IF(!m_pPager, "Data discarded by flush operation as no pager is attached."); - // Flushing will remove the most accessed block, so invalidate the pointer. - m_pLastAccessedBlock = 0; + // Clear this pointer so it doesn't hang on to any blocks. + m_pLastAccessedBlock = nullptr; - //Replaced the for loop here as the call to - //eraseBlock was invalidating the iterator. - /*while (m_pRecentlyUsedBlocks.size() > 0) - { - eraseBlock(m_pRecentlyUsedBlocks.begin()); - }*/ + // Erase all the most recently used blocks. + m_pRecentlyUsedBlocks.clear(); + + // Remove deleted blocks from the list of all loaded blocks. + purgeNullPtrsFromAllBlocks(); + + // If there are still some blocks left then this is a cause for concern. Perhaps samplers are holding on to them? + POLYVOX_LOG_WARNING_IF(m_pAllBlocks.size() > 0, "Blocks still exist after performing flushAll()! Perhaps you have samplers attached?"); } //////////////////////////////////////////////////////////////////////////////// @@ -419,7 +421,10 @@ namespace PolyVox template void LargeVolume::flush(Region regFlush) { - /*POLYVOX_THROW_IF(!m_pPager, invalid_operation, "You cannot flush data out of the volume because it was created without a pager attached."); + /*POLYVOX_LOG_WARNING_IF(!m_pPager, "Data discarded by flush operation as no pager is attached."); + + // Clear this pointer so it doesn't hang on to any blocks. + m_pLastAccessedBlock = nullptr; Vector3DInt32 v3dStart; for(int i = 0; i < 3; i++) From e88466ad4d112db6835ae08fe2f3b96000b69806 Mon Sep 17 00:00:00 2001 From: David Williams Date: Thu, 18 Sep 2014 14:14:08 +0200 Subject: [PATCH 18/42] Tidying flush function. --- .../include/PolyVoxCore/LargeVolume.inl | 29 +++++++------------ 1 file changed, 11 insertions(+), 18 deletions(-) diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl index 5be837b7..10973f35 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl @@ -421,11 +421,12 @@ namespace PolyVox template void LargeVolume::flush(Region regFlush) { - /*POLYVOX_LOG_WARNING_IF(!m_pPager, "Data discarded by flush operation as no pager is attached."); + POLYVOX_LOG_WARNING_IF(!m_pPager, "Data discarded by flush operation as no pager is attached."); // Clear this pointer so it doesn't hang on to any blocks. m_pLastAccessedBlock = nullptr; + // Convert the start and end positions into block space coordinates Vector3DInt32 v3dStart; for(int i = 0; i < 3; i++) { @@ -438,28 +439,20 @@ namespace PolyVox v3dEnd.setElement(i, regFlush.getUpperCorner().getElement(i) >> m_uBlockSideLengthPower); } + // Loops over the specified positions and delete the corresponding blocks. for(int32_t x = v3dStart.getX(); x <= v3dEnd.getX(); x++) { for(int32_t y = v3dStart.getY(); y <= v3dEnd.getY(); y++) { for(int32_t z = v3dStart.getZ(); z <= v3dEnd.getZ(); z++) { - Vector3DInt32 pos(x,y,z); - typename SharedPtrBlockMap::iterator itBlock = m_pRecentlyUsedBlocks.find(pos); - if (itBlock == m_pRecentlyUsedBlocks.end()) - { - // not loaded, not unloading - continue; - } - eraseBlock(itBlock); - // eraseBlock might cause a call to getUncompressedBlock, which again sets m_pLastAccessedBlock - if(m_v3dLastAccessedBlockPos == pos) - { - m_pLastAccessedBlock = 0; - } - } // for z - } // for y - } // for x*/ + m_pRecentlyUsedBlocks.erase(Vector3DInt32(x, y, z)); m_pLastAccessedBlock = 0; + } + } + } + + // We might now have so null pointers in the 'all blocks' list so clean them up. + purgeNullPtrsFromAllBlocks(); } //////////////////////////////////////////////////////////////////////////////// @@ -541,7 +534,7 @@ namespace PolyVox { // We've found an entry in the 'all blocks' list, but it can be null. This happens if a sampler was the // last thing to be keeping it alive and then the sampler let it go. In this case we remove it from the - // list, and it will get added again soon when we pag e it in and fil it with valid data. + // list, and it will get added again soon when we page it in and fill it with valid data. if (itWeakUncompressedBlock->second.expired()) { m_pAllBlocks.erase(itWeakUncompressedBlock); From 38b876412928db203508097d4571b36becbb9441 Mon Sep 17 00:00:00 2001 From: David Williams Date: Thu, 18 Sep 2014 14:22:44 +0200 Subject: [PATCH 19/42] Tidying up prefetch function. --- .../include/PolyVoxCore/LargeVolume.inl | 42 ++++--------------- 1 file changed, 9 insertions(+), 33 deletions(-) diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl index 10973f35..fcf13e20 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl @@ -340,6 +340,7 @@ namespace PolyVox template void LargeVolume::prefetch(Region regPrefetch) { + // Convert the start and end positions into block space coordinates Vector3DInt32 v3dStart; for(int i = 0; i < 3; i++) { @@ -352,46 +353,21 @@ namespace PolyVox v3dEnd.setElement(i, regPrefetch.getUpperCorner().getElement(i) >> m_uBlockSideLengthPower); } - Vector3DInt32 v3dSize = v3dEnd - v3dStart + Vector3DInt32(1,1,1); - uint32_t numblocks = static_cast(v3dSize.getX() * v3dSize.getY() * v3dSize.getZ()); + Region region(v3dStart, v3dEnd); + uint32_t uNoOfBlocks = static_cast(region.getWidthInVoxels() * region.getHeightInVoxels() * region.getDepthInVoxels()); + POLYVOX_LOG_WARNING_IF(uNoOfBlocks > m_uMaxNumberOfUncompressedBlocks, "Attempting to prefetch more than the maximum number of blocks."); + uNoOfBlocks = (std::min)(uNoOfBlocks, m_uMaxNumberOfUncompressedBlocks); - // FIXME - reinstate some logic to handle when the prefetched region is too large. - - /*if(numblocks > m_uMaxNumberOfBlocksInMemory) - { - // cannot support the amount of blocks... so only load the maximum possible - numblocks = m_uMaxNumberOfBlocksInMemory; - }*/ for(int32_t x = v3dStart.getX(); x <= v3dEnd.getX(); x++) { for(int32_t y = v3dStart.getY(); y <= v3dEnd.getY(); y++) { for(int32_t z = v3dStart.getZ(); z <= v3dEnd.getZ(); z++) - { - Vector3DInt32 pos(x,y,z); - /*typename CompressedBlockMap::iterator itBlock = m_pRecentlyUsedBlocks.find(pos); - - if(itBlock != m_pRecentlyUsedBlocks.end()) - { - // If the block is already loaded then we don't load it again. This means it does not get uncompressed, - // whereas if we were to call getUncompressedBlock() regardless then it would also get uncompressed. - // This might be nice, but on the prefetch region could be bigger than the uncompressed cache size. - // This would limit the amount of prefetching we could do. - continue; - }*/ - - if(numblocks == 0) - { - // Loading any more blocks would attempt to overflow the memory and therefore erase blocks - // we loaded in the beginning. This wouldn't cause logic problems but would be wasteful. - return; - } - // load a block - numblocks--; + { getUncompressedBlock(x,y,z); - } // for z - } // for y - } // for x + } + } + } } //////////////////////////////////////////////////////////////////////////////// From 8dd026e095d257a6852fd85adabb4728dbcd609d Mon Sep 17 00:00:00 2001 From: David Williams Date: Thu, 18 Sep 2014 14:51:43 +0200 Subject: [PATCH 20/42] Work on limiting maximum memory usage. --- examples/Paging/main.cpp | 4 +-- .../include/PolyVoxCore/LargeVolume.h | 2 +- .../include/PolyVoxCore/LargeVolume.inl | 30 ++++++++++++------- tests/testvolume.cpp | 3 +- 4 files changed, 23 insertions(+), 16 deletions(-) diff --git a/examples/Paging/main.cpp b/examples/Paging/main.cpp index a01779fb..1fd22aab 100644 --- a/examples/Paging/main.cpp +++ b/examples/Paging/main.cpp @@ -158,10 +158,8 @@ int main(int argc, char *argv[]) PerlinNoisePager* pager = new PerlinNoisePager(); LargeVolume volData(PolyVox::Region::MaxRegion, pager, 256); - //volData.setMaxNumberOfBlocksInMemory(4096); - volData.setMaxNumberOfUncompressedBlocks(64); + volData.setTargetMemoryUsage(2 * 1024 * 1024); // 2Mb - //volData.setMaxNumberOfUncompressedBlocks(4096); //createSphereInVolume(volData, 30); //createPerlinTerrain(volData); //createPerlinVolumeSlow(volData); diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h index 2bd177ea..01008dcd 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h @@ -256,7 +256,7 @@ namespace PolyVox POLYVOX_DEPRECATED VoxelType getVoxelAt(const Vector3DInt32& v3dPos) const; /// Sets the number of blocks for which uncompressed data is stored - void setMaxNumberOfUncompressedBlocks(uint32_t uMaxNumberOfUncompressedBlocks); + void setTargetMemoryUsage(uint32_t uTargetMemoryUsageInBytes); /// Sets the voxel at the position given by x,y,z coordinates void setVoxel(int32_t uXPos, int32_t uYPos, int32_t uZPos, VoxelType tValue, WrapMode eWrapMode = WrapModes::Validate); /// Sets the voxel at the position given by a 3D vector diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl index fcf13e20..86461937 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl @@ -225,22 +225,26 @@ namespace PolyVox /// \param uMaxNumberOfUncompressedBlocks The number of blocks for which uncompressed data can be cached. //////////////////////////////////////////////////////////////////////////////// template - void LargeVolume::setMaxNumberOfUncompressedBlocks(uint32_t uMaxNumberOfUncompressedBlocks) + void LargeVolume::setTargetMemoryUsage(uint32_t uTargetMemoryUsageInBytes) { POLYVOX_THROW_IF(!m_pPager, invalid_operation, "You cannot limit the memory usage of the volume because it was created without a pager attached."); - //clearBlockCache(); + uint32_t uUncompressedBlockSizeInBytes = m_uBlockSideLength * m_uBlockSideLength * m_uBlockSideLength * sizeof(VoxelType); - /*if (m_pRecentlyUsedBlocks.size() > uMaxNumberOfBlocksInMemory) + m_uMaxNumberOfUncompressedBlocks = uTargetMemoryUsageInBytes / uUncompressedBlockSizeInBytes; + + const uint32_t uMinPracticalNoOfBlocks = 4; + POLYVOX_LOG_WARNING_IF(m_uMaxNumberOfUncompressedBlocks < uMinPracticalNoOfBlocks, "The target memory usage is set too low and cannot be adhered to."); + m_uMaxNumberOfUncompressedBlocks = (std::max)(m_uMaxNumberOfUncompressedBlocks, uMinPracticalNoOfBlocks); + + + if (m_pRecentlyUsedBlocks.size() > m_uMaxNumberOfUncompressedBlocks) { flushAll(); - }*/ + } - m_uMaxNumberOfUncompressedBlocks = uMaxNumberOfUncompressedBlocks; - - uint32_t uUncompressedBlockSizeInBytes = m_uBlockSideLength * m_uBlockSideLength * m_uBlockSideLength * sizeof(VoxelType); - POLYVOX_LOG_DEBUG("The maximum number of uncompresed blocks has been set to " << m_uMaxNumberOfUncompressedBlocks - << ", which is " << m_uMaxNumberOfUncompressedBlocks * uUncompressedBlockSizeInBytes << " bytes"); + POLYVOX_LOG_DEBUG("Target memory usage for volume set to " << uTargetMemoryUsageInBytes << "bytes (" + << m_uMaxNumberOfUncompressedBlocks << " blocks of " << uUncompressedBlockSizeInBytes << "bytes each)."); } //////////////////////////////////////////////////////////////////////////////// @@ -353,11 +357,13 @@ namespace PolyVox v3dEnd.setElement(i, regPrefetch.getUpperCorner().getElement(i) >> m_uBlockSideLengthPower); } + // Ensure we don't page in more blocks than the volume can hold. Region region(v3dStart, v3dEnd); uint32_t uNoOfBlocks = static_cast(region.getWidthInVoxels() * region.getHeightInVoxels() * region.getDepthInVoxels()); POLYVOX_LOG_WARNING_IF(uNoOfBlocks > m_uMaxNumberOfUncompressedBlocks, "Attempting to prefetch more than the maximum number of blocks."); uNoOfBlocks = (std::min)(uNoOfBlocks, m_uMaxNumberOfUncompressedBlocks); + // Loops over the specified positions and touch the corresponding blocks. for(int32_t x = v3dStart.getX(); x <= v3dEnd.getX(); x++) { for(int32_t y = v3dStart.getY(); y <= v3dEnd.getY(); y++) @@ -598,12 +604,16 @@ namespace PolyVox template void LargeVolume::purgeNullPtrsFromAllBlocks(void) const { - for (auto blockIter = m_pAllBlocks.begin(); blockIter != m_pAllBlocks.end(); blockIter++) + for (auto blockIter = m_pAllBlocks.begin(); blockIter != m_pAllBlocks.end();) { if (blockIter->second.expired()) { blockIter = m_pAllBlocks.erase(blockIter); } + else + { + blockIter++; + } } } diff --git a/tests/testvolume.cpp b/tests/testvolume.cpp index 3d737c0d..ae4df9e8 100644 --- a/tests/testvolume.cpp +++ b/tests/testvolume.cpp @@ -277,8 +277,7 @@ TestVolume::TestVolume() m_pSimpleVolume = new SimpleVolume(region); m_pLargeVolume = new LargeVolume(region, m_pFilePager, 32); - //m_pLargeVolume->setMaxNumberOfBlocksInMemory(32); - m_pLargeVolume->setMaxNumberOfUncompressedBlocks(64); + m_pLargeVolume->setTargetMemoryUsage(1 * 1024 * 1024); //Fill the volume with some data for(int z = region.getLowerZ(); z <= region.getUpperZ(); z++) From d2bbd6beba36fe4fbda3250a016f6fdc1317b75f Mon Sep 17 00:00:00 2001 From: David Williams Date: Thu, 18 Sep 2014 14:58:17 +0200 Subject: [PATCH 21/42] Some renaming. --- examples/Paging/main.cpp | 2 +- .../include/PolyVoxCore/LargeVolume.h | 4 +-- .../include/PolyVoxCore/LargeVolume.inl | 28 +++++++++---------- tests/testvolume.cpp | 2 +- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/examples/Paging/main.cpp b/examples/Paging/main.cpp index 1fd22aab..886764d3 100644 --- a/examples/Paging/main.cpp +++ b/examples/Paging/main.cpp @@ -158,7 +158,7 @@ int main(int argc, char *argv[]) PerlinNoisePager* pager = new PerlinNoisePager(); LargeVolume volData(PolyVox::Region::MaxRegion, pager, 256); - volData.setTargetMemoryUsage(2 * 1024 * 1024); // 2Mb + volData.setMemoryUsageLimit(2 * 1024 * 1024); // 2Mb //createSphereInVolume(volData, 30); //createPerlinTerrain(volData); diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h index 01008dcd..fc8f94c2 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h @@ -256,7 +256,7 @@ namespace PolyVox POLYVOX_DEPRECATED VoxelType getVoxelAt(const Vector3DInt32& v3dPos) const; /// Sets the number of blocks for which uncompressed data is stored - void setTargetMemoryUsage(uint32_t uTargetMemoryUsageInBytes); + void setMemoryUsageLimit(uint32_t uMemoryUsageInBytes); /// Sets the voxel at the position given by x,y,z coordinates void setVoxel(int32_t uXPos, int32_t uYPos, int32_t uZPos, VoxelType tValue, WrapMode eWrapMode = WrapModes::Validate); /// Sets the voxel at the position given by a 3D vector @@ -324,7 +324,7 @@ namespace PolyVox mutable uint32_t m_uTimestamper; mutable Vector3DInt32 m_v3dLastAccessedBlockPos; mutable std::shared_ptr< UncompressedBlock > m_pLastAccessedBlock; - uint32_t m_uMaxNumberOfUncompressedBlocks; + uint32_t m_uBlockCountLimit; // The size of the volume Region m_regValidRegionInBlocks; diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl index 86461937..425fac7c 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl @@ -52,13 +52,13 @@ namespace PolyVox if (m_pPager) { // If a pager is available then we can set a sensible limit on our memory usage. - m_uMaxNumberOfUncompressedBlocks = 256; + m_uBlockCountLimit = 256; } else { // If there is no pager provided then we set the block limit to the maximum // value to ensure the system never attempts to page blocks out of memory. - m_uMaxNumberOfUncompressedBlocks = (std::numeric_limits::max)(); + m_uBlockCountLimit = (std::numeric_limits::max)(); } initialise(); @@ -225,26 +225,26 @@ namespace PolyVox /// \param uMaxNumberOfUncompressedBlocks The number of blocks for which uncompressed data can be cached. //////////////////////////////////////////////////////////////////////////////// template - void LargeVolume::setTargetMemoryUsage(uint32_t uTargetMemoryUsageInBytes) + void LargeVolume::setMemoryUsageLimit(uint32_t uMemoryUsageInBytes) { POLYVOX_THROW_IF(!m_pPager, invalid_operation, "You cannot limit the memory usage of the volume because it was created without a pager attached."); uint32_t uUncompressedBlockSizeInBytes = m_uBlockSideLength * m_uBlockSideLength * m_uBlockSideLength * sizeof(VoxelType); - m_uMaxNumberOfUncompressedBlocks = uTargetMemoryUsageInBytes / uUncompressedBlockSizeInBytes; + m_uBlockCountLimit = uMemoryUsageInBytes / uUncompressedBlockSizeInBytes; const uint32_t uMinPracticalNoOfBlocks = 4; - POLYVOX_LOG_WARNING_IF(m_uMaxNumberOfUncompressedBlocks < uMinPracticalNoOfBlocks, "The target memory usage is set too low and cannot be adhered to."); - m_uMaxNumberOfUncompressedBlocks = (std::max)(m_uMaxNumberOfUncompressedBlocks, uMinPracticalNoOfBlocks); + POLYVOX_LOG_WARNING_IF(m_uBlockCountLimit < uMinPracticalNoOfBlocks, "The memory usage limit is set too low and cannot be adhered to."); + m_uBlockCountLimit = (std::max)(m_uBlockCountLimit, uMinPracticalNoOfBlocks); - if (m_pRecentlyUsedBlocks.size() > m_uMaxNumberOfUncompressedBlocks) + if (m_pRecentlyUsedBlocks.size() > m_uBlockCountLimit) { flushAll(); } - POLYVOX_LOG_DEBUG("Target memory usage for volume set to " << uTargetMemoryUsageInBytes << "bytes (" - << m_uMaxNumberOfUncompressedBlocks << " blocks of " << uUncompressedBlockSizeInBytes << "bytes each)."); + POLYVOX_LOG_DEBUG("Target memory usage for volume set to " << uMemoryUsageInBytes << "bytes (" + << m_uBlockCountLimit << " blocks of " << uUncompressedBlockSizeInBytes << "bytes each)."); } //////////////////////////////////////////////////////////////////////////////// @@ -360,8 +360,8 @@ namespace PolyVox // Ensure we don't page in more blocks than the volume can hold. Region region(v3dStart, v3dEnd); uint32_t uNoOfBlocks = static_cast(region.getWidthInVoxels() * region.getHeightInVoxels() * region.getDepthInVoxels()); - POLYVOX_LOG_WARNING_IF(uNoOfBlocks > m_uMaxNumberOfUncompressedBlocks, "Attempting to prefetch more than the maximum number of blocks."); - uNoOfBlocks = (std::min)(uNoOfBlocks, m_uMaxNumberOfUncompressedBlocks); + POLYVOX_LOG_WARNING_IF(uNoOfBlocks > m_uBlockCountLimit, "Attempting to prefetch more than the maximum number of blocks."); + uNoOfBlocks = (std::min)(uNoOfBlocks, m_uBlockCountLimit); // Loops over the specified positions and touch the corresponding blocks. for(int32_t x = v3dStart.getX(); x <= v3dEnd.getX(); x++) @@ -455,7 +455,7 @@ namespace PolyVox } m_uTimestamper = 0; - //m_uMaxNumberOfUncompressedBlocks = 16; + //m_uBlockCountLimit = 16; //m_uMaxNumberOfBlocksInMemory = 1024; m_v3dLastAccessedBlockPos = Vector3DInt32(0,0,0); //There are no invalid positions, but initially the m_pLastAccessedBlock pointer will be null; m_pLastAccessedBlock = 0; @@ -470,7 +470,7 @@ namespace PolyVox m_regValidRegionInBlocks.setUpperY(this->m_regValidRegion.getUpperY() >> m_uBlockSideLengthPower); m_regValidRegionInBlocks.setUpperZ(this->m_regValidRegion.getUpperZ() >> m_uBlockSideLengthPower); - //setMaxNumberOfUncompressedBlocks(m_uMaxNumberOfUncompressedBlocks); + //setMaxNumberOfUncompressedBlocks(m_uBlockCountLimit); //Clear the previous data m_pRecentlyUsedBlocks.clear(); @@ -538,7 +538,7 @@ namespace PolyVox // As we are loading a new block we should try to ensure we don't go over our target memory usage. bool erasedBlock = false; - while (m_pRecentlyUsedBlocks.size() + 1 > m_uMaxNumberOfUncompressedBlocks) // +1 ready for new block we will add next. + while (m_pRecentlyUsedBlocks.size() + 1 > m_uBlockCountLimit) // +1 ready for new block we will add next. { // This should never hit, because it should not have been possible for // the user to limit the number of blocks if they did not provide a pager. diff --git a/tests/testvolume.cpp b/tests/testvolume.cpp index ae4df9e8..28703eb3 100644 --- a/tests/testvolume.cpp +++ b/tests/testvolume.cpp @@ -277,7 +277,7 @@ TestVolume::TestVolume() m_pSimpleVolume = new SimpleVolume(region); m_pLargeVolume = new LargeVolume(region, m_pFilePager, 32); - m_pLargeVolume->setTargetMemoryUsage(1 * 1024 * 1024); + m_pLargeVolume->setMemoryUsageLimit(1 * 1024 * 1024); //Fill the volume with some data for(int z = region.getLowerZ(); z <= region.getUpperZ(); z++) From b08974c197a77da83ec828ece17a2881b65bd156 Mon Sep 17 00:00:00 2001 From: David Williams Date: Thu, 18 Sep 2014 16:26:47 +0200 Subject: [PATCH 22/42] Work on limiting memory usage. --- examples/Paging/main.cpp | 4 +- .../include/PolyVoxCore/LargeVolume.h | 5 ++ .../include/PolyVoxCore/LargeVolume.inl | 64 +++++++++++++------ .../include/PolyVoxCore/UncompressedBlock.h | 1 + .../include/PolyVoxCore/UncompressedBlock.inl | 9 ++- 5 files changed, 59 insertions(+), 24 deletions(-) diff --git a/examples/Paging/main.cpp b/examples/Paging/main.cpp index 886764d3..d9359087 100644 --- a/examples/Paging/main.cpp +++ b/examples/Paging/main.cpp @@ -157,8 +157,8 @@ int main(int argc, char *argv[]) openGLWidget.show(); PerlinNoisePager* pager = new PerlinNoisePager(); - LargeVolume volData(PolyVox::Region::MaxRegion, pager, 256); - volData.setMemoryUsageLimit(2 * 1024 * 1024); // 2Mb + LargeVolume volData(PolyVox::Region::MaxRegion, pager, 64); + volData.setMemoryUsageLimit(8 * 1024 * 1024); // 8Mb //createSphereInVolume(volData, 30); //createPerlinTerrain(volData); diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h index fc8f94c2..1fa9a5b6 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h @@ -334,6 +334,11 @@ namespace PolyVox uint8_t m_uBlockSideLengthPower; Pager* m_pPager; + + // Enough to make sure a blocks and it's neighbours can be loaded, with a few to spare. + static const uint32_t uMinPracticalNoOfBlocks = 32; + // Should preent multi-gigabyte volumes with reasonable block sizes. + static const uint32_t uMaxPracticalNoOfBlocks = 32768; }; } diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl index 425fac7c..b4ee5feb 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl @@ -51,8 +51,25 @@ namespace PolyVox if (m_pPager) { - // If a pager is available then we can set a sensible limit on our memory usage. - m_uBlockCountLimit = 256; + // If the user is creating a vast (almost infinite) volume then we can bet they will be + // expecting a high memory usage and will want a fair number of blocks to play around with. + if (regValid == Region::MaxRegion) + { + m_uBlockCountLimit = 1024; + } + else + { + // Otherwise we try to choose a block count to avoid too much thrashing, particularly when iterating + // over the whole volume. This means at least enough blocks to cover one edge of the volume, and ideally + // enough for a whole face. Which face? Longest edge by shortest edge seems like a reasonable guess. + uint32_t longestSide = (std::max)(regValid.getWidthInVoxels(), (std::max)(regValid.getHeightInVoxels(), regValid.getDepthInVoxels())); + uint32_t shortestSide = (std::min)(regValid.getWidthInVoxels(), (std::min)(regValid.getHeightInVoxels(), regValid.getDepthInVoxels())); + + longestSide /= m_uBlockSideLength; + shortestSide /= m_uBlockSideLength; + + m_uBlockCountLimit = longestSide * shortestSide; + } } else { @@ -61,6 +78,14 @@ namespace PolyVox m_uBlockCountLimit = (std::numeric_limits::max)(); } + // Make sure the calculated block limit is within practical bounds + m_uBlockCountLimit = (std::max)(m_uBlockCountLimit, uMinPracticalNoOfBlocks); + m_uBlockCountLimit = (std::min)(m_uBlockCountLimit, uMaxPracticalNoOfBlocks); + + uint32_t uUncompressedBlockSizeInBytes = UncompressedBlock::calculateSizeInBytes(m_uBlockSideLength); + POLYVOX_LOG_DEBUG("Memory usage limit for volume initially set to " << (m_uBlockCountLimit * uUncompressedBlockSizeInBytes) / (1024 * 1024) + << "Mb (" << m_uBlockCountLimit << " blocks of " << uUncompressedBlockSizeInBytes / 1024 << "Kb each)."); + initialise(); } @@ -229,22 +254,24 @@ namespace PolyVox { POLYVOX_THROW_IF(!m_pPager, invalid_operation, "You cannot limit the memory usage of the volume because it was created without a pager attached."); - uint32_t uUncompressedBlockSizeInBytes = m_uBlockSideLength * m_uBlockSideLength * m_uBlockSideLength * sizeof(VoxelType); - + // Calculate the number of blocks based on the memory limit and the size of each block. + uint32_t uUncompressedBlockSizeInBytes = UncompressedBlock::calculateSizeInBytes(m_uBlockSideLength); m_uBlockCountLimit = uMemoryUsageInBytes / uUncompressedBlockSizeInBytes; - const uint32_t uMinPracticalNoOfBlocks = 4; - POLYVOX_LOG_WARNING_IF(m_uBlockCountLimit < uMinPracticalNoOfBlocks, "The memory usage limit is set too low and cannot be adhered to."); + // We need at least a few blocks available to avoid thrashing, and in pratice there will probably be hundreds. + POLYVOX_LOG_WARNING_IF(m_uBlockCountLimit < uMinPracticalNoOfBlocks, "Requested memory usage limit of " + << uMemoryUsageInBytes / (1024 * 1024) << "Mb is too low and cannot be adhered to."); m_uBlockCountLimit = (std::max)(m_uBlockCountLimit, uMinPracticalNoOfBlocks); + m_uBlockCountLimit = (std::min)(m_uBlockCountLimit, uMaxPracticalNoOfBlocks); - + // If the new limit is less than the number of blocks already loaded then the easiest solution is to flush and start loading again. if (m_pRecentlyUsedBlocks.size() > m_uBlockCountLimit) { flushAll(); } - POLYVOX_LOG_DEBUG("Target memory usage for volume set to " << uMemoryUsageInBytes << "bytes (" - << m_uBlockCountLimit << " blocks of " << uUncompressedBlockSizeInBytes << "bytes each)."); + POLYVOX_LOG_DEBUG("Memory usage limit for volume now set to " << (m_uBlockCountLimit * uUncompressedBlockSizeInBytes) / (1024 * 1024) + << "Mb (" << m_uBlockCountLimit << " blocks of " << uUncompressedBlockSizeInBytes / 1024 << "Kb each)."); } //////////////////////////////////////////////////////////////////////////////// @@ -579,24 +606,19 @@ namespace PolyVox } //////////////////////////////////////////////////////////////////////////////// - /// Note: This function needs reviewing for accuracy... + /// Calculate the memory usage of the volume. //////////////////////////////////////////////////////////////////////////////// template uint32_t LargeVolume::calculateSizeInBytes(void) { + // Purge null blocks so we know that all blocks are used. + purgeNullPtrsFromAllBlocks(); + + // We include the size of the LargeVolume class but it should be insignificant. uint32_t uSizeInBytes = sizeof(LargeVolume); - //Memory used by the blocks - typename SharedPtrBlockMap::iterator i; - for (i = m_pRecentlyUsedBlocks.begin(); i != m_pRecentlyUsedBlocks.end(); i++) - { - //Inaccurate - account for rest of loaded block. - uSizeInBytes += i->second->calculateSizeInBytes(); - } - - //Memory used by the block cache. - //uSizeInBytes += m_vecBlocksWithUncompressedData.capacity() * sizeof(Block); - //uSizeInBytes += m_vecBlocksWithUncompressedData.size() * m_uBlockSideLength * m_uBlockSideLength * m_uBlockSideLength * sizeof(VoxelType); + // Memory used by the blocks + uSizeInBytes += UncompressedBlock::calculateSizeInBytes(m_uBlockSideLength) * m_pAllBlocks.size(); return uSizeInBytes; } diff --git a/library/PolyVoxCore/include/PolyVoxCore/UncompressedBlock.h b/library/PolyVoxCore/include/PolyVoxCore/UncompressedBlock.h index c8479322..f65b0dc2 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/UncompressedBlock.h +++ b/library/PolyVoxCore/include/PolyVoxCore/UncompressedBlock.h @@ -64,6 +64,7 @@ namespace PolyVox // Made this private for consistancy with CompressedBlock. // Users shouldn't really need this for UncompressedBlock anyway. uint32_t calculateSizeInBytes(void); + static uint32_t calculateSizeInBytes(uint32_t uSideLength); VoxelType* m_tData; uint16_t m_uSideLength; diff --git a/library/PolyVoxCore/include/PolyVoxCore/UncompressedBlock.inl b/library/PolyVoxCore/include/PolyVoxCore/UncompressedBlock.inl index 401ff40e..6f73d2f1 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/UncompressedBlock.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/UncompressedBlock.inl @@ -140,9 +140,16 @@ namespace PolyVox template uint32_t UncompressedBlock::calculateSizeInBytes(void) + { + // Call through to the static version + return calculateSizeInBytes(m_uSideLength); + } + + template + uint32_t UncompressedBlock::calculateSizeInBytes(uint32_t uSideLength) { // Returns the size of this class plus the size of the uncompressed data. - uint32_t uSizeInBytes = sizeof(UncompressedBlock) + (m_uSideLength * m_uSideLength * m_uSideLength * sizeof(VoxelType)); + uint32_t uSizeInBytes = sizeof(UncompressedBlock) + (uSideLength * uSideLength * uSideLength * sizeof(VoxelType)); return uSizeInBytes; } } From 2602b0010347b383bffb00f6c0c6c4b133d190af Mon Sep 17 00:00:00 2001 From: David Williams Date: Thu, 18 Sep 2014 16:54:14 +0200 Subject: [PATCH 23/42] Simplifying code. --- .../PolyVoxCore/include/PolyVoxCore/LargeVolume.inl | 10 +++------- .../include/PolyVoxCore/UncompressedBlock.inl | 5 +++-- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl index b4ee5feb..25588e69 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl @@ -614,13 +614,9 @@ namespace PolyVox // Purge null blocks so we know that all blocks are used. purgeNullPtrsFromAllBlocks(); - // We include the size of the LargeVolume class but it should be insignificant. - uint32_t uSizeInBytes = sizeof(LargeVolume); - - // Memory used by the blocks - uSizeInBytes += UncompressedBlock::calculateSizeInBytes(m_uBlockSideLength) * m_pAllBlocks.size(); - - return uSizeInBytes; + // Note: We disregard the size of the other class members as they are likely to be very small compared to the size of the + // allocated voxel data. This also keeps the reported size as a power of two, which makes other memory calculations easier. + return UncompressedBlock::calculateSizeInBytes(m_uBlockSideLength) * m_pAllBlocks.size(); } template diff --git a/library/PolyVoxCore/include/PolyVoxCore/UncompressedBlock.inl b/library/PolyVoxCore/include/PolyVoxCore/UncompressedBlock.inl index 6f73d2f1..083d763e 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/UncompressedBlock.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/UncompressedBlock.inl @@ -148,8 +148,9 @@ namespace PolyVox template uint32_t UncompressedBlock::calculateSizeInBytes(uint32_t uSideLength) { - // Returns the size of this class plus the size of the uncompressed data. - uint32_t uSizeInBytes = sizeof(UncompressedBlock) + (uSideLength * uSideLength * uSideLength * sizeof(VoxelType)); + // Note: We disregard the size of the other class members as they are likely to be very small compared to the size of the + // allocated voxel data. This also keeps the reported size as a power of two, which makes other memory calculations easier. + uint32_t uSizeInBytes = uSideLength * uSideLength * uSideLength * sizeof(VoxelType); return uSizeInBytes; } } From 3a08487dc22c0c1f0c16b87e4605209c683fd8de Mon Sep 17 00:00:00 2001 From: David Williams Date: Thu, 18 Sep 2014 23:35:16 +0200 Subject: [PATCH 24/42] Blocks now stored in an unordered_map (hash based) rather than a map. --- .../include/PolyVoxCore/LargeVolume.h | 21 +++---------------- .../PolyVoxCore/include/PolyVoxCore/Vector.h | 14 +++++++++++++ 2 files changed, 17 insertions(+), 18 deletions(-) diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h index 1fa9a5b6..968f82da 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h @@ -33,6 +33,7 @@ freely, subject to the following restrictions: #include #include //For abort() #include //For memcpy +#include #include #include #include @@ -284,24 +285,8 @@ namespace PolyVox private: - struct BlockPositionCompare - { - bool operator() (const PolyVox::Vector3DInt32& a, const PolyVox::Vector3DInt32& b) const - { - const uint32_t size = 3; - for(uint32_t ct = 0; ct < size; ++ct) - { - if (a.getElement(ct) < b.getElement(ct)) - return true; - if (b.getElement(ct) < a.getElement(ct)) - return false; - } - return false; - } - }; - - typedef std::map >, BlockPositionCompare> SharedPtrBlockMap; - typedef std::map >, BlockPositionCompare> WeakPtrBlockMap; + typedef std::unordered_map > > SharedPtrBlockMap; + typedef std::unordered_map > > WeakPtrBlockMap; void initialise(); diff --git a/library/PolyVoxCore/include/PolyVoxCore/Vector.h b/library/PolyVoxCore/include/PolyVoxCore/Vector.h index db3da0f8..7a7ff7ed 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/Vector.h +++ b/library/PolyVoxCore/include/PolyVoxCore/Vector.h @@ -31,6 +31,7 @@ freely, subject to the following restrictions: #include #include +#include #include namespace PolyVox @@ -228,6 +229,19 @@ namespace PolyVox }//namespace PolyVox +namespace std +{ + template <> + struct hash + { + std::size_t operator()(const PolyVox::Vector3DInt32& vec) const + { + return ((vec.getX() & 0xFF)) | ((vec.getY() & 0xFF) << 8) | ((vec.getZ() & 0xFF) << 16); + } + }; +} + + #include "PolyVoxCore/Vector.inl" #endif From 25a1d953875883e33462444664573d436a4d637f Mon Sep 17 00:00:00 2001 From: David Williams Date: Fri, 19 Sep 2014 16:50:24 +0200 Subject: [PATCH 25/42] Renamed LargeVolume to PagedVolume, deleted SimpleVolume, and set up typedefs pointing LargeVolume and SimpleVolume to PagedVolume for backwards compatibility. --- library/PolyVoxCore/CMakeLists.txt | 8 +- .../include/PolyVoxCore/BaseVolume.h | 2 +- .../include/PolyVoxCore/BaseVolume.inl | 2 +- .../include/PolyVoxCore/LargeVolume.h | 332 +--- .../include/PolyVoxCore/PagedVolume.h | 333 ++++ .../{LargeVolume.inl => PagedVolume.inl} | 1398 ++++++++--------- ...lumeSampler.inl => PagedVolumeSampler.inl} | 1124 ++++++------- .../PolyVoxCore/PolyVoxForwardDeclarations.h | 19 +- .../include/PolyVoxCore/SimpleVolume.h | 228 +-- .../include/PolyVoxCore/SimpleVolume.inl | 441 ------ .../include/PolyVoxCore/SimpleVolumeBlock.inl | 130 -- .../PolyVoxCore/SimpleVolumeSampler.inl | 581 ------- .../include/PolyVoxCore/UncompressedBlock.h | 4 +- 13 files changed, 1616 insertions(+), 2986 deletions(-) create mode 100644 library/PolyVoxCore/include/PolyVoxCore/PagedVolume.h rename library/PolyVoxCore/include/PolyVoxCore/{LargeVolume.inl => PagedVolume.inl} (92%) rename library/PolyVoxCore/include/PolyVoxCore/{LargeVolumeSampler.inl => PagedVolumeSampler.inl} (84%) delete mode 100644 library/PolyVoxCore/include/PolyVoxCore/SimpleVolume.inl delete mode 100644 library/PolyVoxCore/include/PolyVoxCore/SimpleVolumeBlock.inl delete mode 100644 library/PolyVoxCore/include/PolyVoxCore/SimpleVolumeSampler.inl diff --git a/library/PolyVoxCore/CMakeLists.txt b/library/PolyVoxCore/CMakeLists.txt index 486cdf97..4e18c75e 100644 --- a/library/PolyVoxCore/CMakeLists.txt +++ b/library/PolyVoxCore/CMakeLists.txt @@ -50,8 +50,6 @@ SET(CORE_INC_FILES include/PolyVoxCore/IteratorController.h include/PolyVoxCore/IteratorController.inl include/PolyVoxCore/LargeVolume.h - include/PolyVoxCore/LargeVolume.inl - include/PolyVoxCore/LargeVolumeSampler.inl include/PolyVoxCore/LowPassFilter.h include/PolyVoxCore/LowPassFilter.inl include/PolyVoxCore/MarchingCubesSurfaceExtractor.h @@ -60,6 +58,9 @@ SET(CORE_INC_FILES include/PolyVoxCore/MaterialDensityPair.h include/PolyVoxCore/Mesh.h include/PolyVoxCore/Mesh.inl + include/PolyVoxCore/PagedVolume.h + include/PolyVoxCore/PagedVolume.inl + include/PolyVoxCore/PagedVolumeSampler.inl include/PolyVoxCore/Pager.h include/PolyVoxCore/PolyVoxForwardDeclarations.h include/PolyVoxCore/Picking.h @@ -71,9 +72,6 @@ SET(CORE_INC_FILES include/PolyVoxCore/Raycast.inl include/PolyVoxCore/Region.h include/PolyVoxCore/SimpleVolume.h - include/PolyVoxCore/SimpleVolume.inl - include/PolyVoxCore/SimpleVolumeBlock.inl - include/PolyVoxCore/SimpleVolumeSampler.inl include/PolyVoxCore/UncompressedBlock.h include/PolyVoxCore/UncompressedBlock.inl include/PolyVoxCore/Vector.h diff --git a/library/PolyVoxCore/include/PolyVoxCore/BaseVolume.h b/library/PolyVoxCore/include/PolyVoxCore/BaseVolume.h index eeb3c50d..3f5f1c0f 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/BaseVolume.h +++ b/library/PolyVoxCore/include/PolyVoxCore/BaseVolume.h @@ -32,7 +32,7 @@ freely, subject to the following restrictions: namespace PolyVox { /// The BaseVolume class provides common functionality and an interface for other volume classes to implement. You should not try to create an instance of this - /// class directly. Instead you should use RawVolume, SimpleVolume, or LargeVolume. + /// class directly. Instead you should use RawVolume, SimpleVolume, or PagedVolume. //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// /// More details to come... //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/library/PolyVoxCore/include/PolyVoxCore/BaseVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/BaseVolume.inl index dc280ee2..7669a4ca 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/BaseVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/BaseVolume.inl @@ -26,7 +26,7 @@ namespace PolyVox //////////////////////////////////////////////////////////////////////////////// /// This is protected because you should never create a BaseVolume directly, you should instead use one of the derived classes. /// - /// \sa RawVolume, SimpleVolume, LargeVolume + /// \sa RawVolume, SimpleVolume, PagedVolume //////////////////////////////////////////////////////////////////////////////// template BaseVolume::BaseVolume(const Region& regValid) diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h index 968f82da..ff60e49f 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h @@ -1,333 +1,7 @@ -/******************************************************************************* -Copyright (c) 2005-2009 David Williams - -This software is provided 'as-is', without any express or implied -warranty. In no event will the authors be held liable for any damages -arising from the use of this software. - -Permission is granted to anyone to use this software for any purpose, -including commercial applications, and to alter it and redistribute it -freely, subject to the following restrictions: - - 1. The origin of this software must not be misrepresented; you must not - claim that you wrote the original software. If you use this software - in a product, an acknowledgment in the product documentation would be - appreciated but is not required. - - 2. Altered source versions must be plainly marked as such, and must not be - misrepresented as being the original software. - - 3. This notice may not be removed or altered from any source - distribution. -*******************************************************************************/ - #ifndef __PolyVox_LargeVolume_H__ #define __PolyVox_LargeVolume_H__ -#include "PolyVoxCore/BaseVolume.h" -#include "PolyVoxCore/Pager.h" -#include "PolyVoxCore/Region.h" -#include "PolyVoxCore/UncompressedBlock.h" -#include "PolyVoxCore/Vector.h" +#include "PagedVolume.h" +#include "PolyVoxForwardDeclarations.h" -#include -#include //For abort() -#include //For memcpy -#include -#include -#include -#include -#include //For invalid_argument -#include - -namespace PolyVox -{ - /// The LargeVolume class provides a memory efficient method of storing voxel data while also allowing fast access and modification. - //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - /// A LargeVolume is essentially a 3D array in which each element (or voxel) is identified by a three dimensional (x,y,z) coordinate. - /// We use the LargeVolume class to store our data in an efficient way, and it is the input to many of the algorithms (such as the surface - /// extractors) which form the heart of PolyVox. The LargeVolume class is templatised so that different types of data can be stored within each voxel. - /// - /// Basic usage - /// ----------- - /// - /// The following code snippet shows how to construct a volume and demonstrates basic usage: - /// - /// \code - /// LargeVolume volume(Region(Vector3DInt32(0,0,0), Vector3DInt32(63,127,255))); - /// volume.setVoxelAt(15, 90, 42, Material8(5)); - /// std::cout << "Voxel at (15, 90, 42) has value: " << volume.getVoxelAt(15, 90, 42).getMaterial() << std::endl; - /// std::cout << "Width = " << volume.getWidth() << ", Height = " << volume.getHeight() << ", Depth = " << volume.getDepth() << std::endl; - /// \endcode - /// - /// In this particular example each voxel in the LargeVolume is of type 'Material8', as specified by the template parameter. This is one of several - /// predefined voxel types, and it is also possible to define your own. The Material8 type simply holds an integer value where zero represents - /// empty space and any other value represents a solid material. - /// - /// The LargeVolume constructor takes a Region as a parameter. This specifies the valid range of voxels which can be held in the volume, so in this - /// particular case the valid voxel positions are (0,0,0) to (63, 127, 255). Attempts to access voxels outside this range will result is accessing the - /// border value (see getBorderValue() and setBorderValue()). PolyVox also has support for near infinite volumes which will be discussed later. - /// - /// Access to individual voxels is provided via the setVoxelAt() and getVoxelAt() member functions. Advanced users may also be interested in - /// the Sampler class for faster read-only access to a large number of voxels. - /// - /// Lastly the example prints out some properties of the LargeVolume. Note that the dimentsions getWidth(), getHeight(), and getDepth() are inclusive, such - /// that the width is 64 when the range of valid x coordinates goes from 0 to 63. - /// - /// Data Representaion - /// ------------------ - /// If stored carelessly, volume data can take up a huge amount of memory. For example, a volume of dimensions 1024x1024x1024 with - /// 1 byte per voxel will require 1GB of memory if stored in an uncompressed form. Natuarally our LargeVolume class is much more efficient - /// than this and it is worth understanding (at least at a high level) the approach which is used. - /// - /// Essentially, the LargeVolume class stores its data as a collection of blocks. Each of these block is much smaller than the whole volume, - /// for example a typical size might be 32x32x32 voxels (though is is configurable by the user). In this case, a 256x512x1024 volume - /// would contain 8x16x32 = 4096 blocks. The data for each block is stored in a compressed form, which uses only a small amout of - /// memory but it is hard to modify the data. Therefore, before any given voxel can be modified, its corresponding block must be uncompressed. - /// - /// The compression and decompression of block is a relatively slow process and so we aim to do this as rarely as possible. In order - /// to achive this, the volume class stores a cache of recently used blocks and their associated uncompressed data. Each time a voxel - /// is touched a timestamp is updated on the corresponding block. When the cache becomes full the block with the oldest timestamp is - /// recompressed and moved out of the cache. - /// - /// Achieving high compression rates - /// -------------------------------- - /// The compression rates which can be achieved can vary significantly depending the nature of the data you are storing, but you can - /// encourage high compression rates by making your data as homogenous as possible. If you are simply storing a material with each - /// voxel then this will probably happen naturally. Games such as Minecraft which use this approach will typically involve large areas - /// of the same material which will compress down well. - /// - /// However, if you are storing density values then you may want to take some care. The advantage of storing smoothly changing values - /// is that you can get smooth surfaces extracted, but storing smoothly changing values inside or outside objects (rather than just - /// on the boundary) does not benefit the surface and is very hard to compress effectively. You may wish to apply some thresholding to - /// your density values to reduce this problem (this threasholding should only be applied to voxels who don't contribute to the surface). - /// - /// Paging large volumes - /// -------------------- - /// The compression scheme described previously will typically allow you to load several billion voxels into a few hundred megabytes of memory, - /// though as explained the exact compression rate is highly dependant on your data. If you have more data than this then PolyVox provides a - /// mechanism by which parts of the volume can be paged out of memory by calling user supplied callback functions. This mechanism allows a - /// potentially unlimited amount of data to be loaded, provided the user is able to take responsibility for storing any data which PolyVox - /// cannot fit in memory, and then returning it back to PolyVox on demand. For example, the user might choose to temporarily store this data - /// on disk or stream it to a remote database. - /// - /// You can construct such a LargeVolume as follows: - /// - /// \code - /// void myDataRequiredHandler(const ConstVolumeProxy& volume, const PolyVox::Region& reg) - /// { - /// //This function is being called because part of the data is missing from memory and needs to be supplied. The parameter - /// //'volume' provides access to the volume data, and the parameter 'reg' indicates which region of the volume you need fill. - /// } - /// - /// void myDataOverflowHandler(const ConstVolumeProxy& vol, const PolyVox::Region& reg) - /// { - /// //This function is being called because part of the data is about to be removed from memory. The parameter 'volume' - /// //provides access to the volume data, and the parameter 'reg' indicates which region of the volume you need to store. - /// } - /// - /// LargeVolumevolData(&myDataRequiredHandler, &myDataOverflowHandler); - /// \endcode - /// - /// Essentially you are providing an extension to the LargeVolume class - a way for data to be stored once PolyVox has run out of memory for it. Note - /// that you don't actually have to do anything with the data - you could simply decide that once it gets removed from memory it doesn't matter - /// anymore. But you still need to be ready to then provide something to PolyVox (even if it's just default data) in the event that it is requested. - /// - /// Cache-aware traversal - /// --------------------- - /// You might be suprised at just how many cache misses can occur when you traverse the volume in a naive manner. Consider a 1024x1024x1024 volume - /// with blocks of size 32x32x32. And imagine you iterate over this volume with a simple three-level for loop which iterates over x, the y, then z. - /// If you start at position (0,0,0) then ny the time you reach position (1023,0,0) you have touched 1024 voxels along one edge of the volume and - /// have pulled 32 blocks into the cache. By the time you reach (1023,1023,0) you have hit 1024x1024 voxels and pulled 32x32 blocks into the cache. - /// You are now ready to touch voxel (0,0,1) which is right nect to where you started, but unless your cache is at least 32x32 blocks large then this - /// initial block has already been cleared from the cache. - /// - /// Ensuring you have a large enough cache size can obviously help the above situation, but you might also consider iterating over the voxels in a - /// different order. For example, if you replace your three-level loop with a six-level loop then you can first process all the voxels between (0,0,0) - /// and (31,31,31), then process all the voxels between (32,0,0) and (63,0,0), and so forth. Using this approach you will have no cache misses even - /// is your cache sise is only one. Of course the logic is more complex, but writing code in such a cache-aware manner may be beneficial in some situations. - /// - /// Threading - /// --------- - /// The LargeVolume class does not make any guarentees about thread safety. You should ensure that all accesses are performed from the same thread. - /// This is true even if you are only reading data from the volume, as concurrently reading from different threads can invalidate the contents - /// of the block cache (amoung other problems). - //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - template - class LargeVolume : public BaseVolume - { - public: - //There seems to be some descrepency between Visual Studio and GCC about how the following class should be declared. - //There is a work around (see also See http://goo.gl/qu1wn) given below which appears to work on VS2010 and GCC, but - //which seems to cause internal compiler errors on VS2008 when building with the /Gm 'Enable Minimal Rebuild' compiler - //option. For now it seems best to 'fix' it with the preprocessor insstead, but maybe the workaround can be reinstated - //in the future - //typedef Volume VolumeOfVoxelType; //Workaround for GCC/VS2010 differences. - //class Sampler : public VolumeOfVoxelType::template Sampler< LargeVolume > - #ifndef SWIG -#if defined(_MSC_VER) - class Sampler : public BaseVolume::Sampler< LargeVolume > //This line works on VS2010 -#else - class Sampler : public BaseVolume::template Sampler< LargeVolume > //This line works on GCC -#endif - { - public: - Sampler(LargeVolume* volume); - ~Sampler(); - - /// \deprecated - POLYVOX_DEPRECATED VoxelType getSubSampledVoxel(uint8_t uLevel) const; - inline VoxelType getVoxel(void) const; - - void setPosition(const Vector3DInt32& v3dNewPos); - void setPosition(int32_t xPos, int32_t yPos, int32_t zPos); - inline bool setVoxel(VoxelType tValue); - - void movePositiveX(void); - void movePositiveY(void); - void movePositiveZ(void); - - void moveNegativeX(void); - void moveNegativeY(void); - void moveNegativeZ(void); - - inline VoxelType peekVoxel1nx1ny1nz(void) const; - inline VoxelType peekVoxel1nx1ny0pz(void) const; - inline VoxelType peekVoxel1nx1ny1pz(void) const; - inline VoxelType peekVoxel1nx0py1nz(void) const; - inline VoxelType peekVoxel1nx0py0pz(void) const; - inline VoxelType peekVoxel1nx0py1pz(void) const; - inline VoxelType peekVoxel1nx1py1nz(void) const; - inline VoxelType peekVoxel1nx1py0pz(void) const; - inline VoxelType peekVoxel1nx1py1pz(void) const; - - inline VoxelType peekVoxel0px1ny1nz(void) const; - inline VoxelType peekVoxel0px1ny0pz(void) const; - inline VoxelType peekVoxel0px1ny1pz(void) const; - inline VoxelType peekVoxel0px0py1nz(void) const; - inline VoxelType peekVoxel0px0py0pz(void) const; - inline VoxelType peekVoxel0px0py1pz(void) const; - inline VoxelType peekVoxel0px1py1nz(void) const; - inline VoxelType peekVoxel0px1py0pz(void) const; - inline VoxelType peekVoxel0px1py1pz(void) const; - - inline VoxelType peekVoxel1px1ny1nz(void) const; - inline VoxelType peekVoxel1px1ny0pz(void) const; - inline VoxelType peekVoxel1px1ny1pz(void) const; - inline VoxelType peekVoxel1px0py1nz(void) const; - inline VoxelType peekVoxel1px0py0pz(void) const; - inline VoxelType peekVoxel1px0py1pz(void) const; - inline VoxelType peekVoxel1px1py1nz(void) const; - inline VoxelType peekVoxel1px1py0pz(void) const; - inline VoxelType peekVoxel1px1py1pz(void) const; - - private: - //Other current position information - VoxelType* mCurrentVoxel; - }; - - #endif - - public: - /// Constructor for creating a fixed size volume. - LargeVolume - ( - const Region& regValid, - Pager* pPager = nullptr, - uint16_t uBlockSideLength = 32 - ); - /// Destructor - ~LargeVolume(); - - /// Gets a voxel at the position given by x,y,z coordinates - template - VoxelType getVoxel(int32_t uXPos, int32_t uYPos, int32_t uZPos, VoxelType tBorder = VoxelType()) const; - /// Gets a voxel at the position given by a 3D vector - template - VoxelType getVoxel(const Vector3DInt32& v3dPos, VoxelType tBorder = VoxelType()) const; - - /// Gets a voxel at the position given by x,y,z coordinates - VoxelType getVoxel(int32_t uXPos, int32_t uYPos, int32_t uZPos, WrapMode eWrapMode = WrapModes::Validate, VoxelType tBorder = VoxelType()) const; - /// Gets a voxel at the position given by a 3D vector - VoxelType getVoxel(const Vector3DInt32& v3dPos, WrapMode eWrapMode = WrapModes::Validate, VoxelType tBorder = VoxelType()) const; - - /// Gets a voxel at the position given by x,y,z coordinates - POLYVOX_DEPRECATED VoxelType getVoxelAt(int32_t uXPos, int32_t uYPos, int32_t uZPos) const; - /// Gets a voxel at the position given by a 3D vector - POLYVOX_DEPRECATED VoxelType getVoxelAt(const Vector3DInt32& v3dPos) const; - - /// Sets the number of blocks for which uncompressed data is stored - void setMemoryUsageLimit(uint32_t uMemoryUsageInBytes); - /// Sets the voxel at the position given by x,y,z coordinates - void setVoxel(int32_t uXPos, int32_t uYPos, int32_t uZPos, VoxelType tValue, WrapMode eWrapMode = WrapModes::Validate); - /// Sets the voxel at the position given by a 3D vector - void setVoxel(const Vector3DInt32& v3dPos, VoxelType tValue, WrapMode eWrapMode = WrapModes::Validate); - /// Sets the voxel at the position given by x,y,z coordinates - bool setVoxelAt(int32_t uXPos, int32_t uYPos, int32_t uZPos, VoxelType tValue); - /// Sets the voxel at the position given by a 3D vector - bool setVoxelAt(const Vector3DInt32& v3dPos, VoxelType tValue); - /// Tries to ensure that the voxels within the specified Region are loaded into memory. - void prefetch(Region regPrefetch); - /// Ensures that any voxels within the specified Region are removed from memory. - void flush(Region regFlush); - /// Removes all voxels from memory - void flushAll(); - - /// Calculates approximatly how many bytes of memory the volume is currently using. - uint32_t calculateSizeInBytes(void); - - protected: - /// Copy constructor - LargeVolume(const LargeVolume& rhs); - - /// Assignment operator - LargeVolume& operator=(const LargeVolume& rhs); - - private: - - typedef std::unordered_map > > SharedPtrBlockMap; - typedef std::unordered_map > > WeakPtrBlockMap; - - void initialise(); - - // A trick to implement specialization of template member functions in template classes. See http://stackoverflow.com/a/4951057 - template - VoxelType getVoxelImpl(int32_t uXPos, int32_t uYPos, int32_t uZPos, WrapModeType, VoxelType tBorder) const; - VoxelType getVoxelImpl(int32_t uXPos, int32_t uYPos, int32_t uZPos, WrapModeType, VoxelType tBorder) const; - VoxelType getVoxelImpl(int32_t uXPos, int32_t uYPos, int32_t uZPos, WrapModeType, VoxelType tBorder) const; - VoxelType getVoxelImpl(int32_t uXPos, int32_t uYPos, int32_t uZPos, WrapModeType, VoxelType tBorder) const; - VoxelType getVoxelImpl(int32_t uXPos, int32_t uYPos, int32_t uZPos, WrapModeType, VoxelType tBorder) const; - - std::shared_ptr< UncompressedBlock > getUncompressedBlock(int32_t uBlockX, int32_t uBlockY, int32_t uBlockZ) const; - - void purgeNullPtrsFromAllBlocks(void) const; - - // The block data - mutable WeakPtrBlockMap m_pAllBlocks; - mutable SharedPtrBlockMap m_pRecentlyUsedBlocks; - - mutable uint32_t m_uTimestamper; - mutable Vector3DInt32 m_v3dLastAccessedBlockPos; - mutable std::shared_ptr< UncompressedBlock > m_pLastAccessedBlock; - uint32_t m_uBlockCountLimit; - - // The size of the volume - Region m_regValidRegionInBlocks; - - // The size of the blocks - uint16_t m_uBlockSideLength; - uint8_t m_uBlockSideLengthPower; - - Pager* m_pPager; - - // Enough to make sure a blocks and it's neighbours can be loaded, with a few to spare. - static const uint32_t uMinPracticalNoOfBlocks = 32; - // Should preent multi-gigabyte volumes with reasonable block sizes. - static const uint32_t uMaxPracticalNoOfBlocks = 32768; - }; -} - -#include "PolyVoxCore/LargeVolume.inl" -#include "PolyVoxCore/LargeVolumeSampler.inl" - -#endif //__PolyVox_LargeVolume_H__ +#endif //__PolyVox_LargeVolume_H__ \ No newline at end of file diff --git a/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.h b/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.h new file mode 100644 index 00000000..4c08721c --- /dev/null +++ b/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.h @@ -0,0 +1,333 @@ +/******************************************************************************* +Copyright (c) 2005-2009 David Williams + +This software is provided 'as-is', without any express or implied +warranty. In no event will the authors be held liable for any damages +arising from the use of this software. + +Permission is granted to anyone to use this software for any purpose, +including commercial applications, and to alter it and redistribute it +freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + + 3. This notice may not be removed or altered from any source + distribution. +*******************************************************************************/ + +#ifndef __PolyVox_PagedVolume_H__ +#define __PolyVox_PagedVolume_H__ + +#include "PolyVoxCore/BaseVolume.h" +#include "PolyVoxCore/Pager.h" +#include "PolyVoxCore/Region.h" +#include "PolyVoxCore/UncompressedBlock.h" +#include "PolyVoxCore/Vector.h" + +#include +#include //For abort() +#include //For memcpy +#include +#include +#include +#include +#include //For invalid_argument +#include + +namespace PolyVox +{ + /// The PagedVolume class provides a memory efficient method of storing voxel data while also allowing fast access and modification. + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + /// A PagedVolume is essentially a 3D array in which each element (or voxel) is identified by a three dimensional (x,y,z) coordinate. + /// We use the PagedVolume class to store our data in an efficient way, and it is the input to many of the algorithms (such as the surface + /// extractors) which form the heart of PolyVox. The PagedVolume class is templatised so that different types of data can be stored within each voxel. + /// + /// Basic usage + /// ----------- + /// + /// The following code snippet shows how to construct a volume and demonstrates basic usage: + /// + /// \code + /// PagedVolume volume(Region(Vector3DInt32(0,0,0), Vector3DInt32(63,127,255))); + /// volume.setVoxelAt(15, 90, 42, Material8(5)); + /// std::cout << "Voxel at (15, 90, 42) has value: " << volume.getVoxelAt(15, 90, 42).getMaterial() << std::endl; + /// std::cout << "Width = " << volume.getWidth() << ", Height = " << volume.getHeight() << ", Depth = " << volume.getDepth() << std::endl; + /// \endcode + /// + /// In this particular example each voxel in the PagedVolume is of type 'Material8', as specified by the template parameter. This is one of several + /// predefined voxel types, and it is also possible to define your own. The Material8 type simply holds an integer value where zero represents + /// empty space and any other value represents a solid material. + /// + /// The PagedVolume constructor takes a Region as a parameter. This specifies the valid range of voxels which can be held in the volume, so in this + /// particular case the valid voxel positions are (0,0,0) to (63, 127, 255). Attempts to access voxels outside this range will result is accessing the + /// border value (see getBorderValue() and setBorderValue()). PolyVox also has support for near infinite volumes which will be discussed later. + /// + /// Access to individual voxels is provided via the setVoxelAt() and getVoxelAt() member functions. Advanced users may also be interested in + /// the Sampler class for faster read-only access to a large number of voxels. + /// + /// Lastly the example prints out some properties of the PagedVolume. Note that the dimentsions getWidth(), getHeight(), and getDepth() are inclusive, such + /// that the width is 64 when the range of valid x coordinates goes from 0 to 63. + /// + /// Data Representaion + /// ------------------ + /// If stored carelessly, volume data can take up a huge amount of memory. For example, a volume of dimensions 1024x1024x1024 with + /// 1 byte per voxel will require 1GB of memory if stored in an uncompressed form. Natuarally our PagedVolume class is much more efficient + /// than this and it is worth understanding (at least at a high level) the approach which is used. + /// + /// Essentially, the PagedVolume class stores its data as a collection of blocks. Each of these block is much smaller than the whole volume, + /// for example a typical size might be 32x32x32 voxels (though is is configurable by the user). In this case, a 256x512x1024 volume + /// would contain 8x16x32 = 4096 blocks. The data for each block is stored in a compressed form, which uses only a small amout of + /// memory but it is hard to modify the data. Therefore, before any given voxel can be modified, its corresponding block must be uncompressed. + /// + /// The compression and decompression of block is a relatively slow process and so we aim to do this as rarely as possible. In order + /// to achive this, the volume class stores a cache of recently used blocks and their associated uncompressed data. Each time a voxel + /// is touched a timestamp is updated on the corresponding block. When the cache becomes full the block with the oldest timestamp is + /// recompressed and moved out of the cache. + /// + /// Achieving high compression rates + /// -------------------------------- + /// The compression rates which can be achieved can vary significantly depending the nature of the data you are storing, but you can + /// encourage high compression rates by making your data as homogenous as possible. If you are simply storing a material with each + /// voxel then this will probably happen naturally. Games such as Minecraft which use this approach will typically involve large areas + /// of the same material which will compress down well. + /// + /// However, if you are storing density values then you may want to take some care. The advantage of storing smoothly changing values + /// is that you can get smooth surfaces extracted, but storing smoothly changing values inside or outside objects (rather than just + /// on the boundary) does not benefit the surface and is very hard to compress effectively. You may wish to apply some thresholding to + /// your density values to reduce this problem (this threasholding should only be applied to voxels who don't contribute to the surface). + /// + /// Paging large volumes + /// -------------------- + /// The compression scheme described previously will typically allow you to load several billion voxels into a few hundred megabytes of memory, + /// though as explained the exact compression rate is highly dependant on your data. If you have more data than this then PolyVox provides a + /// mechanism by which parts of the volume can be paged out of memory by calling user supplied callback functions. This mechanism allows a + /// potentially unlimited amount of data to be loaded, provided the user is able to take responsibility for storing any data which PolyVox + /// cannot fit in memory, and then returning it back to PolyVox on demand. For example, the user might choose to temporarily store this data + /// on disk or stream it to a remote database. + /// + /// You can construct such a PagedVolume as follows: + /// + /// \code + /// void myDataRequiredHandler(const ConstVolumeProxy& volume, const PolyVox::Region& reg) + /// { + /// //This function is being called because part of the data is missing from memory and needs to be supplied. The parameter + /// //'volume' provides access to the volume data, and the parameter 'reg' indicates which region of the volume you need fill. + /// } + /// + /// void myDataOverflowHandler(const ConstVolumeProxy& vol, const PolyVox::Region& reg) + /// { + /// //This function is being called because part of the data is about to be removed from memory. The parameter 'volume' + /// //provides access to the volume data, and the parameter 'reg' indicates which region of the volume you need to store. + /// } + /// + /// PagedVolumevolData(&myDataRequiredHandler, &myDataOverflowHandler); + /// \endcode + /// + /// Essentially you are providing an extension to the PagedVolume class - a way for data to be stored once PolyVox has run out of memory for it. Note + /// that you don't actually have to do anything with the data - you could simply decide that once it gets removed from memory it doesn't matter + /// anymore. But you still need to be ready to then provide something to PolyVox (even if it's just default data) in the event that it is requested. + /// + /// Cache-aware traversal + /// --------------------- + /// You might be suprised at just how many cache misses can occur when you traverse the volume in a naive manner. Consider a 1024x1024x1024 volume + /// with blocks of size 32x32x32. And imagine you iterate over this volume with a simple three-level for loop which iterates over x, the y, then z. + /// If you start at position (0,0,0) then ny the time you reach position (1023,0,0) you have touched 1024 voxels along one edge of the volume and + /// have pulled 32 blocks into the cache. By the time you reach (1023,1023,0) you have hit 1024x1024 voxels and pulled 32x32 blocks into the cache. + /// You are now ready to touch voxel (0,0,1) which is right nect to where you started, but unless your cache is at least 32x32 blocks large then this + /// initial block has already been cleared from the cache. + /// + /// Ensuring you have a large enough cache size can obviously help the above situation, but you might also consider iterating over the voxels in a + /// different order. For example, if you replace your three-level loop with a six-level loop then you can first process all the voxels between (0,0,0) + /// and (31,31,31), then process all the voxels between (32,0,0) and (63,0,0), and so forth. Using this approach you will have no cache misses even + /// is your cache sise is only one. Of course the logic is more complex, but writing code in such a cache-aware manner may be beneficial in some situations. + /// + /// Threading + /// --------- + /// The PagedVolume class does not make any guarentees about thread safety. You should ensure that all accesses are performed from the same thread. + /// This is true even if you are only reading data from the volume, as concurrently reading from different threads can invalidate the contents + /// of the block cache (amoung other problems). + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + template + class PagedVolume : public BaseVolume + { + public: + //There seems to be some descrepency between Visual Studio and GCC about how the following class should be declared. + //There is a work around (see also See http://goo.gl/qu1wn) given below which appears to work on VS2010 and GCC, but + //which seems to cause internal compiler errors on VS2008 when building with the /Gm 'Enable Minimal Rebuild' compiler + //option. For now it seems best to 'fix' it with the preprocessor insstead, but maybe the workaround can be reinstated + //in the future + //typedef Volume VolumeOfVoxelType; //Workaround for GCC/VS2010 differences. + //class Sampler : public VolumeOfVoxelType::template Sampler< PagedVolume > + #ifndef SWIG +#if defined(_MSC_VER) + class Sampler : public BaseVolume::Sampler< PagedVolume > //This line works on VS2010 +#else + class Sampler : public BaseVolume::template Sampler< PagedVolume > //This line works on GCC +#endif + { + public: + Sampler(PagedVolume* volume); + ~Sampler(); + + /// \deprecated + POLYVOX_DEPRECATED VoxelType getSubSampledVoxel(uint8_t uLevel) const; + inline VoxelType getVoxel(void) const; + + void setPosition(const Vector3DInt32& v3dNewPos); + void setPosition(int32_t xPos, int32_t yPos, int32_t zPos); + inline bool setVoxel(VoxelType tValue); + + void movePositiveX(void); + void movePositiveY(void); + void movePositiveZ(void); + + void moveNegativeX(void); + void moveNegativeY(void); + void moveNegativeZ(void); + + inline VoxelType peekVoxel1nx1ny1nz(void) const; + inline VoxelType peekVoxel1nx1ny0pz(void) const; + inline VoxelType peekVoxel1nx1ny1pz(void) const; + inline VoxelType peekVoxel1nx0py1nz(void) const; + inline VoxelType peekVoxel1nx0py0pz(void) const; + inline VoxelType peekVoxel1nx0py1pz(void) const; + inline VoxelType peekVoxel1nx1py1nz(void) const; + inline VoxelType peekVoxel1nx1py0pz(void) const; + inline VoxelType peekVoxel1nx1py1pz(void) const; + + inline VoxelType peekVoxel0px1ny1nz(void) const; + inline VoxelType peekVoxel0px1ny0pz(void) const; + inline VoxelType peekVoxel0px1ny1pz(void) const; + inline VoxelType peekVoxel0px0py1nz(void) const; + inline VoxelType peekVoxel0px0py0pz(void) const; + inline VoxelType peekVoxel0px0py1pz(void) const; + inline VoxelType peekVoxel0px1py1nz(void) const; + inline VoxelType peekVoxel0px1py0pz(void) const; + inline VoxelType peekVoxel0px1py1pz(void) const; + + inline VoxelType peekVoxel1px1ny1nz(void) const; + inline VoxelType peekVoxel1px1ny0pz(void) const; + inline VoxelType peekVoxel1px1ny1pz(void) const; + inline VoxelType peekVoxel1px0py1nz(void) const; + inline VoxelType peekVoxel1px0py0pz(void) const; + inline VoxelType peekVoxel1px0py1pz(void) const; + inline VoxelType peekVoxel1px1py1nz(void) const; + inline VoxelType peekVoxel1px1py0pz(void) const; + inline VoxelType peekVoxel1px1py1pz(void) const; + + private: + //Other current position information + VoxelType* mCurrentVoxel; + }; + + #endif + + public: + /// Constructor for creating a fixed size volume. + PagedVolume + ( + const Region& regValid, + Pager* pPager = nullptr, + uint16_t uBlockSideLength = 32 + ); + /// Destructor + ~PagedVolume(); + + /// Gets a voxel at the position given by x,y,z coordinates + template + VoxelType getVoxel(int32_t uXPos, int32_t uYPos, int32_t uZPos, VoxelType tBorder = VoxelType()) const; + /// Gets a voxel at the position given by a 3D vector + template + VoxelType getVoxel(const Vector3DInt32& v3dPos, VoxelType tBorder = VoxelType()) const; + + /// Gets a voxel at the position given by x,y,z coordinates + VoxelType getVoxel(int32_t uXPos, int32_t uYPos, int32_t uZPos, WrapMode eWrapMode = WrapModes::Validate, VoxelType tBorder = VoxelType()) const; + /// Gets a voxel at the position given by a 3D vector + VoxelType getVoxel(const Vector3DInt32& v3dPos, WrapMode eWrapMode = WrapModes::Validate, VoxelType tBorder = VoxelType()) const; + + /// Gets a voxel at the position given by x,y,z coordinates + POLYVOX_DEPRECATED VoxelType getVoxelAt(int32_t uXPos, int32_t uYPos, int32_t uZPos) const; + /// Gets a voxel at the position given by a 3D vector + POLYVOX_DEPRECATED VoxelType getVoxelAt(const Vector3DInt32& v3dPos) const; + + /// Sets the number of blocks for which uncompressed data is stored + void setMemoryUsageLimit(uint32_t uMemoryUsageInBytes); + /// Sets the voxel at the position given by x,y,z coordinates + void setVoxel(int32_t uXPos, int32_t uYPos, int32_t uZPos, VoxelType tValue, WrapMode eWrapMode = WrapModes::Validate); + /// Sets the voxel at the position given by a 3D vector + void setVoxel(const Vector3DInt32& v3dPos, VoxelType tValue, WrapMode eWrapMode = WrapModes::Validate); + /// Sets the voxel at the position given by x,y,z coordinates + bool setVoxelAt(int32_t uXPos, int32_t uYPos, int32_t uZPos, VoxelType tValue); + /// Sets the voxel at the position given by a 3D vector + bool setVoxelAt(const Vector3DInt32& v3dPos, VoxelType tValue); + /// Tries to ensure that the voxels within the specified Region are loaded into memory. + void prefetch(Region regPrefetch); + /// Ensures that any voxels within the specified Region are removed from memory. + void flush(Region regFlush); + /// Removes all voxels from memory + void flushAll(); + + /// Calculates approximatly how many bytes of memory the volume is currently using. + uint32_t calculateSizeInBytes(void); + + protected: + /// Copy constructor + PagedVolume(const PagedVolume& rhs); + + /// Assignment operator + PagedVolume& operator=(const PagedVolume& rhs); + + private: + + typedef std::unordered_map > > SharedPtrBlockMap; + typedef std::unordered_map > > WeakPtrBlockMap; + + void initialise(); + + // A trick to implement specialization of template member functions in template classes. See http://stackoverflow.com/a/4951057 + template + VoxelType getVoxelImpl(int32_t uXPos, int32_t uYPos, int32_t uZPos, WrapModeType, VoxelType tBorder) const; + VoxelType getVoxelImpl(int32_t uXPos, int32_t uYPos, int32_t uZPos, WrapModeType, VoxelType tBorder) const; + VoxelType getVoxelImpl(int32_t uXPos, int32_t uYPos, int32_t uZPos, WrapModeType, VoxelType tBorder) const; + VoxelType getVoxelImpl(int32_t uXPos, int32_t uYPos, int32_t uZPos, WrapModeType, VoxelType tBorder) const; + VoxelType getVoxelImpl(int32_t uXPos, int32_t uYPos, int32_t uZPos, WrapModeType, VoxelType tBorder) const; + + std::shared_ptr< UncompressedBlock > getUncompressedBlock(int32_t uBlockX, int32_t uBlockY, int32_t uBlockZ) const; + + void purgeNullPtrsFromAllBlocks(void) const; + + // The block data + mutable WeakPtrBlockMap m_pAllBlocks; + mutable SharedPtrBlockMap m_pRecentlyUsedBlocks; + + mutable uint32_t m_uTimestamper; + mutable Vector3DInt32 m_v3dLastAccessedBlockPos; + mutable std::shared_ptr< UncompressedBlock > m_pLastAccessedBlock; + uint32_t m_uBlockCountLimit; + + // The size of the volume + Region m_regValidRegionInBlocks; + + // The size of the blocks + uint16_t m_uBlockSideLength; + uint8_t m_uBlockSideLengthPower; + + Pager* m_pPager; + + // Enough to make sure a blocks and it's neighbours can be loaded, with a few to spare. + static const uint32_t uMinPracticalNoOfBlocks = 32; + // Should preent multi-gigabyte volumes with reasonable block sizes. + static const uint32_t uMaxPracticalNoOfBlocks = 32768; + }; +} + +#include "PolyVoxCore/PagedVolume.inl" +#include "PolyVoxCore/PagedVolumeSampler.inl" + +#endif //__PolyVox_PagedVolume_H__ diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.inl similarity index 92% rename from library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl rename to library/PolyVoxCore/include/PolyVoxCore/PagedVolume.inl index 25588e69..07c0e23e 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.inl @@ -1,699 +1,699 @@ -/******************************************************************************* -Copyright (c) 2005-2009 David Williams - -This software is provided 'as-is', without any express or implied -warranty. In no event will the authors be held liable for any damages -arising from the use of this software. - -Permission is granted to anyone to use this software for any purpose, -including commercial applications, and to alter it and redistribute it -freely, subject to the following restrictions: - - 1. The origin of this software must not be misrepresented; you must not - claim that you wrote the original software. If you use this software - in a product, an acknowledgment in the product documentation would be - appreciated but is not required. - - 2. Altered source versions must be plainly marked as such, and must not be - misrepresented as being the original software. - - 3. This notice may not be removed or altered from any source - distribution. -*******************************************************************************/ - -#include "PolyVoxCore/Impl/ErrorHandling.h" - -#include -#include - -namespace PolyVox -{ - //////////////////////////////////////////////////////////////////////////////// - /// This constructor creates a volume with a fixed size which is specified as a parameter. By default this constructor will not enable paging but you can override this if desired. If you do wish to enable paging then you are required to provide the call back function (see the other LargeVolume constructor). - /// \param regValid Specifies the minimum and maximum valid voxel positions. - /// \param pBlockCompressor An implementation of the Compressor interface which is used to compress blocks in memory. - /// \param dataRequiredHandler The callback function which will be called when PolyVox tries to use data which is not currently in momory. - /// \param dataOverflowHandler The callback function which will be called when PolyVox has too much data and needs to remove some from memory. - /// \param bPagingEnabled Controls whether or not paging is enabled for this LargeVolume. - /// \param uBlockSideLength The size of the blocks making up the volume. Small blocks will compress/decompress faster, but there will also be more of them meaning voxel access could be slower. - //////////////////////////////////////////////////////////////////////////////// - template - LargeVolume::LargeVolume - ( - const Region& regValid, - Pager* pPager, - uint16_t uBlockSideLength - ) - :BaseVolume(regValid) - { - m_uBlockSideLength = uBlockSideLength; - m_pPager = pPager; - - if (m_pPager) - { - // If the user is creating a vast (almost infinite) volume then we can bet they will be - // expecting a high memory usage and will want a fair number of blocks to play around with. - if (regValid == Region::MaxRegion) - { - m_uBlockCountLimit = 1024; - } - else - { - // Otherwise we try to choose a block count to avoid too much thrashing, particularly when iterating - // over the whole volume. This means at least enough blocks to cover one edge of the volume, and ideally - // enough for a whole face. Which face? Longest edge by shortest edge seems like a reasonable guess. - uint32_t longestSide = (std::max)(regValid.getWidthInVoxels(), (std::max)(regValid.getHeightInVoxels(), regValid.getDepthInVoxels())); - uint32_t shortestSide = (std::min)(regValid.getWidthInVoxels(), (std::min)(regValid.getHeightInVoxels(), regValid.getDepthInVoxels())); - - longestSide /= m_uBlockSideLength; - shortestSide /= m_uBlockSideLength; - - m_uBlockCountLimit = longestSide * shortestSide; - } - } - else - { - // If there is no pager provided then we set the block limit to the maximum - // value to ensure the system never attempts to page blocks out of memory. - m_uBlockCountLimit = (std::numeric_limits::max)(); - } - - // Make sure the calculated block limit is within practical bounds - m_uBlockCountLimit = (std::max)(m_uBlockCountLimit, uMinPracticalNoOfBlocks); - m_uBlockCountLimit = (std::min)(m_uBlockCountLimit, uMaxPracticalNoOfBlocks); - - uint32_t uUncompressedBlockSizeInBytes = UncompressedBlock::calculateSizeInBytes(m_uBlockSideLength); - POLYVOX_LOG_DEBUG("Memory usage limit for volume initially set to " << (m_uBlockCountLimit * uUncompressedBlockSizeInBytes) / (1024 * 1024) - << "Mb (" << m_uBlockCountLimit << " blocks of " << uUncompressedBlockSizeInBytes / 1024 << "Kb each)."); - - initialise(); - } - - //////////////////////////////////////////////////////////////////////////////// - /// This function should never be called. Copying volumes by value would be expensive, and we want to prevent users from doing - /// it by accident (such as when passing them as paramenters to functions). That said, there are times when you really do want to - /// make a copy of a volume and in this case you should look at the VolumeResampler. - /// - /// \sa VolumeResampler - //////////////////////////////////////////////////////////////////////////////// - template - LargeVolume::LargeVolume(const LargeVolume& /*rhs*/) - { - POLYVOX_THROW(not_implemented, "Volume copy constructor not implemented for performance reasons."); - } - - //////////////////////////////////////////////////////////////////////////////// - /// Destroys the volume The destructor will call flushAll() to ensure that a paging volume has the chance to save it's data via the dataOverflowHandler() if desired. - //////////////////////////////////////////////////////////////////////////////// - template - LargeVolume::~LargeVolume() - { - flushAll(); - } - - //////////////////////////////////////////////////////////////////////////////// - /// This function should never be called. Copying volumes by value would be expensive, and we want to prevent users from doing - /// it by accident (such as when passing them as paramenters to functions). That said, there are times when you really do want to - /// make a copy of a volume and in this case you should look at the Volumeresampler. - /// - /// \sa VolumeResampler - //////////////////////////////////////////////////////////////////////////////// - template - LargeVolume& LargeVolume::operator=(const LargeVolume& /*rhs*/) - { - POLYVOX_THROW(not_implemented, "Volume assignment operator not implemented for performance reasons."); - } - - //////////////////////////////////////////////////////////////////////////////// - /// This version of the function requires the wrap mode to be specified as a - /// template parameter, which can provide better performance. - /// \param uXPos The \c x position of the voxel - /// \param uYPos The \c y position of the voxel - /// \param uZPos The \c z position of the voxel - /// \tparam eWrapMode Specifies the behaviour when the requested position is outside of the volume. - /// \param tBorder The border value to use if the wrap mode is set to 'Border'. - /// \return The voxel value - //////////////////////////////////////////////////////////////////////////////// - template - template - VoxelType LargeVolume::getVoxel(int32_t uXPos, int32_t uYPos, int32_t uZPos, VoxelType tBorder) const - { - // Simply call through to the real implementation - return getVoxelImpl(uXPos, uYPos, uZPos, WrapModeType(), tBorder); - } - - //////////////////////////////////////////////////////////////////////////////// - /// This version of the function requires the wrap mode to be specified as a - /// template parameter, which can provide better performance. - /// \param uXPos The \c x position of the voxel - /// \param uYPos The \c y position of the voxel - /// \param uZPos The \c z position of the voxel - /// \tparam eWrapMode Specifies the behaviour when the requested position is outside of the volume. - /// \param tBorder The border value to use if the wrap mode is set to 'Border'. - /// \return The voxel value - //////////////////////////////////////////////////////////////////////////////// - template - template - VoxelType LargeVolume::getVoxel(const Vector3DInt32& v3dPos, VoxelType tBorder) const - { - // Simply call through to the real implementation - return getVoxel(v3dPos.getX(), v3dPos.getY(), v3dPos.getZ(), tBorder); - } - - //////////////////////////////////////////////////////////////////////////////// - /// This version of the function is provided so that the wrap mode does not need - /// to be specified as a template parameter, as it may be confusing to some users. - /// \param uXPos The \c x position of the voxel - /// \param uYPos The \c y position of the voxel - /// \param uZPos The \c z position of the voxel - /// \param eWrapMode Specifies the behaviour when the requested position is outside of the volume. - /// \param tBorder The border value to use if the wrap mode is set to 'Border'. - /// \return The voxel value - //////////////////////////////////////////////////////////////////////////////// - template - VoxelType LargeVolume::getVoxel(int32_t uXPos, int32_t uYPos, int32_t uZPos, WrapMode eWrapMode, VoxelType tBorder) const - { - switch(eWrapMode) - { - case WrapModes::Validate: - return getVoxelImpl(uXPos, uYPos, uZPos, WrapModeType(), tBorder); - case WrapModes::Clamp: - return getVoxelImpl(uXPos, uYPos, uZPos, WrapModeType(), tBorder); - case WrapModes::Border: - return getVoxelImpl(uXPos, uYPos, uZPos, WrapModeType(), tBorder); - case WrapModes::AssumeValid: - return getVoxelImpl(uXPos, uYPos, uZPos, WrapModeType(), tBorder); - default: - // Should never happen - POLYVOX_ASSERT(false, "Invalid wrap mode"); - return VoxelType(); - } - } - - //////////////////////////////////////////////////////////////////////////////// - /// This version of the function is provided so that the wrap mode does not need - /// to be specified as a template parameter, as it may be confusing to some users. - /// \param v3dPos The 3D position of the voxel - /// \param eWrapMode Specifies the behaviour when the requested position is outside of the volume. - /// \param tBorder The border value to use if the wrap mode is set to 'Border'. - /// \return The voxel value - //////////////////////////////////////////////////////////////////////////////// - template - VoxelType LargeVolume::getVoxel(const Vector3DInt32& v3dPos, WrapMode eWrapMode, VoxelType tBorder) const - { - return getVoxel(v3dPos.getX(), v3dPos.getY(), v3dPos.getZ(), eWrapMode, tBorder); - } - - //////////////////////////////////////////////////////////////////////////////// - /// \param uXPos The \c x position of the voxel - /// \param uYPos The \c y position of the voxel - /// \param uZPos The \c z position of the voxel - /// \return The voxel value - //////////////////////////////////////////////////////////////////////////////// - template - VoxelType LargeVolume::getVoxelAt(int32_t uXPos, int32_t uYPos, int32_t uZPos) const - { - if(this->m_regValidRegion.containsPoint(Vector3DInt32(uXPos, uYPos, uZPos))) - { - const int32_t blockX = uXPos >> m_uBlockSideLengthPower; - const int32_t blockY = uYPos >> m_uBlockSideLengthPower; - const int32_t blockZ = uZPos >> m_uBlockSideLengthPower; - - const uint16_t xOffset = static_cast(uXPos - (blockX << m_uBlockSideLengthPower)); - const uint16_t yOffset = static_cast(uYPos - (blockY << m_uBlockSideLengthPower)); - const uint16_t zOffset = static_cast(uZPos - (blockZ << m_uBlockSideLengthPower)); - - auto pUncompressedBlock = getUncompressedBlock(blockX, blockY, blockZ); - - return pUncompressedBlock->getVoxel(xOffset, yOffset, zOffset); - } - else - { - return this->getBorderValue(); - } - } - - //////////////////////////////////////////////////////////////////////////////// - /// \param v3dPos The 3D position of the voxel - /// \return The voxel value - //////////////////////////////////////////////////////////////////////////////// - template - VoxelType LargeVolume::getVoxelAt(const Vector3DInt32& v3dPos) const - { - return getVoxelAt(v3dPos.getX(), v3dPos.getY(), v3dPos.getZ()); - } - - //////////////////////////////////////////////////////////////////////////////// - /// Increasing the size of the block cache will increase memory but may improve performance. - /// You may want to set this to a large value (e.g. 1024) when you are first loading your - /// volume data and then set it to a smaller value (e.g.64) for general processing. - /// \param uMaxNumberOfUncompressedBlocks The number of blocks for which uncompressed data can be cached. - //////////////////////////////////////////////////////////////////////////////// - template - void LargeVolume::setMemoryUsageLimit(uint32_t uMemoryUsageInBytes) - { - POLYVOX_THROW_IF(!m_pPager, invalid_operation, "You cannot limit the memory usage of the volume because it was created without a pager attached."); - - // Calculate the number of blocks based on the memory limit and the size of each block. - uint32_t uUncompressedBlockSizeInBytes = UncompressedBlock::calculateSizeInBytes(m_uBlockSideLength); - m_uBlockCountLimit = uMemoryUsageInBytes / uUncompressedBlockSizeInBytes; - - // We need at least a few blocks available to avoid thrashing, and in pratice there will probably be hundreds. - POLYVOX_LOG_WARNING_IF(m_uBlockCountLimit < uMinPracticalNoOfBlocks, "Requested memory usage limit of " - << uMemoryUsageInBytes / (1024 * 1024) << "Mb is too low and cannot be adhered to."); - m_uBlockCountLimit = (std::max)(m_uBlockCountLimit, uMinPracticalNoOfBlocks); - m_uBlockCountLimit = (std::min)(m_uBlockCountLimit, uMaxPracticalNoOfBlocks); - - // If the new limit is less than the number of blocks already loaded then the easiest solution is to flush and start loading again. - if (m_pRecentlyUsedBlocks.size() > m_uBlockCountLimit) - { - flushAll(); - } - - POLYVOX_LOG_DEBUG("Memory usage limit for volume now set to " << (m_uBlockCountLimit * uUncompressedBlockSizeInBytes) / (1024 * 1024) - << "Mb (" << m_uBlockCountLimit << " blocks of " << uUncompressedBlockSizeInBytes / 1024 << "Kb each)."); - } - - //////////////////////////////////////////////////////////////////////////////// - /// \param uXPos the \c x position of the voxel - /// \param uYPos the \c y position of the voxel - /// \param uZPos the \c z position of the voxel - /// \param tValue the value to which the voxel will be set - /// \param eWrapMode Specifies the behaviour when the requested position is outside of the volume. - /// This must be set to 'None' or 'DontCheck'. Other wrap modes cannot be used when writing to volume data. - //////////////////////////////////////////////////////////////////////////////// - template - void LargeVolume::setVoxel(int32_t uXPos, int32_t uYPos, int32_t uZPos, VoxelType tValue, WrapMode eWrapMode) - { - if((eWrapMode != WrapModes::Validate) && (eWrapMode != WrapModes::AssumeValid)) - { - POLYVOX_THROW(std::invalid_argument, "Invalid wrap mode in call to setVoxel(). It must be 'None' or 'DontCheck'."); - } - - // This validation is skipped if the wrap mode is 'DontCheck' - if(eWrapMode == WrapModes::Validate) - { - if(this->m_regValidRegion.containsPoint(Vector3DInt32(uXPos, uYPos, uZPos)) == false) - { - POLYVOX_THROW(std::out_of_range, "Position is outside valid region"); - } - } - - const int32_t blockX = uXPos >> m_uBlockSideLengthPower; - const int32_t blockY = uYPos >> m_uBlockSideLengthPower; - const int32_t blockZ = uZPos >> m_uBlockSideLengthPower; - - const uint16_t xOffset = static_cast(uXPos - (blockX << m_uBlockSideLengthPower)); - const uint16_t yOffset = static_cast(uYPos - (blockY << m_uBlockSideLengthPower)); - const uint16_t zOffset = static_cast(uZPos - (blockZ << m_uBlockSideLengthPower)); - - auto pUncompressedBlock = getUncompressedBlock(blockX, blockY, blockZ); - pUncompressedBlock->setVoxelAt(xOffset, yOffset, zOffset, tValue); - } - - //////////////////////////////////////////////////////////////////////////////// - /// \param v3dPos the 3D position of the voxel - /// \param tValue the value to which the voxel will be set - /// \param eWrapMode Specifies the behaviour when the requested position is outside of the volume. - /// This must be set to 'None' or 'DontCheck'. Other wrap modes cannot be used when writing to volume data. - //////////////////////////////////////////////////////////////////////////////// - template - void LargeVolume::setVoxel(const Vector3DInt32& v3dPos, VoxelType tValue, WrapMode eWrapMode) - { - setVoxel(v3dPos.getX(), v3dPos.getY(), v3dPos.getZ(), tValue, eWrapMode); - } - - //////////////////////////////////////////////////////////////////////////////// - /// \param uXPos the \c x position of the voxel - /// \param uYPos the \c y position of the voxel - /// \param uZPos the \c z position of the voxel - /// \param tValue the value to which the voxel will be set - /// \return whether the requested position is inside the volume - //////////////////////////////////////////////////////////////////////////////// - template - bool LargeVolume::setVoxelAt(int32_t uXPos, int32_t uYPos, int32_t uZPos, VoxelType tValue) - { - // PolyVox does not throw an exception when a voxel is out of range. Please see 'Error Handling' in the User Manual. - POLYVOX_ASSERT(this->m_regValidRegion.containsPoint(Vector3DInt32(uXPos, uYPos, uZPos)), "Position is outside valid region"); - - const int32_t blockX = uXPos >> m_uBlockSideLengthPower; - const int32_t blockY = uYPos >> m_uBlockSideLengthPower; - const int32_t blockZ = uZPos >> m_uBlockSideLengthPower; - - const uint16_t xOffset = static_cast(uXPos - (blockX << m_uBlockSideLengthPower)); - const uint16_t yOffset = static_cast(uYPos - (blockY << m_uBlockSideLengthPower)); - const uint16_t zOffset = static_cast(uZPos - (blockZ << m_uBlockSideLengthPower)); - - auto pUncompressedBlock = getUncompressedBlock(blockX, blockY, blockZ); - - pUncompressedBlock->setVoxelAt(xOffset, yOffset, zOffset, tValue); - - //Return true to indicate that we modified a voxel. - return true; - } - - //////////////////////////////////////////////////////////////////////////////// - /// \param v3dPos the 3D position of the voxel - /// \param tValue the value to which the voxel will be set - /// \return whether the requested position is inside the volume - //////////////////////////////////////////////////////////////////////////////// - template - bool LargeVolume::setVoxelAt(const Vector3DInt32& v3dPos, VoxelType tValue) - { - return setVoxelAt(v3dPos.getX(), v3dPos.getY(), v3dPos.getZ(), tValue); - } - - - //////////////////////////////////////////////////////////////////////////////// - /// Note that if *NOTE - update docs - MaxNumberOfBlocksInMemory no longer exists* MaxNumberOfBlocksInMemory is not large enough to support the region this function will only load part of the region. In this case it is undefined which parts will actually be loaded. If all the voxels in the given region are already loaded, this function will not do anything. Other voxels might be unloaded to make space for the new voxels. - /// \param regPrefetch The Region of voxels to prefetch into memory. - //////////////////////////////////////////////////////////////////////////////// - template - void LargeVolume::prefetch(Region regPrefetch) - { - // Convert the start and end positions into block space coordinates - Vector3DInt32 v3dStart; - for(int i = 0; i < 3; i++) - { - v3dStart.setElement(i, regPrefetch.getLowerCorner().getElement(i) >> m_uBlockSideLengthPower); - } - - Vector3DInt32 v3dEnd; - for(int i = 0; i < 3; i++) - { - v3dEnd.setElement(i, regPrefetch.getUpperCorner().getElement(i) >> m_uBlockSideLengthPower); - } - - // Ensure we don't page in more blocks than the volume can hold. - Region region(v3dStart, v3dEnd); - uint32_t uNoOfBlocks = static_cast(region.getWidthInVoxels() * region.getHeightInVoxels() * region.getDepthInVoxels()); - POLYVOX_LOG_WARNING_IF(uNoOfBlocks > m_uBlockCountLimit, "Attempting to prefetch more than the maximum number of blocks."); - uNoOfBlocks = (std::min)(uNoOfBlocks, m_uBlockCountLimit); - - // Loops over the specified positions and touch the corresponding blocks. - for(int32_t x = v3dStart.getX(); x <= v3dEnd.getX(); x++) - { - for(int32_t y = v3dStart.getY(); y <= v3dEnd.getY(); y++) - { - for(int32_t z = v3dStart.getZ(); z <= v3dEnd.getZ(); z++) - { - getUncompressedBlock(x,y,z); - } - } - } - } - - //////////////////////////////////////////////////////////////////////////////// - /// Removes all voxels from memory, and calls dataOverflowHandler() to ensure the application has a chance to store the data. - //////////////////////////////////////////////////////////////////////////////// - template - void LargeVolume::flushAll() - { - POLYVOX_LOG_WARNING_IF(!m_pPager, "Data discarded by flush operation as no pager is attached."); - - // Clear this pointer so it doesn't hang on to any blocks. - m_pLastAccessedBlock = nullptr; - - // Erase all the most recently used blocks. - m_pRecentlyUsedBlocks.clear(); - - // Remove deleted blocks from the list of all loaded blocks. - purgeNullPtrsFromAllBlocks(); - - // If there are still some blocks left then this is a cause for concern. Perhaps samplers are holding on to them? - POLYVOX_LOG_WARNING_IF(m_pAllBlocks.size() > 0, "Blocks still exist after performing flushAll()! Perhaps you have samplers attached?"); - } - - //////////////////////////////////////////////////////////////////////////////// - /// Removes all voxels in the specified Region from memory, and calls dataOverflowHandler() to ensure the application has a chance to store the data. It is possible that there are no voxels loaded in the Region, in which case the function will have no effect. - //////////////////////////////////////////////////////////////////////////////// - template - void LargeVolume::flush(Region regFlush) - { - POLYVOX_LOG_WARNING_IF(!m_pPager, "Data discarded by flush operation as no pager is attached."); - - // Clear this pointer so it doesn't hang on to any blocks. - m_pLastAccessedBlock = nullptr; - - // Convert the start and end positions into block space coordinates - Vector3DInt32 v3dStart; - for(int i = 0; i < 3; i++) - { - v3dStart.setElement(i, regFlush.getLowerCorner().getElement(i) >> m_uBlockSideLengthPower); - } - - Vector3DInt32 v3dEnd; - for(int i = 0; i < 3; i++) - { - v3dEnd.setElement(i, regFlush.getUpperCorner().getElement(i) >> m_uBlockSideLengthPower); - } - - // Loops over the specified positions and delete the corresponding blocks. - for(int32_t x = v3dStart.getX(); x <= v3dEnd.getX(); x++) - { - for(int32_t y = v3dStart.getY(); y <= v3dEnd.getY(); y++) - { - for(int32_t z = v3dStart.getZ(); z <= v3dEnd.getZ(); z++) - { - m_pRecentlyUsedBlocks.erase(Vector3DInt32(x, y, z)); m_pLastAccessedBlock = 0; - } - } - } - - // We might now have so null pointers in the 'all blocks' list so clean them up. - purgeNullPtrsFromAllBlocks(); - } - - //////////////////////////////////////////////////////////////////////////////// - /// This function should probably be made internal... - //////////////////////////////////////////////////////////////////////////////// - template - void LargeVolume::initialise() - { - //Validate parameters - if(m_uBlockSideLength == 0) - { - POLYVOX_THROW(std::invalid_argument, "Block side length cannot be zero."); - } - - if(!isPowerOf2(m_uBlockSideLength)) - { - POLYVOX_THROW(std::invalid_argument, "Block side length must be a power of two."); - } - - m_uTimestamper = 0; - //m_uBlockCountLimit = 16; - //m_uMaxNumberOfBlocksInMemory = 1024; - m_v3dLastAccessedBlockPos = Vector3DInt32(0,0,0); //There are no invalid positions, but initially the m_pLastAccessedBlock pointer will be null; - m_pLastAccessedBlock = 0; - - //Compute the block side length - m_uBlockSideLengthPower = logBase2(m_uBlockSideLength); - - m_regValidRegionInBlocks.setLowerX(this->m_regValidRegion.getLowerX() >> m_uBlockSideLengthPower); - m_regValidRegionInBlocks.setLowerY(this->m_regValidRegion.getLowerY() >> m_uBlockSideLengthPower); - m_regValidRegionInBlocks.setLowerZ(this->m_regValidRegion.getLowerZ() >> m_uBlockSideLengthPower); - m_regValidRegionInBlocks.setUpperX(this->m_regValidRegion.getUpperX() >> m_uBlockSideLengthPower); - m_regValidRegionInBlocks.setUpperY(this->m_regValidRegion.getUpperY() >> m_uBlockSideLengthPower); - m_regValidRegionInBlocks.setUpperZ(this->m_regValidRegion.getUpperZ() >> m_uBlockSideLengthPower); - - //setMaxNumberOfUncompressedBlocks(m_uBlockCountLimit); - - //Clear the previous data - m_pRecentlyUsedBlocks.clear(); - - //Other properties we might find useful later - this->m_uLongestSideLength = (std::max)((std::max)(this->getWidth(),this->getHeight()),this->getDepth()); - this->m_uShortestSideLength = (std::min)((std::min)(this->getWidth(),this->getHeight()),this->getDepth()); - this->m_fDiagonalLength = sqrtf(static_cast(this->getWidth() * this->getWidth() + this->getHeight() * this->getHeight() + this->getDepth() * this->getDepth())); - } - - template - std::shared_ptr< UncompressedBlock > LargeVolume::getUncompressedBlock(int32_t uBlockX, int32_t uBlockY, int32_t uBlockZ) const - { - Vector3DInt32 v3dBlockPos(uBlockX, uBlockY, uBlockZ); - - //Check if we have the same block as last time, if so there's no need to even update - //the time stamp. If we updated it everytime then that would be every time we touched - //a voxel, which would overflow a uint32_t and require us to use a uint64_t instead. - //This check should also provide a significant speed boost as usually it is true. - if((v3dBlockPos == m_v3dLastAccessedBlockPos) && (m_pLastAccessedBlock != 0)) - { - return m_pLastAccessedBlock; - } - - // The block was not the same as last time, but we can now hope it is in the set of most recently used blocks. - std::shared_ptr< UncompressedBlock > pUncompressedBlock = nullptr; - typename SharedPtrBlockMap::iterator itUncompressedBlock = m_pRecentlyUsedBlocks.find(v3dBlockPos); - - // Check whether the block was found. - if ((itUncompressedBlock) != m_pRecentlyUsedBlocks.end()) - { - // The block was found so we can use it. - pUncompressedBlock = itUncompressedBlock->second; - POLYVOX_ASSERT(pUncompressedBlock, "Recent block list shold never contain a null pointer."); - } - - if (!pUncompressedBlock) - { - // Although it's not in our recently use blocks, there's some (slim) chance that it - // exists in the list of all loaded blocks, because a sampler may be holding on to it. - typename WeakPtrBlockMap::iterator itWeakUncompressedBlock = m_pAllBlocks.find(v3dBlockPos); - if (itWeakUncompressedBlock != m_pAllBlocks.end()) - { - // We've found an entry in the 'all blocks' list, but it can be null. This happens if a sampler was the - // last thing to be keeping it alive and then the sampler let it go. In this case we remove it from the - // list, and it will get added again soon when we page it in and fill it with valid data. - if (itWeakUncompressedBlock->second.expired()) - { - m_pAllBlocks.erase(itWeakUncompressedBlock); - } - else - { - // The block is valid. We know it's not in the recently used list (we checked earlier) so it should be added. - pUncompressedBlock = std::shared_ptr< UncompressedBlock >(itWeakUncompressedBlock->second); - m_pRecentlyUsedBlocks.insert(std::make_pair(v3dBlockPos, pUncompressedBlock)); - } - } - } - - // If we still haven't found the block then it's time to create a new one and page it in from disk. - if (!pUncompressedBlock) - { - // The block was not found so we will create a new one. - pUncompressedBlock = std::make_shared< UncompressedBlock >(v3dBlockPos, m_uBlockSideLength, m_pPager); - - // As we are loading a new block we should try to ensure we don't go over our target memory usage. - bool erasedBlock = false; - while (m_pRecentlyUsedBlocks.size() + 1 > m_uBlockCountLimit) // +1 ready for new block we will add next. - { - // This should never hit, because it should not have been possible for - // the user to limit the number of blocks if they did not provide a pager. - POLYVOX_ASSERT(m_pPager, "A valid pager is required to limit number of blocks"); - - // Find the least recently used block. Hopefully this isn't too slow. - typename SharedPtrBlockMap::iterator itUnloadBlock = m_pRecentlyUsedBlocks.begin(); - for (typename SharedPtrBlockMap::iterator i = m_pRecentlyUsedBlocks.begin(); i != m_pRecentlyUsedBlocks.end(); i++) - { - if (i->second->m_uBlockLastAccessed < itUnloadBlock->second->m_uBlockLastAccessed) - { - itUnloadBlock = i; - } - } - - // Erase the least recently used block - m_pRecentlyUsedBlocks.erase(itUnloadBlock); - erasedBlock = true; - } - - // If we've deleted any blocks from the recently used list then this - // seems like a good place to purge the 'all blocks' list as well. - if (erasedBlock) - { - purgeNullPtrsFromAllBlocks(); - } - - // Add our new block to the maps. - m_pAllBlocks.insert(std::make_pair(v3dBlockPos, pUncompressedBlock)); - m_pRecentlyUsedBlocks.insert(std::make_pair(v3dBlockPos, pUncompressedBlock)); - } - - pUncompressedBlock->m_uBlockLastAccessed = ++m_uTimestamper; - m_pLastAccessedBlock = pUncompressedBlock; - m_v3dLastAccessedBlockPos = v3dBlockPos; - - return pUncompressedBlock; - } - - //////////////////////////////////////////////////////////////////////////////// - /// Calculate the memory usage of the volume. - //////////////////////////////////////////////////////////////////////////////// - template - uint32_t LargeVolume::calculateSizeInBytes(void) - { - // Purge null blocks so we know that all blocks are used. - purgeNullPtrsFromAllBlocks(); - - // Note: We disregard the size of the other class members as they are likely to be very small compared to the size of the - // allocated voxel data. This also keeps the reported size as a power of two, which makes other memory calculations easier. - return UncompressedBlock::calculateSizeInBytes(m_uBlockSideLength) * m_pAllBlocks.size(); - } - - template - void LargeVolume::purgeNullPtrsFromAllBlocks(void) const - { - for (auto blockIter = m_pAllBlocks.begin(); blockIter != m_pAllBlocks.end();) - { - if (blockIter->second.expired()) - { - blockIter = m_pAllBlocks.erase(blockIter); - } - else - { - blockIter++; - } - } - } - - template - template - VoxelType LargeVolume::getVoxelImpl(int32_t uXPos, int32_t uYPos, int32_t uZPos, WrapModeType, VoxelType tBorder) const - { - // This function should never be called because one of the specialisations should always match. - POLYVOX_ASSERT(false, "This function is not implemented and should never be called!"); - } - - template - VoxelType LargeVolume::getVoxelImpl(int32_t uXPos, int32_t uYPos, int32_t uZPos, WrapModeType, VoxelType tBorder) const - { - if(this->m_regValidRegion.containsPoint(Vector3DInt32(uXPos, uYPos, uZPos)) == false) - { - POLYVOX_THROW(std::out_of_range, "Position is outside valid region"); - } - - return getVoxelImpl(uXPos, uYPos, uZPos, WrapModeType(), tBorder); // No wrapping as we've just validated the position. - } - - template - VoxelType LargeVolume::getVoxelImpl(int32_t uXPos, int32_t uYPos, int32_t uZPos, WrapModeType, VoxelType tBorder) const - { - //Perform clamping - uXPos = (std::max)(uXPos, this->m_regValidRegion.getLowerX()); - uYPos = (std::max)(uYPos, this->m_regValidRegion.getLowerY()); - uZPos = (std::max)(uZPos, this->m_regValidRegion.getLowerZ()); - uXPos = (std::min)(uXPos, this->m_regValidRegion.getUpperX()); - uYPos = (std::min)(uYPos, this->m_regValidRegion.getUpperY()); - uZPos = (std::min)(uZPos, this->m_regValidRegion.getUpperZ()); - - return getVoxelImpl(uXPos, uYPos, uZPos, WrapModeType(), tBorder); // No wrapping as we've just validated the position. - } - - template - VoxelType LargeVolume::getVoxelImpl(int32_t uXPos, int32_t uYPos, int32_t uZPos, WrapModeType, VoxelType tBorder) const - { - if(this->m_regValidRegion.containsPoint(uXPos, uYPos, uZPos)) - { - return getVoxelImpl(uXPos, uYPos, uZPos, WrapModeType(), tBorder); // No wrapping as we've just validated the position. - } - else - { - return tBorder; - } - } - - template - VoxelType LargeVolume::getVoxelImpl(int32_t uXPos, int32_t uYPos, int32_t uZPos, WrapModeType, VoxelType /*tBorder*/) const - { - const int32_t blockX = uXPos >> m_uBlockSideLengthPower; - const int32_t blockY = uYPos >> m_uBlockSideLengthPower; - const int32_t blockZ = uZPos >> m_uBlockSideLengthPower; - - const uint16_t xOffset = static_cast(uXPos - (blockX << m_uBlockSideLengthPower)); - const uint16_t yOffset = static_cast(uYPos - (blockY << m_uBlockSideLengthPower)); - const uint16_t zOffset = static_cast(uZPos - (blockZ << m_uBlockSideLengthPower)); - - auto pUncompressedBlock = getUncompressedBlock(blockX, blockY, blockZ); - return pUncompressedBlock->getVoxel(xOffset, yOffset, zOffset); - } -} - +/******************************************************************************* +Copyright (c) 2005-2009 David Williams + +This software is provided 'as-is', without any express or implied +warranty. In no event will the authors be held liable for any damages +arising from the use of this software. + +Permission is granted to anyone to use this software for any purpose, +including commercial applications, and to alter it and redistribute it +freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + + 3. This notice may not be removed or altered from any source + distribution. +*******************************************************************************/ + +#include "PolyVoxCore/Impl/ErrorHandling.h" + +#include +#include + +namespace PolyVox +{ + //////////////////////////////////////////////////////////////////////////////// + /// This constructor creates a volume with a fixed size which is specified as a parameter. By default this constructor will not enable paging but you can override this if desired. If you do wish to enable paging then you are required to provide the call back function (see the other PagedVolume constructor). + /// \param regValid Specifies the minimum and maximum valid voxel positions. + /// \param pBlockCompressor An implementation of the Compressor interface which is used to compress blocks in memory. + /// \param dataRequiredHandler The callback function which will be called when PolyVox tries to use data which is not currently in momory. + /// \param dataOverflowHandler The callback function which will be called when PolyVox has too much data and needs to remove some from memory. + /// \param bPagingEnabled Controls whether or not paging is enabled for this PagedVolume. + /// \param uBlockSideLength The size of the blocks making up the volume. Small blocks will compress/decompress faster, but there will also be more of them meaning voxel access could be slower. + //////////////////////////////////////////////////////////////////////////////// + template + PagedVolume::PagedVolume + ( + const Region& regValid, + Pager* pPager, + uint16_t uBlockSideLength + ) + :BaseVolume(regValid) + { + m_uBlockSideLength = uBlockSideLength; + m_pPager = pPager; + + if (m_pPager) + { + // If the user is creating a vast (almost infinite) volume then we can bet they will be + // expecting a high memory usage and will want a fair number of blocks to play around with. + if (regValid == Region::MaxRegion) + { + m_uBlockCountLimit = 1024; + } + else + { + // Otherwise we try to choose a block count to avoid too much thrashing, particularly when iterating + // over the whole volume. This means at least enough blocks to cover one edge of the volume, and ideally + // enough for a whole face. Which face? Longest edge by shortest edge seems like a reasonable guess. + uint32_t longestSide = (std::max)(regValid.getWidthInVoxels(), (std::max)(regValid.getHeightInVoxels(), regValid.getDepthInVoxels())); + uint32_t shortestSide = (std::min)(regValid.getWidthInVoxels(), (std::min)(regValid.getHeightInVoxels(), regValid.getDepthInVoxels())); + + longestSide /= m_uBlockSideLength; + shortestSide /= m_uBlockSideLength; + + m_uBlockCountLimit = longestSide * shortestSide; + } + } + else + { + // If there is no pager provided then we set the block limit to the maximum + // value to ensure the system never attempts to page blocks out of memory. + m_uBlockCountLimit = (std::numeric_limits::max)(); + } + + // Make sure the calculated block limit is within practical bounds + m_uBlockCountLimit = (std::max)(m_uBlockCountLimit, uMinPracticalNoOfBlocks); + m_uBlockCountLimit = (std::min)(m_uBlockCountLimit, uMaxPracticalNoOfBlocks); + + uint32_t uUncompressedBlockSizeInBytes = UncompressedBlock::calculateSizeInBytes(m_uBlockSideLength); + POLYVOX_LOG_DEBUG("Memory usage limit for volume initially set to " << (m_uBlockCountLimit * uUncompressedBlockSizeInBytes) / (1024 * 1024) + << "Mb (" << m_uBlockCountLimit << " blocks of " << uUncompressedBlockSizeInBytes / 1024 << "Kb each)."); + + initialise(); + } + + //////////////////////////////////////////////////////////////////////////////// + /// This function should never be called. Copying volumes by value would be expensive, and we want to prevent users from doing + /// it by accident (such as when passing them as paramenters to functions). That said, there are times when you really do want to + /// make a copy of a volume and in this case you should look at the VolumeResampler. + /// + /// \sa VolumeResampler + //////////////////////////////////////////////////////////////////////////////// + template + PagedVolume::PagedVolume(const PagedVolume& /*rhs*/) + { + POLYVOX_THROW(not_implemented, "Volume copy constructor not implemented for performance reasons."); + } + + //////////////////////////////////////////////////////////////////////////////// + /// Destroys the volume The destructor will call flushAll() to ensure that a paging volume has the chance to save it's data via the dataOverflowHandler() if desired. + //////////////////////////////////////////////////////////////////////////////// + template + PagedVolume::~PagedVolume() + { + flushAll(); + } + + //////////////////////////////////////////////////////////////////////////////// + /// This function should never be called. Copying volumes by value would be expensive, and we want to prevent users from doing + /// it by accident (such as when passing them as paramenters to functions). That said, there are times when you really do want to + /// make a copy of a volume and in this case you should look at the Volumeresampler. + /// + /// \sa VolumeResampler + //////////////////////////////////////////////////////////////////////////////// + template + PagedVolume& PagedVolume::operator=(const PagedVolume& /*rhs*/) + { + POLYVOX_THROW(not_implemented, "Volume assignment operator not implemented for performance reasons."); + } + + //////////////////////////////////////////////////////////////////////////////// + /// This version of the function requires the wrap mode to be specified as a + /// template parameter, which can provide better performance. + /// \param uXPos The \c x position of the voxel + /// \param uYPos The \c y position of the voxel + /// \param uZPos The \c z position of the voxel + /// \tparam eWrapMode Specifies the behaviour when the requested position is outside of the volume. + /// \param tBorder The border value to use if the wrap mode is set to 'Border'. + /// \return The voxel value + //////////////////////////////////////////////////////////////////////////////// + template + template + VoxelType PagedVolume::getVoxel(int32_t uXPos, int32_t uYPos, int32_t uZPos, VoxelType tBorder) const + { + // Simply call through to the real implementation + return getVoxelImpl(uXPos, uYPos, uZPos, WrapModeType(), tBorder); + } + + //////////////////////////////////////////////////////////////////////////////// + /// This version of the function requires the wrap mode to be specified as a + /// template parameter, which can provide better performance. + /// \param uXPos The \c x position of the voxel + /// \param uYPos The \c y position of the voxel + /// \param uZPos The \c z position of the voxel + /// \tparam eWrapMode Specifies the behaviour when the requested position is outside of the volume. + /// \param tBorder The border value to use if the wrap mode is set to 'Border'. + /// \return The voxel value + //////////////////////////////////////////////////////////////////////////////// + template + template + VoxelType PagedVolume::getVoxel(const Vector3DInt32& v3dPos, VoxelType tBorder) const + { + // Simply call through to the real implementation + return getVoxel(v3dPos.getX(), v3dPos.getY(), v3dPos.getZ(), tBorder); + } + + //////////////////////////////////////////////////////////////////////////////// + /// This version of the function is provided so that the wrap mode does not need + /// to be specified as a template parameter, as it may be confusing to some users. + /// \param uXPos The \c x position of the voxel + /// \param uYPos The \c y position of the voxel + /// \param uZPos The \c z position of the voxel + /// \param eWrapMode Specifies the behaviour when the requested position is outside of the volume. + /// \param tBorder The border value to use if the wrap mode is set to 'Border'. + /// \return The voxel value + //////////////////////////////////////////////////////////////////////////////// + template + VoxelType PagedVolume::getVoxel(int32_t uXPos, int32_t uYPos, int32_t uZPos, WrapMode eWrapMode, VoxelType tBorder) const + { + switch(eWrapMode) + { + case WrapModes::Validate: + return getVoxelImpl(uXPos, uYPos, uZPos, WrapModeType(), tBorder); + case WrapModes::Clamp: + return getVoxelImpl(uXPos, uYPos, uZPos, WrapModeType(), tBorder); + case WrapModes::Border: + return getVoxelImpl(uXPos, uYPos, uZPos, WrapModeType(), tBorder); + case WrapModes::AssumeValid: + return getVoxelImpl(uXPos, uYPos, uZPos, WrapModeType(), tBorder); + default: + // Should never happen + POLYVOX_ASSERT(false, "Invalid wrap mode"); + return VoxelType(); + } + } + + //////////////////////////////////////////////////////////////////////////////// + /// This version of the function is provided so that the wrap mode does not need + /// to be specified as a template parameter, as it may be confusing to some users. + /// \param v3dPos The 3D position of the voxel + /// \param eWrapMode Specifies the behaviour when the requested position is outside of the volume. + /// \param tBorder The border value to use if the wrap mode is set to 'Border'. + /// \return The voxel value + //////////////////////////////////////////////////////////////////////////////// + template + VoxelType PagedVolume::getVoxel(const Vector3DInt32& v3dPos, WrapMode eWrapMode, VoxelType tBorder) const + { + return getVoxel(v3dPos.getX(), v3dPos.getY(), v3dPos.getZ(), eWrapMode, tBorder); + } + + //////////////////////////////////////////////////////////////////////////////// + /// \param uXPos The \c x position of the voxel + /// \param uYPos The \c y position of the voxel + /// \param uZPos The \c z position of the voxel + /// \return The voxel value + //////////////////////////////////////////////////////////////////////////////// + template + VoxelType PagedVolume::getVoxelAt(int32_t uXPos, int32_t uYPos, int32_t uZPos) const + { + if(this->m_regValidRegion.containsPoint(Vector3DInt32(uXPos, uYPos, uZPos))) + { + const int32_t blockX = uXPos >> m_uBlockSideLengthPower; + const int32_t blockY = uYPos >> m_uBlockSideLengthPower; + const int32_t blockZ = uZPos >> m_uBlockSideLengthPower; + + const uint16_t xOffset = static_cast(uXPos - (blockX << m_uBlockSideLengthPower)); + const uint16_t yOffset = static_cast(uYPos - (blockY << m_uBlockSideLengthPower)); + const uint16_t zOffset = static_cast(uZPos - (blockZ << m_uBlockSideLengthPower)); + + auto pUncompressedBlock = getUncompressedBlock(blockX, blockY, blockZ); + + return pUncompressedBlock->getVoxel(xOffset, yOffset, zOffset); + } + else + { + return this->getBorderValue(); + } + } + + //////////////////////////////////////////////////////////////////////////////// + /// \param v3dPos The 3D position of the voxel + /// \return The voxel value + //////////////////////////////////////////////////////////////////////////////// + template + VoxelType PagedVolume::getVoxelAt(const Vector3DInt32& v3dPos) const + { + return getVoxelAt(v3dPos.getX(), v3dPos.getY(), v3dPos.getZ()); + } + + //////////////////////////////////////////////////////////////////////////////// + /// Increasing the size of the block cache will increase memory but may improve performance. + /// You may want to set this to a large value (e.g. 1024) when you are first loading your + /// volume data and then set it to a smaller value (e.g.64) for general processing. + /// \param uMaxNumberOfUncompressedBlocks The number of blocks for which uncompressed data can be cached. + //////////////////////////////////////////////////////////////////////////////// + template + void PagedVolume::setMemoryUsageLimit(uint32_t uMemoryUsageInBytes) + { + POLYVOX_THROW_IF(!m_pPager, invalid_operation, "You cannot limit the memory usage of the volume because it was created without a pager attached."); + + // Calculate the number of blocks based on the memory limit and the size of each block. + uint32_t uUncompressedBlockSizeInBytes = UncompressedBlock::calculateSizeInBytes(m_uBlockSideLength); + m_uBlockCountLimit = uMemoryUsageInBytes / uUncompressedBlockSizeInBytes; + + // We need at least a few blocks available to avoid thrashing, and in pratice there will probably be hundreds. + POLYVOX_LOG_WARNING_IF(m_uBlockCountLimit < uMinPracticalNoOfBlocks, "Requested memory usage limit of " + << uMemoryUsageInBytes / (1024 * 1024) << "Mb is too low and cannot be adhered to."); + m_uBlockCountLimit = (std::max)(m_uBlockCountLimit, uMinPracticalNoOfBlocks); + m_uBlockCountLimit = (std::min)(m_uBlockCountLimit, uMaxPracticalNoOfBlocks); + + // If the new limit is less than the number of blocks already loaded then the easiest solution is to flush and start loading again. + if (m_pRecentlyUsedBlocks.size() > m_uBlockCountLimit) + { + flushAll(); + } + + POLYVOX_LOG_DEBUG("Memory usage limit for volume now set to " << (m_uBlockCountLimit * uUncompressedBlockSizeInBytes) / (1024 * 1024) + << "Mb (" << m_uBlockCountLimit << " blocks of " << uUncompressedBlockSizeInBytes / 1024 << "Kb each)."); + } + + //////////////////////////////////////////////////////////////////////////////// + /// \param uXPos the \c x position of the voxel + /// \param uYPos the \c y position of the voxel + /// \param uZPos the \c z position of the voxel + /// \param tValue the value to which the voxel will be set + /// \param eWrapMode Specifies the behaviour when the requested position is outside of the volume. + /// This must be set to 'None' or 'DontCheck'. Other wrap modes cannot be used when writing to volume data. + //////////////////////////////////////////////////////////////////////////////// + template + void PagedVolume::setVoxel(int32_t uXPos, int32_t uYPos, int32_t uZPos, VoxelType tValue, WrapMode eWrapMode) + { + if((eWrapMode != WrapModes::Validate) && (eWrapMode != WrapModes::AssumeValid)) + { + POLYVOX_THROW(std::invalid_argument, "Invalid wrap mode in call to setVoxel(). It must be 'None' or 'DontCheck'."); + } + + // This validation is skipped if the wrap mode is 'DontCheck' + if(eWrapMode == WrapModes::Validate) + { + if(this->m_regValidRegion.containsPoint(Vector3DInt32(uXPos, uYPos, uZPos)) == false) + { + POLYVOX_THROW(std::out_of_range, "Position is outside valid region"); + } + } + + const int32_t blockX = uXPos >> m_uBlockSideLengthPower; + const int32_t blockY = uYPos >> m_uBlockSideLengthPower; + const int32_t blockZ = uZPos >> m_uBlockSideLengthPower; + + const uint16_t xOffset = static_cast(uXPos - (blockX << m_uBlockSideLengthPower)); + const uint16_t yOffset = static_cast(uYPos - (blockY << m_uBlockSideLengthPower)); + const uint16_t zOffset = static_cast(uZPos - (blockZ << m_uBlockSideLengthPower)); + + auto pUncompressedBlock = getUncompressedBlock(blockX, blockY, blockZ); + pUncompressedBlock->setVoxelAt(xOffset, yOffset, zOffset, tValue); + } + + //////////////////////////////////////////////////////////////////////////////// + /// \param v3dPos the 3D position of the voxel + /// \param tValue the value to which the voxel will be set + /// \param eWrapMode Specifies the behaviour when the requested position is outside of the volume. + /// This must be set to 'None' or 'DontCheck'. Other wrap modes cannot be used when writing to volume data. + //////////////////////////////////////////////////////////////////////////////// + template + void PagedVolume::setVoxel(const Vector3DInt32& v3dPos, VoxelType tValue, WrapMode eWrapMode) + { + setVoxel(v3dPos.getX(), v3dPos.getY(), v3dPos.getZ(), tValue, eWrapMode); + } + + //////////////////////////////////////////////////////////////////////////////// + /// \param uXPos the \c x position of the voxel + /// \param uYPos the \c y position of the voxel + /// \param uZPos the \c z position of the voxel + /// \param tValue the value to which the voxel will be set + /// \return whether the requested position is inside the volume + //////////////////////////////////////////////////////////////////////////////// + template + bool PagedVolume::setVoxelAt(int32_t uXPos, int32_t uYPos, int32_t uZPos, VoxelType tValue) + { + // PolyVox does not throw an exception when a voxel is out of range. Please see 'Error Handling' in the User Manual. + POLYVOX_ASSERT(this->m_regValidRegion.containsPoint(Vector3DInt32(uXPos, uYPos, uZPos)), "Position is outside valid region"); + + const int32_t blockX = uXPos >> m_uBlockSideLengthPower; + const int32_t blockY = uYPos >> m_uBlockSideLengthPower; + const int32_t blockZ = uZPos >> m_uBlockSideLengthPower; + + const uint16_t xOffset = static_cast(uXPos - (blockX << m_uBlockSideLengthPower)); + const uint16_t yOffset = static_cast(uYPos - (blockY << m_uBlockSideLengthPower)); + const uint16_t zOffset = static_cast(uZPos - (blockZ << m_uBlockSideLengthPower)); + + auto pUncompressedBlock = getUncompressedBlock(blockX, blockY, blockZ); + + pUncompressedBlock->setVoxelAt(xOffset, yOffset, zOffset, tValue); + + //Return true to indicate that we modified a voxel. + return true; + } + + //////////////////////////////////////////////////////////////////////////////// + /// \param v3dPos the 3D position of the voxel + /// \param tValue the value to which the voxel will be set + /// \return whether the requested position is inside the volume + //////////////////////////////////////////////////////////////////////////////// + template + bool PagedVolume::setVoxelAt(const Vector3DInt32& v3dPos, VoxelType tValue) + { + return setVoxelAt(v3dPos.getX(), v3dPos.getY(), v3dPos.getZ(), tValue); + } + + + //////////////////////////////////////////////////////////////////////////////// + /// Note that if *NOTE - update docs - MaxNumberOfBlocksInMemory no longer exists* MaxNumberOfBlocksInMemory is not large enough to support the region this function will only load part of the region. In this case it is undefined which parts will actually be loaded. If all the voxels in the given region are already loaded, this function will not do anything. Other voxels might be unloaded to make space for the new voxels. + /// \param regPrefetch The Region of voxels to prefetch into memory. + //////////////////////////////////////////////////////////////////////////////// + template + void PagedVolume::prefetch(Region regPrefetch) + { + // Convert the start and end positions into block space coordinates + Vector3DInt32 v3dStart; + for(int i = 0; i < 3; i++) + { + v3dStart.setElement(i, regPrefetch.getLowerCorner().getElement(i) >> m_uBlockSideLengthPower); + } + + Vector3DInt32 v3dEnd; + for(int i = 0; i < 3; i++) + { + v3dEnd.setElement(i, regPrefetch.getUpperCorner().getElement(i) >> m_uBlockSideLengthPower); + } + + // Ensure we don't page in more blocks than the volume can hold. + Region region(v3dStart, v3dEnd); + uint32_t uNoOfBlocks = static_cast(region.getWidthInVoxels() * region.getHeightInVoxels() * region.getDepthInVoxels()); + POLYVOX_LOG_WARNING_IF(uNoOfBlocks > m_uBlockCountLimit, "Attempting to prefetch more than the maximum number of blocks."); + uNoOfBlocks = (std::min)(uNoOfBlocks, m_uBlockCountLimit); + + // Loops over the specified positions and touch the corresponding blocks. + for(int32_t x = v3dStart.getX(); x <= v3dEnd.getX(); x++) + { + for(int32_t y = v3dStart.getY(); y <= v3dEnd.getY(); y++) + { + for(int32_t z = v3dStart.getZ(); z <= v3dEnd.getZ(); z++) + { + getUncompressedBlock(x,y,z); + } + } + } + } + + //////////////////////////////////////////////////////////////////////////////// + /// Removes all voxels from memory, and calls dataOverflowHandler() to ensure the application has a chance to store the data. + //////////////////////////////////////////////////////////////////////////////// + template + void PagedVolume::flushAll() + { + POLYVOX_LOG_WARNING_IF(!m_pPager, "Data discarded by flush operation as no pager is attached."); + + // Clear this pointer so it doesn't hang on to any blocks. + m_pLastAccessedBlock = nullptr; + + // Erase all the most recently used blocks. + m_pRecentlyUsedBlocks.clear(); + + // Remove deleted blocks from the list of all loaded blocks. + purgeNullPtrsFromAllBlocks(); + + // If there are still some blocks left then this is a cause for concern. Perhaps samplers are holding on to them? + POLYVOX_LOG_WARNING_IF(m_pAllBlocks.size() > 0, "Blocks still exist after performing flushAll()! Perhaps you have samplers attached?"); + } + + //////////////////////////////////////////////////////////////////////////////// + /// Removes all voxels in the specified Region from memory, and calls dataOverflowHandler() to ensure the application has a chance to store the data. It is possible that there are no voxels loaded in the Region, in which case the function will have no effect. + //////////////////////////////////////////////////////////////////////////////// + template + void PagedVolume::flush(Region regFlush) + { + POLYVOX_LOG_WARNING_IF(!m_pPager, "Data discarded by flush operation as no pager is attached."); + + // Clear this pointer so it doesn't hang on to any blocks. + m_pLastAccessedBlock = nullptr; + + // Convert the start and end positions into block space coordinates + Vector3DInt32 v3dStart; + for(int i = 0; i < 3; i++) + { + v3dStart.setElement(i, regFlush.getLowerCorner().getElement(i) >> m_uBlockSideLengthPower); + } + + Vector3DInt32 v3dEnd; + for(int i = 0; i < 3; i++) + { + v3dEnd.setElement(i, regFlush.getUpperCorner().getElement(i) >> m_uBlockSideLengthPower); + } + + // Loops over the specified positions and delete the corresponding blocks. + for(int32_t x = v3dStart.getX(); x <= v3dEnd.getX(); x++) + { + for(int32_t y = v3dStart.getY(); y <= v3dEnd.getY(); y++) + { + for(int32_t z = v3dStart.getZ(); z <= v3dEnd.getZ(); z++) + { + m_pRecentlyUsedBlocks.erase(Vector3DInt32(x, y, z)); m_pLastAccessedBlock = 0; + } + } + } + + // We might now have so null pointers in the 'all blocks' list so clean them up. + purgeNullPtrsFromAllBlocks(); + } + + //////////////////////////////////////////////////////////////////////////////// + /// This function should probably be made internal... + //////////////////////////////////////////////////////////////////////////////// + template + void PagedVolume::initialise() + { + //Validate parameters + if(m_uBlockSideLength == 0) + { + POLYVOX_THROW(std::invalid_argument, "Block side length cannot be zero."); + } + + if(!isPowerOf2(m_uBlockSideLength)) + { + POLYVOX_THROW(std::invalid_argument, "Block side length must be a power of two."); + } + + m_uTimestamper = 0; + //m_uBlockCountLimit = 16; + //m_uMaxNumberOfBlocksInMemory = 1024; + m_v3dLastAccessedBlockPos = Vector3DInt32(0,0,0); //There are no invalid positions, but initially the m_pLastAccessedBlock pointer will be null; + m_pLastAccessedBlock = 0; + + //Compute the block side length + m_uBlockSideLengthPower = logBase2(m_uBlockSideLength); + + m_regValidRegionInBlocks.setLowerX(this->m_regValidRegion.getLowerX() >> m_uBlockSideLengthPower); + m_regValidRegionInBlocks.setLowerY(this->m_regValidRegion.getLowerY() >> m_uBlockSideLengthPower); + m_regValidRegionInBlocks.setLowerZ(this->m_regValidRegion.getLowerZ() >> m_uBlockSideLengthPower); + m_regValidRegionInBlocks.setUpperX(this->m_regValidRegion.getUpperX() >> m_uBlockSideLengthPower); + m_regValidRegionInBlocks.setUpperY(this->m_regValidRegion.getUpperY() >> m_uBlockSideLengthPower); + m_regValidRegionInBlocks.setUpperZ(this->m_regValidRegion.getUpperZ() >> m_uBlockSideLengthPower); + + //setMaxNumberOfUncompressedBlocks(m_uBlockCountLimit); + + //Clear the previous data + m_pRecentlyUsedBlocks.clear(); + + //Other properties we might find useful later + this->m_uLongestSideLength = (std::max)((std::max)(this->getWidth(),this->getHeight()),this->getDepth()); + this->m_uShortestSideLength = (std::min)((std::min)(this->getWidth(),this->getHeight()),this->getDepth()); + this->m_fDiagonalLength = sqrtf(static_cast(this->getWidth() * this->getWidth() + this->getHeight() * this->getHeight() + this->getDepth() * this->getDepth())); + } + + template + std::shared_ptr< UncompressedBlock > PagedVolume::getUncompressedBlock(int32_t uBlockX, int32_t uBlockY, int32_t uBlockZ) const + { + Vector3DInt32 v3dBlockPos(uBlockX, uBlockY, uBlockZ); + + //Check if we have the same block as last time, if so there's no need to even update + //the time stamp. If we updated it everytime then that would be every time we touched + //a voxel, which would overflow a uint32_t and require us to use a uint64_t instead. + //This check should also provide a significant speed boost as usually it is true. + if((v3dBlockPos == m_v3dLastAccessedBlockPos) && (m_pLastAccessedBlock != 0)) + { + return m_pLastAccessedBlock; + } + + // The block was not the same as last time, but we can now hope it is in the set of most recently used blocks. + std::shared_ptr< UncompressedBlock > pUncompressedBlock = nullptr; + typename SharedPtrBlockMap::iterator itUncompressedBlock = m_pRecentlyUsedBlocks.find(v3dBlockPos); + + // Check whether the block was found. + if ((itUncompressedBlock) != m_pRecentlyUsedBlocks.end()) + { + // The block was found so we can use it. + pUncompressedBlock = itUncompressedBlock->second; + POLYVOX_ASSERT(pUncompressedBlock, "Recent block list shold never contain a null pointer."); + } + + if (!pUncompressedBlock) + { + // Although it's not in our recently use blocks, there's some (slim) chance that it + // exists in the list of all loaded blocks, because a sampler may be holding on to it. + typename WeakPtrBlockMap::iterator itWeakUncompressedBlock = m_pAllBlocks.find(v3dBlockPos); + if (itWeakUncompressedBlock != m_pAllBlocks.end()) + { + // We've found an entry in the 'all blocks' list, but it can be null. This happens if a sampler was the + // last thing to be keeping it alive and then the sampler let it go. In this case we remove it from the + // list, and it will get added again soon when we page it in and fill it with valid data. + if (itWeakUncompressedBlock->second.expired()) + { + m_pAllBlocks.erase(itWeakUncompressedBlock); + } + else + { + // The block is valid. We know it's not in the recently used list (we checked earlier) so it should be added. + pUncompressedBlock = std::shared_ptr< UncompressedBlock >(itWeakUncompressedBlock->second); + m_pRecentlyUsedBlocks.insert(std::make_pair(v3dBlockPos, pUncompressedBlock)); + } + } + } + + // If we still haven't found the block then it's time to create a new one and page it in from disk. + if (!pUncompressedBlock) + { + // The block was not found so we will create a new one. + pUncompressedBlock = std::make_shared< UncompressedBlock >(v3dBlockPos, m_uBlockSideLength, m_pPager); + + // As we are loading a new block we should try to ensure we don't go over our target memory usage. + bool erasedBlock = false; + while (m_pRecentlyUsedBlocks.size() + 1 > m_uBlockCountLimit) // +1 ready for new block we will add next. + { + // This should never hit, because it should not have been possible for + // the user to limit the number of blocks if they did not provide a pager. + POLYVOX_ASSERT(m_pPager, "A valid pager is required to limit number of blocks"); + + // Find the least recently used block. Hopefully this isn't too slow. + typename SharedPtrBlockMap::iterator itUnloadBlock = m_pRecentlyUsedBlocks.begin(); + for (typename SharedPtrBlockMap::iterator i = m_pRecentlyUsedBlocks.begin(); i != m_pRecentlyUsedBlocks.end(); i++) + { + if (i->second->m_uBlockLastAccessed < itUnloadBlock->second->m_uBlockLastAccessed) + { + itUnloadBlock = i; + } + } + + // Erase the least recently used block + m_pRecentlyUsedBlocks.erase(itUnloadBlock); + erasedBlock = true; + } + + // If we've deleted any blocks from the recently used list then this + // seems like a good place to purge the 'all blocks' list as well. + if (erasedBlock) + { + purgeNullPtrsFromAllBlocks(); + } + + // Add our new block to the maps. + m_pAllBlocks.insert(std::make_pair(v3dBlockPos, pUncompressedBlock)); + m_pRecentlyUsedBlocks.insert(std::make_pair(v3dBlockPos, pUncompressedBlock)); + } + + pUncompressedBlock->m_uBlockLastAccessed = ++m_uTimestamper; + m_pLastAccessedBlock = pUncompressedBlock; + m_v3dLastAccessedBlockPos = v3dBlockPos; + + return pUncompressedBlock; + } + + //////////////////////////////////////////////////////////////////////////////// + /// Calculate the memory usage of the volume. + //////////////////////////////////////////////////////////////////////////////// + template + uint32_t PagedVolume::calculateSizeInBytes(void) + { + // Purge null blocks so we know that all blocks are used. + purgeNullPtrsFromAllBlocks(); + + // Note: We disregard the size of the other class members as they are likely to be very small compared to the size of the + // allocated voxel data. This also keeps the reported size as a power of two, which makes other memory calculations easier. + return UncompressedBlock::calculateSizeInBytes(m_uBlockSideLength) * m_pAllBlocks.size(); + } + + template + void PagedVolume::purgeNullPtrsFromAllBlocks(void) const + { + for (auto blockIter = m_pAllBlocks.begin(); blockIter != m_pAllBlocks.end();) + { + if (blockIter->second.expired()) + { + blockIter = m_pAllBlocks.erase(blockIter); + } + else + { + blockIter++; + } + } + } + + template + template + VoxelType PagedVolume::getVoxelImpl(int32_t uXPos, int32_t uYPos, int32_t uZPos, WrapModeType, VoxelType tBorder) const + { + // This function should never be called because one of the specialisations should always match. + POLYVOX_ASSERT(false, "This function is not implemented and should never be called!"); + } + + template + VoxelType PagedVolume::getVoxelImpl(int32_t uXPos, int32_t uYPos, int32_t uZPos, WrapModeType, VoxelType tBorder) const + { + if(this->m_regValidRegion.containsPoint(Vector3DInt32(uXPos, uYPos, uZPos)) == false) + { + POLYVOX_THROW(std::out_of_range, "Position is outside valid region"); + } + + return getVoxelImpl(uXPos, uYPos, uZPos, WrapModeType(), tBorder); // No wrapping as we've just validated the position. + } + + template + VoxelType PagedVolume::getVoxelImpl(int32_t uXPos, int32_t uYPos, int32_t uZPos, WrapModeType, VoxelType tBorder) const + { + //Perform clamping + uXPos = (std::max)(uXPos, this->m_regValidRegion.getLowerX()); + uYPos = (std::max)(uYPos, this->m_regValidRegion.getLowerY()); + uZPos = (std::max)(uZPos, this->m_regValidRegion.getLowerZ()); + uXPos = (std::min)(uXPos, this->m_regValidRegion.getUpperX()); + uYPos = (std::min)(uYPos, this->m_regValidRegion.getUpperY()); + uZPos = (std::min)(uZPos, this->m_regValidRegion.getUpperZ()); + + return getVoxelImpl(uXPos, uYPos, uZPos, WrapModeType(), tBorder); // No wrapping as we've just validated the position. + } + + template + VoxelType PagedVolume::getVoxelImpl(int32_t uXPos, int32_t uYPos, int32_t uZPos, WrapModeType, VoxelType tBorder) const + { + if(this->m_regValidRegion.containsPoint(uXPos, uYPos, uZPos)) + { + return getVoxelImpl(uXPos, uYPos, uZPos, WrapModeType(), tBorder); // No wrapping as we've just validated the position. + } + else + { + return tBorder; + } + } + + template + VoxelType PagedVolume::getVoxelImpl(int32_t uXPos, int32_t uYPos, int32_t uZPos, WrapModeType, VoxelType /*tBorder*/) const + { + const int32_t blockX = uXPos >> m_uBlockSideLengthPower; + const int32_t blockY = uYPos >> m_uBlockSideLengthPower; + const int32_t blockZ = uZPos >> m_uBlockSideLengthPower; + + const uint16_t xOffset = static_cast(uXPos - (blockX << m_uBlockSideLengthPower)); + const uint16_t yOffset = static_cast(uYPos - (blockY << m_uBlockSideLengthPower)); + const uint16_t zOffset = static_cast(uZPos - (blockZ << m_uBlockSideLengthPower)); + + auto pUncompressedBlock = getUncompressedBlock(blockX, blockY, blockZ); + return pUncompressedBlock->getVoxel(xOffset, yOffset, zOffset); + } +} + diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolumeSampler.inl b/library/PolyVoxCore/include/PolyVoxCore/PagedVolumeSampler.inl similarity index 84% rename from library/PolyVoxCore/include/PolyVoxCore/LargeVolumeSampler.inl rename to library/PolyVoxCore/include/PolyVoxCore/PagedVolumeSampler.inl index df07bb77..5c55e8ae 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolumeSampler.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/PagedVolumeSampler.inl @@ -1,562 +1,562 @@ -/******************************************************************************* -Copyright (c) 2005-2009 David Williams - -This software is provided 'as-is', without any express or implied -warranty. In no event will the authors be held liable for any damages -arising from the use of this software. - -Permission is granted to anyone to use this software for any purpose, -including commercial applications, and to alter it and redistribute it -freely, subject to the following restrictions: - - 1. The origin of this software must not be misrepresented; you must not - claim that you wrote the original software. If you use this software - in a product, an acknowledgment in the product documentation would be - appreciated but is not required. - - 2. Altered source versions must be plainly marked as such, and must not be - misrepresented as being the original software. - - 3. This notice may not be removed or altered from any source - distribution. -*******************************************************************************/ - -#define CAN_GO_NEG_X(val) ((val > this->mVolume->getEnclosingRegion().getLowerX()) && (val % this->mVolume->m_uBlockSideLength != 0)) -#define CAN_GO_POS_X(val) ((val < this->mVolume->getEnclosingRegion().getUpperX()) && ((val + 1) % this->mVolume->m_uBlockSideLength != 0)) -#define CAN_GO_NEG_Y(val) ((val > this->mVolume->getEnclosingRegion().getLowerY()) && (val % this->mVolume->m_uBlockSideLength != 0)) -#define CAN_GO_POS_Y(val) ((val < this->mVolume->getEnclosingRegion().getUpperY()) && ((val + 1) % this->mVolume->m_uBlockSideLength != 0)) -#define CAN_GO_NEG_Z(val) ((val > this->mVolume->getEnclosingRegion().getLowerZ()) && (val % this->mVolume->m_uBlockSideLength != 0)) -#define CAN_GO_POS_Z(val) ((val < this->mVolume->getEnclosingRegion().getUpperZ()) && ((val + 1) % this->mVolume->m_uBlockSideLength != 0)) - -namespace PolyVox -{ - template - LargeVolume::Sampler::Sampler(LargeVolume* volume) - :BaseVolume::template Sampler< LargeVolume >(volume) - { - } - - template - LargeVolume::Sampler::~Sampler() - { - } - - template - VoxelType LargeVolume::Sampler::getSubSampledVoxel(uint8_t uLevel) const - { - if(uLevel == 0) - { - return getVoxel(); - } - else if(uLevel == 1) - { - VoxelType tValue = getVoxel(); - tValue = (std::min)(tValue, peekVoxel1px0py0pz()); - tValue = (std::min)(tValue, peekVoxel0px1py0pz()); - tValue = (std::min)(tValue, peekVoxel1px1py0pz()); - tValue = (std::min)(tValue, peekVoxel0px0py1pz()); - tValue = (std::min)(tValue, peekVoxel1px0py1pz()); - tValue = (std::min)(tValue, peekVoxel0px1py1pz()); - tValue = (std::min)(tValue, peekVoxel1px1py1pz()); - return tValue; - } - else - { - const uint8_t uSize = 1 << uLevel; - - VoxelType tValue = (std::numeric_limits::max)(); - for(uint8_t z = 0; z < uSize; ++z) - { - for(uint8_t y = 0; y < uSize; ++y) - { - for(uint8_t x = 0; x < uSize; ++x) - { - tValue = (std::min)(tValue, this->mVolume->getVoxelAt(this->mXPosInVolume + x, this->mYPosInVolume + y, this->mZPosInVolume + z)); - } - } - } - return tValue; - } - } - - template - VoxelType LargeVolume::Sampler::getVoxel(void) const - { - if(this->isCurrentPositionValid()) - { - return *mCurrentVoxel; - } - else - { - return this->getVoxelImpl(this->mXPosInVolume, this->mYPosInVolume, this->mZPosInVolume); - } - } - - template - void LargeVolume::Sampler::setPosition(const Vector3DInt32& v3dNewPos) - { - setPosition(v3dNewPos.getX(), v3dNewPos.getY(), v3dNewPos.getZ()); - } - - template - void LargeVolume::Sampler::setPosition(int32_t xPos, int32_t yPos, int32_t zPos) - { - // Base version updates position and validity flags. - BaseVolume::template Sampler< LargeVolume >::setPosition(xPos, yPos, zPos); - - // Then we update the voxel pointer - if(this->isCurrentPositionValid()) - { - const int32_t uXBlock = this->mXPosInVolume >> this->mVolume->m_uBlockSideLengthPower; - const int32_t uYBlock = this->mYPosInVolume >> this->mVolume->m_uBlockSideLengthPower; - const int32_t uZBlock = this->mZPosInVolume >> this->mVolume->m_uBlockSideLengthPower; - - const uint16_t uXPosInBlock = static_cast(this->mXPosInVolume - (uXBlock << this->mVolume->m_uBlockSideLengthPower)); - const uint16_t uYPosInBlock = static_cast(this->mYPosInVolume - (uYBlock << this->mVolume->m_uBlockSideLengthPower)); - const uint16_t uZPosInBlock = static_cast(this->mZPosInVolume - (uZBlock << this->mVolume->m_uBlockSideLengthPower)); - - const uint32_t uVoxelIndexInBlock = uXPosInBlock + - uYPosInBlock * this->mVolume->m_uBlockSideLength + - uZPosInBlock * this->mVolume->m_uBlockSideLength * this->mVolume->m_uBlockSideLength; - - auto pUncompressedCurrentBlock = this->mVolume->getUncompressedBlock(uXBlock, uYBlock, uZBlock); - - mCurrentVoxel = pUncompressedCurrentBlock->m_tData + uVoxelIndexInBlock; - } - else - { - mCurrentVoxel = 0; - } - } - - template - bool LargeVolume::Sampler::setVoxel(VoxelType tValue) - { - /*if(m_bIsCurrentPositionValidInX && m_bIsCurrentPositionValidInY && m_bIsCurrentPositionValidInZ) - { - *mCurrentVoxel = tValue; - return true; - } - else - { - return false; - }*/ - - //Need to think what effect this has on any existing iterators. - POLYVOX_THROW(not_implemented, "This function cannot be used on LargeVolume samplers."); - return false; - } - - template - void LargeVolume::Sampler::movePositiveX(void) - { - // We'll need this in a moment... - bool bIsOldPositionValid = this->isCurrentPositionValid(); - - // Base version updates position and validity flags. - BaseVolume::template Sampler< LargeVolume >::movePositiveX(); - - // Then we update the voxel pointer - if((this->isCurrentPositionValid()) && bIsOldPositionValid && ((this->mXPosInVolume) % this->mVolume->m_uBlockSideLength != 0)) - { - //No need to compute new block. - ++mCurrentVoxel; - } - else - { - //We've hit the block boundary. Just calling setPosition() is the easiest way to resolve this. - setPosition(this->mXPosInVolume, this->mYPosInVolume, this->mZPosInVolume); - } - } - - template - void LargeVolume::Sampler::movePositiveY(void) - { - // We'll need this in a moment... - bool bIsOldPositionValid = this->isCurrentPositionValid(); - - // Base version updates position and validity flags. - BaseVolume::template Sampler< LargeVolume >::movePositiveY(); - - // Then we update the voxel pointer - if((this->isCurrentPositionValid()) && bIsOldPositionValid && ((this->mYPosInVolume) % this->mVolume->m_uBlockSideLength != 0)) - { - //No need to compute new block. - mCurrentVoxel += this->mVolume->m_uBlockSideLength; - } - else - { - //We've hit the block boundary. Just calling setPosition() is the easiest way to resolve this. - setPosition(this->mXPosInVolume, this->mYPosInVolume, this->mZPosInVolume); - } - } - - template - void LargeVolume::Sampler::movePositiveZ(void) - { - // We'll need this in a moment... - bool bIsOldPositionValid = this->isCurrentPositionValid(); - - // Base version updates position and validity flags. - BaseVolume::template Sampler< LargeVolume >::movePositiveZ(); - - // Then we update the voxel pointer - if((this->isCurrentPositionValid()) && bIsOldPositionValid && ((this->mZPosInVolume) % this->mVolume->m_uBlockSideLength != 0)) - { - //No need to compute new block. - mCurrentVoxel += this->mVolume->m_uBlockSideLength * this->mVolume->m_uBlockSideLength; - } - else - { - //We've hit the block boundary. Just calling setPosition() is the easiest way to resolve this. - setPosition(this->mXPosInVolume, this->mYPosInVolume, this->mZPosInVolume); - } - } - - template - void LargeVolume::Sampler::moveNegativeX(void) - { - // We'll need this in a moment... - bool bIsOldPositionValid = this->isCurrentPositionValid(); - - // Base version updates position and validity flags. - BaseVolume::template Sampler< LargeVolume >::moveNegativeX(); - - // Then we update the voxel pointer - if((this->isCurrentPositionValid()) && bIsOldPositionValid && ((this->mXPosInVolume + 1) % this->mVolume->m_uBlockSideLength != 0)) - { - //No need to compute new block. - --mCurrentVoxel; - } - else - { - //We've hit the block boundary. Just calling setPosition() is the easiest way to resolve this. - setPosition(this->mXPosInVolume, this->mYPosInVolume, this->mZPosInVolume); - } - } - - template - void LargeVolume::Sampler::moveNegativeY(void) - { - // We'll need this in a moment... - bool bIsOldPositionValid = this->isCurrentPositionValid(); - - // Base version updates position and validity flags. - BaseVolume::template Sampler< LargeVolume >::moveNegativeY(); - - // Then we update the voxel pointer - if((this->isCurrentPositionValid()) && bIsOldPositionValid && ((this->mYPosInVolume + 1) % this->mVolume->m_uBlockSideLength != 0)) - { - //No need to compute new block. - mCurrentVoxel -= this->mVolume->m_uBlockSideLength; - } - else - { - //We've hit the block boundary. Just calling setPosition() is the easiest way to resolve this. - setPosition(this->mXPosInVolume, this->mYPosInVolume, this->mZPosInVolume); - } - } - - template - void LargeVolume::Sampler::moveNegativeZ(void) - { - // We'll need this in a moment... - bool bIsOldPositionValid = this->isCurrentPositionValid(); - - // Base version updates position and validity flags. - BaseVolume::template Sampler< LargeVolume >::moveNegativeZ(); - - // Then we update the voxel pointer - if((this->isCurrentPositionValid()) && bIsOldPositionValid && ((this->mZPosInVolume + 1) % this->mVolume->m_uBlockSideLength != 0)) - { - //No need to compute new block. - mCurrentVoxel -= this->mVolume->m_uBlockSideLength * this->mVolume->m_uBlockSideLength; - } - else - { - //We've hit the block boundary. Just calling setPosition() is the easiest way to resolve this. - setPosition(this->mXPosInVolume, this->mYPosInVolume, this->mZPosInVolume); - } - } - - template - VoxelType LargeVolume::Sampler::peekVoxel1nx1ny1nz(void) const - { - if((this->isCurrentPositionValid()) && CAN_GO_NEG_X(this->mXPosInVolume) && CAN_GO_NEG_Y(this->mYPosInVolume) && CAN_GO_NEG_Z(this->mZPosInVolume) ) - { - return *(mCurrentVoxel - 1 - this->mVolume->m_uBlockSideLength - this->mVolume->m_uBlockSideLength*this->mVolume->m_uBlockSideLength); - } - return this->mVolume->getVoxel(this->mXPosInVolume-1,this->mYPosInVolume-1,this->mZPosInVolume-1,this->m_eWrapMode, this->m_tBorder); - } - - template - VoxelType LargeVolume::Sampler::peekVoxel1nx1ny0pz(void) const - { - if((this->isCurrentPositionValid()) && CAN_GO_NEG_X(this->mXPosInVolume) && CAN_GO_NEG_Y(this->mYPosInVolume) ) - { - return *(mCurrentVoxel - 1 - this->mVolume->m_uBlockSideLength); - } - return this->mVolume->getVoxel(this->mXPosInVolume-1,this->mYPosInVolume-1,this->mZPosInVolume,this->m_eWrapMode, this->m_tBorder); - } - - template - VoxelType LargeVolume::Sampler::peekVoxel1nx1ny1pz(void) const - { - if((this->isCurrentPositionValid()) && CAN_GO_NEG_X(this->mXPosInVolume) && CAN_GO_NEG_Y(this->mYPosInVolume) && CAN_GO_POS_Z(this->mZPosInVolume) ) - { - return *(mCurrentVoxel - 1 - this->mVolume->m_uBlockSideLength + this->mVolume->m_uBlockSideLength*this->mVolume->m_uBlockSideLength); - } - return this->mVolume->getVoxel(this->mXPosInVolume-1,this->mYPosInVolume-1,this->mZPosInVolume+1,this->m_eWrapMode, this->m_tBorder); - } - - template - VoxelType LargeVolume::Sampler::peekVoxel1nx0py1nz(void) const - { - if((this->isCurrentPositionValid()) && CAN_GO_NEG_X(this->mXPosInVolume) && CAN_GO_NEG_Z(this->mZPosInVolume) ) - { - return *(mCurrentVoxel - 1 - this->mVolume->m_uBlockSideLength*this->mVolume->m_uBlockSideLength); - } - return this->mVolume->getVoxel(this->mXPosInVolume-1,this->mYPosInVolume,this->mZPosInVolume-1,this->m_eWrapMode, this->m_tBorder); - } - - template - VoxelType LargeVolume::Sampler::peekVoxel1nx0py0pz(void) const - { - if((this->isCurrentPositionValid()) && CAN_GO_NEG_X(this->mXPosInVolume) ) - { - return *(mCurrentVoxel - 1); - } - return this->mVolume->getVoxel(this->mXPosInVolume-1,this->mYPosInVolume,this->mZPosInVolume,this->m_eWrapMode, this->m_tBorder); - } - - template - VoxelType LargeVolume::Sampler::peekVoxel1nx0py1pz(void) const - { - if((this->isCurrentPositionValid()) && CAN_GO_NEG_X(this->mXPosInVolume) && CAN_GO_POS_Z(this->mZPosInVolume) ) - { - return *(mCurrentVoxel - 1 + this->mVolume->m_uBlockSideLength*this->mVolume->m_uBlockSideLength); - } - return this->mVolume->getVoxel(this->mXPosInVolume-1,this->mYPosInVolume,this->mZPosInVolume+1,this->m_eWrapMode, this->m_tBorder); - } - - template - VoxelType LargeVolume::Sampler::peekVoxel1nx1py1nz(void) const - { - if((this->isCurrentPositionValid()) && CAN_GO_NEG_X(this->mXPosInVolume) && CAN_GO_POS_Y(this->mYPosInVolume) && CAN_GO_NEG_Z(this->mZPosInVolume) ) - { - return *(mCurrentVoxel - 1 + this->mVolume->m_uBlockSideLength - this->mVolume->m_uBlockSideLength*this->mVolume->m_uBlockSideLength); - } - return this->mVolume->getVoxel(this->mXPosInVolume-1,this->mYPosInVolume+1,this->mZPosInVolume-1,this->m_eWrapMode, this->m_tBorder); - } - - template - VoxelType LargeVolume::Sampler::peekVoxel1nx1py0pz(void) const - { - if((this->isCurrentPositionValid()) && CAN_GO_NEG_X(this->mXPosInVolume) && CAN_GO_POS_Y(this->mYPosInVolume) ) - { - return *(mCurrentVoxel - 1 + this->mVolume->m_uBlockSideLength); - } - return this->mVolume->getVoxel(this->mXPosInVolume-1,this->mYPosInVolume+1,this->mZPosInVolume,this->m_eWrapMode, this->m_tBorder); - } - - template - VoxelType LargeVolume::Sampler::peekVoxel1nx1py1pz(void) const - { - if((this->isCurrentPositionValid()) && CAN_GO_NEG_X(this->mXPosInVolume) && CAN_GO_POS_Y(this->mYPosInVolume) && CAN_GO_POS_Z(this->mZPosInVolume) ) - { - return *(mCurrentVoxel - 1 + this->mVolume->m_uBlockSideLength + this->mVolume->m_uBlockSideLength*this->mVolume->m_uBlockSideLength); - } - return this->mVolume->getVoxel(this->mXPosInVolume-1,this->mYPosInVolume+1,this->mZPosInVolume+1,this->m_eWrapMode, this->m_tBorder); - } - - ////////////////////////////////////////////////////////////////////////// - - template - VoxelType LargeVolume::Sampler::peekVoxel0px1ny1nz(void) const - { - if((this->isCurrentPositionValid()) && CAN_GO_NEG_Y(this->mYPosInVolume) && CAN_GO_NEG_Z(this->mZPosInVolume) ) - { - return *(mCurrentVoxel - this->mVolume->m_uBlockSideLength - this->mVolume->m_uBlockSideLength*this->mVolume->m_uBlockSideLength); - } - return this->mVolume->getVoxel(this->mXPosInVolume,this->mYPosInVolume-1,this->mZPosInVolume-1,this->m_eWrapMode, this->m_tBorder); - } - - template - VoxelType LargeVolume::Sampler::peekVoxel0px1ny0pz(void) const - { - if((this->isCurrentPositionValid()) && CAN_GO_NEG_Y(this->mYPosInVolume) ) - { - return *(mCurrentVoxel - this->mVolume->m_uBlockSideLength); - } - return this->mVolume->getVoxel(this->mXPosInVolume,this->mYPosInVolume-1,this->mZPosInVolume,this->m_eWrapMode, this->m_tBorder); - } - - template - VoxelType LargeVolume::Sampler::peekVoxel0px1ny1pz(void) const - { - if((this->isCurrentPositionValid()) && CAN_GO_NEG_Y(this->mYPosInVolume) && CAN_GO_POS_Z(this->mZPosInVolume) ) - { - return *(mCurrentVoxel - this->mVolume->m_uBlockSideLength + this->mVolume->m_uBlockSideLength*this->mVolume->m_uBlockSideLength); - } - return this->mVolume->getVoxel(this->mXPosInVolume,this->mYPosInVolume-1,this->mZPosInVolume+1,this->m_eWrapMode, this->m_tBorder); - } - - template - VoxelType LargeVolume::Sampler::peekVoxel0px0py1nz(void) const - { - if((this->isCurrentPositionValid()) && CAN_GO_NEG_Z(this->mZPosInVolume) ) - { - return *(mCurrentVoxel - this->mVolume->m_uBlockSideLength*this->mVolume->m_uBlockSideLength); - } - return this->mVolume->getVoxel(this->mXPosInVolume,this->mYPosInVolume,this->mZPosInVolume-1,this->m_eWrapMode, this->m_tBorder); - } - - template - VoxelType LargeVolume::Sampler::peekVoxel0px0py0pz(void) const - { - if((this->isCurrentPositionValid())) - { - return *mCurrentVoxel; - } - return this->mVolume->getVoxel(this->mXPosInVolume, this->mYPosInVolume, this->mZPosInVolume,this->m_eWrapMode, this->m_tBorder); - } - - template - VoxelType LargeVolume::Sampler::peekVoxel0px0py1pz(void) const - { - if((this->isCurrentPositionValid()) && CAN_GO_POS_Z(this->mZPosInVolume) ) - { - return *(mCurrentVoxel + this->mVolume->m_uBlockSideLength*this->mVolume->m_uBlockSideLength); - } - return this->mVolume->getVoxel(this->mXPosInVolume,this->mYPosInVolume,this->mZPosInVolume+1,this->m_eWrapMode, this->m_tBorder); - } - - template - VoxelType LargeVolume::Sampler::peekVoxel0px1py1nz(void) const - { - if((this->isCurrentPositionValid()) && CAN_GO_POS_Y(this->mYPosInVolume) && CAN_GO_NEG_Z(this->mZPosInVolume) ) - { - return *(mCurrentVoxel + this->mVolume->m_uBlockSideLength - this->mVolume->m_uBlockSideLength*this->mVolume->m_uBlockSideLength); - } - return this->mVolume->getVoxel(this->mXPosInVolume,this->mYPosInVolume+1,this->mZPosInVolume-1,this->m_eWrapMode, this->m_tBorder); - } - - template - VoxelType LargeVolume::Sampler::peekVoxel0px1py0pz(void) const - { - if((this->isCurrentPositionValid()) && CAN_GO_POS_Y(this->mYPosInVolume) ) - { - return *(mCurrentVoxel + this->mVolume->m_uBlockSideLength); - } - return this->mVolume->getVoxel(this->mXPosInVolume,this->mYPosInVolume+1,this->mZPosInVolume,this->m_eWrapMode, this->m_tBorder); - } - - template - VoxelType LargeVolume::Sampler::peekVoxel0px1py1pz(void) const - { - if((this->isCurrentPositionValid()) && CAN_GO_POS_Y(this->mYPosInVolume) && CAN_GO_POS_Z(this->mZPosInVolume) ) - { - return *(mCurrentVoxel + this->mVolume->m_uBlockSideLength + this->mVolume->m_uBlockSideLength*this->mVolume->m_uBlockSideLength); - } - return this->mVolume->getVoxel(this->mXPosInVolume,this->mYPosInVolume+1,this->mZPosInVolume+1,this->m_eWrapMode, this->m_tBorder); - } - - ////////////////////////////////////////////////////////////////////////// - - template - VoxelType LargeVolume::Sampler::peekVoxel1px1ny1nz(void) const - { - if((this->isCurrentPositionValid()) && CAN_GO_POS_X(this->mXPosInVolume) && CAN_GO_NEG_Y(this->mYPosInVolume) && CAN_GO_NEG_Z(this->mZPosInVolume) ) - { - return *(mCurrentVoxel + 1 - this->mVolume->m_uBlockSideLength - this->mVolume->m_uBlockSideLength*this->mVolume->m_uBlockSideLength); - } - return this->mVolume->getVoxel(this->mXPosInVolume+1,this->mYPosInVolume-1,this->mZPosInVolume-1,this->m_eWrapMode, this->m_tBorder); - } - - template - VoxelType LargeVolume::Sampler::peekVoxel1px1ny0pz(void) const - { - if((this->isCurrentPositionValid()) && CAN_GO_POS_X(this->mXPosInVolume) && CAN_GO_NEG_Y(this->mYPosInVolume) ) - { - return *(mCurrentVoxel + 1 - this->mVolume->m_uBlockSideLength); - } - return this->mVolume->getVoxel(this->mXPosInVolume+1,this->mYPosInVolume-1,this->mZPosInVolume,this->m_eWrapMode, this->m_tBorder); - } - - template - VoxelType LargeVolume::Sampler::peekVoxel1px1ny1pz(void) const - { - if((this->isCurrentPositionValid()) && CAN_GO_POS_X(this->mXPosInVolume) && CAN_GO_NEG_Y(this->mYPosInVolume) && CAN_GO_POS_Z(this->mZPosInVolume) ) - { - return *(mCurrentVoxel + 1 - this->mVolume->m_uBlockSideLength + this->mVolume->m_uBlockSideLength*this->mVolume->m_uBlockSideLength); - } - return this->mVolume->getVoxel(this->mXPosInVolume+1,this->mYPosInVolume-1,this->mZPosInVolume+1,this->m_eWrapMode, this->m_tBorder); - } - - template - VoxelType LargeVolume::Sampler::peekVoxel1px0py1nz(void) const - { - if((this->isCurrentPositionValid()) && CAN_GO_POS_X(this->mXPosInVolume) && CAN_GO_NEG_Z(this->mZPosInVolume) ) - { - return *(mCurrentVoxel + 1 - this->mVolume->m_uBlockSideLength*this->mVolume->m_uBlockSideLength); - } - return this->mVolume->getVoxel(this->mXPosInVolume+1,this->mYPosInVolume,this->mZPosInVolume-1,this->m_eWrapMode, this->m_tBorder); - } - - template - VoxelType LargeVolume::Sampler::peekVoxel1px0py0pz(void) const - { - if((this->isCurrentPositionValid()) && CAN_GO_POS_X(this->mXPosInVolume) ) - { - return *(mCurrentVoxel + 1); - } - return this->mVolume->getVoxel(this->mXPosInVolume+1,this->mYPosInVolume,this->mZPosInVolume,this->m_eWrapMode, this->m_tBorder); - } - - template - VoxelType LargeVolume::Sampler::peekVoxel1px0py1pz(void) const - { - if((this->isCurrentPositionValid()) && CAN_GO_POS_X(this->mXPosInVolume) && CAN_GO_POS_Z(this->mZPosInVolume) ) - { - return *(mCurrentVoxel + 1 + this->mVolume->m_uBlockSideLength*this->mVolume->m_uBlockSideLength); - } - return this->mVolume->getVoxel(this->mXPosInVolume+1,this->mYPosInVolume,this->mZPosInVolume+1,this->m_eWrapMode, this->m_tBorder); - } - - template - VoxelType LargeVolume::Sampler::peekVoxel1px1py1nz(void) const - { - if((this->isCurrentPositionValid()) && CAN_GO_POS_X(this->mXPosInVolume) && CAN_GO_POS_Y(this->mYPosInVolume) && CAN_GO_NEG_Z(this->mZPosInVolume) ) - { - return *(mCurrentVoxel + 1 + this->mVolume->m_uBlockSideLength - this->mVolume->m_uBlockSideLength*this->mVolume->m_uBlockSideLength); - } - return this->mVolume->getVoxel(this->mXPosInVolume+1,this->mYPosInVolume+1,this->mZPosInVolume-1,this->m_eWrapMode, this->m_tBorder); - } - - template - VoxelType LargeVolume::Sampler::peekVoxel1px1py0pz(void) const - { - if((this->isCurrentPositionValid()) && CAN_GO_POS_X(this->mXPosInVolume) && CAN_GO_POS_Y(this->mYPosInVolume) ) - { - return *(mCurrentVoxel + 1 + this->mVolume->m_uBlockSideLength); - } - return this->mVolume->getVoxel(this->mXPosInVolume+1,this->mYPosInVolume+1,this->mZPosInVolume,this->m_eWrapMode, this->m_tBorder); - } - - template - VoxelType LargeVolume::Sampler::peekVoxel1px1py1pz(void) const - { - if((this->isCurrentPositionValid()) && CAN_GO_POS_X(this->mXPosInVolume) && CAN_GO_POS_Y(this->mYPosInVolume) && CAN_GO_POS_Z(this->mZPosInVolume) ) - { - return *(mCurrentVoxel + 1 + this->mVolume->m_uBlockSideLength + this->mVolume->m_uBlockSideLength*this->mVolume->m_uBlockSideLength); - } - return this->mVolume->getVoxel(this->mXPosInVolume+1,this->mYPosInVolume+1,this->mZPosInVolume+1,this->m_eWrapMode, this->m_tBorder); - } -} - -#undef CAN_GO_NEG_X -#undef CAN_GO_POS_X -#undef CAN_GO_NEG_Y -#undef CAN_GO_POS_Y -#undef CAN_GO_NEG_Z -#undef CAN_GO_POS_Z +/******************************************************************************* +Copyright (c) 2005-2009 David Williams + +This software is provided 'as-is', without any express or implied +warranty. In no event will the authors be held liable for any damages +arising from the use of this software. + +Permission is granted to anyone to use this software for any purpose, +including commercial applications, and to alter it and redistribute it +freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + + 3. This notice may not be removed or altered from any source + distribution. +*******************************************************************************/ + +#define CAN_GO_NEG_X(val) ((val > this->mVolume->getEnclosingRegion().getLowerX()) && (val % this->mVolume->m_uBlockSideLength != 0)) +#define CAN_GO_POS_X(val) ((val < this->mVolume->getEnclosingRegion().getUpperX()) && ((val + 1) % this->mVolume->m_uBlockSideLength != 0)) +#define CAN_GO_NEG_Y(val) ((val > this->mVolume->getEnclosingRegion().getLowerY()) && (val % this->mVolume->m_uBlockSideLength != 0)) +#define CAN_GO_POS_Y(val) ((val < this->mVolume->getEnclosingRegion().getUpperY()) && ((val + 1) % this->mVolume->m_uBlockSideLength != 0)) +#define CAN_GO_NEG_Z(val) ((val > this->mVolume->getEnclosingRegion().getLowerZ()) && (val % this->mVolume->m_uBlockSideLength != 0)) +#define CAN_GO_POS_Z(val) ((val < this->mVolume->getEnclosingRegion().getUpperZ()) && ((val + 1) % this->mVolume->m_uBlockSideLength != 0)) + +namespace PolyVox +{ + template + PagedVolume::Sampler::Sampler(PagedVolume* volume) + :BaseVolume::template Sampler< PagedVolume >(volume) + { + } + + template + PagedVolume::Sampler::~Sampler() + { + } + + template + VoxelType PagedVolume::Sampler::getSubSampledVoxel(uint8_t uLevel) const + { + if(uLevel == 0) + { + return getVoxel(); + } + else if(uLevel == 1) + { + VoxelType tValue = getVoxel(); + tValue = (std::min)(tValue, peekVoxel1px0py0pz()); + tValue = (std::min)(tValue, peekVoxel0px1py0pz()); + tValue = (std::min)(tValue, peekVoxel1px1py0pz()); + tValue = (std::min)(tValue, peekVoxel0px0py1pz()); + tValue = (std::min)(tValue, peekVoxel1px0py1pz()); + tValue = (std::min)(tValue, peekVoxel0px1py1pz()); + tValue = (std::min)(tValue, peekVoxel1px1py1pz()); + return tValue; + } + else + { + const uint8_t uSize = 1 << uLevel; + + VoxelType tValue = (std::numeric_limits::max)(); + for(uint8_t z = 0; z < uSize; ++z) + { + for(uint8_t y = 0; y < uSize; ++y) + { + for(uint8_t x = 0; x < uSize; ++x) + { + tValue = (std::min)(tValue, this->mVolume->getVoxelAt(this->mXPosInVolume + x, this->mYPosInVolume + y, this->mZPosInVolume + z)); + } + } + } + return tValue; + } + } + + template + VoxelType PagedVolume::Sampler::getVoxel(void) const + { + if(this->isCurrentPositionValid()) + { + return *mCurrentVoxel; + } + else + { + return this->getVoxelImpl(this->mXPosInVolume, this->mYPosInVolume, this->mZPosInVolume); + } + } + + template + void PagedVolume::Sampler::setPosition(const Vector3DInt32& v3dNewPos) + { + setPosition(v3dNewPos.getX(), v3dNewPos.getY(), v3dNewPos.getZ()); + } + + template + void PagedVolume::Sampler::setPosition(int32_t xPos, int32_t yPos, int32_t zPos) + { + // Base version updates position and validity flags. + BaseVolume::template Sampler< PagedVolume >::setPosition(xPos, yPos, zPos); + + // Then we update the voxel pointer + if(this->isCurrentPositionValid()) + { + const int32_t uXBlock = this->mXPosInVolume >> this->mVolume->m_uBlockSideLengthPower; + const int32_t uYBlock = this->mYPosInVolume >> this->mVolume->m_uBlockSideLengthPower; + const int32_t uZBlock = this->mZPosInVolume >> this->mVolume->m_uBlockSideLengthPower; + + const uint16_t uXPosInBlock = static_cast(this->mXPosInVolume - (uXBlock << this->mVolume->m_uBlockSideLengthPower)); + const uint16_t uYPosInBlock = static_cast(this->mYPosInVolume - (uYBlock << this->mVolume->m_uBlockSideLengthPower)); + const uint16_t uZPosInBlock = static_cast(this->mZPosInVolume - (uZBlock << this->mVolume->m_uBlockSideLengthPower)); + + const uint32_t uVoxelIndexInBlock = uXPosInBlock + + uYPosInBlock * this->mVolume->m_uBlockSideLength + + uZPosInBlock * this->mVolume->m_uBlockSideLength * this->mVolume->m_uBlockSideLength; + + auto pUncompressedCurrentBlock = this->mVolume->getUncompressedBlock(uXBlock, uYBlock, uZBlock); + + mCurrentVoxel = pUncompressedCurrentBlock->m_tData + uVoxelIndexInBlock; + } + else + { + mCurrentVoxel = 0; + } + } + + template + bool PagedVolume::Sampler::setVoxel(VoxelType tValue) + { + /*if(m_bIsCurrentPositionValidInX && m_bIsCurrentPositionValidInY && m_bIsCurrentPositionValidInZ) + { + *mCurrentVoxel = tValue; + return true; + } + else + { + return false; + }*/ + + //Need to think what effect this has on any existing iterators. + POLYVOX_THROW(not_implemented, "This function cannot be used on PagedVolume samplers."); + return false; + } + + template + void PagedVolume::Sampler::movePositiveX(void) + { + // We'll need this in a moment... + bool bIsOldPositionValid = this->isCurrentPositionValid(); + + // Base version updates position and validity flags. + BaseVolume::template Sampler< PagedVolume >::movePositiveX(); + + // Then we update the voxel pointer + if((this->isCurrentPositionValid()) && bIsOldPositionValid && ((this->mXPosInVolume) % this->mVolume->m_uBlockSideLength != 0)) + { + //No need to compute new block. + ++mCurrentVoxel; + } + else + { + //We've hit the block boundary. Just calling setPosition() is the easiest way to resolve this. + setPosition(this->mXPosInVolume, this->mYPosInVolume, this->mZPosInVolume); + } + } + + template + void PagedVolume::Sampler::movePositiveY(void) + { + // We'll need this in a moment... + bool bIsOldPositionValid = this->isCurrentPositionValid(); + + // Base version updates position and validity flags. + BaseVolume::template Sampler< PagedVolume >::movePositiveY(); + + // Then we update the voxel pointer + if((this->isCurrentPositionValid()) && bIsOldPositionValid && ((this->mYPosInVolume) % this->mVolume->m_uBlockSideLength != 0)) + { + //No need to compute new block. + mCurrentVoxel += this->mVolume->m_uBlockSideLength; + } + else + { + //We've hit the block boundary. Just calling setPosition() is the easiest way to resolve this. + setPosition(this->mXPosInVolume, this->mYPosInVolume, this->mZPosInVolume); + } + } + + template + void PagedVolume::Sampler::movePositiveZ(void) + { + // We'll need this in a moment... + bool bIsOldPositionValid = this->isCurrentPositionValid(); + + // Base version updates position and validity flags. + BaseVolume::template Sampler< PagedVolume >::movePositiveZ(); + + // Then we update the voxel pointer + if((this->isCurrentPositionValid()) && bIsOldPositionValid && ((this->mZPosInVolume) % this->mVolume->m_uBlockSideLength != 0)) + { + //No need to compute new block. + mCurrentVoxel += this->mVolume->m_uBlockSideLength * this->mVolume->m_uBlockSideLength; + } + else + { + //We've hit the block boundary. Just calling setPosition() is the easiest way to resolve this. + setPosition(this->mXPosInVolume, this->mYPosInVolume, this->mZPosInVolume); + } + } + + template + void PagedVolume::Sampler::moveNegativeX(void) + { + // We'll need this in a moment... + bool bIsOldPositionValid = this->isCurrentPositionValid(); + + // Base version updates position and validity flags. + BaseVolume::template Sampler< PagedVolume >::moveNegativeX(); + + // Then we update the voxel pointer + if((this->isCurrentPositionValid()) && bIsOldPositionValid && ((this->mXPosInVolume + 1) % this->mVolume->m_uBlockSideLength != 0)) + { + //No need to compute new block. + --mCurrentVoxel; + } + else + { + //We've hit the block boundary. Just calling setPosition() is the easiest way to resolve this. + setPosition(this->mXPosInVolume, this->mYPosInVolume, this->mZPosInVolume); + } + } + + template + void PagedVolume::Sampler::moveNegativeY(void) + { + // We'll need this in a moment... + bool bIsOldPositionValid = this->isCurrentPositionValid(); + + // Base version updates position and validity flags. + BaseVolume::template Sampler< PagedVolume >::moveNegativeY(); + + // Then we update the voxel pointer + if((this->isCurrentPositionValid()) && bIsOldPositionValid && ((this->mYPosInVolume + 1) % this->mVolume->m_uBlockSideLength != 0)) + { + //No need to compute new block. + mCurrentVoxel -= this->mVolume->m_uBlockSideLength; + } + else + { + //We've hit the block boundary. Just calling setPosition() is the easiest way to resolve this. + setPosition(this->mXPosInVolume, this->mYPosInVolume, this->mZPosInVolume); + } + } + + template + void PagedVolume::Sampler::moveNegativeZ(void) + { + // We'll need this in a moment... + bool bIsOldPositionValid = this->isCurrentPositionValid(); + + // Base version updates position and validity flags. + BaseVolume::template Sampler< PagedVolume >::moveNegativeZ(); + + // Then we update the voxel pointer + if((this->isCurrentPositionValid()) && bIsOldPositionValid && ((this->mZPosInVolume + 1) % this->mVolume->m_uBlockSideLength != 0)) + { + //No need to compute new block. + mCurrentVoxel -= this->mVolume->m_uBlockSideLength * this->mVolume->m_uBlockSideLength; + } + else + { + //We've hit the block boundary. Just calling setPosition() is the easiest way to resolve this. + setPosition(this->mXPosInVolume, this->mYPosInVolume, this->mZPosInVolume); + } + } + + template + VoxelType PagedVolume::Sampler::peekVoxel1nx1ny1nz(void) const + { + if((this->isCurrentPositionValid()) && CAN_GO_NEG_X(this->mXPosInVolume) && CAN_GO_NEG_Y(this->mYPosInVolume) && CAN_GO_NEG_Z(this->mZPosInVolume) ) + { + return *(mCurrentVoxel - 1 - this->mVolume->m_uBlockSideLength - this->mVolume->m_uBlockSideLength*this->mVolume->m_uBlockSideLength); + } + return this->mVolume->getVoxel(this->mXPosInVolume-1,this->mYPosInVolume-1,this->mZPosInVolume-1,this->m_eWrapMode, this->m_tBorder); + } + + template + VoxelType PagedVolume::Sampler::peekVoxel1nx1ny0pz(void) const + { + if((this->isCurrentPositionValid()) && CAN_GO_NEG_X(this->mXPosInVolume) && CAN_GO_NEG_Y(this->mYPosInVolume) ) + { + return *(mCurrentVoxel - 1 - this->mVolume->m_uBlockSideLength); + } + return this->mVolume->getVoxel(this->mXPosInVolume-1,this->mYPosInVolume-1,this->mZPosInVolume,this->m_eWrapMode, this->m_tBorder); + } + + template + VoxelType PagedVolume::Sampler::peekVoxel1nx1ny1pz(void) const + { + if((this->isCurrentPositionValid()) && CAN_GO_NEG_X(this->mXPosInVolume) && CAN_GO_NEG_Y(this->mYPosInVolume) && CAN_GO_POS_Z(this->mZPosInVolume) ) + { + return *(mCurrentVoxel - 1 - this->mVolume->m_uBlockSideLength + this->mVolume->m_uBlockSideLength*this->mVolume->m_uBlockSideLength); + } + return this->mVolume->getVoxel(this->mXPosInVolume-1,this->mYPosInVolume-1,this->mZPosInVolume+1,this->m_eWrapMode, this->m_tBorder); + } + + template + VoxelType PagedVolume::Sampler::peekVoxel1nx0py1nz(void) const + { + if((this->isCurrentPositionValid()) && CAN_GO_NEG_X(this->mXPosInVolume) && CAN_GO_NEG_Z(this->mZPosInVolume) ) + { + return *(mCurrentVoxel - 1 - this->mVolume->m_uBlockSideLength*this->mVolume->m_uBlockSideLength); + } + return this->mVolume->getVoxel(this->mXPosInVolume-1,this->mYPosInVolume,this->mZPosInVolume-1,this->m_eWrapMode, this->m_tBorder); + } + + template + VoxelType PagedVolume::Sampler::peekVoxel1nx0py0pz(void) const + { + if((this->isCurrentPositionValid()) && CAN_GO_NEG_X(this->mXPosInVolume) ) + { + return *(mCurrentVoxel - 1); + } + return this->mVolume->getVoxel(this->mXPosInVolume-1,this->mYPosInVolume,this->mZPosInVolume,this->m_eWrapMode, this->m_tBorder); + } + + template + VoxelType PagedVolume::Sampler::peekVoxel1nx0py1pz(void) const + { + if((this->isCurrentPositionValid()) && CAN_GO_NEG_X(this->mXPosInVolume) && CAN_GO_POS_Z(this->mZPosInVolume) ) + { + return *(mCurrentVoxel - 1 + this->mVolume->m_uBlockSideLength*this->mVolume->m_uBlockSideLength); + } + return this->mVolume->getVoxel(this->mXPosInVolume-1,this->mYPosInVolume,this->mZPosInVolume+1,this->m_eWrapMode, this->m_tBorder); + } + + template + VoxelType PagedVolume::Sampler::peekVoxel1nx1py1nz(void) const + { + if((this->isCurrentPositionValid()) && CAN_GO_NEG_X(this->mXPosInVolume) && CAN_GO_POS_Y(this->mYPosInVolume) && CAN_GO_NEG_Z(this->mZPosInVolume) ) + { + return *(mCurrentVoxel - 1 + this->mVolume->m_uBlockSideLength - this->mVolume->m_uBlockSideLength*this->mVolume->m_uBlockSideLength); + } + return this->mVolume->getVoxel(this->mXPosInVolume-1,this->mYPosInVolume+1,this->mZPosInVolume-1,this->m_eWrapMode, this->m_tBorder); + } + + template + VoxelType PagedVolume::Sampler::peekVoxel1nx1py0pz(void) const + { + if((this->isCurrentPositionValid()) && CAN_GO_NEG_X(this->mXPosInVolume) && CAN_GO_POS_Y(this->mYPosInVolume) ) + { + return *(mCurrentVoxel - 1 + this->mVolume->m_uBlockSideLength); + } + return this->mVolume->getVoxel(this->mXPosInVolume-1,this->mYPosInVolume+1,this->mZPosInVolume,this->m_eWrapMode, this->m_tBorder); + } + + template + VoxelType PagedVolume::Sampler::peekVoxel1nx1py1pz(void) const + { + if((this->isCurrentPositionValid()) && CAN_GO_NEG_X(this->mXPosInVolume) && CAN_GO_POS_Y(this->mYPosInVolume) && CAN_GO_POS_Z(this->mZPosInVolume) ) + { + return *(mCurrentVoxel - 1 + this->mVolume->m_uBlockSideLength + this->mVolume->m_uBlockSideLength*this->mVolume->m_uBlockSideLength); + } + return this->mVolume->getVoxel(this->mXPosInVolume-1,this->mYPosInVolume+1,this->mZPosInVolume+1,this->m_eWrapMode, this->m_tBorder); + } + + ////////////////////////////////////////////////////////////////////////// + + template + VoxelType PagedVolume::Sampler::peekVoxel0px1ny1nz(void) const + { + if((this->isCurrentPositionValid()) && CAN_GO_NEG_Y(this->mYPosInVolume) && CAN_GO_NEG_Z(this->mZPosInVolume) ) + { + return *(mCurrentVoxel - this->mVolume->m_uBlockSideLength - this->mVolume->m_uBlockSideLength*this->mVolume->m_uBlockSideLength); + } + return this->mVolume->getVoxel(this->mXPosInVolume,this->mYPosInVolume-1,this->mZPosInVolume-1,this->m_eWrapMode, this->m_tBorder); + } + + template + VoxelType PagedVolume::Sampler::peekVoxel0px1ny0pz(void) const + { + if((this->isCurrentPositionValid()) && CAN_GO_NEG_Y(this->mYPosInVolume) ) + { + return *(mCurrentVoxel - this->mVolume->m_uBlockSideLength); + } + return this->mVolume->getVoxel(this->mXPosInVolume,this->mYPosInVolume-1,this->mZPosInVolume,this->m_eWrapMode, this->m_tBorder); + } + + template + VoxelType PagedVolume::Sampler::peekVoxel0px1ny1pz(void) const + { + if((this->isCurrentPositionValid()) && CAN_GO_NEG_Y(this->mYPosInVolume) && CAN_GO_POS_Z(this->mZPosInVolume) ) + { + return *(mCurrentVoxel - this->mVolume->m_uBlockSideLength + this->mVolume->m_uBlockSideLength*this->mVolume->m_uBlockSideLength); + } + return this->mVolume->getVoxel(this->mXPosInVolume,this->mYPosInVolume-1,this->mZPosInVolume+1,this->m_eWrapMode, this->m_tBorder); + } + + template + VoxelType PagedVolume::Sampler::peekVoxel0px0py1nz(void) const + { + if((this->isCurrentPositionValid()) && CAN_GO_NEG_Z(this->mZPosInVolume) ) + { + return *(mCurrentVoxel - this->mVolume->m_uBlockSideLength*this->mVolume->m_uBlockSideLength); + } + return this->mVolume->getVoxel(this->mXPosInVolume,this->mYPosInVolume,this->mZPosInVolume-1,this->m_eWrapMode, this->m_tBorder); + } + + template + VoxelType PagedVolume::Sampler::peekVoxel0px0py0pz(void) const + { + if((this->isCurrentPositionValid())) + { + return *mCurrentVoxel; + } + return this->mVolume->getVoxel(this->mXPosInVolume, this->mYPosInVolume, this->mZPosInVolume,this->m_eWrapMode, this->m_tBorder); + } + + template + VoxelType PagedVolume::Sampler::peekVoxel0px0py1pz(void) const + { + if((this->isCurrentPositionValid()) && CAN_GO_POS_Z(this->mZPosInVolume) ) + { + return *(mCurrentVoxel + this->mVolume->m_uBlockSideLength*this->mVolume->m_uBlockSideLength); + } + return this->mVolume->getVoxel(this->mXPosInVolume,this->mYPosInVolume,this->mZPosInVolume+1,this->m_eWrapMode, this->m_tBorder); + } + + template + VoxelType PagedVolume::Sampler::peekVoxel0px1py1nz(void) const + { + if((this->isCurrentPositionValid()) && CAN_GO_POS_Y(this->mYPosInVolume) && CAN_GO_NEG_Z(this->mZPosInVolume) ) + { + return *(mCurrentVoxel + this->mVolume->m_uBlockSideLength - this->mVolume->m_uBlockSideLength*this->mVolume->m_uBlockSideLength); + } + return this->mVolume->getVoxel(this->mXPosInVolume,this->mYPosInVolume+1,this->mZPosInVolume-1,this->m_eWrapMode, this->m_tBorder); + } + + template + VoxelType PagedVolume::Sampler::peekVoxel0px1py0pz(void) const + { + if((this->isCurrentPositionValid()) && CAN_GO_POS_Y(this->mYPosInVolume) ) + { + return *(mCurrentVoxel + this->mVolume->m_uBlockSideLength); + } + return this->mVolume->getVoxel(this->mXPosInVolume,this->mYPosInVolume+1,this->mZPosInVolume,this->m_eWrapMode, this->m_tBorder); + } + + template + VoxelType PagedVolume::Sampler::peekVoxel0px1py1pz(void) const + { + if((this->isCurrentPositionValid()) && CAN_GO_POS_Y(this->mYPosInVolume) && CAN_GO_POS_Z(this->mZPosInVolume) ) + { + return *(mCurrentVoxel + this->mVolume->m_uBlockSideLength + this->mVolume->m_uBlockSideLength*this->mVolume->m_uBlockSideLength); + } + return this->mVolume->getVoxel(this->mXPosInVolume,this->mYPosInVolume+1,this->mZPosInVolume+1,this->m_eWrapMode, this->m_tBorder); + } + + ////////////////////////////////////////////////////////////////////////// + + template + VoxelType PagedVolume::Sampler::peekVoxel1px1ny1nz(void) const + { + if((this->isCurrentPositionValid()) && CAN_GO_POS_X(this->mXPosInVolume) && CAN_GO_NEG_Y(this->mYPosInVolume) && CAN_GO_NEG_Z(this->mZPosInVolume) ) + { + return *(mCurrentVoxel + 1 - this->mVolume->m_uBlockSideLength - this->mVolume->m_uBlockSideLength*this->mVolume->m_uBlockSideLength); + } + return this->mVolume->getVoxel(this->mXPosInVolume+1,this->mYPosInVolume-1,this->mZPosInVolume-1,this->m_eWrapMode, this->m_tBorder); + } + + template + VoxelType PagedVolume::Sampler::peekVoxel1px1ny0pz(void) const + { + if((this->isCurrentPositionValid()) && CAN_GO_POS_X(this->mXPosInVolume) && CAN_GO_NEG_Y(this->mYPosInVolume) ) + { + return *(mCurrentVoxel + 1 - this->mVolume->m_uBlockSideLength); + } + return this->mVolume->getVoxel(this->mXPosInVolume+1,this->mYPosInVolume-1,this->mZPosInVolume,this->m_eWrapMode, this->m_tBorder); + } + + template + VoxelType PagedVolume::Sampler::peekVoxel1px1ny1pz(void) const + { + if((this->isCurrentPositionValid()) && CAN_GO_POS_X(this->mXPosInVolume) && CAN_GO_NEG_Y(this->mYPosInVolume) && CAN_GO_POS_Z(this->mZPosInVolume) ) + { + return *(mCurrentVoxel + 1 - this->mVolume->m_uBlockSideLength + this->mVolume->m_uBlockSideLength*this->mVolume->m_uBlockSideLength); + } + return this->mVolume->getVoxel(this->mXPosInVolume+1,this->mYPosInVolume-1,this->mZPosInVolume+1,this->m_eWrapMode, this->m_tBorder); + } + + template + VoxelType PagedVolume::Sampler::peekVoxel1px0py1nz(void) const + { + if((this->isCurrentPositionValid()) && CAN_GO_POS_X(this->mXPosInVolume) && CAN_GO_NEG_Z(this->mZPosInVolume) ) + { + return *(mCurrentVoxel + 1 - this->mVolume->m_uBlockSideLength*this->mVolume->m_uBlockSideLength); + } + return this->mVolume->getVoxel(this->mXPosInVolume+1,this->mYPosInVolume,this->mZPosInVolume-1,this->m_eWrapMode, this->m_tBorder); + } + + template + VoxelType PagedVolume::Sampler::peekVoxel1px0py0pz(void) const + { + if((this->isCurrentPositionValid()) && CAN_GO_POS_X(this->mXPosInVolume) ) + { + return *(mCurrentVoxel + 1); + } + return this->mVolume->getVoxel(this->mXPosInVolume+1,this->mYPosInVolume,this->mZPosInVolume,this->m_eWrapMode, this->m_tBorder); + } + + template + VoxelType PagedVolume::Sampler::peekVoxel1px0py1pz(void) const + { + if((this->isCurrentPositionValid()) && CAN_GO_POS_X(this->mXPosInVolume) && CAN_GO_POS_Z(this->mZPosInVolume) ) + { + return *(mCurrentVoxel + 1 + this->mVolume->m_uBlockSideLength*this->mVolume->m_uBlockSideLength); + } + return this->mVolume->getVoxel(this->mXPosInVolume+1,this->mYPosInVolume,this->mZPosInVolume+1,this->m_eWrapMode, this->m_tBorder); + } + + template + VoxelType PagedVolume::Sampler::peekVoxel1px1py1nz(void) const + { + if((this->isCurrentPositionValid()) && CAN_GO_POS_X(this->mXPosInVolume) && CAN_GO_POS_Y(this->mYPosInVolume) && CAN_GO_NEG_Z(this->mZPosInVolume) ) + { + return *(mCurrentVoxel + 1 + this->mVolume->m_uBlockSideLength - this->mVolume->m_uBlockSideLength*this->mVolume->m_uBlockSideLength); + } + return this->mVolume->getVoxel(this->mXPosInVolume+1,this->mYPosInVolume+1,this->mZPosInVolume-1,this->m_eWrapMode, this->m_tBorder); + } + + template + VoxelType PagedVolume::Sampler::peekVoxel1px1py0pz(void) const + { + if((this->isCurrentPositionValid()) && CAN_GO_POS_X(this->mXPosInVolume) && CAN_GO_POS_Y(this->mYPosInVolume) ) + { + return *(mCurrentVoxel + 1 + this->mVolume->m_uBlockSideLength); + } + return this->mVolume->getVoxel(this->mXPosInVolume+1,this->mYPosInVolume+1,this->mZPosInVolume,this->m_eWrapMode, this->m_tBorder); + } + + template + VoxelType PagedVolume::Sampler::peekVoxel1px1py1pz(void) const + { + if((this->isCurrentPositionValid()) && CAN_GO_POS_X(this->mXPosInVolume) && CAN_GO_POS_Y(this->mYPosInVolume) && CAN_GO_POS_Z(this->mZPosInVolume) ) + { + return *(mCurrentVoxel + 1 + this->mVolume->m_uBlockSideLength + this->mVolume->m_uBlockSideLength*this->mVolume->m_uBlockSideLength); + } + return this->mVolume->getVoxel(this->mXPosInVolume+1,this->mYPosInVolume+1,this->mZPosInVolume+1,this->m_eWrapMode, this->m_tBorder); + } +} + +#undef CAN_GO_NEG_X +#undef CAN_GO_POS_X +#undef CAN_GO_NEG_Y +#undef CAN_GO_POS_Y +#undef CAN_GO_NEG_Z +#undef CAN_GO_POS_Z diff --git a/library/PolyVoxCore/include/PolyVoxCore/PolyVoxForwardDeclarations.h b/library/PolyVoxCore/include/PolyVoxCore/PolyVoxForwardDeclarations.h index 4c63906d..449ff4a9 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/PolyVoxForwardDeclarations.h +++ b/library/PolyVoxCore/include/PolyVoxCore/PolyVoxForwardDeclarations.h @@ -103,11 +103,6 @@ namespace PolyVox //////////////////////////////////////////////////////////////////////////////// template class FilePager; - //////////////////////////////////////////////////////////////////////////////// - // LargeVolume - //////////////////////////////////////////////////////////////////////////////// - template class LargeVolume; - //////////////////////////////////////////////////////////////////////////////// // MarchingCubesSurfaceExtractor //////////////////////////////////////////////////////////////////////////////// @@ -145,6 +140,15 @@ namespace PolyVox typedef uint32_t DefaultIndexType; template class Mesh; + //////////////////////////////////////////////////////////////////////////////// + // PagedVolume + //////////////////////////////////////////////////////////////////////////////// + template class PagedVolume; + template + using LargeVolume = PagedVolume; + template + using SimpleVolume = PagedVolume; + //////////////////////////////////////////////////////////////////////////////// // Pager //////////////////////////////////////////////////////////////////////////////// @@ -160,11 +164,6 @@ namespace PolyVox //////////////////////////////////////////////////////////////////////////////// class Region; - //////////////////////////////////////////////////////////////////////////////// - // SimpleVolume - //////////////////////////////////////////////////////////////////////////////// - template class SimpleVolume; - //////////////////////////////////////////////////////////////////////////////// // Vector //////////////////////////////////////////////////////////////////////////////// diff --git a/library/PolyVoxCore/include/PolyVoxCore/SimpleVolume.h b/library/PolyVoxCore/include/PolyVoxCore/SimpleVolume.h index e8dda3e3..9967d874 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/SimpleVolume.h +++ b/library/PolyVoxCore/include/PolyVoxCore/SimpleVolume.h @@ -1,229 +1,7 @@ -/******************************************************************************* -Copyright (c) 2005-2009 David Williams - -This software is provided 'as-is', without any express or implied -warranty. In no event will the authors be held liable for any damages -arising from the use of this software. - -Permission is granted to anyone to use this software for any purpose, -including commercial applications, and to alter it and redistribute it -freely, subject to the following restrictions: - - 1. The origin of this software must not be misrepresented; you must not - claim that you wrote the original software. If you use this software - in a product, an acknowledgment in the product documentation would be - appreciated but is not required. - - 2. Altered source versions must be plainly marked as such, and must not be - misrepresented as being the original software. - - 3. This notice may not be removed or altered from any source - distribution. -*******************************************************************************/ - #ifndef __PolyVox_SimpleVolume_H__ #define __PolyVox_SimpleVolume_H__ -#include "Impl/Utility.h" +#include "PagedVolume.h" +#include "PolyVoxForwardDeclarations.h" -#include "PolyVoxCore/BaseVolume.h" -#include "PolyVoxCore/Region.h" -#include "PolyVoxCore/Vector.h" - -#include //For abort() -#include //For memcpy -#include -#include -#include //For invalid_argument - -namespace PolyVox -{ - template - class SimpleVolume : public BaseVolume - { - public: - #ifndef SWIG - //Could be made private? - class Block - { - public: - Block(uint16_t uSideLength = 0); - ~Block(); - - uint16_t getSideLength(void) const; - VoxelType getVoxelAt(uint16_t uXPos, uint16_t uYPos, uint16_t uZPos) const; - VoxelType getVoxelAt(const Vector3DUint16& v3dPos) const; - - void setVoxelAt(uint16_t uXPos, uint16_t uYPos, uint16_t uZPos, VoxelType tValue); - void setVoxelAt(const Vector3DUint16& v3dPos, VoxelType tValue); - - void fill(VoxelType tValue); - void initialise(uint16_t uSideLength); - uint32_t calculateSizeInBytes(void); - - public: - VoxelType* m_tUncompressedData; - uint16_t m_uSideLength; - uint8_t m_uSideLengthPower; - }; - - //There seems to be some descrepency between Visual Studio and GCC about how the following class should be declared. - //There is a work around (see also See http://goo.gl/qu1wn) given below which appears to work on VS2010 and GCC, but - //which seems to cause internal compiler errors on VS2008 when building with the /Gm 'Enable Minimal Rebuild' compiler - //option. For now it seems best to 'fix' it with the preprocessor insstead, but maybe the workaround can be reinstated - //in the future - //typedef Volume VolumeOfVoxelType; //Workaround for GCC/VS2010 differences. - //class Sampler : public VolumeOfVoxelType::template Sampler< SimpleVolume > -#if defined(_MSC_VER) - class Sampler : public BaseVolume::Sampler< SimpleVolume > //This line works on VS2010 -#else - class Sampler : public BaseVolume::template Sampler< SimpleVolume > //This line works on GCC -#endif - { - public: - /// Construct a new Sampler - Sampler(SimpleVolume* volume); - ~Sampler(); - - /// \deprecated - POLYVOX_DEPRECATED VoxelType getSubSampledVoxel(uint8_t uLevel) const; - /// Get the value of the current voxel - inline VoxelType getVoxel(void) const; - - /// Set the current voxel position - void setPosition(const Vector3DInt32& v3dNewPos); - /// Set the current voxel position - void setPosition(int32_t xPos, int32_t yPos, int32_t zPos); - /// Set the value of the current voxel - inline bool setVoxel(VoxelType tValue); - - /// Increase the \a x position by \a 1 - void movePositiveX(void); - /// Increase the \a y position by \a 1 - void movePositiveY(void); - /// Increase the \a z position by \a 1 - void movePositiveZ(void); - - /// Decrease the \a x position by \a 1 - void moveNegativeX(void); - /// Decrease the \a y position by \a 1 - void moveNegativeY(void); - /// Decrease the \a z position by \a 1 - void moveNegativeZ(void); - - inline VoxelType peekVoxel1nx1ny1nz(void) const; - inline VoxelType peekVoxel1nx1ny0pz(void) const; - inline VoxelType peekVoxel1nx1ny1pz(void) const; - inline VoxelType peekVoxel1nx0py1nz(void) const; - inline VoxelType peekVoxel1nx0py0pz(void) const; - inline VoxelType peekVoxel1nx0py1pz(void) const; - inline VoxelType peekVoxel1nx1py1nz(void) const; - inline VoxelType peekVoxel1nx1py0pz(void) const; - inline VoxelType peekVoxel1nx1py1pz(void) const; - - inline VoxelType peekVoxel0px1ny1nz(void) const; - inline VoxelType peekVoxel0px1ny0pz(void) const; - inline VoxelType peekVoxel0px1ny1pz(void) const; - inline VoxelType peekVoxel0px0py1nz(void) const; - inline VoxelType peekVoxel0px0py0pz(void) const; - inline VoxelType peekVoxel0px0py1pz(void) const; - inline VoxelType peekVoxel0px1py1nz(void) const; - inline VoxelType peekVoxel0px1py0pz(void) const; - inline VoxelType peekVoxel0px1py1pz(void) const; - - inline VoxelType peekVoxel1px1ny1nz(void) const; - inline VoxelType peekVoxel1px1ny0pz(void) const; - inline VoxelType peekVoxel1px1ny1pz(void) const; - inline VoxelType peekVoxel1px0py1nz(void) const; - inline VoxelType peekVoxel1px0py0pz(void) const; - inline VoxelType peekVoxel1px0py1pz(void) const; - inline VoxelType peekVoxel1px1py1nz(void) const; - inline VoxelType peekVoxel1px1py0pz(void) const; - inline VoxelType peekVoxel1px1py1pz(void) const; - - private: - //Other current position information - VoxelType* mCurrentVoxel; - }; - #endif - - public: - /// Constructor for creating a fixed size volume. - SimpleVolume(const Region& regValid, uint16_t uBlockSideLength = 32); - - /// Destructor - ~SimpleVolume(); - - /// Gets a voxel at the position given by x,y,z coordinates - template - VoxelType getVoxel(int32_t uXPos, int32_t uYPos, int32_t uZPos, VoxelType tBorder = VoxelType()) const; - /// Gets a voxel at the position given by a 3D vector - template - VoxelType getVoxel(const Vector3DInt32& v3dPos, VoxelType tBorder = VoxelType()) const; - - /// Gets a voxel at the position given by x,y,z coordinates - VoxelType getVoxel(int32_t uXPos, int32_t uYPos, int32_t uZPos, WrapMode eWrapMode = WrapModes::Validate, VoxelType tBorder = VoxelType()) const; - /// Gets a voxel at the position given by a 3D vector - VoxelType getVoxel(const Vector3DInt32& v3dPos, WrapMode eWrapMode = WrapModes::Validate, VoxelType tBorder = VoxelType()) const; - - /// Gets a voxel at the position given by x,y,z coordinates - POLYVOX_DEPRECATED VoxelType getVoxelAt(int32_t uXPos, int32_t uYPos, int32_t uZPos) const; - /// Gets a voxel at the position given by a 3D vector - POLYVOX_DEPRECATED VoxelType getVoxelAt(const Vector3DInt32& v3dPos) const; - - /// Sets the voxel at the position given by x,y,z coordinates - void setVoxel(int32_t uXPos, int32_t uYPos, int32_t uZPos, VoxelType tValue, WrapMode eWrapMode = WrapModes::Validate); - /// Sets the voxel at the position given by a 3D vector - void setVoxel(const Vector3DInt32& v3dPos, VoxelType tValue, WrapMode eWrapMode = WrapModes::Validate); - /// Sets the voxel at the position given by x,y,z coordinates - bool setVoxelAt(int32_t uXPos, int32_t uYPos, int32_t uZPos, VoxelType tValue); - /// Sets the voxel at the position given by a 3D vector - bool setVoxelAt(const Vector3DInt32& v3dPos, VoxelType tValue); - - /// Calculates approximatly how many bytes of memory the volume is currently using. - uint32_t calculateSizeInBytes(void); - - protected: - /// Copy constructor - SimpleVolume(const SimpleVolume& rhs); - - /// Assignment operator - SimpleVolume& operator=(const SimpleVolume& rhs); - - private: - void initialise(const Region& regValidRegion, uint16_t uBlockSideLength); - - // A trick to implement specialization of template member functions in template classes. See http://stackoverflow.com/a/4951057 - template - VoxelType getVoxelImpl(int32_t uXPos, int32_t uYPos, int32_t uZPos, WrapModeType, VoxelType tBorder) const; - VoxelType getVoxelImpl(int32_t uXPos, int32_t uYPos, int32_t uZPos, WrapModeType, VoxelType tBorder) const; - VoxelType getVoxelImpl(int32_t uXPos, int32_t uYPos, int32_t uZPos, WrapModeType, VoxelType tBorder) const; - VoxelType getVoxelImpl(int32_t uXPos, int32_t uYPos, int32_t uZPos, WrapModeType, VoxelType tBorder) const; - VoxelType getVoxelImpl(int32_t uXPos, int32_t uYPos, int32_t uZPos, WrapModeType, VoxelType tBorder) const; - - Block* getUncompressedBlock(int32_t uBlockX, int32_t uBlockY, int32_t uBlockZ) const; - - //The block data - Block* m_pBlocks; - - //The size of the volume in vlocks - Region m_regValidRegionInBlocks; - - //Volume size measured in blocks. - uint32_t m_uNoOfBlocksInVolume; - uint16_t m_uWidthInBlocks; - uint16_t m_uHeightInBlocks; - uint16_t m_uDepthInBlocks; - - //The size of the blocks - uint32_t m_uNoOfVoxelsPerBlock; - uint16_t m_uBlockSideLength; - uint8_t m_uBlockSideLengthPower; - }; -} - -#include "PolyVoxCore/SimpleVolumeBlock.inl" -#include "PolyVoxCore/SimpleVolume.inl" -#include "PolyVoxCore/SimpleVolumeSampler.inl" - -#endif //__PolyVox_SimpleVolume_H__ +#endif //__PolyVox_SimpleVolume_H__ \ No newline at end of file diff --git a/library/PolyVoxCore/include/PolyVoxCore/SimpleVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/SimpleVolume.inl deleted file mode 100644 index f32aacdd..00000000 --- a/library/PolyVoxCore/include/PolyVoxCore/SimpleVolume.inl +++ /dev/null @@ -1,441 +0,0 @@ -/******************************************************************************* -Copyright (c) 2005-2009 David Williams - -This software is provided 'as-is', without any express or implied -warranty. In no event will the authors be held liable for any damages -arising from the use of this software. - -Permission is granted to anyone to use this software for any purpose, -including commercial applications, and to alter it and redistribute it -freely, subject to the following restrictions: - - 1. The origin of this software must not be misrepresented; you must not - claim that you wrote the original software. If you use this software - in a product, an acknowledgment in the product documentation would be - appreciated but is not required. - - 2. Altered source versions must be plainly marked as such, and must not be - misrepresented as being the original software. - - 3. This notice may not be removed or altered from any source - distribution. -*******************************************************************************/ - -#include "PolyVoxCore/Impl/ErrorHandling.h" - -namespace PolyVox -{ - //////////////////////////////////////////////////////////////////////////////// - /// This constructor creates a volume with a fixed size which is specified as a parameter. - /// \param regValid Specifies the minimum and maximum valid voxel positions. - /// \param uBlockSideLength The size of the block to use within the volume - //////////////////////////////////////////////////////////////////////////////// - template - SimpleVolume::SimpleVolume(const Region& regValid, uint16_t uBlockSideLength) - :BaseVolume(regValid) - { - //Create a volume of the right size. - initialise(regValid,uBlockSideLength); - } - - //////////////////////////////////////////////////////////////////////////////// - /// This function should never be called. Copying volumes by value would be expensive, and we want to prevent users from doing - /// it by accident (such as when passing them as paramenters to functions). That said, there are times when you really do want to - /// make a copy of a volume and in this case you should look at the Volumeresampler. - /// - /// \sa VolumeResampler - //////////////////////////////////////////////////////////////////////////////// - template - SimpleVolume::SimpleVolume(const SimpleVolume& /*rhs*/) - { - POLYVOX_THROW(not_implemented, "Volume copy constructor not implemented for performance reasons."); - } - - //////////////////////////////////////////////////////////////////////////////// - /// Destroys the volume - //////////////////////////////////////////////////////////////////////////////// - template - SimpleVolume::~SimpleVolume() - { - delete[] m_pBlocks; - } - - //////////////////////////////////////////////////////////////////////////////// - /// This function should never be called. Copying volumes by value would be expensive, and we want to prevent users from doing - /// it by accident (such as when passing them as paramenters to functions). That said, there are times when you really do want to - /// make a copy of a volume and in this case you should look at the Volumeresampler. - /// - /// \sa VolumeResampler - //////////////////////////////////////////////////////////////////////////////// - template - SimpleVolume& SimpleVolume::operator=(const SimpleVolume& /*rhs*/) - { - POLYVOX_THROW(not_implemented, "Volume assignment operator not implemented for performance reasons."); - } - - //////////////////////////////////////////////////////////////////////////////// - /// This version of the function requires the wrap mode to be specified as a - /// template parameter, which can provide better performance. - /// \param uXPos The \c x position of the voxel - /// \param uYPos The \c y position of the voxel - /// \param uZPos The \c z position of the voxel - /// \tparam eWrapMode Specifies the behaviour when the requested position is outside of the volume. - /// \param tBorder The border value to use if the wrap mode is set to 'Border'. - /// \return The voxel value - //////////////////////////////////////////////////////////////////////////////// - template - template - VoxelType SimpleVolume::getVoxel(int32_t uXPos, int32_t uYPos, int32_t uZPos, VoxelType tBorder) const - { - // Simply call through to the real implementation - return getVoxelImpl(uXPos, uYPos, uZPos, WrapModeType(), tBorder); - } - - //////////////////////////////////////////////////////////////////////////////// - /// This version of the function requires the wrap mode to be specified as a - /// template parameter, which can provide better performance. - /// \param uXPos The \c x position of the voxel - /// \param uYPos The \c y position of the voxel - /// \param uZPos The \c z position of the voxel - /// \tparam eWrapMode Specifies the behaviour when the requested position is outside of the volume. - /// \param tBorder The border value to use if the wrap mode is set to 'Border'. - /// \return The voxel value - //////////////////////////////////////////////////////////////////////////////// - template - template - VoxelType SimpleVolume::getVoxel(const Vector3DInt32& v3dPos, VoxelType tBorder) const - { - // Simply call through to the real implementation - return getVoxel(v3dPos.getX(), v3dPos.getY(), v3dPos.getZ(), tBorder); - } - - //////////////////////////////////////////////////////////////////////////////// - /// This version of the function is provided so that the wrap mode does not need - /// to be specified as a template parameter, as it may be confusing to some users. - /// \param uXPos The \c x position of the voxel - /// \param uYPos The \c y position of the voxel - /// \param uZPos The \c z position of the voxel - /// \param eWrapMode Specifies the behaviour when the requested position is outside of the volume. - /// \param tBorder The border value to use if the wrap mode is set to 'Border'. - /// \return The voxel value - //////////////////////////////////////////////////////////////////////////////// - template - VoxelType SimpleVolume::getVoxel(int32_t uXPos, int32_t uYPos, int32_t uZPos, WrapMode eWrapMode, VoxelType tBorder) const - { - switch(eWrapMode) - { - case WrapModes::Validate: - return getVoxelImpl(uXPos, uYPos, uZPos, WrapModeType(), tBorder); - case WrapModes::Clamp: - return getVoxelImpl(uXPos, uYPos, uZPos, WrapModeType(), tBorder); - case WrapModes::Border: - return getVoxelImpl(uXPos, uYPos, uZPos, WrapModeType(), tBorder); - case WrapModes::AssumeValid: - return getVoxelImpl(uXPos, uYPos, uZPos, WrapModeType(), tBorder); - default: - // Should never happen - POLYVOX_ASSERT(false, "Invalid wrap mode"); - return VoxelType(); - } - } - - //////////////////////////////////////////////////////////////////////////////// - /// This version of the function is provided so that the wrap mode does not need - /// to be specified as a template parameter, as it may be confusing to some users. - /// \param v3dPos The 3D position of the voxel - /// \param eWrapMode Specifies the behaviour when the requested position is outside of the volume. - /// \param tBorder The border value to use if the wrap mode is set to 'Border'. - /// \return The voxel value - //////////////////////////////////////////////////////////////////////////////// - template - VoxelType SimpleVolume::getVoxel(const Vector3DInt32& v3dPos, WrapMode eWrapMode, VoxelType tBorder) const - { - return getVoxel(v3dPos.getX(), v3dPos.getY(), v3dPos.getZ(), eWrapMode, tBorder); - } - - //////////////////////////////////////////////////////////////////////////////// - /// \param uXPos The \c x position of the voxel - /// \param uYPos The \c y position of the voxel - /// \param uZPos The \c z position of the voxel - /// \return The voxel value - //////////////////////////////////////////////////////////////////////////////// - template - VoxelType SimpleVolume::getVoxelAt(int32_t uXPos, int32_t uYPos, int32_t uZPos) const - { - if(this->m_regValidRegion.containsPoint(Vector3DInt32(uXPos, uYPos, uZPos))) - { - const int32_t blockX = uXPos >> m_uBlockSideLengthPower; - const int32_t blockY = uYPos >> m_uBlockSideLengthPower; - const int32_t blockZ = uZPos >> m_uBlockSideLengthPower; - - const uint16_t xOffset = static_cast(uXPos - (blockX << m_uBlockSideLengthPower)); - const uint16_t yOffset = static_cast(uYPos - (blockY << m_uBlockSideLengthPower)); - const uint16_t zOffset = static_cast(uZPos - (blockZ << m_uBlockSideLengthPower)); - - typename SimpleVolume::Block* pUncompressedBlock = getUncompressedBlock(blockX, blockY, blockZ); - - return pUncompressedBlock->getVoxelAt(xOffset,yOffset,zOffset); - } - else - { - return this->getBorderValue(); - } - } - - //////////////////////////////////////////////////////////////////////////////// - /// \param v3dPos The 3D position of the voxel - /// \return The voxel value - //////////////////////////////////////////////////////////////////////////////// - template - VoxelType SimpleVolume::getVoxelAt(const Vector3DInt32& v3dPos) const - { - return getVoxelAt(v3dPos.getX(), v3dPos.getY(), v3dPos.getZ()); - } - - //////////////////////////////////////////////////////////////////////////////// - /// \param uXPos the \c x position of the voxel - /// \param uYPos the \c y position of the voxel - /// \param uZPos the \c z position of the voxel - /// \param tValue the value to which the voxel will be set - /// \param eWrapMode Specifies the behaviour when the requested position is outside of the volume. - /// This must be set to 'None' or 'DontCheck'. Other wrap modes cannot be used when writing to volume data. - //////////////////////////////////////////////////////////////////////////////// - template - void SimpleVolume::setVoxel(int32_t uXPos, int32_t uYPos, int32_t uZPos, VoxelType tValue, WrapMode eWrapMode) - { - if((eWrapMode != WrapModes::Validate) && (eWrapMode != WrapModes::AssumeValid)) - { - POLYVOX_THROW(std::invalid_argument, "Invalid wrap mode in call to setVoxel(). It must be 'None' or 'DontCheck'."); - } - - // This validation is skipped if the wrap mode is 'DontCheck' - if(eWrapMode == WrapModes::Validate) - { - if(this->m_regValidRegion.containsPoint(Vector3DInt32(uXPos, uYPos, uZPos)) == false) - { - POLYVOX_THROW(std::out_of_range, "Position is outside valid region"); - } - } - - const int32_t blockX = uXPos >> m_uBlockSideLengthPower; - const int32_t blockY = uYPos >> m_uBlockSideLengthPower; - const int32_t blockZ = uZPos >> m_uBlockSideLengthPower; - - const uint16_t xOffset = uXPos - (blockX << m_uBlockSideLengthPower); - const uint16_t yOffset = uYPos - (blockY << m_uBlockSideLengthPower); - const uint16_t zOffset = uZPos - (blockZ << m_uBlockSideLengthPower); - - typename SimpleVolume::Block* pUncompressedBlock = getUncompressedBlock(blockX, blockY, blockZ); - - pUncompressedBlock->setVoxelAt(xOffset,yOffset,zOffset, tValue); - } - - //////////////////////////////////////////////////////////////////////////////// - /// \param v3dPos the 3D position of the voxel - /// \param tValue the value to which the voxel will be set - /// \param eWrapMode Specifies the behaviour when the requested position is outside of the volume. - /// This must be set to 'None' or 'DontCheck'. Other wrap modes cannot be used when writing to volume data. - //////////////////////////////////////////////////////////////////////////////// - template - void SimpleVolume::setVoxel(const Vector3DInt32& v3dPos, VoxelType tValue, WrapMode eWrapMode) - { - setVoxel(v3dPos.getX(), v3dPos.getY(), v3dPos.getZ(), tValue, eWrapMode); - } - - //////////////////////////////////////////////////////////////////////////////// - /// \param uXPos the \c x position of the voxel - /// \param uYPos the \c y position of the voxel - /// \param uZPos the \c z position of the voxel - /// \param tValue the value to which the voxel will be set - /// \return whether the requested position is inside the volume - //////////////////////////////////////////////////////////////////////////////// - template - bool SimpleVolume::setVoxelAt(int32_t uXPos, int32_t uYPos, int32_t uZPos, VoxelType tValue) - { - // PolyVox does not throw an exception when a voxel is out of range. Please see 'Error Handling' in the User Manual. - POLYVOX_ASSERT(this->m_regValidRegion.containsPoint(Vector3DInt32(uXPos, uYPos, uZPos)), "Position is outside valid region"); - - const int32_t blockX = uXPos >> m_uBlockSideLengthPower; - const int32_t blockY = uYPos >> m_uBlockSideLengthPower; - const int32_t blockZ = uZPos >> m_uBlockSideLengthPower; - - const uint16_t xOffset = uXPos - (blockX << m_uBlockSideLengthPower); - const uint16_t yOffset = uYPos - (blockY << m_uBlockSideLengthPower); - const uint16_t zOffset = uZPos - (blockZ << m_uBlockSideLengthPower); - - typename SimpleVolume::Block* pUncompressedBlock = getUncompressedBlock(blockX, blockY, blockZ); - - pUncompressedBlock->setVoxelAt(xOffset,yOffset,zOffset, tValue); - - //Return true to indicate that we modified a voxel. - return true; - } - - //////////////////////////////////////////////////////////////////////////////// - /// \param v3dPos the 3D position of the voxel - /// \param tValue the value to which the voxel will be set - /// \return whether the requested position is inside the volume - //////////////////////////////////////////////////////////////////////////////// - template - bool SimpleVolume::setVoxelAt(const Vector3DInt32& v3dPos, VoxelType tValue) - { - return setVoxelAt(v3dPos.getX(), v3dPos.getY(), v3dPos.getZ(), tValue); - } - - //////////////////////////////////////////////////////////////////////////////// - /// This function should probably be made internal... - //////////////////////////////////////////////////////////////////////////////// - template - void SimpleVolume::initialise(const Region& regValidRegion, uint16_t uBlockSideLength) - { - //Release mode validation - if(uBlockSideLength < 8) - { - POLYVOX_THROW(std::invalid_argument, "Block side length should be at least 8"); - } - if(uBlockSideLength > 256) - { - POLYVOX_THROW(std::invalid_argument, "Block side length should not be more than 256"); - } - if(!isPowerOf2(uBlockSideLength)) - { - POLYVOX_THROW(std::invalid_argument, "Block side length must be a power of two."); - } - - this->m_regValidRegion = regValidRegion; - - //Compute the block side length - m_uBlockSideLength = uBlockSideLength; - m_uBlockSideLengthPower = logBase2(m_uBlockSideLength); - m_uNoOfVoxelsPerBlock = m_uBlockSideLength * m_uBlockSideLength * m_uBlockSideLength; - - m_regValidRegionInBlocks.setLowerX(this->m_regValidRegion.getLowerX() >> m_uBlockSideLengthPower); - m_regValidRegionInBlocks.setLowerY(this->m_regValidRegion.getLowerY() >> m_uBlockSideLengthPower); - m_regValidRegionInBlocks.setLowerZ(this->m_regValidRegion.getLowerZ() >> m_uBlockSideLengthPower); - m_regValidRegionInBlocks.setUpperX(this->m_regValidRegion.getUpperX() >> m_uBlockSideLengthPower); - m_regValidRegionInBlocks.setUpperY(this->m_regValidRegion.getUpperY() >> m_uBlockSideLengthPower); - m_regValidRegionInBlocks.setUpperZ(this->m_regValidRegion.getUpperZ() >> m_uBlockSideLengthPower); - - //Compute the size of the volume in blocks (and note +1 at the end) - m_uWidthInBlocks = m_regValidRegionInBlocks.getUpperX() - m_regValidRegionInBlocks.getLowerX() + 1; - m_uHeightInBlocks = m_regValidRegionInBlocks.getUpperY() - m_regValidRegionInBlocks.getLowerY() + 1; - m_uDepthInBlocks = m_regValidRegionInBlocks.getUpperZ() - m_regValidRegionInBlocks.getLowerZ() + 1; - m_uNoOfBlocksInVolume = m_uWidthInBlocks * m_uHeightInBlocks * m_uDepthInBlocks; - - //Allocate the data - m_pBlocks = new Block[m_uNoOfBlocksInVolume]; - for(uint32_t i = 0; i < m_uNoOfBlocksInVolume; ++i) - { - m_pBlocks[i].initialise(m_uBlockSideLength); - } - - //Other properties we might find useful later - this->m_uLongestSideLength = (std::max)((std::max)(this->getWidth(),this->getHeight()),this->getDepth()); - this->m_uShortestSideLength = (std::min)((std::min)(this->getWidth(),this->getHeight()),this->getDepth()); - this->m_fDiagonalLength = sqrtf(static_cast(this->getWidth() * this->getWidth() + this->getHeight() * this->getHeight() + this->getDepth() * this->getDepth())); - } - - template - typename SimpleVolume::Block* SimpleVolume::getUncompressedBlock(int32_t uBlockX, int32_t uBlockY, int32_t uBlockZ) const - { - //The lower left corner of the volume could be - //anywhere, but array indices need to start at zero. - uBlockX -= m_regValidRegionInBlocks.getLowerX(); - uBlockY -= m_regValidRegionInBlocks.getLowerY(); - uBlockZ -= m_regValidRegionInBlocks.getLowerZ(); - - POLYVOX_ASSERT(uBlockX >= 0, "Block coordinate must not be negative."); - POLYVOX_ASSERT(uBlockY >= 0, "Block coordinate must not be negative."); - POLYVOX_ASSERT(uBlockZ >= 0, "Block coordinate must not be negative."); - - //Compute the block index - uint32_t uBlockIndex = - uBlockX + - uBlockY * m_uWidthInBlocks + - uBlockZ * m_uWidthInBlocks * m_uHeightInBlocks; - - //Return the block - return &(m_pBlocks[uBlockIndex]); - } - - //////////////////////////////////////////////////////////////////////////////// - /// \todo This function needs reviewing for accuracy... - /// - /// \return The number of bytes used - //////////////////////////////////////////////////////////////////////////////// - template - uint32_t SimpleVolume::calculateSizeInBytes(void) - { - uint32_t uSizeInBytes = sizeof(SimpleVolume); - - uint32_t uSizeOfBlockInBytes = m_uNoOfVoxelsPerBlock * sizeof(VoxelType); - - //Memory used by the blocks - uSizeInBytes += uSizeOfBlockInBytes * (m_uNoOfBlocksInVolume); - - return uSizeInBytes; - } - - template - template - VoxelType SimpleVolume::getVoxelImpl(int32_t uXPos, int32_t uYPos, int32_t uZPos, WrapModeType, VoxelType tBorder) const - { - // This function should never be called because one of the specialisations should always match. - POLYVOX_ASSERT(false, "This function is not implemented and should never be called!"); - } - - template - VoxelType SimpleVolume::getVoxelImpl(int32_t uXPos, int32_t uYPos, int32_t uZPos, WrapModeType, VoxelType tBorder) const - { - if(this->m_regValidRegion.containsPoint(Vector3DInt32(uXPos, uYPos, uZPos)) == false) - { - POLYVOX_THROW(std::out_of_range, "Position is outside valid region"); - } - - return getVoxelImpl(uXPos, uYPos, uZPos, WrapModeType(), tBorder); // No wrapping as we've just validated the position. - } - - template - VoxelType SimpleVolume::getVoxelImpl(int32_t uXPos, int32_t uYPos, int32_t uZPos, WrapModeType, VoxelType tBorder) const - { - //Perform clamping - uXPos = (std::max)(uXPos, this->m_regValidRegion.getLowerX()); - uYPos = (std::max)(uYPos, this->m_regValidRegion.getLowerY()); - uZPos = (std::max)(uZPos, this->m_regValidRegion.getLowerZ()); - uXPos = (std::min)(uXPos, this->m_regValidRegion.getUpperX()); - uYPos = (std::min)(uYPos, this->m_regValidRegion.getUpperY()); - uZPos = (std::min)(uZPos, this->m_regValidRegion.getUpperZ()); - - return getVoxelImpl(uXPos, uYPos, uZPos, WrapModeType(), tBorder); // No wrapping as we've just validated the position. - } - - template - VoxelType SimpleVolume::getVoxelImpl(int32_t uXPos, int32_t uYPos, int32_t uZPos, WrapModeType, VoxelType tBorder) const - { - if(this->m_regValidRegion.containsPoint(uXPos, uYPos, uZPos)) - { - return getVoxelImpl(uXPos, uYPos, uZPos, WrapModeType(), tBorder); // No wrapping as we've just validated the position. - } - else - { - return tBorder; - } - } - - template - VoxelType SimpleVolume::getVoxelImpl(int32_t uXPos, int32_t uYPos, int32_t uZPos, WrapModeType, VoxelType /*tBorder*/) const - { - const int32_t blockX = uXPos >> m_uBlockSideLengthPower; - const int32_t blockY = uYPos >> m_uBlockSideLengthPower; - const int32_t blockZ = uZPos >> m_uBlockSideLengthPower; - - const uint16_t xOffset = static_cast(uXPos - (blockX << m_uBlockSideLengthPower)); - const uint16_t yOffset = static_cast(uYPos - (blockY << m_uBlockSideLengthPower)); - const uint16_t zOffset = static_cast(uZPos - (blockZ << m_uBlockSideLengthPower)); - - typename SimpleVolume::Block* pUncompressedBlock = getUncompressedBlock(blockX, blockY, blockZ); - - return pUncompressedBlock->getVoxelAt(xOffset,yOffset,zOffset); - } -} - diff --git a/library/PolyVoxCore/include/PolyVoxCore/SimpleVolumeBlock.inl b/library/PolyVoxCore/include/PolyVoxCore/SimpleVolumeBlock.inl deleted file mode 100644 index 08e06c1e..00000000 --- a/library/PolyVoxCore/include/PolyVoxCore/SimpleVolumeBlock.inl +++ /dev/null @@ -1,130 +0,0 @@ -/******************************************************************************* -Copyright (c) 2005-2009 David Williams - -This software is provided 'as-is', without any express or implied -warranty. In no event will the authors be held liable for any damages -arising from the use of this software. - -Permission is granted to anyone to use this software for any purpose, -including commercial applications, and to alter it and redistribute it -freely, subject to the following restrictions: - - 1. The origin of this software must not be misrepresented; you must not - claim that you wrote the original software. If you use this software - in a product, an acknowledgment in the product documentation would be - appreciated but is not required. - - 2. Altered source versions must be plainly marked as such, and must not be - misrepresented as being the original software. - - 3. This notice may not be removed or altered from any source - distribution. -*******************************************************************************/ - -#include "PolyVoxCore/Impl/ErrorHandling.h" - -namespace PolyVox -{ - template - SimpleVolume::Block::Block(uint16_t uSideLength) - :m_tUncompressedData(0) - ,m_uSideLength(0) - ,m_uSideLengthPower(0) - { - if(uSideLength != 0) - { - initialise(uSideLength); - } - } - - template - SimpleVolume::Block::~Block() - { - delete[] m_tUncompressedData; - } - - template - uint16_t SimpleVolume::Block::getSideLength(void) const - { - return m_uSideLength; - } - - template - VoxelType SimpleVolume::Block::getVoxelAt(uint16_t uXPos, uint16_t uYPos, uint16_t uZPos) const - { - // This is internal code not directly called by the user. For efficiency we assert rather than throwing. - POLYVOX_ASSERT(uXPos < m_uSideLength, "Position is outside of the block."); - POLYVOX_ASSERT(uYPos < m_uSideLength, "Position is outside of the block."); - POLYVOX_ASSERT(uZPos < m_uSideLength, "Position is outside of the block."); - POLYVOX_ASSERT(m_tUncompressedData, "No uncompressed data available"); - - return m_tUncompressedData - [ - uXPos + - uYPos * m_uSideLength + - uZPos * m_uSideLength * m_uSideLength - ]; - } - - template - VoxelType SimpleVolume::Block::getVoxelAt(const Vector3DUint16& v3dPos) const - { - return getVoxelAt(v3dPos.getX(), v3dPos.getY(), v3dPos.getZ()); - } - - template - void SimpleVolume::Block::setVoxelAt(uint16_t uXPos, uint16_t uYPos, uint16_t uZPos, VoxelType tValue) - { - // This is internal code not directly called by the user. For efficiency we assert rather than throwing. - POLYVOX_ASSERT(uXPos < m_uSideLength, "Position is outside of the block."); - POLYVOX_ASSERT(uYPos < m_uSideLength, "Position is outside of the block."); - POLYVOX_ASSERT(uZPos < m_uSideLength, "Position is outside of the block."); - POLYVOX_ASSERT(m_tUncompressedData, "No uncompressed data available"); - - m_tUncompressedData - [ - uXPos + - uYPos * m_uSideLength + - uZPos * m_uSideLength * m_uSideLength - ] = tValue; - } - - template - void SimpleVolume::Block::setVoxelAt(const Vector3DUint16& v3dPos, VoxelType tValue) - { - setVoxelAt(v3dPos.getX(), v3dPos.getY(), v3dPos.getZ(), tValue); - } - - template - void SimpleVolume::Block::fill(VoxelType tValue) - { - const uint32_t uNoOfVoxels = m_uSideLength * m_uSideLength * m_uSideLength; - std::fill(m_tUncompressedData, m_tUncompressedData + uNoOfVoxels, tValue); - } - - template - void SimpleVolume::Block::initialise(uint16_t uSideLength) - { - //Release mode validation - if(!isPowerOf2(uSideLength)) - { - POLYVOX_THROW(std::invalid_argument, "Block side length must be a power of two."); - } - - //Compute the side length - m_uSideLength = uSideLength; - m_uSideLengthPower = logBase2(uSideLength); - - m_tUncompressedData = new VoxelType[m_uSideLength * m_uSideLength * m_uSideLength]; - - SimpleVolume::Block::fill(VoxelType()); - } - - template - uint32_t SimpleVolume::Block::calculateSizeInBytes(void) - { - uint32_t uSizeInBytes = sizeof(Block); - uSizeInBytes += sizeof(VoxelType) * m_uSideLength * m_uSideLength * m_uSideLength; - return uSizeInBytes; - } -} diff --git a/library/PolyVoxCore/include/PolyVoxCore/SimpleVolumeSampler.inl b/library/PolyVoxCore/include/PolyVoxCore/SimpleVolumeSampler.inl deleted file mode 100644 index 6cf16bef..00000000 --- a/library/PolyVoxCore/include/PolyVoxCore/SimpleVolumeSampler.inl +++ /dev/null @@ -1,581 +0,0 @@ -/******************************************************************************* -Copyright (c) 2005-2009 David Williams - -This software is provided 'as-is', without any express or implied -warranty. In no event will the authors be held liable for any damages -arising from the use of this software. - -Permission is granted to anyone to use this software for any purpose, -including commercial applications, and to alter it and redistribute it -freely, subject to the following restrictions: - - 1. The origin of this software must not be misrepresented; you must not - claim that you wrote the original software. If you use this software - in a product, an acknowledgment in the product documentation would be - appreciated but is not required. - - 2. Altered source versions must be plainly marked as such, and must not be - misrepresented as being the original software. - - 3. This notice may not be removed or altered from any source - distribution. -*******************************************************************************/ - -#define CAN_GO_NEG_X(val) ((val > this->mVolume->getEnclosingRegion().getLowerX()) && (val % this->mVolume->m_uBlockSideLength != 0)) -#define CAN_GO_POS_X(val) ((val < this->mVolume->getEnclosingRegion().getUpperX()) && ((val + 1) % this->mVolume->m_uBlockSideLength != 0)) -#define CAN_GO_NEG_Y(val) ((val > this->mVolume->getEnclosingRegion().getLowerY()) && (val % this->mVolume->m_uBlockSideLength != 0)) -#define CAN_GO_POS_Y(val) ((val < this->mVolume->getEnclosingRegion().getUpperY()) && ((val + 1) % this->mVolume->m_uBlockSideLength != 0)) -#define CAN_GO_NEG_Z(val) ((val > this->mVolume->getEnclosingRegion().getLowerZ()) && (val % this->mVolume->m_uBlockSideLength != 0)) -#define CAN_GO_POS_Z(val) ((val < this->mVolume->getEnclosingRegion().getUpperZ()) && ((val + 1) % this->mVolume->m_uBlockSideLength != 0)) - -namespace PolyVox -{ - /** - * \param volume The SimpleVolume you want to sample - */ - template - SimpleVolume::Sampler::Sampler(SimpleVolume* volume) - :BaseVolume::template Sampler< SimpleVolume >(volume) - { - } - - template - SimpleVolume::Sampler::~Sampler() - { - } - - template - VoxelType SimpleVolume::Sampler::getSubSampledVoxel(uint8_t uLevel) const - { - if(uLevel == 0) - { - return getVoxel(); - } - else if(uLevel == 1) - { - VoxelType tValue = getVoxel(); - tValue = (std::min)(tValue, peekVoxel1px0py0pz()); - tValue = (std::min)(tValue, peekVoxel0px1py0pz()); - tValue = (std::min)(tValue, peekVoxel1px1py0pz()); - tValue = (std::min)(tValue, peekVoxel0px0py1pz()); - tValue = (std::min)(tValue, peekVoxel1px0py1pz()); - tValue = (std::min)(tValue, peekVoxel0px1py1pz()); - tValue = (std::min)(tValue, peekVoxel1px1py1pz()); - return tValue; - } - else - { - const uint8_t uSize = 1 << uLevel; - - VoxelType tValue = (std::numeric_limits::max)(); - for(uint8_t z = 0; z < uSize; ++z) - { - for(uint8_t y = 0; y < uSize; ++y) - { - for(uint8_t x = 0; x < uSize; ++x) - { - tValue = (std::min)(tValue, this->mVolume->getVoxelAt(this->mXPosInVolume + x, this->mYPosInVolume + y, this->mZPosInVolume + z)); - } - } - } - return tValue; - } - } - - /** - * \return The current voxel - */ - template - VoxelType SimpleVolume::Sampler::getVoxel(void) const - { - if(this->isCurrentPositionValid()) - { - return *mCurrentVoxel; - } - else - { - return this->getVoxelImpl(this->mXPosInVolume, this->mYPosInVolume, this->mZPosInVolume); - } - } - - /** - * \param v3dNewPos The position to set to - */ - template - void SimpleVolume::Sampler::setPosition(const Vector3DInt32& v3dNewPos) - { - setPosition(v3dNewPos.getX(), v3dNewPos.getY(), v3dNewPos.getZ()); - } - - /** - * \param xPos The \a x position to set to - * \param yPos The \a y position to set to - * \param zPos The \a z position to set to - */ - template - void SimpleVolume::Sampler::setPosition(int32_t xPos, int32_t yPos, int32_t zPos) - { - // Base version updates position and validity flags. - BaseVolume::template Sampler< SimpleVolume >::setPosition(xPos, yPos, zPos); - - // Then we update the voxel pointer - if(this->isCurrentPositionValid()) - { - const int32_t uXBlock = this->mXPosInVolume >> this->mVolume->m_uBlockSideLengthPower; - const int32_t uYBlock = this->mYPosInVolume >> this->mVolume->m_uBlockSideLengthPower; - const int32_t uZBlock = this->mZPosInVolume >> this->mVolume->m_uBlockSideLengthPower; - - const uint16_t uXPosInBlock = static_cast(this->mXPosInVolume - (uXBlock << this->mVolume->m_uBlockSideLengthPower)); - const uint16_t uYPosInBlock = static_cast(this->mYPosInVolume - (uYBlock << this->mVolume->m_uBlockSideLengthPower)); - const uint16_t uZPosInBlock = static_cast(this->mZPosInVolume - (uZBlock << this->mVolume->m_uBlockSideLengthPower)); - - const uint32_t uVoxelIndexInBlock = uXPosInBlock + - uYPosInBlock * this->mVolume->m_uBlockSideLength + - uZPosInBlock * this->mVolume->m_uBlockSideLength * this->mVolume->m_uBlockSideLength; - - Block* pUncompressedCurrentBlock = this->mVolume->getUncompressedBlock(uXBlock, uYBlock, uZBlock); - - mCurrentVoxel = pUncompressedCurrentBlock->m_tUncompressedData + uVoxelIndexInBlock; - } - else - { - mCurrentVoxel = 0; - } - } - - /** - * \details - * - * This function checks that the current voxel position that you're trying - * to set is not outside the volume. If it is, this function returns - * \a false, otherwise it will return \a true. - * - * \param tValue The value to set to voxel to - */ - template - bool SimpleVolume::Sampler::setVoxel(VoxelType tValue) - { - if(this->m_bIsCurrentPositionValidInX && this->m_bIsCurrentPositionValidInY && this->m_bIsCurrentPositionValidInZ) - { - *mCurrentVoxel = tValue; - return true; - } - else - { - return false; - } - } - - template - void SimpleVolume::Sampler::movePositiveX(void) - { - // We'll need this in a moment... - bool bIsOldPositionValid = this->isCurrentPositionValid(); - - // Base version updates position and validity flags. - BaseVolume::template Sampler< SimpleVolume >::movePositiveX(); - - // Then we update the voxel pointer - if((this->isCurrentPositionValid()) && bIsOldPositionValid && ((this->mXPosInVolume) % this->mVolume->m_uBlockSideLength != 0)) - { - //No need to compute new block. - ++mCurrentVoxel; - } - else - { - //We've hit the block boundary. Just calling setPosition() is the easiest way to resolve this. - setPosition(this->mXPosInVolume, this->mYPosInVolume, this->mZPosInVolume); - } - } - - template - void SimpleVolume::Sampler::movePositiveY(void) - { - // We'll need this in a moment... - bool bIsOldPositionValid = this->isCurrentPositionValid(); - - // Base version updates position and validity flags. - BaseVolume::template Sampler< SimpleVolume >::movePositiveY(); - - // Then we update the voxel pointer - if((this->isCurrentPositionValid()) && bIsOldPositionValid && ((this->mYPosInVolume) % this->mVolume->m_uBlockSideLength != 0)) - { - //No need to compute new block. - mCurrentVoxel += this->mVolume->m_uBlockSideLength; - } - else - { - //We've hit the block boundary. Just calling setPosition() is the easiest way to resolve this. - setPosition(this->mXPosInVolume, this->mYPosInVolume, this->mZPosInVolume); - } - } - - template - void SimpleVolume::Sampler::movePositiveZ(void) - { - // We'll need this in a moment... - bool bIsOldPositionValid = this->isCurrentPositionValid(); - - // Base version updates position and validity flags. - BaseVolume::template Sampler< SimpleVolume >::movePositiveZ(); - - // Then we update the voxel pointer - if((this->isCurrentPositionValid()) && bIsOldPositionValid && ((this->mZPosInVolume) % this->mVolume->m_uBlockSideLength != 0)) - { - //No need to compute new block. - mCurrentVoxel += this->mVolume->m_uBlockSideLength * this->mVolume->m_uBlockSideLength; - } - else - { - //We've hit the block boundary. Just calling setPosition() is the easiest way to resolve this. - setPosition(this->mXPosInVolume, this->mYPosInVolume, this->mZPosInVolume); - } - } - - template - void SimpleVolume::Sampler::moveNegativeX(void) - { - // We'll need this in a moment... - bool bIsOldPositionValid = this->isCurrentPositionValid(); - - // Base version updates position and validity flags. - BaseVolume::template Sampler< SimpleVolume >::moveNegativeX(); - - // Then we update the voxel pointer - if((this->isCurrentPositionValid()) && bIsOldPositionValid && ((this->mXPosInVolume + 1) % this->mVolume->m_uBlockSideLength != 0)) - { - //No need to compute new block. - --mCurrentVoxel; - } - else - { - //We've hit the block boundary. Just calling setPosition() is the easiest way to resolve this. - setPosition(this->mXPosInVolume, this->mYPosInVolume, this->mZPosInVolume); - } - } - - template - void SimpleVolume::Sampler::moveNegativeY(void) - { - // We'll need this in a moment... - bool bIsOldPositionValid = this->isCurrentPositionValid(); - - // Base version updates position and validity flags. - BaseVolume::template Sampler< SimpleVolume >::moveNegativeY(); - - // Then we update the voxel pointer - if((this->isCurrentPositionValid()) && bIsOldPositionValid && ((this->mYPosInVolume + 1) % this->mVolume->m_uBlockSideLength != 0)) - { - //No need to compute new block. - mCurrentVoxel -= this->mVolume->m_uBlockSideLength; - } - else - { - //We've hit the block boundary. Just calling setPosition() is the easiest way to resolve this. - setPosition(this->mXPosInVolume, this->mYPosInVolume, this->mZPosInVolume); - } - } - - template - void SimpleVolume::Sampler::moveNegativeZ(void) - { - // We'll need this in a moment... - bool bIsOldPositionValid = this->isCurrentPositionValid(); - - // Base version updates position and validity flags. - BaseVolume::template Sampler< SimpleVolume >::moveNegativeZ(); - - // Then we update the voxel pointer - if((this->isCurrentPositionValid()) && bIsOldPositionValid && ((this->mZPosInVolume + 1) % this->mVolume->m_uBlockSideLength != 0)) - { - //No need to compute new block. - mCurrentVoxel -= this->mVolume->m_uBlockSideLength * this->mVolume->m_uBlockSideLength; - } - else - { - //We've hit the block boundary. Just calling setPosition() is the easiest way to resolve this. - setPosition(this->mXPosInVolume, this->mYPosInVolume, this->mZPosInVolume); - } - } - - template - VoxelType SimpleVolume::Sampler::peekVoxel1nx1ny1nz(void) const - { - if((this->isCurrentPositionValid()) && CAN_GO_NEG_X(this->mXPosInVolume) && CAN_GO_NEG_Y(this->mYPosInVolume) && CAN_GO_NEG_Z(this->mZPosInVolume) ) - { - return *(mCurrentVoxel - 1 - this->mVolume->m_uBlockSideLength - this->mVolume->m_uBlockSideLength*this->mVolume->m_uBlockSideLength); - } - return this->mVolume->getVoxel(this->mXPosInVolume-1,this->mYPosInVolume-1,this->mZPosInVolume-1,this->m_eWrapMode, this->m_tBorder); - } - - template - VoxelType SimpleVolume::Sampler::peekVoxel1nx1ny0pz(void) const - { - if((this->isCurrentPositionValid()) && CAN_GO_NEG_X(this->mXPosInVolume) && CAN_GO_NEG_Y(this->mYPosInVolume) ) - { - return *(mCurrentVoxel - 1 - this->mVolume->m_uBlockSideLength); - } - return this->mVolume->getVoxel(this->mXPosInVolume-1,this->mYPosInVolume-1,this->mZPosInVolume,this->m_eWrapMode, this->m_tBorder); - } - - template - VoxelType SimpleVolume::Sampler::peekVoxel1nx1ny1pz(void) const - { - if((this->isCurrentPositionValid()) && CAN_GO_NEG_X(this->mXPosInVolume) && CAN_GO_NEG_Y(this->mYPosInVolume) && CAN_GO_POS_Z(this->mZPosInVolume) ) - { - return *(mCurrentVoxel - 1 - this->mVolume->m_uBlockSideLength + this->mVolume->m_uBlockSideLength*this->mVolume->m_uBlockSideLength); - } - return this->mVolume->getVoxel(this->mXPosInVolume-1,this->mYPosInVolume-1,this->mZPosInVolume+1,this->m_eWrapMode, this->m_tBorder); - } - - template - VoxelType SimpleVolume::Sampler::peekVoxel1nx0py1nz(void) const - { - if((this->isCurrentPositionValid()) && CAN_GO_NEG_X(this->mXPosInVolume) && CAN_GO_NEG_Z(this->mZPosInVolume) ) - { - return *(mCurrentVoxel - 1 - this->mVolume->m_uBlockSideLength*this->mVolume->m_uBlockSideLength); - } - return this->mVolume->getVoxel(this->mXPosInVolume-1,this->mYPosInVolume,this->mZPosInVolume-1,this->m_eWrapMode, this->m_tBorder); - } - - template - VoxelType SimpleVolume::Sampler::peekVoxel1nx0py0pz(void) const - { - if((this->isCurrentPositionValid()) && CAN_GO_NEG_X(this->mXPosInVolume) ) - { - return *(mCurrentVoxel - 1); - } - return this->mVolume->getVoxel(this->mXPosInVolume-1,this->mYPosInVolume,this->mZPosInVolume,this->m_eWrapMode, this->m_tBorder); - } - - template - VoxelType SimpleVolume::Sampler::peekVoxel1nx0py1pz(void) const - { - if((this->isCurrentPositionValid()) && CAN_GO_NEG_X(this->mXPosInVolume) && CAN_GO_POS_Z(this->mZPosInVolume) ) - { - return *(mCurrentVoxel - 1 + this->mVolume->m_uBlockSideLength*this->mVolume->m_uBlockSideLength); - } - return this->mVolume->getVoxel(this->mXPosInVolume-1,this->mYPosInVolume,this->mZPosInVolume+1,this->m_eWrapMode, this->m_tBorder); - } - - template - VoxelType SimpleVolume::Sampler::peekVoxel1nx1py1nz(void) const - { - if((this->isCurrentPositionValid()) && CAN_GO_NEG_X(this->mXPosInVolume) && CAN_GO_POS_Y(this->mYPosInVolume) && CAN_GO_NEG_Z(this->mZPosInVolume) ) - { - return *(mCurrentVoxel - 1 + this->mVolume->m_uBlockSideLength - this->mVolume->m_uBlockSideLength*this->mVolume->m_uBlockSideLength); - } - return this->mVolume->getVoxel(this->mXPosInVolume-1,this->mYPosInVolume+1,this->mZPosInVolume-1,this->m_eWrapMode, this->m_tBorder); - } - - template - VoxelType SimpleVolume::Sampler::peekVoxel1nx1py0pz(void) const - { - if((this->isCurrentPositionValid()) && CAN_GO_NEG_X(this->mXPosInVolume) && CAN_GO_POS_Y(this->mYPosInVolume) ) - { - return *(mCurrentVoxel - 1 + this->mVolume->m_uBlockSideLength); - } - return this->mVolume->getVoxel(this->mXPosInVolume-1,this->mYPosInVolume+1,this->mZPosInVolume,this->m_eWrapMode, this->m_tBorder); - } - - template - VoxelType SimpleVolume::Sampler::peekVoxel1nx1py1pz(void) const - { - if((this->isCurrentPositionValid()) && CAN_GO_NEG_X(this->mXPosInVolume) && CAN_GO_POS_Y(this->mYPosInVolume) && CAN_GO_POS_Z(this->mZPosInVolume) ) - { - return *(mCurrentVoxel - 1 + this->mVolume->m_uBlockSideLength + this->mVolume->m_uBlockSideLength*this->mVolume->m_uBlockSideLength); - } - return this->mVolume->getVoxel(this->mXPosInVolume-1,this->mYPosInVolume+1,this->mZPosInVolume+1,this->m_eWrapMode, this->m_tBorder); - } - - ////////////////////////////////////////////////////////////////////////// - - template - VoxelType SimpleVolume::Sampler::peekVoxel0px1ny1nz(void) const - { - if((this->isCurrentPositionValid()) && CAN_GO_NEG_Y(this->mYPosInVolume) && CAN_GO_NEG_Z(this->mZPosInVolume) ) - { - return *(mCurrentVoxel - this->mVolume->m_uBlockSideLength - this->mVolume->m_uBlockSideLength*this->mVolume->m_uBlockSideLength); - } - return this->mVolume->getVoxel(this->mXPosInVolume,this->mYPosInVolume-1,this->mZPosInVolume-1,this->m_eWrapMode, this->m_tBorder); - } - - template - VoxelType SimpleVolume::Sampler::peekVoxel0px1ny0pz(void) const - { - if((this->isCurrentPositionValid()) && CAN_GO_NEG_Y(this->mYPosInVolume) ) - { - return *(mCurrentVoxel - this->mVolume->m_uBlockSideLength); - } - return this->mVolume->getVoxel(this->mXPosInVolume,this->mYPosInVolume-1,this->mZPosInVolume,this->m_eWrapMode, this->m_tBorder); - } - - template - VoxelType SimpleVolume::Sampler::peekVoxel0px1ny1pz(void) const - { - if((this->isCurrentPositionValid()) && CAN_GO_NEG_Y(this->mYPosInVolume) && CAN_GO_POS_Z(this->mZPosInVolume) ) - { - return *(mCurrentVoxel - this->mVolume->m_uBlockSideLength + this->mVolume->m_uBlockSideLength*this->mVolume->m_uBlockSideLength); - } - return this->mVolume->getVoxel(this->mXPosInVolume,this->mYPosInVolume-1,this->mZPosInVolume+1,this->m_eWrapMode, this->m_tBorder); - } - - template - VoxelType SimpleVolume::Sampler::peekVoxel0px0py1nz(void) const - { - if((this->isCurrentPositionValid()) && CAN_GO_NEG_Z(this->mZPosInVolume) ) - { - return *(mCurrentVoxel - this->mVolume->m_uBlockSideLength*this->mVolume->m_uBlockSideLength); - } - return this->mVolume->getVoxel(this->mXPosInVolume,this->mYPosInVolume,this->mZPosInVolume-1,this->m_eWrapMode, this->m_tBorder); - } - - template - VoxelType SimpleVolume::Sampler::peekVoxel0px0py0pz(void) const - { - if((this->isCurrentPositionValid())) - { - return *mCurrentVoxel; - } - return this->mVolume->getVoxel(this->mXPosInVolume, this->mYPosInVolume, this->mZPosInVolume,this->m_eWrapMode, this->m_tBorder); - } - - template - VoxelType SimpleVolume::Sampler::peekVoxel0px0py1pz(void) const - { - if((this->isCurrentPositionValid()) && CAN_GO_POS_Z(this->mZPosInVolume) ) - { - return *(mCurrentVoxel + this->mVolume->m_uBlockSideLength*this->mVolume->m_uBlockSideLength); - } - return this->mVolume->getVoxel(this->mXPosInVolume,this->mYPosInVolume,this->mZPosInVolume+1,this->m_eWrapMode, this->m_tBorder); - } - - template - VoxelType SimpleVolume::Sampler::peekVoxel0px1py1nz(void) const - { - if((this->isCurrentPositionValid()) && CAN_GO_POS_Y(this->mYPosInVolume) && CAN_GO_NEG_Z(this->mZPosInVolume) ) - { - return *(mCurrentVoxel + this->mVolume->m_uBlockSideLength - this->mVolume->m_uBlockSideLength*this->mVolume->m_uBlockSideLength); - } - return this->mVolume->getVoxel(this->mXPosInVolume,this->mYPosInVolume+1,this->mZPosInVolume-1,this->m_eWrapMode, this->m_tBorder); - } - - template - VoxelType SimpleVolume::Sampler::peekVoxel0px1py0pz(void) const - { - if((this->isCurrentPositionValid()) && CAN_GO_POS_Y(this->mYPosInVolume) ) - { - return *(mCurrentVoxel + this->mVolume->m_uBlockSideLength); - } - return this->mVolume->getVoxel(this->mXPosInVolume,this->mYPosInVolume+1,this->mZPosInVolume,this->m_eWrapMode, this->m_tBorder); - } - - template - VoxelType SimpleVolume::Sampler::peekVoxel0px1py1pz(void) const - { - if((this->isCurrentPositionValid()) && CAN_GO_POS_Y(this->mYPosInVolume) && CAN_GO_POS_Z(this->mZPosInVolume) ) - { - return *(mCurrentVoxel + this->mVolume->m_uBlockSideLength + this->mVolume->m_uBlockSideLength*this->mVolume->m_uBlockSideLength); - } - return this->mVolume->getVoxel(this->mXPosInVolume,this->mYPosInVolume+1,this->mZPosInVolume+1,this->m_eWrapMode, this->m_tBorder); - } - - ////////////////////////////////////////////////////////////////////////// - - template - VoxelType SimpleVolume::Sampler::peekVoxel1px1ny1nz(void) const - { - if((this->isCurrentPositionValid()) && CAN_GO_POS_X(this->mXPosInVolume) && CAN_GO_NEG_Y(this->mYPosInVolume) && CAN_GO_NEG_Z(this->mZPosInVolume) ) - { - return *(mCurrentVoxel + 1 - this->mVolume->m_uBlockSideLength - this->mVolume->m_uBlockSideLength*this->mVolume->m_uBlockSideLength); - } - return this->mVolume->getVoxel(this->mXPosInVolume+1,this->mYPosInVolume-1,this->mZPosInVolume-1,this->m_eWrapMode, this->m_tBorder); - } - - template - VoxelType SimpleVolume::Sampler::peekVoxel1px1ny0pz(void) const - { - if((this->isCurrentPositionValid()) && CAN_GO_POS_X(this->mXPosInVolume) && CAN_GO_NEG_Y(this->mYPosInVolume) ) - { - return *(mCurrentVoxel + 1 - this->mVolume->m_uBlockSideLength); - } - return this->mVolume->getVoxel(this->mXPosInVolume+1,this->mYPosInVolume-1,this->mZPosInVolume,this->m_eWrapMode, this->m_tBorder); - } - - template - VoxelType SimpleVolume::Sampler::peekVoxel1px1ny1pz(void) const - { - if((this->isCurrentPositionValid()) && CAN_GO_POS_X(this->mXPosInVolume) && CAN_GO_NEG_Y(this->mYPosInVolume) && CAN_GO_POS_Z(this->mZPosInVolume) ) - { - return *(mCurrentVoxel + 1 - this->mVolume->m_uBlockSideLength + this->mVolume->m_uBlockSideLength*this->mVolume->m_uBlockSideLength); - } - return this->mVolume->getVoxel(this->mXPosInVolume+1,this->mYPosInVolume-1,this->mZPosInVolume+1,this->m_eWrapMode, this->m_tBorder); - } - - template - VoxelType SimpleVolume::Sampler::peekVoxel1px0py1nz(void) const - { - if((this->isCurrentPositionValid()) && CAN_GO_POS_X(this->mXPosInVolume) && CAN_GO_NEG_Z(this->mZPosInVolume) ) - { - return *(mCurrentVoxel + 1 - this->mVolume->m_uBlockSideLength*this->mVolume->m_uBlockSideLength); - } - return this->mVolume->getVoxel(this->mXPosInVolume+1,this->mYPosInVolume,this->mZPosInVolume-1,this->m_eWrapMode, this->m_tBorder); - } - - template - VoxelType SimpleVolume::Sampler::peekVoxel1px0py0pz(void) const - { - if((this->isCurrentPositionValid()) && CAN_GO_POS_X(this->mXPosInVolume) ) - { - return *(mCurrentVoxel + 1); - } - return this->mVolume->getVoxel(this->mXPosInVolume+1,this->mYPosInVolume,this->mZPosInVolume,this->m_eWrapMode, this->m_tBorder); - } - - template - VoxelType SimpleVolume::Sampler::peekVoxel1px0py1pz(void) const - { - if((this->isCurrentPositionValid()) && CAN_GO_POS_X(this->mXPosInVolume) && CAN_GO_POS_Z(this->mZPosInVolume) ) - { - return *(mCurrentVoxel + 1 + this->mVolume->m_uBlockSideLength*this->mVolume->m_uBlockSideLength); - } - return this->mVolume->getVoxel(this->mXPosInVolume+1,this->mYPosInVolume,this->mZPosInVolume+1,this->m_eWrapMode, this->m_tBorder); - } - - template - VoxelType SimpleVolume::Sampler::peekVoxel1px1py1nz(void) const - { - if((this->isCurrentPositionValid()) && CAN_GO_POS_X(this->mXPosInVolume) && CAN_GO_POS_Y(this->mYPosInVolume) && CAN_GO_NEG_Z(this->mZPosInVolume) ) - { - return *(mCurrentVoxel + 1 + this->mVolume->m_uBlockSideLength - this->mVolume->m_uBlockSideLength*this->mVolume->m_uBlockSideLength); - } - return this->mVolume->getVoxel(this->mXPosInVolume+1,this->mYPosInVolume+1,this->mZPosInVolume-1,this->m_eWrapMode, this->m_tBorder); - } - - template - VoxelType SimpleVolume::Sampler::peekVoxel1px1py0pz(void) const - { - if((this->isCurrentPositionValid()) && CAN_GO_POS_X(this->mXPosInVolume) && CAN_GO_POS_Y(this->mYPosInVolume) ) - { - return *(mCurrentVoxel + 1 + this->mVolume->m_uBlockSideLength); - } - return this->mVolume->getVoxel(this->mXPosInVolume+1,this->mYPosInVolume+1,this->mZPosInVolume,this->m_eWrapMode, this->m_tBorder); - } - - template - VoxelType SimpleVolume::Sampler::peekVoxel1px1py1pz(void) const - { - if((this->isCurrentPositionValid()) && CAN_GO_POS_X(this->mXPosInVolume) && CAN_GO_POS_Y(this->mYPosInVolume) && CAN_GO_POS_Z(this->mZPosInVolume) ) - { - return *(mCurrentVoxel + 1 + this->mVolume->m_uBlockSideLength + this->mVolume->m_uBlockSideLength*this->mVolume->m_uBlockSideLength); - } - return this->mVolume->getVoxel(this->mXPosInVolume+1,this->mYPosInVolume+1,this->mZPosInVolume+1,this->m_eWrapMode, this->m_tBorder); - } -} - -#undef CAN_GO_NEG_X -#undef CAN_GO_POS_X -#undef CAN_GO_NEG_Y -#undef CAN_GO_POS_Y -#undef CAN_GO_NEG_Z -#undef CAN_GO_POS_Z diff --git a/library/PolyVoxCore/include/PolyVoxCore/UncompressedBlock.h b/library/PolyVoxCore/include/PolyVoxCore/UncompressedBlock.h index f65b0dc2..abc3ca63 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/UncompressedBlock.h +++ b/library/PolyVoxCore/include/PolyVoxCore/UncompressedBlock.h @@ -32,7 +32,7 @@ namespace PolyVox template class UncompressedBlock { - friend class LargeVolume; + friend class PagedVolume; public: UncompressedBlock(Vector3DInt32 v3dPosition, uint16_t uSideLength, Pager* pPager = nullptr); @@ -54,7 +54,7 @@ namespace PolyVox /// Private assignment operator to prevent accisdental copying UncompressedBlock& operator=(const UncompressedBlock& /*rhs*/) {}; - // This is updated by the LargeVolume and used to discard the least recently used blocks. + // This is updated by the PagedVolume and used to discard the least recently used blocks. uint32_t m_uBlockLastAccessed; // This is so we can tell whether a uncompressed block has to be recompressed and whether From 6f08184fcbcff04ad0efa5401462947a537abb0a Mon Sep 17 00:00:00 2001 From: David Williams Date: Fri, 19 Sep 2014 17:02:44 +0200 Subject: [PATCH 26/42] Fixed initialization of blocks. --- .../PolyVoxCore/include/PolyVoxCore/UncompressedBlock.inl | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/library/PolyVoxCore/include/PolyVoxCore/UncompressedBlock.inl b/library/PolyVoxCore/include/PolyVoxCore/UncompressedBlock.inl index 083d763e..4e038024 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/UncompressedBlock.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/UncompressedBlock.inl @@ -54,6 +54,11 @@ namespace PolyVox // Page the data in m_pPager->pageIn(reg, this); } + else + { + // Just fill with zeros + std::fill(m_tData, m_tData + uNoOfVoxels, VoxelType()); + } // We'll use this later to decide if data needs to be paged out again. m_bDataModified = false; From 4779f917871562f585085a5a2dd30015d235775b Mon Sep 17 00:00:00 2001 From: David Williams Date: Sat, 20 Sep 2014 14:49:12 +0200 Subject: [PATCH 27/42] TestVolume unit test now tests PagedVolume rather than SimpleVolume and LargeVolume. --- tests/testvolume.cpp | 141 ++++++++----------------------------------- tests/testvolume.h | 31 +++------- 2 files changed, 34 insertions(+), 138 deletions(-) diff --git a/tests/testvolume.cpp b/tests/testvolume.cpp index 28703eb3..4007c6ed 100644 --- a/tests/testvolume.cpp +++ b/tests/testvolume.cpp @@ -24,9 +24,8 @@ freely, subject to the following restrictions: #include "testvolume.h" #include "PolyVoxCore/FilePager.h" -#include "PolyVoxCore/LargeVolume.h" +#include "PolyVoxCore/PagedVolume.h" #include "PolyVoxCore/RawVolume.h" -#include "PolyVoxCore/SimpleVolume.h" #include #include @@ -274,10 +273,9 @@ TestVolume::TestVolume() //Create the volumes m_pRawVolume = new RawVolume(region); - m_pSimpleVolume = new SimpleVolume(region); - m_pLargeVolume = new LargeVolume(region, m_pFilePager, 32); + m_pPagedVolume = new LargeVolume(region, m_pFilePager, 32); - m_pLargeVolume->setMemoryUsageLimit(1 * 1024 * 1024); + m_pPagedVolume->setMemoryUsageLimit(1 * 1024 * 1024); //Fill the volume with some data for(int z = region.getLowerZ(); z <= region.getUpperZ(); z++) @@ -288,8 +286,7 @@ TestVolume::TestVolume() { int32_t value = x + y + z; m_pRawVolume->setVoxelAt(x, y, z, value); - m_pSimpleVolume->setVoxelAt(x, y, z, value); - m_pLargeVolume->setVoxelAt(x, y, z, value); + m_pPagedVolume->setVoxelAt(x, y, z, value); } } } @@ -298,18 +295,16 @@ TestVolume::TestVolume() TestVolume::~TestVolume() { delete m_pRawVolume; - delete m_pSimpleVolume; - delete m_pLargeVolume; + delete m_pPagedVolume; delete m_pFilePager; - delete m_pBlockCompressor; } /* * RawVolume Tests */ -/*void TestVolume::testRawVolumeDirectAccessAllInternalForwards() +void TestVolume::testRawVolumeDirectAccessAllInternalForwards() { int32_t result = 0; @@ -395,176 +390,88 @@ void TestVolume::testRawVolumeSamplersWithExternalBackwards() result = testSamplersWithWrappingBackwards(m_pRawVolume, -1, -3, -2, 2, 5, 4); } QCOMPARE(result, static_cast(-769775893)); -}*/ +} /* - * SimpleVolume Tests + * PagedVolume Tests */ -/*void TestVolume::testSimpleVolumeDirectAccessAllInternalForwards() +void TestVolume::testPagedVolumeDirectAccessAllInternalForwards() { int32_t result = 0; QBENCHMARK { - result = testDirectAccessWithWrappingForwards(m_pSimpleVolume, 4, 2, 2, -3, -1, -2); + result = testDirectAccessWithWrappingForwards(m_pPagedVolume, 4, 2, 2, -3, -1, -2); } QCOMPARE(result, static_cast(1004598054)); } -void TestVolume::testSimpleVolumeSamplersAllInternalForwards() +void TestVolume::testPagedVolumeSamplersAllInternalForwards() { int32_t result = 0; QBENCHMARK { - result = testSamplersWithWrappingForwards(m_pSimpleVolume, 4, 2, 2, -3, -1, -2); + result = testSamplersWithWrappingForwards(m_pPagedVolume, 4, 2, 2, -3, -1, -2); } QCOMPARE(result, static_cast(1004598054)); } -void TestVolume::testSimpleVolumeDirectAccessWithExternalForwards() +void TestVolume::testPagedVolumeDirectAccessWithExternalForwards() { int32_t result = 0; QBENCHMARK { - result = testDirectAccessWithWrappingForwards(m_pSimpleVolume, -1, -3, -2, 2, 5, 4); + result = testDirectAccessWithWrappingForwards(m_pPagedVolume, -1, -3, -2, 2, 5, 4); } QCOMPARE(result, static_cast(-928601007)); } -void TestVolume::testSimpleVolumeSamplersWithExternalForwards() +void TestVolume::testPagedVolumeSamplersWithExternalForwards() { int32_t result = 0; QBENCHMARK { - result = testSamplersWithWrappingForwards(m_pSimpleVolume, -1, -3, -2, 2, 5, 4); + result = testSamplersWithWrappingForwards(m_pPagedVolume, -1, -3, -2, 2, 5, 4); } QCOMPARE(result, static_cast(-928601007)); } -void TestVolume::testSimpleVolumeDirectAccessAllInternalBackwards() +void TestVolume::testPagedVolumeDirectAccessAllInternalBackwards() { int32_t result = 0; QBENCHMARK { - result = testDirectAccessWithWrappingBackwards(m_pSimpleVolume, 4, 2, 2, -3, -1, -2); + result = testDirectAccessWithWrappingBackwards(m_pPagedVolume, 4, 2, 2, -3, -1, -2); } QCOMPARE(result, static_cast(-269366578)); } -void TestVolume::testSimpleVolumeSamplersAllInternalBackwards() +void TestVolume::testPagedVolumeSamplersAllInternalBackwards() { int32_t result = 0; QBENCHMARK { - result = testSamplersWithWrappingBackwards(m_pSimpleVolume, 4, 2, 2, -3, -1, -2); + result = testSamplersWithWrappingBackwards(m_pPagedVolume, 4, 2, 2, -3, -1, -2); } QCOMPARE(result, static_cast(-269366578)); } -void TestVolume::testSimpleVolumeDirectAccessWithExternalBackwards() +void TestVolume::testPagedVolumeDirectAccessWithExternalBackwards() { int32_t result = 0; QBENCHMARK { - result = testDirectAccessWithWrappingBackwards(m_pSimpleVolume, -1, -3, -2, 2, 5, 4); + result = testDirectAccessWithWrappingBackwards(m_pPagedVolume, -1, -3, -2, 2, 5, 4); } QCOMPARE(result, static_cast(-769775893)); } -void TestVolume::testSimpleVolumeSamplersWithExternalBackwards() +void TestVolume::testPagedVolumeSamplersWithExternalBackwards() { int32_t result = 0; QBENCHMARK { - result = testSamplersWithWrappingBackwards(m_pSimpleVolume, -1, -3, -2, 2, 5, 4); - } - QCOMPARE(result, static_cast(-769775893)); -}*/ - -/* - * LargeVolume Tests - */ - -void TestVolume::testLargeVolumeDirectAccessAllInternalForwards() -{ - int32_t result = 0; - QBENCHMARK - { - result = testDirectAccessWithWrappingForwards(m_pLargeVolume, 4, 2, 2, -3, -1, -2); - } - QCOMPARE(result, static_cast(1004598054)); -} - -void TestVolume::testLargeVolumeSamplersAllInternalForwards() -{ - int32_t result = 0; - - QBENCHMARK - { - result = testSamplersWithWrappingForwards(m_pLargeVolume, 4, 2, 2, -3, -1, -2); - } - QCOMPARE(result, static_cast(1004598054)); -} - -void TestVolume::testLargeVolumeDirectAccessWithExternalForwards() -{ - int32_t result = 0; - QBENCHMARK - { - result = testDirectAccessWithWrappingForwards(m_pLargeVolume, -1, -3, -2, 2, 5, 4); - } - QCOMPARE(result, static_cast(-928601007)); -} - -void TestVolume::testLargeVolumeSamplersWithExternalForwards() -{ - int32_t result = 0; - - QBENCHMARK - { - result = testSamplersWithWrappingForwards(m_pLargeVolume, -1, -3, -2, 2, 5, 4); - } - QCOMPARE(result, static_cast(-928601007)); -} - -void TestVolume::testLargeVolumeDirectAccessAllInternalBackwards() -{ - int32_t result = 0; - QBENCHMARK - { - result = testDirectAccessWithWrappingBackwards(m_pLargeVolume, 4, 2, 2, -3, -1, -2); - } - QCOMPARE(result, static_cast(-269366578)); -} - -void TestVolume::testLargeVolumeSamplersAllInternalBackwards() -{ - int32_t result = 0; - - QBENCHMARK - { - result = testSamplersWithWrappingBackwards(m_pLargeVolume, 4, 2, 2, -3, -1, -2); - } - QCOMPARE(result, static_cast(-269366578)); -} - -void TestVolume::testLargeVolumeDirectAccessWithExternalBackwards() -{ - int32_t result = 0; - QBENCHMARK - { - result = testDirectAccessWithWrappingBackwards(m_pLargeVolume, -1, -3, -2, 2, 5, 4); - } - QCOMPARE(result, static_cast(-769775893)); -} - -void TestVolume::testLargeVolumeSamplersWithExternalBackwards() -{ - int32_t result = 0; - - QBENCHMARK - { - result = testSamplersWithWrappingBackwards(m_pLargeVolume, -1, -3, -2, 2, 5, 4); + result = testSamplersWithWrappingBackwards(m_pPagedVolume, -1, -3, -2, 2, 5, 4); } QCOMPARE(result, static_cast(-769775893)); } diff --git a/tests/testvolume.h b/tests/testvolume.h index 57390641..ea1a1d0b 100644 --- a/tests/testvolume.h +++ b/tests/testvolume.h @@ -37,7 +37,7 @@ public: ~TestVolume(); private slots: - /*void testRawVolumeDirectAccessAllInternalForwards(); + void testRawVolumeDirectAccessAllInternalForwards(); void testRawVolumeSamplersAllInternalForwards(); void testRawVolumeDirectAccessWithExternalForwards(); void testRawVolumeSamplersWithExternalForwards(); @@ -46,31 +46,20 @@ private slots: void testRawVolumeDirectAccessWithExternalBackwards(); void testRawVolumeSamplersWithExternalBackwards(); - void testSimpleVolumeDirectAccessAllInternalForwards(); - void testSimpleVolumeSamplersAllInternalForwards(); - void testSimpleVolumeDirectAccessWithExternalForwards(); - void testSimpleVolumeSamplersWithExternalForwards(); - void testSimpleVolumeDirectAccessAllInternalBackwards(); - void testSimpleVolumeSamplersAllInternalBackwards(); - void testSimpleVolumeDirectAccessWithExternalBackwards(); - void testSimpleVolumeSamplersWithExternalBackwards();*/ - - void testLargeVolumeDirectAccessAllInternalForwards(); - void testLargeVolumeSamplersAllInternalForwards(); - void testLargeVolumeDirectAccessWithExternalForwards(); - void testLargeVolumeSamplersWithExternalForwards(); - void testLargeVolumeDirectAccessAllInternalBackwards(); - void testLargeVolumeSamplersAllInternalBackwards(); - void testLargeVolumeDirectAccessWithExternalBackwards(); - void testLargeVolumeSamplersWithExternalBackwards(); + void testPagedVolumeDirectAccessAllInternalForwards(); + void testPagedVolumeSamplersAllInternalForwards(); + void testPagedVolumeDirectAccessWithExternalForwards(); + void testPagedVolumeSamplersWithExternalForwards(); + void testPagedVolumeDirectAccessAllInternalBackwards(); + void testPagedVolumeSamplersAllInternalBackwards(); + void testPagedVolumeDirectAccessWithExternalBackwards(); + void testPagedVolumeSamplersWithExternalBackwards(); private: - PolyVox::BlockCompressor* m_pBlockCompressor; PolyVox::FilePager* m_pFilePager; PolyVox::RawVolume* m_pRawVolume; - PolyVox::SimpleVolume* m_pSimpleVolume; - PolyVox::LargeVolume* m_pLargeVolume; + PolyVox::PagedVolume* m_pPagedVolume; }; #endif From 880dcd8645f21eb9638c74bfc0ffceb01c5a50ba Mon Sep 17 00:00:00 2001 From: David Williams Date: Sat, 20 Sep 2014 17:17:18 +0200 Subject: [PATCH 28/42] Fixed incorrect messages about temp files not being deleted. --- examples/OpenGL/main.cpp | 2 +- library/PolyVoxCore/include/PolyVoxCore/FilePager.h | 11 +++++++++-- tests/TestSurfaceExtractor.cpp | 4 ++-- tests/testvolume.cpp | 2 +- 4 files changed, 13 insertions(+), 6 deletions(-) diff --git a/examples/OpenGL/main.cpp b/examples/OpenGL/main.cpp index 1c51f511..9a36fac8 100644 --- a/examples/OpenGL/main.cpp +++ b/examples/OpenGL/main.cpp @@ -50,7 +50,7 @@ const int32_t g_uVolumeSideLength = 128; int main(int argc, char *argv[]) { - FilePager* pager = new FilePager("./"); + FilePager* pager = new FilePager("."); LargeVolume volData(PolyVox::Region(Vector3DInt32(0, 0, 0), Vector3DInt32(g_uVolumeSideLength - 1, g_uVolumeSideLength - 1, g_uVolumeSideLength - 1)), pager); //Make our volume contain a sphere in the center. diff --git a/library/PolyVoxCore/include/PolyVoxCore/FilePager.h b/library/PolyVoxCore/include/PolyVoxCore/FilePager.h index 8e3e36ed..6f466204 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/FilePager.h +++ b/library/PolyVoxCore/include/PolyVoxCore/FilePager.h @@ -46,10 +46,17 @@ namespace PolyVox { public: /// Constructor - FilePager(const std::string& strFolderName) + FilePager(const std::string& strFolderName = ".") :Pager() ,m_strFolderName(strFolderName) { + // Add the trailing slash, assuming the user dind't already do it. + if ((m_strFolderName.back() != '/') && (m_strFolderName.back() != '\\')) + { + m_strFolderName.append("/"); + } + + // Build a unique prefix to avoid multiple pagers using the same filenames. srand(static_cast(time(0))); int iRandomValue = rand(); @@ -63,7 +70,7 @@ namespace PolyVox { for(std::vector::iterator iter = m_vecCreatedFiles.begin(); iter < m_vecCreatedFiles.end(); iter++) { - POLYVOX_LOG_WARNING_IF(!std::remove(iter->c_str()), "Failed to delete '" << *iter << "' when destroying FilePager"); + POLYVOX_LOG_WARNING_IF(std::remove(iter->c_str()) != 0, "Failed to delete '" << *iter << "' when destroying FilePager"); } m_vecCreatedFiles.clear(); diff --git a/tests/TestSurfaceExtractor.cpp b/tests/TestSurfaceExtractor.cpp index fc7f92ba..599281bb 100644 --- a/tests/TestSurfaceExtractor.cpp +++ b/tests/TestSurfaceExtractor.cpp @@ -102,7 +102,7 @@ VolumeType* createAndFillVolume(void) { const int32_t uVolumeSideLength = 64; - FilePager* pager = new FilePager("./"); + FilePager* pager = new FilePager("."); //Create empty volume VolumeType* volData = new VolumeType(Region(Vector3DInt32(0, 0, 0), Vector3DInt32(uVolumeSideLength - 1, uVolumeSideLength - 1, uVolumeSideLength - 1)), pager); @@ -139,7 +139,7 @@ float randomFloat(float a, float b) template VolumeType* createAndFillVolumeWithNoise(int32_t iVolumeSideLength, float minValue, float maxValue) { - FilePager* pager = new FilePager("./"); + FilePager* pager = new FilePager("."); //Create empty volume VolumeType* volData = new VolumeType(Region(Vector3DInt32(0, 0, 0), Vector3DInt32(iVolumeSideLength - 1, iVolumeSideLength - 1, iVolumeSideLength - 1)), pager); diff --git a/tests/testvolume.cpp b/tests/testvolume.cpp index 4007c6ed..a8f8202b 100644 --- a/tests/testvolume.cpp +++ b/tests/testvolume.cpp @@ -269,7 +269,7 @@ TestVolume::TestVolume() { Region region(-57, -31, 12, 64, 96, 131); // Deliberatly awkward size - m_pFilePager = new FilePager("./"); + m_pFilePager = new FilePager("."); //Create the volumes m_pRawVolume = new RawVolume(region); From ede35435a0a8d046185b16215a3559352cdf7e8b Mon Sep 17 00:00:00 2001 From: David Williams Date: Sat, 20 Sep 2014 17:26:57 +0200 Subject: [PATCH 29/42] Renamed UncompressedBlock to Chunk. --- examples/Paging/main.cpp | 6 +- library/PolyVoxCore/CMakeLists.txt | 4 +- .../{UncompressedBlock.h => Chunk.h} | 18 ++--- .../{UncompressedBlock.inl => Chunk.inl} | 20 ++--- .../include/PolyVoxCore/FilePager.h | 4 +- .../include/PolyVoxCore/PagedVolume.h | 10 +-- .../include/PolyVoxCore/PagedVolume.inl | 78 +++++++++---------- .../PolyVoxCore/PagedVolumeSampler.inl | 2 +- .../PolyVoxCore/include/PolyVoxCore/Pager.h | 6 +- 9 files changed, 74 insertions(+), 74 deletions(-) rename library/PolyVoxCore/include/PolyVoxCore/{UncompressedBlock.h => Chunk.h} (83%) rename library/PolyVoxCore/include/PolyVoxCore/{UncompressedBlock.inl => Chunk.inl} (85%) diff --git a/examples/Paging/main.cpp b/examples/Paging/main.cpp index d9359087..576bfb67 100644 --- a/examples/Paging/main.cpp +++ b/examples/Paging/main.cpp @@ -90,11 +90,11 @@ public: /// Destructor virtual ~PerlinNoisePager() {}; - virtual void pageIn(const PolyVox::Region& region, UncompressedBlock* pBlockData) + virtual void pageIn(const PolyVox::Region& region, Chunk* pBlockData) { // FIXME - this isn't a great example... it's a shame we have to hard clode the block size and also create/destroy // a compressor each time. These could at least be moved outside somewhere if we can't fix it in a better way... - //UncompressedBlock block(256); + //Chunk block(256); Perlin perlin(2,2,1,234); @@ -143,7 +143,7 @@ public: //delete compressor; } - virtual void pageOut(const PolyVox::Region& region, UncompressedBlock* /*pBlockData*/) + virtual void pageOut(const PolyVox::Region& region, Chunk* /*pBlockData*/) { std::cout << "warning unloading region: " << region.getLowerCorner() << " -> " << region.getUpperCorner() << std::endl; } diff --git a/library/PolyVoxCore/CMakeLists.txt b/library/PolyVoxCore/CMakeLists.txt index 4e18c75e..286f5e14 100644 --- a/library/PolyVoxCore/CMakeLists.txt +++ b/library/PolyVoxCore/CMakeLists.txt @@ -38,6 +38,8 @@ SET(CORE_INC_FILES include/PolyVoxCore/BaseVolume.h include/PolyVoxCore/BaseVolume.inl include/PolyVoxCore/BaseVolumeSampler.inl + include/PolyVoxCore/Chunk.h + include/PolyVoxCore/Chunk.inl include/PolyVoxCore/CubicSurfaceExtractor.h include/PolyVoxCore/CubicSurfaceExtractor.inl include/PolyVoxCore/DefaultIsQuadNeeded.h @@ -72,8 +74,6 @@ SET(CORE_INC_FILES include/PolyVoxCore/Raycast.inl include/PolyVoxCore/Region.h include/PolyVoxCore/SimpleVolume.h - include/PolyVoxCore/UncompressedBlock.h - include/PolyVoxCore/UncompressedBlock.inl include/PolyVoxCore/Vector.h include/PolyVoxCore/Vector.inl include/PolyVoxCore/Vertex.h diff --git a/library/PolyVoxCore/include/PolyVoxCore/UncompressedBlock.h b/library/PolyVoxCore/include/PolyVoxCore/Chunk.h similarity index 83% rename from library/PolyVoxCore/include/PolyVoxCore/UncompressedBlock.h rename to library/PolyVoxCore/include/PolyVoxCore/Chunk.h index abc3ca63..4f92e458 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/UncompressedBlock.h +++ b/library/PolyVoxCore/include/PolyVoxCore/Chunk.h @@ -21,8 +21,8 @@ freely, subject to the following restrictions: distribution. *******************************************************************************/ -#ifndef __PolyVox_UncompressedBlock_H__ -#define __PolyVox_UncompressedBlock_H__ +#ifndef __PolyVox_Chunk_H__ +#define __PolyVox_Chunk_H__ #include "PolyVoxCore/PolyVoxForwardDeclarations.h" #include "PolyVoxCore/Vector.h" @@ -30,13 +30,13 @@ freely, subject to the following restrictions: namespace PolyVox { template - class UncompressedBlock + class Chunk { friend class PagedVolume; public: - UncompressedBlock(Vector3DInt32 v3dPosition, uint16_t uSideLength, Pager* pPager = nullptr); - ~UncompressedBlock(); + Chunk(Vector3DInt32 v3dPosition, uint16_t uSideLength, Pager* pPager = nullptr); + ~Chunk(); VoxelType* getData(void) const; uint32_t getDataSizeInBytes(void) const; @@ -49,10 +49,10 @@ namespace PolyVox private: /// Private copy constructor to prevent accisdental copying - UncompressedBlock(const UncompressedBlock& /*rhs*/) {}; + Chunk(const Chunk& /*rhs*/) {}; /// Private assignment operator to prevent accisdental copying - UncompressedBlock& operator=(const UncompressedBlock& /*rhs*/) {}; + Chunk& operator=(const Chunk& /*rhs*/) {}; // This is updated by the PagedVolume and used to discard the least recently used blocks. uint32_t m_uBlockLastAccessed; @@ -62,7 +62,7 @@ namespace PolyVox bool m_bDataModified; // Made this private for consistancy with CompressedBlock. - // Users shouldn't really need this for UncompressedBlock anyway. + // Users shouldn't really need this for Chunk anyway. uint32_t calculateSizeInBytes(void); static uint32_t calculateSizeInBytes(uint32_t uSideLength); @@ -74,6 +74,6 @@ namespace PolyVox }; } -#include "PolyVoxCore/UncompressedBlock.inl" +#include "PolyVoxCore/Chunk.inl" #endif diff --git a/library/PolyVoxCore/include/PolyVoxCore/UncompressedBlock.inl b/library/PolyVoxCore/include/PolyVoxCore/Chunk.inl similarity index 85% rename from library/PolyVoxCore/include/PolyVoxCore/UncompressedBlock.inl rename to library/PolyVoxCore/include/PolyVoxCore/Chunk.inl index 4e038024..ad8e255a 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/UncompressedBlock.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/Chunk.inl @@ -26,7 +26,7 @@ freely, subject to the following restrictions: namespace PolyVox { template - UncompressedBlock::UncompressedBlock(Vector3DInt32 v3dPosition, uint16_t uSideLength, Pager* pPager) + Chunk::Chunk(Vector3DInt32 v3dPosition, uint16_t uSideLength, Pager* pPager) :m_uBlockLastAccessed(0) ,m_bDataModified(true) ,m_tData(0) @@ -65,7 +65,7 @@ namespace PolyVox } template - UncompressedBlock::~UncompressedBlock() + Chunk::~Chunk() { if (m_pPager && m_bDataModified) { @@ -82,19 +82,19 @@ namespace PolyVox } template - VoxelType* UncompressedBlock::getData(void) const + VoxelType* Chunk::getData(void) const { return m_tData; } template - uint32_t UncompressedBlock::getDataSizeInBytes(void) const + uint32_t Chunk::getDataSizeInBytes(void) const { return m_uSideLength * m_uSideLength * m_uSideLength * sizeof(VoxelType); } template - VoxelType UncompressedBlock::getVoxel(uint16_t uXPos, uint16_t uYPos, uint16_t uZPos) const + VoxelType Chunk::getVoxel(uint16_t uXPos, uint16_t uYPos, uint16_t uZPos) const { // This code is not usually expected to be called by the user, with the exception of when implementing paging // of uncompressed data. It's a performance critical code path so we use asserts rather than exceptions. @@ -112,13 +112,13 @@ namespace PolyVox } template - VoxelType UncompressedBlock::getVoxel(const Vector3DUint16& v3dPos) const + VoxelType Chunk::getVoxel(const Vector3DUint16& v3dPos) const { return getVoxel(v3dPos.getX(), v3dPos.getY(), v3dPos.getZ()); } template - void UncompressedBlock::setVoxelAt(uint16_t uXPos, uint16_t uYPos, uint16_t uZPos, VoxelType tValue) + void Chunk::setVoxelAt(uint16_t uXPos, uint16_t uYPos, uint16_t uZPos, VoxelType tValue) { // This code is not usually expected to be called by the user, with the exception of when implementing paging // of uncompressed data. It's a performance critical code path so we use asserts rather than exceptions. @@ -138,20 +138,20 @@ namespace PolyVox } template - void UncompressedBlock::setVoxelAt(const Vector3DUint16& v3dPos, VoxelType tValue) + void Chunk::setVoxelAt(const Vector3DUint16& v3dPos, VoxelType tValue) { setVoxelAt(v3dPos.getX(), v3dPos.getY(), v3dPos.getZ(), tValue); } template - uint32_t UncompressedBlock::calculateSizeInBytes(void) + uint32_t Chunk::calculateSizeInBytes(void) { // Call through to the static version return calculateSizeInBytes(m_uSideLength); } template - uint32_t UncompressedBlock::calculateSizeInBytes(uint32_t uSideLength) + uint32_t Chunk::calculateSizeInBytes(uint32_t uSideLength) { // Note: We disregard the size of the other class members as they are likely to be very small compared to the size of the // allocated voxel data. This also keeps the reported size as a power of two, which makes other memory calculations easier. diff --git a/library/PolyVoxCore/include/PolyVoxCore/FilePager.h b/library/PolyVoxCore/include/PolyVoxCore/FilePager.h index 6f466204..83ae5047 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/FilePager.h +++ b/library/PolyVoxCore/include/PolyVoxCore/FilePager.h @@ -76,7 +76,7 @@ namespace PolyVox m_vecCreatedFiles.clear(); } - virtual void pageIn(const Region& region, UncompressedBlock* pBlockData) + virtual void pageIn(const Region& region, Chunk* pBlockData) { POLYVOX_ASSERT(pBlockData, "Attempting to page in NULL block"); //POLYVOX_ASSERT(pBlockData->hasUncompressedData() == false, "Block should not have uncompressed data"); @@ -120,7 +120,7 @@ namespace PolyVox } } - virtual void pageOut(const Region& region, UncompressedBlock* pBlockData) + virtual void pageOut(const Region& region, Chunk* pBlockData) { POLYVOX_ASSERT(pBlockData, "Attempting to page out NULL block"); //POLYVOX_ASSERT(pBlockData->hasUncompressedData() == false, "Block should not have uncompressed data"); diff --git a/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.h b/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.h index 4c08721c..215f2349 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.h +++ b/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.h @@ -27,7 +27,7 @@ freely, subject to the following restrictions: #include "PolyVoxCore/BaseVolume.h" #include "PolyVoxCore/Pager.h" #include "PolyVoxCore/Region.h" -#include "PolyVoxCore/UncompressedBlock.h" +#include "PolyVoxCore/Chunk.h" #include "PolyVoxCore/Vector.h" #include @@ -285,8 +285,8 @@ namespace PolyVox private: - typedef std::unordered_map > > SharedPtrBlockMap; - typedef std::unordered_map > > WeakPtrBlockMap; + typedef std::unordered_map > > SharedPtrBlockMap; + typedef std::unordered_map > > WeakPtrBlockMap; void initialise(); @@ -298,7 +298,7 @@ namespace PolyVox VoxelType getVoxelImpl(int32_t uXPos, int32_t uYPos, int32_t uZPos, WrapModeType, VoxelType tBorder) const; VoxelType getVoxelImpl(int32_t uXPos, int32_t uYPos, int32_t uZPos, WrapModeType, VoxelType tBorder) const; - std::shared_ptr< UncompressedBlock > getUncompressedBlock(int32_t uBlockX, int32_t uBlockY, int32_t uBlockZ) const; + std::shared_ptr< Chunk > getChunk(int32_t uBlockX, int32_t uBlockY, int32_t uBlockZ) const; void purgeNullPtrsFromAllBlocks(void) const; @@ -308,7 +308,7 @@ namespace PolyVox mutable uint32_t m_uTimestamper; mutable Vector3DInt32 m_v3dLastAccessedBlockPos; - mutable std::shared_ptr< UncompressedBlock > m_pLastAccessedBlock; + mutable std::shared_ptr< Chunk > m_pLastAccessedBlock; uint32_t m_uBlockCountLimit; // The size of the volume diff --git a/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.inl index 07c0e23e..56a8f36e 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.inl @@ -82,9 +82,9 @@ namespace PolyVox m_uBlockCountLimit = (std::max)(m_uBlockCountLimit, uMinPracticalNoOfBlocks); m_uBlockCountLimit = (std::min)(m_uBlockCountLimit, uMaxPracticalNoOfBlocks); - uint32_t uUncompressedBlockSizeInBytes = UncompressedBlock::calculateSizeInBytes(m_uBlockSideLength); - POLYVOX_LOG_DEBUG("Memory usage limit for volume initially set to " << (m_uBlockCountLimit * uUncompressedBlockSizeInBytes) / (1024 * 1024) - << "Mb (" << m_uBlockCountLimit << " blocks of " << uUncompressedBlockSizeInBytes / 1024 << "Kb each)."); + uint32_t uChunkSizeInBytes = Chunk::calculateSizeInBytes(m_uBlockSideLength); + POLYVOX_LOG_DEBUG("Memory usage limit for volume initially set to " << (m_uBlockCountLimit * uChunkSizeInBytes) / (1024 * 1024) + << "Mb (" << m_uBlockCountLimit << " blocks of " << uChunkSizeInBytes / 1024 << "Kb each)."); initialise(); } @@ -223,9 +223,9 @@ namespace PolyVox const uint16_t yOffset = static_cast(uYPos - (blockY << m_uBlockSideLengthPower)); const uint16_t zOffset = static_cast(uZPos - (blockZ << m_uBlockSideLengthPower)); - auto pUncompressedBlock = getUncompressedBlock(blockX, blockY, blockZ); + auto pChunk = getChunk(blockX, blockY, blockZ); - return pUncompressedBlock->getVoxel(xOffset, yOffset, zOffset); + return pChunk->getVoxel(xOffset, yOffset, zOffset); } else { @@ -247,7 +247,7 @@ namespace PolyVox /// Increasing the size of the block cache will increase memory but may improve performance. /// You may want to set this to a large value (e.g. 1024) when you are first loading your /// volume data and then set it to a smaller value (e.g.64) for general processing. - /// \param uMaxNumberOfUncompressedBlocks The number of blocks for which uncompressed data can be cached. + /// \param uMaxNumberOfChunks The number of blocks for which uncompressed data can be cached. //////////////////////////////////////////////////////////////////////////////// template void PagedVolume::setMemoryUsageLimit(uint32_t uMemoryUsageInBytes) @@ -255,8 +255,8 @@ namespace PolyVox POLYVOX_THROW_IF(!m_pPager, invalid_operation, "You cannot limit the memory usage of the volume because it was created without a pager attached."); // Calculate the number of blocks based on the memory limit and the size of each block. - uint32_t uUncompressedBlockSizeInBytes = UncompressedBlock::calculateSizeInBytes(m_uBlockSideLength); - m_uBlockCountLimit = uMemoryUsageInBytes / uUncompressedBlockSizeInBytes; + uint32_t uChunkSizeInBytes = Chunk::calculateSizeInBytes(m_uBlockSideLength); + m_uBlockCountLimit = uMemoryUsageInBytes / uChunkSizeInBytes; // We need at least a few blocks available to avoid thrashing, and in pratice there will probably be hundreds. POLYVOX_LOG_WARNING_IF(m_uBlockCountLimit < uMinPracticalNoOfBlocks, "Requested memory usage limit of " @@ -270,8 +270,8 @@ namespace PolyVox flushAll(); } - POLYVOX_LOG_DEBUG("Memory usage limit for volume now set to " << (m_uBlockCountLimit * uUncompressedBlockSizeInBytes) / (1024 * 1024) - << "Mb (" << m_uBlockCountLimit << " blocks of " << uUncompressedBlockSizeInBytes / 1024 << "Kb each)."); + POLYVOX_LOG_DEBUG("Memory usage limit for volume now set to " << (m_uBlockCountLimit * uChunkSizeInBytes) / (1024 * 1024) + << "Mb (" << m_uBlockCountLimit << " blocks of " << uChunkSizeInBytes / 1024 << "Kb each)."); } //////////////////////////////////////////////////////////////////////////////// @@ -307,8 +307,8 @@ namespace PolyVox const uint16_t yOffset = static_cast(uYPos - (blockY << m_uBlockSideLengthPower)); const uint16_t zOffset = static_cast(uZPos - (blockZ << m_uBlockSideLengthPower)); - auto pUncompressedBlock = getUncompressedBlock(blockX, blockY, blockZ); - pUncompressedBlock->setVoxelAt(xOffset, yOffset, zOffset, tValue); + auto pChunk = getChunk(blockX, blockY, blockZ); + pChunk->setVoxelAt(xOffset, yOffset, zOffset, tValue); } //////////////////////////////////////////////////////////////////////////////// @@ -344,9 +344,9 @@ namespace PolyVox const uint16_t yOffset = static_cast(uYPos - (blockY << m_uBlockSideLengthPower)); const uint16_t zOffset = static_cast(uZPos - (blockZ << m_uBlockSideLengthPower)); - auto pUncompressedBlock = getUncompressedBlock(blockX, blockY, blockZ); + auto pChunk = getChunk(blockX, blockY, blockZ); - pUncompressedBlock->setVoxelAt(xOffset, yOffset, zOffset, tValue); + pChunk->setVoxelAt(xOffset, yOffset, zOffset, tValue); //Return true to indicate that we modified a voxel. return true; @@ -397,7 +397,7 @@ namespace PolyVox { for(int32_t z = v3dStart.getZ(); z <= v3dEnd.getZ(); z++) { - getUncompressedBlock(x,y,z); + getChunk(x,y,z); } } } @@ -497,7 +497,7 @@ namespace PolyVox m_regValidRegionInBlocks.setUpperY(this->m_regValidRegion.getUpperY() >> m_uBlockSideLengthPower); m_regValidRegionInBlocks.setUpperZ(this->m_regValidRegion.getUpperZ() >> m_uBlockSideLengthPower); - //setMaxNumberOfUncompressedBlocks(m_uBlockCountLimit); + //setMaxNumberOfChunks(m_uBlockCountLimit); //Clear the previous data m_pRecentlyUsedBlocks.clear(); @@ -509,7 +509,7 @@ namespace PolyVox } template - std::shared_ptr< UncompressedBlock > PagedVolume::getUncompressedBlock(int32_t uBlockX, int32_t uBlockY, int32_t uBlockZ) const + std::shared_ptr< Chunk > PagedVolume::getChunk(int32_t uBlockX, int32_t uBlockY, int32_t uBlockZ) const { Vector3DInt32 v3dBlockPos(uBlockX, uBlockY, uBlockZ); @@ -523,45 +523,45 @@ namespace PolyVox } // The block was not the same as last time, but we can now hope it is in the set of most recently used blocks. - std::shared_ptr< UncompressedBlock > pUncompressedBlock = nullptr; - typename SharedPtrBlockMap::iterator itUncompressedBlock = m_pRecentlyUsedBlocks.find(v3dBlockPos); + std::shared_ptr< Chunk > pChunk = nullptr; + typename SharedPtrBlockMap::iterator itChunk = m_pRecentlyUsedBlocks.find(v3dBlockPos); // Check whether the block was found. - if ((itUncompressedBlock) != m_pRecentlyUsedBlocks.end()) + if ((itChunk) != m_pRecentlyUsedBlocks.end()) { // The block was found so we can use it. - pUncompressedBlock = itUncompressedBlock->second; - POLYVOX_ASSERT(pUncompressedBlock, "Recent block list shold never contain a null pointer."); + pChunk = itChunk->second; + POLYVOX_ASSERT(pChunk, "Recent block list shold never contain a null pointer."); } - if (!pUncompressedBlock) + if (!pChunk) { // Although it's not in our recently use blocks, there's some (slim) chance that it // exists in the list of all loaded blocks, because a sampler may be holding on to it. - typename WeakPtrBlockMap::iterator itWeakUncompressedBlock = m_pAllBlocks.find(v3dBlockPos); - if (itWeakUncompressedBlock != m_pAllBlocks.end()) + typename WeakPtrBlockMap::iterator itWeakChunk = m_pAllBlocks.find(v3dBlockPos); + if (itWeakChunk != m_pAllBlocks.end()) { // We've found an entry in the 'all blocks' list, but it can be null. This happens if a sampler was the // last thing to be keeping it alive and then the sampler let it go. In this case we remove it from the // list, and it will get added again soon when we page it in and fill it with valid data. - if (itWeakUncompressedBlock->second.expired()) + if (itWeakChunk->second.expired()) { - m_pAllBlocks.erase(itWeakUncompressedBlock); + m_pAllBlocks.erase(itWeakChunk); } else { // The block is valid. We know it's not in the recently used list (we checked earlier) so it should be added. - pUncompressedBlock = std::shared_ptr< UncompressedBlock >(itWeakUncompressedBlock->second); - m_pRecentlyUsedBlocks.insert(std::make_pair(v3dBlockPos, pUncompressedBlock)); + pChunk = std::shared_ptr< Chunk >(itWeakChunk->second); + m_pRecentlyUsedBlocks.insert(std::make_pair(v3dBlockPos, pChunk)); } } } // If we still haven't found the block then it's time to create a new one and page it in from disk. - if (!pUncompressedBlock) + if (!pChunk) { // The block was not found so we will create a new one. - pUncompressedBlock = std::make_shared< UncompressedBlock >(v3dBlockPos, m_uBlockSideLength, m_pPager); + pChunk = std::make_shared< Chunk >(v3dBlockPos, m_uBlockSideLength, m_pPager); // As we are loading a new block we should try to ensure we don't go over our target memory usage. bool erasedBlock = false; @@ -594,15 +594,15 @@ namespace PolyVox } // Add our new block to the maps. - m_pAllBlocks.insert(std::make_pair(v3dBlockPos, pUncompressedBlock)); - m_pRecentlyUsedBlocks.insert(std::make_pair(v3dBlockPos, pUncompressedBlock)); + m_pAllBlocks.insert(std::make_pair(v3dBlockPos, pChunk)); + m_pRecentlyUsedBlocks.insert(std::make_pair(v3dBlockPos, pChunk)); } - pUncompressedBlock->m_uBlockLastAccessed = ++m_uTimestamper; - m_pLastAccessedBlock = pUncompressedBlock; + pChunk->m_uBlockLastAccessed = ++m_uTimestamper; + m_pLastAccessedBlock = pChunk; m_v3dLastAccessedBlockPos = v3dBlockPos; - return pUncompressedBlock; + return pChunk; } //////////////////////////////////////////////////////////////////////////////// @@ -616,7 +616,7 @@ namespace PolyVox // Note: We disregard the size of the other class members as they are likely to be very small compared to the size of the // allocated voxel data. This also keeps the reported size as a power of two, which makes other memory calculations easier. - return UncompressedBlock::calculateSizeInBytes(m_uBlockSideLength) * m_pAllBlocks.size(); + return Chunk::calculateSizeInBytes(m_uBlockSideLength) * m_pAllBlocks.size(); } template @@ -692,8 +692,8 @@ namespace PolyVox const uint16_t yOffset = static_cast(uYPos - (blockY << m_uBlockSideLengthPower)); const uint16_t zOffset = static_cast(uZPos - (blockZ << m_uBlockSideLengthPower)); - auto pUncompressedBlock = getUncompressedBlock(blockX, blockY, blockZ); - return pUncompressedBlock->getVoxel(xOffset, yOffset, zOffset); + auto pChunk = getChunk(blockX, blockY, blockZ); + return pChunk->getVoxel(xOffset, yOffset, zOffset); } } diff --git a/library/PolyVoxCore/include/PolyVoxCore/PagedVolumeSampler.inl b/library/PolyVoxCore/include/PolyVoxCore/PagedVolumeSampler.inl index 5c55e8ae..d6f38554 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/PagedVolumeSampler.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/PagedVolumeSampler.inl @@ -119,7 +119,7 @@ namespace PolyVox uYPosInBlock * this->mVolume->m_uBlockSideLength + uZPosInBlock * this->mVolume->m_uBlockSideLength * this->mVolume->m_uBlockSideLength; - auto pUncompressedCurrentBlock = this->mVolume->getUncompressedBlock(uXBlock, uYBlock, uZBlock); + auto pUncompressedCurrentBlock = this->mVolume->getChunk(uXBlock, uYBlock, uZBlock); mCurrentVoxel = pUncompressedCurrentBlock->m_tData + uVoxelIndexInBlock; } diff --git a/library/PolyVoxCore/include/PolyVoxCore/Pager.h b/library/PolyVoxCore/include/PolyVoxCore/Pager.h index 6b670302..f8edc620 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/Pager.h +++ b/library/PolyVoxCore/include/PolyVoxCore/Pager.h @@ -24,7 +24,7 @@ freely, subject to the following restrictions: #ifndef __PolyVox_Pager_H__ #define __PolyVox_Pager_H__ -#include "PolyVoxCore/UncompressedBlock.h" +#include "PolyVoxCore/Chunk.h" #include "PolyVoxCore/Impl/TypeDef.h" #include @@ -43,8 +43,8 @@ namespace PolyVox /// Destructor virtual ~Pager() {}; - virtual void pageIn(const Region& region, UncompressedBlock* pBlockData) = 0; - virtual void pageOut(const Region& region, UncompressedBlock* pBlockData) = 0; + virtual void pageIn(const Region& region, Chunk* pBlockData) = 0; + virtual void pageOut(const Region& region, Chunk* pBlockData) = 0; }; } From 8d7d88a7df90e6a13c50f0a73bb9e59568334dc1 Mon Sep 17 00:00:00 2001 From: David Williams Date: Sat, 20 Sep 2014 17:39:12 +0200 Subject: [PATCH 30/42] Renamed a bunch of stuff from using 'block' to 'chunk' --- examples/Paging/main.cpp | 8 +- .../PolyVoxCore/include/PolyVoxCore/Chunk.h | 4 +- .../PolyVoxCore/include/PolyVoxCore/Chunk.inl | 20 +-- .../include/PolyVoxCore/FilePager.h | 18 +- .../include/PolyVoxCore/PagedVolume.h | 20 +-- .../include/PolyVoxCore/PagedVolume.inl | 160 +++++++++--------- .../PolyVoxCore/PagedVolumeSampler.inl | 96 +++++------ .../PolyVoxCore/include/PolyVoxCore/Pager.h | 4 +- 8 files changed, 165 insertions(+), 165 deletions(-) diff --git a/examples/Paging/main.cpp b/examples/Paging/main.cpp index 576bfb67..b111a6bf 100644 --- a/examples/Paging/main.cpp +++ b/examples/Paging/main.cpp @@ -90,7 +90,7 @@ public: /// Destructor virtual ~PerlinNoisePager() {}; - virtual void pageIn(const PolyVox::Region& region, Chunk* pBlockData) + virtual void pageIn(const PolyVox::Region& region, Chunk* pChunk) { // FIXME - this isn't a great example... it's a shame we have to hard clode the block size and also create/destroy // a compressor each time. These could at least be moved outside somewhere if we can't fix it in a better way... @@ -132,18 +132,18 @@ public: // Voxel position within a block always start from zero. So if a block represents region (4, 8, 12) to (11, 19, 15) // then the valid block voxels are from (0, 0, 0) to (7, 11, 3). Hence we subtract the lower corner position of the // region from the volume space position in order to get the block space position. - pBlockData->setVoxelAt(x - region.getLowerX(), y - region.getLowerY(), z - region.getLowerZ(), voxel); + pChunk->setVoxelAt(x - region.getLowerX(), y - region.getLowerY(), z - region.getLowerZ(), voxel); } } } // Now compress the computed data into the provided block. //RLEBlockCompressor* compressor = new RLEBlockCompressor(); - //compressor->compress(&block, pBlockData); + //compressor->compress(&block, pChunk); //delete compressor; } - virtual void pageOut(const PolyVox::Region& region, Chunk* /*pBlockData*/) + virtual void pageOut(const PolyVox::Region& region, Chunk* /*pChunk*/) { std::cout << "warning unloading region: " << region.getLowerCorner() << " -> " << region.getUpperCorner() << std::endl; } diff --git a/library/PolyVoxCore/include/PolyVoxCore/Chunk.h b/library/PolyVoxCore/include/PolyVoxCore/Chunk.h index 4f92e458..69df02b3 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/Chunk.h +++ b/library/PolyVoxCore/include/PolyVoxCore/Chunk.h @@ -55,7 +55,7 @@ namespace PolyVox Chunk& operator=(const Chunk& /*rhs*/) {}; // This is updated by the PagedVolume and used to discard the least recently used blocks. - uint32_t m_uBlockLastAccessed; + uint32_t m_uChunkLastAccessed; // This is so we can tell whether a uncompressed block has to be recompressed and whether // a compressed block has to be paged back to disk, or whether they can just be discarded. @@ -70,7 +70,7 @@ namespace PolyVox uint16_t m_uSideLength; uint8_t m_uSideLengthPower; Pager* m_pPager; - Vector3DInt32 m_v3dBlockSpacePosition; + Vector3DInt32 m_v3dChunkSpacePosition; }; } diff --git a/library/PolyVoxCore/include/PolyVoxCore/Chunk.inl b/library/PolyVoxCore/include/PolyVoxCore/Chunk.inl index ad8e255a..16440c7b 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/Chunk.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/Chunk.inl @@ -27,13 +27,13 @@ namespace PolyVox { template Chunk::Chunk(Vector3DInt32 v3dPosition, uint16_t uSideLength, Pager* pPager) - :m_uBlockLastAccessed(0) + :m_uChunkLastAccessed(0) ,m_bDataModified(true) ,m_tData(0) ,m_uSideLength(0) ,m_uSideLengthPower(0) ,m_pPager(pPager) - ,m_v3dBlockSpacePosition(v3dPosition) + ,m_v3dChunkSpacePosition(v3dPosition) { // Compute the side length m_uSideLength = uSideLength; @@ -47,7 +47,7 @@ namespace PolyVox if (m_pPager) { // From the coordinates of the block we deduce the coordinates of the contained voxels. - Vector3DInt32 v3dLower = m_v3dBlockSpacePosition * static_cast(m_uSideLength); + Vector3DInt32 v3dLower = m_v3dChunkSpacePosition * static_cast(m_uSideLength); Vector3DInt32 v3dUpper = v3dLower + Vector3DInt32(m_uSideLength - 1, m_uSideLength - 1, m_uSideLength - 1); Region reg(v3dLower, v3dUpper); @@ -70,7 +70,7 @@ namespace PolyVox if (m_pPager && m_bDataModified) { // From the coordinates of the block we deduce the coordinates of the contained voxels. - Vector3DInt32 v3dLower = m_v3dBlockSpacePosition * static_cast(m_uSideLength); + Vector3DInt32 v3dLower = m_v3dChunkSpacePosition * static_cast(m_uSideLength); Vector3DInt32 v3dUpper = v3dLower + Vector3DInt32(m_uSideLength - 1, m_uSideLength - 1, m_uSideLength - 1); // Page the data out @@ -98,9 +98,9 @@ namespace PolyVox { // This code is not usually expected to be called by the user, with the exception of when implementing paging // of uncompressed data. It's a performance critical code path so we use asserts rather than exceptions. - POLYVOX_ASSERT(uXPos < m_uSideLength, "Supplied position is outside of the block"); - POLYVOX_ASSERT(uYPos < m_uSideLength, "Supplied position is outside of the block"); - POLYVOX_ASSERT(uZPos < m_uSideLength, "Supplied position is outside of the block"); + POLYVOX_ASSERT(uXPos < m_uSideLength, "Supplied position is outside of the chunk"); + POLYVOX_ASSERT(uYPos < m_uSideLength, "Supplied position is outside of the chunk"); + POLYVOX_ASSERT(uZPos < m_uSideLength, "Supplied position is outside of the chunk"); POLYVOX_ASSERT(m_tData, "No uncompressed data - block must be decompressed before accessing voxels."); return m_tData @@ -122,9 +122,9 @@ namespace PolyVox { // This code is not usually expected to be called by the user, with the exception of when implementing paging // of uncompressed data. It's a performance critical code path so we use asserts rather than exceptions. - POLYVOX_ASSERT(uXPos < m_uSideLength, "Supplied position is outside of the block"); - POLYVOX_ASSERT(uYPos < m_uSideLength, "Supplied position is outside of the block"); - POLYVOX_ASSERT(uZPos < m_uSideLength, "Supplied position is outside of the block"); + POLYVOX_ASSERT(uXPos < m_uSideLength, "Supplied position is outside of the chunk"); + POLYVOX_ASSERT(uYPos < m_uSideLength, "Supplied position is outside of the chunk"); + POLYVOX_ASSERT(uZPos < m_uSideLength, "Supplied position is outside of the chunk"); POLYVOX_ASSERT(m_tData, "No uncompressed data - block must be decompressed before accessing voxels."); m_tData diff --git a/library/PolyVoxCore/include/PolyVoxCore/FilePager.h b/library/PolyVoxCore/include/PolyVoxCore/FilePager.h index 83ae5047..671023cc 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/FilePager.h +++ b/library/PolyVoxCore/include/PolyVoxCore/FilePager.h @@ -76,10 +76,10 @@ namespace PolyVox m_vecCreatedFiles.clear(); } - virtual void pageIn(const Region& region, Chunk* pBlockData) + virtual void pageIn(const Region& region, Chunk* pChunk) { - POLYVOX_ASSERT(pBlockData, "Attempting to page in NULL block"); - //POLYVOX_ASSERT(pBlockData->hasUncompressedData() == false, "Block should not have uncompressed data"); + POLYVOX_ASSERT(pChunk, "Attempting to page in NULL block"); + //POLYVOX_ASSERT(pChunk->hasUncompressedData() == false, "Block should not have uncompressed data"); std::stringstream ssFilename; ssFilename << m_strFolderName << "/" << m_strRandomPrefix << "-" @@ -102,10 +102,10 @@ namespace PolyVox uint8_t* buffer = new uint8_t[fileSizeInBytes]; fread(buffer, sizeof(uint8_t), fileSizeInBytes, pFile); - pBlockData->setData(buffer, fileSizeInBytes); + pChunk->setData(buffer, fileSizeInBytes); delete[] buffer;*/ - fread(pBlockData->getData(), sizeof(uint8_t), pBlockData->getDataSizeInBytes(), pFile); + fread(pChunk->getData(), sizeof(uint8_t), pChunk->getDataSizeInBytes(), pFile); if(ferror(pFile)) { @@ -120,10 +120,10 @@ namespace PolyVox } } - virtual void pageOut(const Region& region, Chunk* pBlockData) + virtual void pageOut(const Region& region, Chunk* pChunk) { - POLYVOX_ASSERT(pBlockData, "Attempting to page out NULL block"); - //POLYVOX_ASSERT(pBlockData->hasUncompressedData() == false, "Block should not have uncompressed data"); + POLYVOX_ASSERT(pChunk, "Attempting to page out NULL block"); + //POLYVOX_ASSERT(pChunk->hasUncompressedData() == false, "Block should not have uncompressed data"); POLYVOX_LOG_TRACE("Paging out data for " << region); @@ -146,7 +146,7 @@ namespace PolyVox //The file has been created, so add it to the list to delete on shutdown. m_vecCreatedFiles.push_back(filename); - fwrite(pBlockData->getData(), sizeof(uint8_t), pBlockData->getDataSizeInBytes(), pFile); + fwrite(pChunk->getData(), sizeof(uint8_t), pChunk->getDataSizeInBytes(), pFile); if(ferror(pFile)) { diff --git a/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.h b/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.h index 215f2349..de85418e 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.h +++ b/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.h @@ -234,7 +234,7 @@ namespace PolyVox ( const Region& regValid, Pager* pPager = nullptr, - uint16_t uBlockSideLength = 32 + uint16_t uChunkSideLength = 32 ); /// Destructor ~PagedVolume(); @@ -285,8 +285,8 @@ namespace PolyVox private: - typedef std::unordered_map > > SharedPtrBlockMap; - typedef std::unordered_map > > WeakPtrBlockMap; + typedef std::unordered_map > > SharedPtrChunkMap; + typedef std::unordered_map > > WeakPtrChunkMap; void initialise(); @@ -300,23 +300,23 @@ namespace PolyVox std::shared_ptr< Chunk > getChunk(int32_t uBlockX, int32_t uBlockY, int32_t uBlockZ) const; - void purgeNullPtrsFromAllBlocks(void) const; + void purgeNullPtrsFromAllChunks(void) const; // The block data - mutable WeakPtrBlockMap m_pAllBlocks; - mutable SharedPtrBlockMap m_pRecentlyUsedBlocks; + mutable WeakPtrChunkMap m_pAllChunks; + mutable SharedPtrChunkMap m_pRecentlyUsedChunks; mutable uint32_t m_uTimestamper; - mutable Vector3DInt32 m_v3dLastAccessedBlockPos; - mutable std::shared_ptr< Chunk > m_pLastAccessedBlock; + mutable Vector3DInt32 m_v3dLastAccessedChunkPos; + mutable std::shared_ptr< Chunk > m_pLastAccessedChunk; uint32_t m_uBlockCountLimit; // The size of the volume Region m_regValidRegionInBlocks; // The size of the blocks - uint16_t m_uBlockSideLength; - uint8_t m_uBlockSideLengthPower; + uint16_t m_uChunkSideLength; + uint8_t m_uChunkSideLengthPower; Pager* m_pPager; diff --git a/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.inl index 56a8f36e..48763014 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.inl @@ -35,18 +35,18 @@ namespace PolyVox /// \param dataRequiredHandler The callback function which will be called when PolyVox tries to use data which is not currently in momory. /// \param dataOverflowHandler The callback function which will be called when PolyVox has too much data and needs to remove some from memory. /// \param bPagingEnabled Controls whether or not paging is enabled for this PagedVolume. - /// \param uBlockSideLength The size of the blocks making up the volume. Small blocks will compress/decompress faster, but there will also be more of them meaning voxel access could be slower. + /// \param uChunkSideLength The size of the blocks making up the volume. Small blocks will compress/decompress faster, but there will also be more of them meaning voxel access could be slower. //////////////////////////////////////////////////////////////////////////////// template PagedVolume::PagedVolume ( const Region& regValid, Pager* pPager, - uint16_t uBlockSideLength + uint16_t uChunkSideLength ) :BaseVolume(regValid) { - m_uBlockSideLength = uBlockSideLength; + m_uChunkSideLength = uChunkSideLength; m_pPager = pPager; if (m_pPager) @@ -65,8 +65,8 @@ namespace PolyVox uint32_t longestSide = (std::max)(regValid.getWidthInVoxels(), (std::max)(regValid.getHeightInVoxels(), regValid.getDepthInVoxels())); uint32_t shortestSide = (std::min)(regValid.getWidthInVoxels(), (std::min)(regValid.getHeightInVoxels(), regValid.getDepthInVoxels())); - longestSide /= m_uBlockSideLength; - shortestSide /= m_uBlockSideLength; + longestSide /= m_uChunkSideLength; + shortestSide /= m_uChunkSideLength; m_uBlockCountLimit = longestSide * shortestSide; } @@ -82,7 +82,7 @@ namespace PolyVox m_uBlockCountLimit = (std::max)(m_uBlockCountLimit, uMinPracticalNoOfBlocks); m_uBlockCountLimit = (std::min)(m_uBlockCountLimit, uMaxPracticalNoOfBlocks); - uint32_t uChunkSizeInBytes = Chunk::calculateSizeInBytes(m_uBlockSideLength); + uint32_t uChunkSizeInBytes = Chunk::calculateSizeInBytes(m_uChunkSideLength); POLYVOX_LOG_DEBUG("Memory usage limit for volume initially set to " << (m_uBlockCountLimit * uChunkSizeInBytes) / (1024 * 1024) << "Mb (" << m_uBlockCountLimit << " blocks of " << uChunkSizeInBytes / 1024 << "Kb each)."); @@ -215,13 +215,13 @@ namespace PolyVox { if(this->m_regValidRegion.containsPoint(Vector3DInt32(uXPos, uYPos, uZPos))) { - const int32_t blockX = uXPos >> m_uBlockSideLengthPower; - const int32_t blockY = uYPos >> m_uBlockSideLengthPower; - const int32_t blockZ = uZPos >> m_uBlockSideLengthPower; + const int32_t blockX = uXPos >> m_uChunkSideLengthPower; + const int32_t blockY = uYPos >> m_uChunkSideLengthPower; + const int32_t blockZ = uZPos >> m_uChunkSideLengthPower; - const uint16_t xOffset = static_cast(uXPos - (blockX << m_uBlockSideLengthPower)); - const uint16_t yOffset = static_cast(uYPos - (blockY << m_uBlockSideLengthPower)); - const uint16_t zOffset = static_cast(uZPos - (blockZ << m_uBlockSideLengthPower)); + const uint16_t xOffset = static_cast(uXPos - (blockX << m_uChunkSideLengthPower)); + const uint16_t yOffset = static_cast(uYPos - (blockY << m_uChunkSideLengthPower)); + const uint16_t zOffset = static_cast(uZPos - (blockZ << m_uChunkSideLengthPower)); auto pChunk = getChunk(blockX, blockY, blockZ); @@ -255,7 +255,7 @@ namespace PolyVox POLYVOX_THROW_IF(!m_pPager, invalid_operation, "You cannot limit the memory usage of the volume because it was created without a pager attached."); // Calculate the number of blocks based on the memory limit and the size of each block. - uint32_t uChunkSizeInBytes = Chunk::calculateSizeInBytes(m_uBlockSideLength); + uint32_t uChunkSizeInBytes = Chunk::calculateSizeInBytes(m_uChunkSideLength); m_uBlockCountLimit = uMemoryUsageInBytes / uChunkSizeInBytes; // We need at least a few blocks available to avoid thrashing, and in pratice there will probably be hundreds. @@ -265,7 +265,7 @@ namespace PolyVox m_uBlockCountLimit = (std::min)(m_uBlockCountLimit, uMaxPracticalNoOfBlocks); // If the new limit is less than the number of blocks already loaded then the easiest solution is to flush and start loading again. - if (m_pRecentlyUsedBlocks.size() > m_uBlockCountLimit) + if (m_pRecentlyUsedChunks.size() > m_uBlockCountLimit) { flushAll(); } @@ -299,13 +299,13 @@ namespace PolyVox } } - const int32_t blockX = uXPos >> m_uBlockSideLengthPower; - const int32_t blockY = uYPos >> m_uBlockSideLengthPower; - const int32_t blockZ = uZPos >> m_uBlockSideLengthPower; + const int32_t blockX = uXPos >> m_uChunkSideLengthPower; + const int32_t blockY = uYPos >> m_uChunkSideLengthPower; + const int32_t blockZ = uZPos >> m_uChunkSideLengthPower; - const uint16_t xOffset = static_cast(uXPos - (blockX << m_uBlockSideLengthPower)); - const uint16_t yOffset = static_cast(uYPos - (blockY << m_uBlockSideLengthPower)); - const uint16_t zOffset = static_cast(uZPos - (blockZ << m_uBlockSideLengthPower)); + const uint16_t xOffset = static_cast(uXPos - (blockX << m_uChunkSideLengthPower)); + const uint16_t yOffset = static_cast(uYPos - (blockY << m_uChunkSideLengthPower)); + const uint16_t zOffset = static_cast(uZPos - (blockZ << m_uChunkSideLengthPower)); auto pChunk = getChunk(blockX, blockY, blockZ); pChunk->setVoxelAt(xOffset, yOffset, zOffset, tValue); @@ -336,13 +336,13 @@ namespace PolyVox // PolyVox does not throw an exception when a voxel is out of range. Please see 'Error Handling' in the User Manual. POLYVOX_ASSERT(this->m_regValidRegion.containsPoint(Vector3DInt32(uXPos, uYPos, uZPos)), "Position is outside valid region"); - const int32_t blockX = uXPos >> m_uBlockSideLengthPower; - const int32_t blockY = uYPos >> m_uBlockSideLengthPower; - const int32_t blockZ = uZPos >> m_uBlockSideLengthPower; + const int32_t blockX = uXPos >> m_uChunkSideLengthPower; + const int32_t blockY = uYPos >> m_uChunkSideLengthPower; + const int32_t blockZ = uZPos >> m_uChunkSideLengthPower; - const uint16_t xOffset = static_cast(uXPos - (blockX << m_uBlockSideLengthPower)); - const uint16_t yOffset = static_cast(uYPos - (blockY << m_uBlockSideLengthPower)); - const uint16_t zOffset = static_cast(uZPos - (blockZ << m_uBlockSideLengthPower)); + const uint16_t xOffset = static_cast(uXPos - (blockX << m_uChunkSideLengthPower)); + const uint16_t yOffset = static_cast(uYPos - (blockY << m_uChunkSideLengthPower)); + const uint16_t zOffset = static_cast(uZPos - (blockZ << m_uChunkSideLengthPower)); auto pChunk = getChunk(blockX, blockY, blockZ); @@ -375,13 +375,13 @@ namespace PolyVox Vector3DInt32 v3dStart; for(int i = 0; i < 3; i++) { - v3dStart.setElement(i, regPrefetch.getLowerCorner().getElement(i) >> m_uBlockSideLengthPower); + v3dStart.setElement(i, regPrefetch.getLowerCorner().getElement(i) >> m_uChunkSideLengthPower); } Vector3DInt32 v3dEnd; for(int i = 0; i < 3; i++) { - v3dEnd.setElement(i, regPrefetch.getUpperCorner().getElement(i) >> m_uBlockSideLengthPower); + v3dEnd.setElement(i, regPrefetch.getUpperCorner().getElement(i) >> m_uChunkSideLengthPower); } // Ensure we don't page in more blocks than the volume can hold. @@ -412,16 +412,16 @@ namespace PolyVox POLYVOX_LOG_WARNING_IF(!m_pPager, "Data discarded by flush operation as no pager is attached."); // Clear this pointer so it doesn't hang on to any blocks. - m_pLastAccessedBlock = nullptr; + m_pLastAccessedChunk = nullptr; // Erase all the most recently used blocks. - m_pRecentlyUsedBlocks.clear(); + m_pRecentlyUsedChunks.clear(); // Remove deleted blocks from the list of all loaded blocks. - purgeNullPtrsFromAllBlocks(); + purgeNullPtrsFromAllChunks(); // If there are still some blocks left then this is a cause for concern. Perhaps samplers are holding on to them? - POLYVOX_LOG_WARNING_IF(m_pAllBlocks.size() > 0, "Blocks still exist after performing flushAll()! Perhaps you have samplers attached?"); + POLYVOX_LOG_WARNING_IF(m_pAllChunks.size() > 0, "Blocks still exist after performing flushAll()! Perhaps you have samplers attached?"); } //////////////////////////////////////////////////////////////////////////////// @@ -433,19 +433,19 @@ namespace PolyVox POLYVOX_LOG_WARNING_IF(!m_pPager, "Data discarded by flush operation as no pager is attached."); // Clear this pointer so it doesn't hang on to any blocks. - m_pLastAccessedBlock = nullptr; + m_pLastAccessedChunk = nullptr; // Convert the start and end positions into block space coordinates Vector3DInt32 v3dStart; for(int i = 0; i < 3; i++) { - v3dStart.setElement(i, regFlush.getLowerCorner().getElement(i) >> m_uBlockSideLengthPower); + v3dStart.setElement(i, regFlush.getLowerCorner().getElement(i) >> m_uChunkSideLengthPower); } Vector3DInt32 v3dEnd; for(int i = 0; i < 3; i++) { - v3dEnd.setElement(i, regFlush.getUpperCorner().getElement(i) >> m_uBlockSideLengthPower); + v3dEnd.setElement(i, regFlush.getUpperCorner().getElement(i) >> m_uChunkSideLengthPower); } // Loops over the specified positions and delete the corresponding blocks. @@ -455,13 +455,13 @@ namespace PolyVox { for(int32_t z = v3dStart.getZ(); z <= v3dEnd.getZ(); z++) { - m_pRecentlyUsedBlocks.erase(Vector3DInt32(x, y, z)); m_pLastAccessedBlock = 0; + m_pRecentlyUsedChunks.erase(Vector3DInt32(x, y, z)); m_pLastAccessedChunk = 0; } } } // We might now have so null pointers in the 'all blocks' list so clean them up. - purgeNullPtrsFromAllBlocks(); + purgeNullPtrsFromAllChunks(); } //////////////////////////////////////////////////////////////////////////////// @@ -471,12 +471,12 @@ namespace PolyVox void PagedVolume::initialise() { //Validate parameters - if(m_uBlockSideLength == 0) + if(m_uChunkSideLength == 0) { POLYVOX_THROW(std::invalid_argument, "Block side length cannot be zero."); } - if(!isPowerOf2(m_uBlockSideLength)) + if(!isPowerOf2(m_uChunkSideLength)) { POLYVOX_THROW(std::invalid_argument, "Block side length must be a power of two."); } @@ -484,23 +484,23 @@ namespace PolyVox m_uTimestamper = 0; //m_uBlockCountLimit = 16; //m_uMaxNumberOfBlocksInMemory = 1024; - m_v3dLastAccessedBlockPos = Vector3DInt32(0,0,0); //There are no invalid positions, but initially the m_pLastAccessedBlock pointer will be null; - m_pLastAccessedBlock = 0; + m_v3dLastAccessedChunkPos = Vector3DInt32(0,0,0); //There are no invalid positions, but initially the m_pLastAccessedChunk pointer will be null; + m_pLastAccessedChunk = 0; //Compute the block side length - m_uBlockSideLengthPower = logBase2(m_uBlockSideLength); + m_uChunkSideLengthPower = logBase2(m_uChunkSideLength); - m_regValidRegionInBlocks.setLowerX(this->m_regValidRegion.getLowerX() >> m_uBlockSideLengthPower); - m_regValidRegionInBlocks.setLowerY(this->m_regValidRegion.getLowerY() >> m_uBlockSideLengthPower); - m_regValidRegionInBlocks.setLowerZ(this->m_regValidRegion.getLowerZ() >> m_uBlockSideLengthPower); - m_regValidRegionInBlocks.setUpperX(this->m_regValidRegion.getUpperX() >> m_uBlockSideLengthPower); - m_regValidRegionInBlocks.setUpperY(this->m_regValidRegion.getUpperY() >> m_uBlockSideLengthPower); - m_regValidRegionInBlocks.setUpperZ(this->m_regValidRegion.getUpperZ() >> m_uBlockSideLengthPower); + m_regValidRegionInBlocks.setLowerX(this->m_regValidRegion.getLowerX() >> m_uChunkSideLengthPower); + m_regValidRegionInBlocks.setLowerY(this->m_regValidRegion.getLowerY() >> m_uChunkSideLengthPower); + m_regValidRegionInBlocks.setLowerZ(this->m_regValidRegion.getLowerZ() >> m_uChunkSideLengthPower); + m_regValidRegionInBlocks.setUpperX(this->m_regValidRegion.getUpperX() >> m_uChunkSideLengthPower); + m_regValidRegionInBlocks.setUpperY(this->m_regValidRegion.getUpperY() >> m_uChunkSideLengthPower); + m_regValidRegionInBlocks.setUpperZ(this->m_regValidRegion.getUpperZ() >> m_uChunkSideLengthPower); //setMaxNumberOfChunks(m_uBlockCountLimit); //Clear the previous data - m_pRecentlyUsedBlocks.clear(); + m_pRecentlyUsedChunks.clear(); //Other properties we might find useful later this->m_uLongestSideLength = (std::max)((std::max)(this->getWidth(),this->getHeight()),this->getDepth()); @@ -517,17 +517,17 @@ namespace PolyVox //the time stamp. If we updated it everytime then that would be every time we touched //a voxel, which would overflow a uint32_t and require us to use a uint64_t instead. //This check should also provide a significant speed boost as usually it is true. - if((v3dBlockPos == m_v3dLastAccessedBlockPos) && (m_pLastAccessedBlock != 0)) + if((v3dBlockPos == m_v3dLastAccessedChunkPos) && (m_pLastAccessedChunk != 0)) { - return m_pLastAccessedBlock; + return m_pLastAccessedChunk; } // The block was not the same as last time, but we can now hope it is in the set of most recently used blocks. std::shared_ptr< Chunk > pChunk = nullptr; - typename SharedPtrBlockMap::iterator itChunk = m_pRecentlyUsedBlocks.find(v3dBlockPos); + typename SharedPtrChunkMap::iterator itChunk = m_pRecentlyUsedChunks.find(v3dBlockPos); // Check whether the block was found. - if ((itChunk) != m_pRecentlyUsedBlocks.end()) + if ((itChunk) != m_pRecentlyUsedChunks.end()) { // The block was found so we can use it. pChunk = itChunk->second; @@ -538,21 +538,21 @@ namespace PolyVox { // Although it's not in our recently use blocks, there's some (slim) chance that it // exists in the list of all loaded blocks, because a sampler may be holding on to it. - typename WeakPtrBlockMap::iterator itWeakChunk = m_pAllBlocks.find(v3dBlockPos); - if (itWeakChunk != m_pAllBlocks.end()) + typename WeakPtrChunkMap::iterator itWeakChunk = m_pAllChunks.find(v3dBlockPos); + if (itWeakChunk != m_pAllChunks.end()) { // We've found an entry in the 'all blocks' list, but it can be null. This happens if a sampler was the // last thing to be keeping it alive and then the sampler let it go. In this case we remove it from the // list, and it will get added again soon when we page it in and fill it with valid data. if (itWeakChunk->second.expired()) { - m_pAllBlocks.erase(itWeakChunk); + m_pAllChunks.erase(itWeakChunk); } else { // The block is valid. We know it's not in the recently used list (we checked earlier) so it should be added. pChunk = std::shared_ptr< Chunk >(itWeakChunk->second); - m_pRecentlyUsedBlocks.insert(std::make_pair(v3dBlockPos, pChunk)); + m_pRecentlyUsedChunks.insert(std::make_pair(v3dBlockPos, pChunk)); } } } @@ -561,28 +561,28 @@ namespace PolyVox if (!pChunk) { // The block was not found so we will create a new one. - pChunk = std::make_shared< Chunk >(v3dBlockPos, m_uBlockSideLength, m_pPager); + pChunk = std::make_shared< Chunk >(v3dBlockPos, m_uChunkSideLength, m_pPager); // As we are loading a new block we should try to ensure we don't go over our target memory usage. bool erasedBlock = false; - while (m_pRecentlyUsedBlocks.size() + 1 > m_uBlockCountLimit) // +1 ready for new block we will add next. + while (m_pRecentlyUsedChunks.size() + 1 > m_uBlockCountLimit) // +1 ready for new block we will add next. { // This should never hit, because it should not have been possible for // the user to limit the number of blocks if they did not provide a pager. POLYVOX_ASSERT(m_pPager, "A valid pager is required to limit number of blocks"); // Find the least recently used block. Hopefully this isn't too slow. - typename SharedPtrBlockMap::iterator itUnloadBlock = m_pRecentlyUsedBlocks.begin(); - for (typename SharedPtrBlockMap::iterator i = m_pRecentlyUsedBlocks.begin(); i != m_pRecentlyUsedBlocks.end(); i++) + typename SharedPtrChunkMap::iterator itUnloadBlock = m_pRecentlyUsedChunks.begin(); + for (typename SharedPtrChunkMap::iterator i = m_pRecentlyUsedChunks.begin(); i != m_pRecentlyUsedChunks.end(); i++) { - if (i->second->m_uBlockLastAccessed < itUnloadBlock->second->m_uBlockLastAccessed) + if (i->second->m_uChunkLastAccessed < itUnloadBlock->second->m_uChunkLastAccessed) { itUnloadBlock = i; } } // Erase the least recently used block - m_pRecentlyUsedBlocks.erase(itUnloadBlock); + m_pRecentlyUsedChunks.erase(itUnloadBlock); erasedBlock = true; } @@ -590,17 +590,17 @@ namespace PolyVox // seems like a good place to purge the 'all blocks' list as well. if (erasedBlock) { - purgeNullPtrsFromAllBlocks(); + purgeNullPtrsFromAllChunks(); } // Add our new block to the maps. - m_pAllBlocks.insert(std::make_pair(v3dBlockPos, pChunk)); - m_pRecentlyUsedBlocks.insert(std::make_pair(v3dBlockPos, pChunk)); + m_pAllChunks.insert(std::make_pair(v3dBlockPos, pChunk)); + m_pRecentlyUsedChunks.insert(std::make_pair(v3dBlockPos, pChunk)); } - pChunk->m_uBlockLastAccessed = ++m_uTimestamper; - m_pLastAccessedBlock = pChunk; - m_v3dLastAccessedBlockPos = v3dBlockPos; + pChunk->m_uChunkLastAccessed = ++m_uTimestamper; + m_pLastAccessedChunk = pChunk; + m_v3dLastAccessedChunkPos = v3dBlockPos; return pChunk; } @@ -612,21 +612,21 @@ namespace PolyVox uint32_t PagedVolume::calculateSizeInBytes(void) { // Purge null blocks so we know that all blocks are used. - purgeNullPtrsFromAllBlocks(); + purgeNullPtrsFromAllChunks(); // Note: We disregard the size of the other class members as they are likely to be very small compared to the size of the // allocated voxel data. This also keeps the reported size as a power of two, which makes other memory calculations easier. - return Chunk::calculateSizeInBytes(m_uBlockSideLength) * m_pAllBlocks.size(); + return Chunk::calculateSizeInBytes(m_uChunkSideLength) * m_pAllChunks.size(); } template - void PagedVolume::purgeNullPtrsFromAllBlocks(void) const + void PagedVolume::purgeNullPtrsFromAllChunks(void) const { - for (auto blockIter = m_pAllBlocks.begin(); blockIter != m_pAllBlocks.end();) + for (auto blockIter = m_pAllChunks.begin(); blockIter != m_pAllChunks.end();) { if (blockIter->second.expired()) { - blockIter = m_pAllBlocks.erase(blockIter); + blockIter = m_pAllChunks.erase(blockIter); } else { @@ -684,13 +684,13 @@ namespace PolyVox template VoxelType PagedVolume::getVoxelImpl(int32_t uXPos, int32_t uYPos, int32_t uZPos, WrapModeType, VoxelType /*tBorder*/) const { - const int32_t blockX = uXPos >> m_uBlockSideLengthPower; - const int32_t blockY = uYPos >> m_uBlockSideLengthPower; - const int32_t blockZ = uZPos >> m_uBlockSideLengthPower; + const int32_t blockX = uXPos >> m_uChunkSideLengthPower; + const int32_t blockY = uYPos >> m_uChunkSideLengthPower; + const int32_t blockZ = uZPos >> m_uChunkSideLengthPower; - const uint16_t xOffset = static_cast(uXPos - (blockX << m_uBlockSideLengthPower)); - const uint16_t yOffset = static_cast(uYPos - (blockY << m_uBlockSideLengthPower)); - const uint16_t zOffset = static_cast(uZPos - (blockZ << m_uBlockSideLengthPower)); + const uint16_t xOffset = static_cast(uXPos - (blockX << m_uChunkSideLengthPower)); + const uint16_t yOffset = static_cast(uYPos - (blockY << m_uChunkSideLengthPower)); + const uint16_t zOffset = static_cast(uZPos - (blockZ << m_uChunkSideLengthPower)); auto pChunk = getChunk(blockX, blockY, blockZ); return pChunk->getVoxel(xOffset, yOffset, zOffset); diff --git a/library/PolyVoxCore/include/PolyVoxCore/PagedVolumeSampler.inl b/library/PolyVoxCore/include/PolyVoxCore/PagedVolumeSampler.inl index d6f38554..bd7b8d7f 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/PagedVolumeSampler.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/PagedVolumeSampler.inl @@ -21,12 +21,12 @@ freely, subject to the following restrictions: distribution. *******************************************************************************/ -#define CAN_GO_NEG_X(val) ((val > this->mVolume->getEnclosingRegion().getLowerX()) && (val % this->mVolume->m_uBlockSideLength != 0)) -#define CAN_GO_POS_X(val) ((val < this->mVolume->getEnclosingRegion().getUpperX()) && ((val + 1) % this->mVolume->m_uBlockSideLength != 0)) -#define CAN_GO_NEG_Y(val) ((val > this->mVolume->getEnclosingRegion().getLowerY()) && (val % this->mVolume->m_uBlockSideLength != 0)) -#define CAN_GO_POS_Y(val) ((val < this->mVolume->getEnclosingRegion().getUpperY()) && ((val + 1) % this->mVolume->m_uBlockSideLength != 0)) -#define CAN_GO_NEG_Z(val) ((val > this->mVolume->getEnclosingRegion().getLowerZ()) && (val % this->mVolume->m_uBlockSideLength != 0)) -#define CAN_GO_POS_Z(val) ((val < this->mVolume->getEnclosingRegion().getUpperZ()) && ((val + 1) % this->mVolume->m_uBlockSideLength != 0)) +#define CAN_GO_NEG_X(val) ((val > this->mVolume->getEnclosingRegion().getLowerX()) && (val % this->mVolume->m_uChunkSideLength != 0)) +#define CAN_GO_POS_X(val) ((val < this->mVolume->getEnclosingRegion().getUpperX()) && ((val + 1) % this->mVolume->m_uChunkSideLength != 0)) +#define CAN_GO_NEG_Y(val) ((val > this->mVolume->getEnclosingRegion().getLowerY()) && (val % this->mVolume->m_uChunkSideLength != 0)) +#define CAN_GO_POS_Y(val) ((val < this->mVolume->getEnclosingRegion().getUpperY()) && ((val + 1) % this->mVolume->m_uChunkSideLength != 0)) +#define CAN_GO_NEG_Z(val) ((val > this->mVolume->getEnclosingRegion().getLowerZ()) && (val % this->mVolume->m_uChunkSideLength != 0)) +#define CAN_GO_POS_Z(val) ((val < this->mVolume->getEnclosingRegion().getUpperZ()) && ((val + 1) % this->mVolume->m_uChunkSideLength != 0)) namespace PolyVox { @@ -107,17 +107,17 @@ namespace PolyVox // Then we update the voxel pointer if(this->isCurrentPositionValid()) { - const int32_t uXBlock = this->mXPosInVolume >> this->mVolume->m_uBlockSideLengthPower; - const int32_t uYBlock = this->mYPosInVolume >> this->mVolume->m_uBlockSideLengthPower; - const int32_t uZBlock = this->mZPosInVolume >> this->mVolume->m_uBlockSideLengthPower; + const int32_t uXBlock = this->mXPosInVolume >> this->mVolume->m_uChunkSideLengthPower; + const int32_t uYBlock = this->mYPosInVolume >> this->mVolume->m_uChunkSideLengthPower; + const int32_t uZBlock = this->mZPosInVolume >> this->mVolume->m_uChunkSideLengthPower; - const uint16_t uXPosInBlock = static_cast(this->mXPosInVolume - (uXBlock << this->mVolume->m_uBlockSideLengthPower)); - const uint16_t uYPosInBlock = static_cast(this->mYPosInVolume - (uYBlock << this->mVolume->m_uBlockSideLengthPower)); - const uint16_t uZPosInBlock = static_cast(this->mZPosInVolume - (uZBlock << this->mVolume->m_uBlockSideLengthPower)); + const uint16_t uXPosInBlock = static_cast(this->mXPosInVolume - (uXBlock << this->mVolume->m_uChunkSideLengthPower)); + const uint16_t uYPosInBlock = static_cast(this->mYPosInVolume - (uYBlock << this->mVolume->m_uChunkSideLengthPower)); + const uint16_t uZPosInBlock = static_cast(this->mZPosInVolume - (uZBlock << this->mVolume->m_uChunkSideLengthPower)); const uint32_t uVoxelIndexInBlock = uXPosInBlock + - uYPosInBlock * this->mVolume->m_uBlockSideLength + - uZPosInBlock * this->mVolume->m_uBlockSideLength * this->mVolume->m_uBlockSideLength; + uYPosInBlock * this->mVolume->m_uChunkSideLength + + uZPosInBlock * this->mVolume->m_uChunkSideLength * this->mVolume->m_uChunkSideLength; auto pUncompressedCurrentBlock = this->mVolume->getChunk(uXBlock, uYBlock, uZBlock); @@ -157,7 +157,7 @@ namespace PolyVox BaseVolume::template Sampler< PagedVolume >::movePositiveX(); // Then we update the voxel pointer - if((this->isCurrentPositionValid()) && bIsOldPositionValid && ((this->mXPosInVolume) % this->mVolume->m_uBlockSideLength != 0)) + if((this->isCurrentPositionValid()) && bIsOldPositionValid && ((this->mXPosInVolume) % this->mVolume->m_uChunkSideLength != 0)) { //No need to compute new block. ++mCurrentVoxel; @@ -179,10 +179,10 @@ namespace PolyVox BaseVolume::template Sampler< PagedVolume >::movePositiveY(); // Then we update the voxel pointer - if((this->isCurrentPositionValid()) && bIsOldPositionValid && ((this->mYPosInVolume) % this->mVolume->m_uBlockSideLength != 0)) + if((this->isCurrentPositionValid()) && bIsOldPositionValid && ((this->mYPosInVolume) % this->mVolume->m_uChunkSideLength != 0)) { //No need to compute new block. - mCurrentVoxel += this->mVolume->m_uBlockSideLength; + mCurrentVoxel += this->mVolume->m_uChunkSideLength; } else { @@ -201,10 +201,10 @@ namespace PolyVox BaseVolume::template Sampler< PagedVolume >::movePositiveZ(); // Then we update the voxel pointer - if((this->isCurrentPositionValid()) && bIsOldPositionValid && ((this->mZPosInVolume) % this->mVolume->m_uBlockSideLength != 0)) + if((this->isCurrentPositionValid()) && bIsOldPositionValid && ((this->mZPosInVolume) % this->mVolume->m_uChunkSideLength != 0)) { //No need to compute new block. - mCurrentVoxel += this->mVolume->m_uBlockSideLength * this->mVolume->m_uBlockSideLength; + mCurrentVoxel += this->mVolume->m_uChunkSideLength * this->mVolume->m_uChunkSideLength; } else { @@ -223,7 +223,7 @@ namespace PolyVox BaseVolume::template Sampler< PagedVolume >::moveNegativeX(); // Then we update the voxel pointer - if((this->isCurrentPositionValid()) && bIsOldPositionValid && ((this->mXPosInVolume + 1) % this->mVolume->m_uBlockSideLength != 0)) + if((this->isCurrentPositionValid()) && bIsOldPositionValid && ((this->mXPosInVolume + 1) % this->mVolume->m_uChunkSideLength != 0)) { //No need to compute new block. --mCurrentVoxel; @@ -245,10 +245,10 @@ namespace PolyVox BaseVolume::template Sampler< PagedVolume >::moveNegativeY(); // Then we update the voxel pointer - if((this->isCurrentPositionValid()) && bIsOldPositionValid && ((this->mYPosInVolume + 1) % this->mVolume->m_uBlockSideLength != 0)) + if((this->isCurrentPositionValid()) && bIsOldPositionValid && ((this->mYPosInVolume + 1) % this->mVolume->m_uChunkSideLength != 0)) { //No need to compute new block. - mCurrentVoxel -= this->mVolume->m_uBlockSideLength; + mCurrentVoxel -= this->mVolume->m_uChunkSideLength; } else { @@ -267,10 +267,10 @@ namespace PolyVox BaseVolume::template Sampler< PagedVolume >::moveNegativeZ(); // Then we update the voxel pointer - if((this->isCurrentPositionValid()) && bIsOldPositionValid && ((this->mZPosInVolume + 1) % this->mVolume->m_uBlockSideLength != 0)) + if((this->isCurrentPositionValid()) && bIsOldPositionValid && ((this->mZPosInVolume + 1) % this->mVolume->m_uChunkSideLength != 0)) { //No need to compute new block. - mCurrentVoxel -= this->mVolume->m_uBlockSideLength * this->mVolume->m_uBlockSideLength; + mCurrentVoxel -= this->mVolume->m_uChunkSideLength * this->mVolume->m_uChunkSideLength; } else { @@ -284,7 +284,7 @@ namespace PolyVox { if((this->isCurrentPositionValid()) && CAN_GO_NEG_X(this->mXPosInVolume) && CAN_GO_NEG_Y(this->mYPosInVolume) && CAN_GO_NEG_Z(this->mZPosInVolume) ) { - return *(mCurrentVoxel - 1 - this->mVolume->m_uBlockSideLength - this->mVolume->m_uBlockSideLength*this->mVolume->m_uBlockSideLength); + return *(mCurrentVoxel - 1 - this->mVolume->m_uChunkSideLength - this->mVolume->m_uChunkSideLength*this->mVolume->m_uChunkSideLength); } return this->mVolume->getVoxel(this->mXPosInVolume-1,this->mYPosInVolume-1,this->mZPosInVolume-1,this->m_eWrapMode, this->m_tBorder); } @@ -294,7 +294,7 @@ namespace PolyVox { if((this->isCurrentPositionValid()) && CAN_GO_NEG_X(this->mXPosInVolume) && CAN_GO_NEG_Y(this->mYPosInVolume) ) { - return *(mCurrentVoxel - 1 - this->mVolume->m_uBlockSideLength); + return *(mCurrentVoxel - 1 - this->mVolume->m_uChunkSideLength); } return this->mVolume->getVoxel(this->mXPosInVolume-1,this->mYPosInVolume-1,this->mZPosInVolume,this->m_eWrapMode, this->m_tBorder); } @@ -304,7 +304,7 @@ namespace PolyVox { if((this->isCurrentPositionValid()) && CAN_GO_NEG_X(this->mXPosInVolume) && CAN_GO_NEG_Y(this->mYPosInVolume) && CAN_GO_POS_Z(this->mZPosInVolume) ) { - return *(mCurrentVoxel - 1 - this->mVolume->m_uBlockSideLength + this->mVolume->m_uBlockSideLength*this->mVolume->m_uBlockSideLength); + return *(mCurrentVoxel - 1 - this->mVolume->m_uChunkSideLength + this->mVolume->m_uChunkSideLength*this->mVolume->m_uChunkSideLength); } return this->mVolume->getVoxel(this->mXPosInVolume-1,this->mYPosInVolume-1,this->mZPosInVolume+1,this->m_eWrapMode, this->m_tBorder); } @@ -314,7 +314,7 @@ namespace PolyVox { if((this->isCurrentPositionValid()) && CAN_GO_NEG_X(this->mXPosInVolume) && CAN_GO_NEG_Z(this->mZPosInVolume) ) { - return *(mCurrentVoxel - 1 - this->mVolume->m_uBlockSideLength*this->mVolume->m_uBlockSideLength); + return *(mCurrentVoxel - 1 - this->mVolume->m_uChunkSideLength*this->mVolume->m_uChunkSideLength); } return this->mVolume->getVoxel(this->mXPosInVolume-1,this->mYPosInVolume,this->mZPosInVolume-1,this->m_eWrapMode, this->m_tBorder); } @@ -334,7 +334,7 @@ namespace PolyVox { if((this->isCurrentPositionValid()) && CAN_GO_NEG_X(this->mXPosInVolume) && CAN_GO_POS_Z(this->mZPosInVolume) ) { - return *(mCurrentVoxel - 1 + this->mVolume->m_uBlockSideLength*this->mVolume->m_uBlockSideLength); + return *(mCurrentVoxel - 1 + this->mVolume->m_uChunkSideLength*this->mVolume->m_uChunkSideLength); } return this->mVolume->getVoxel(this->mXPosInVolume-1,this->mYPosInVolume,this->mZPosInVolume+1,this->m_eWrapMode, this->m_tBorder); } @@ -344,7 +344,7 @@ namespace PolyVox { if((this->isCurrentPositionValid()) && CAN_GO_NEG_X(this->mXPosInVolume) && CAN_GO_POS_Y(this->mYPosInVolume) && CAN_GO_NEG_Z(this->mZPosInVolume) ) { - return *(mCurrentVoxel - 1 + this->mVolume->m_uBlockSideLength - this->mVolume->m_uBlockSideLength*this->mVolume->m_uBlockSideLength); + return *(mCurrentVoxel - 1 + this->mVolume->m_uChunkSideLength - this->mVolume->m_uChunkSideLength*this->mVolume->m_uChunkSideLength); } return this->mVolume->getVoxel(this->mXPosInVolume-1,this->mYPosInVolume+1,this->mZPosInVolume-1,this->m_eWrapMode, this->m_tBorder); } @@ -354,7 +354,7 @@ namespace PolyVox { if((this->isCurrentPositionValid()) && CAN_GO_NEG_X(this->mXPosInVolume) && CAN_GO_POS_Y(this->mYPosInVolume) ) { - return *(mCurrentVoxel - 1 + this->mVolume->m_uBlockSideLength); + return *(mCurrentVoxel - 1 + this->mVolume->m_uChunkSideLength); } return this->mVolume->getVoxel(this->mXPosInVolume-1,this->mYPosInVolume+1,this->mZPosInVolume,this->m_eWrapMode, this->m_tBorder); } @@ -364,7 +364,7 @@ namespace PolyVox { if((this->isCurrentPositionValid()) && CAN_GO_NEG_X(this->mXPosInVolume) && CAN_GO_POS_Y(this->mYPosInVolume) && CAN_GO_POS_Z(this->mZPosInVolume) ) { - return *(mCurrentVoxel - 1 + this->mVolume->m_uBlockSideLength + this->mVolume->m_uBlockSideLength*this->mVolume->m_uBlockSideLength); + return *(mCurrentVoxel - 1 + this->mVolume->m_uChunkSideLength + this->mVolume->m_uChunkSideLength*this->mVolume->m_uChunkSideLength); } return this->mVolume->getVoxel(this->mXPosInVolume-1,this->mYPosInVolume+1,this->mZPosInVolume+1,this->m_eWrapMode, this->m_tBorder); } @@ -376,7 +376,7 @@ namespace PolyVox { if((this->isCurrentPositionValid()) && CAN_GO_NEG_Y(this->mYPosInVolume) && CAN_GO_NEG_Z(this->mZPosInVolume) ) { - return *(mCurrentVoxel - this->mVolume->m_uBlockSideLength - this->mVolume->m_uBlockSideLength*this->mVolume->m_uBlockSideLength); + return *(mCurrentVoxel - this->mVolume->m_uChunkSideLength - this->mVolume->m_uChunkSideLength*this->mVolume->m_uChunkSideLength); } return this->mVolume->getVoxel(this->mXPosInVolume,this->mYPosInVolume-1,this->mZPosInVolume-1,this->m_eWrapMode, this->m_tBorder); } @@ -386,7 +386,7 @@ namespace PolyVox { if((this->isCurrentPositionValid()) && CAN_GO_NEG_Y(this->mYPosInVolume) ) { - return *(mCurrentVoxel - this->mVolume->m_uBlockSideLength); + return *(mCurrentVoxel - this->mVolume->m_uChunkSideLength); } return this->mVolume->getVoxel(this->mXPosInVolume,this->mYPosInVolume-1,this->mZPosInVolume,this->m_eWrapMode, this->m_tBorder); } @@ -396,7 +396,7 @@ namespace PolyVox { if((this->isCurrentPositionValid()) && CAN_GO_NEG_Y(this->mYPosInVolume) && CAN_GO_POS_Z(this->mZPosInVolume) ) { - return *(mCurrentVoxel - this->mVolume->m_uBlockSideLength + this->mVolume->m_uBlockSideLength*this->mVolume->m_uBlockSideLength); + return *(mCurrentVoxel - this->mVolume->m_uChunkSideLength + this->mVolume->m_uChunkSideLength*this->mVolume->m_uChunkSideLength); } return this->mVolume->getVoxel(this->mXPosInVolume,this->mYPosInVolume-1,this->mZPosInVolume+1,this->m_eWrapMode, this->m_tBorder); } @@ -406,7 +406,7 @@ namespace PolyVox { if((this->isCurrentPositionValid()) && CAN_GO_NEG_Z(this->mZPosInVolume) ) { - return *(mCurrentVoxel - this->mVolume->m_uBlockSideLength*this->mVolume->m_uBlockSideLength); + return *(mCurrentVoxel - this->mVolume->m_uChunkSideLength*this->mVolume->m_uChunkSideLength); } return this->mVolume->getVoxel(this->mXPosInVolume,this->mYPosInVolume,this->mZPosInVolume-1,this->m_eWrapMode, this->m_tBorder); } @@ -426,7 +426,7 @@ namespace PolyVox { if((this->isCurrentPositionValid()) && CAN_GO_POS_Z(this->mZPosInVolume) ) { - return *(mCurrentVoxel + this->mVolume->m_uBlockSideLength*this->mVolume->m_uBlockSideLength); + return *(mCurrentVoxel + this->mVolume->m_uChunkSideLength*this->mVolume->m_uChunkSideLength); } return this->mVolume->getVoxel(this->mXPosInVolume,this->mYPosInVolume,this->mZPosInVolume+1,this->m_eWrapMode, this->m_tBorder); } @@ -436,7 +436,7 @@ namespace PolyVox { if((this->isCurrentPositionValid()) && CAN_GO_POS_Y(this->mYPosInVolume) && CAN_GO_NEG_Z(this->mZPosInVolume) ) { - return *(mCurrentVoxel + this->mVolume->m_uBlockSideLength - this->mVolume->m_uBlockSideLength*this->mVolume->m_uBlockSideLength); + return *(mCurrentVoxel + this->mVolume->m_uChunkSideLength - this->mVolume->m_uChunkSideLength*this->mVolume->m_uChunkSideLength); } return this->mVolume->getVoxel(this->mXPosInVolume,this->mYPosInVolume+1,this->mZPosInVolume-1,this->m_eWrapMode, this->m_tBorder); } @@ -446,7 +446,7 @@ namespace PolyVox { if((this->isCurrentPositionValid()) && CAN_GO_POS_Y(this->mYPosInVolume) ) { - return *(mCurrentVoxel + this->mVolume->m_uBlockSideLength); + return *(mCurrentVoxel + this->mVolume->m_uChunkSideLength); } return this->mVolume->getVoxel(this->mXPosInVolume,this->mYPosInVolume+1,this->mZPosInVolume,this->m_eWrapMode, this->m_tBorder); } @@ -456,7 +456,7 @@ namespace PolyVox { if((this->isCurrentPositionValid()) && CAN_GO_POS_Y(this->mYPosInVolume) && CAN_GO_POS_Z(this->mZPosInVolume) ) { - return *(mCurrentVoxel + this->mVolume->m_uBlockSideLength + this->mVolume->m_uBlockSideLength*this->mVolume->m_uBlockSideLength); + return *(mCurrentVoxel + this->mVolume->m_uChunkSideLength + this->mVolume->m_uChunkSideLength*this->mVolume->m_uChunkSideLength); } return this->mVolume->getVoxel(this->mXPosInVolume,this->mYPosInVolume+1,this->mZPosInVolume+1,this->m_eWrapMode, this->m_tBorder); } @@ -468,7 +468,7 @@ namespace PolyVox { if((this->isCurrentPositionValid()) && CAN_GO_POS_X(this->mXPosInVolume) && CAN_GO_NEG_Y(this->mYPosInVolume) && CAN_GO_NEG_Z(this->mZPosInVolume) ) { - return *(mCurrentVoxel + 1 - this->mVolume->m_uBlockSideLength - this->mVolume->m_uBlockSideLength*this->mVolume->m_uBlockSideLength); + return *(mCurrentVoxel + 1 - this->mVolume->m_uChunkSideLength - this->mVolume->m_uChunkSideLength*this->mVolume->m_uChunkSideLength); } return this->mVolume->getVoxel(this->mXPosInVolume+1,this->mYPosInVolume-1,this->mZPosInVolume-1,this->m_eWrapMode, this->m_tBorder); } @@ -478,7 +478,7 @@ namespace PolyVox { if((this->isCurrentPositionValid()) && CAN_GO_POS_X(this->mXPosInVolume) && CAN_GO_NEG_Y(this->mYPosInVolume) ) { - return *(mCurrentVoxel + 1 - this->mVolume->m_uBlockSideLength); + return *(mCurrentVoxel + 1 - this->mVolume->m_uChunkSideLength); } return this->mVolume->getVoxel(this->mXPosInVolume+1,this->mYPosInVolume-1,this->mZPosInVolume,this->m_eWrapMode, this->m_tBorder); } @@ -488,7 +488,7 @@ namespace PolyVox { if((this->isCurrentPositionValid()) && CAN_GO_POS_X(this->mXPosInVolume) && CAN_GO_NEG_Y(this->mYPosInVolume) && CAN_GO_POS_Z(this->mZPosInVolume) ) { - return *(mCurrentVoxel + 1 - this->mVolume->m_uBlockSideLength + this->mVolume->m_uBlockSideLength*this->mVolume->m_uBlockSideLength); + return *(mCurrentVoxel + 1 - this->mVolume->m_uChunkSideLength + this->mVolume->m_uChunkSideLength*this->mVolume->m_uChunkSideLength); } return this->mVolume->getVoxel(this->mXPosInVolume+1,this->mYPosInVolume-1,this->mZPosInVolume+1,this->m_eWrapMode, this->m_tBorder); } @@ -498,7 +498,7 @@ namespace PolyVox { if((this->isCurrentPositionValid()) && CAN_GO_POS_X(this->mXPosInVolume) && CAN_GO_NEG_Z(this->mZPosInVolume) ) { - return *(mCurrentVoxel + 1 - this->mVolume->m_uBlockSideLength*this->mVolume->m_uBlockSideLength); + return *(mCurrentVoxel + 1 - this->mVolume->m_uChunkSideLength*this->mVolume->m_uChunkSideLength); } return this->mVolume->getVoxel(this->mXPosInVolume+1,this->mYPosInVolume,this->mZPosInVolume-1,this->m_eWrapMode, this->m_tBorder); } @@ -518,7 +518,7 @@ namespace PolyVox { if((this->isCurrentPositionValid()) && CAN_GO_POS_X(this->mXPosInVolume) && CAN_GO_POS_Z(this->mZPosInVolume) ) { - return *(mCurrentVoxel + 1 + this->mVolume->m_uBlockSideLength*this->mVolume->m_uBlockSideLength); + return *(mCurrentVoxel + 1 + this->mVolume->m_uChunkSideLength*this->mVolume->m_uChunkSideLength); } return this->mVolume->getVoxel(this->mXPosInVolume+1,this->mYPosInVolume,this->mZPosInVolume+1,this->m_eWrapMode, this->m_tBorder); } @@ -528,7 +528,7 @@ namespace PolyVox { if((this->isCurrentPositionValid()) && CAN_GO_POS_X(this->mXPosInVolume) && CAN_GO_POS_Y(this->mYPosInVolume) && CAN_GO_NEG_Z(this->mZPosInVolume) ) { - return *(mCurrentVoxel + 1 + this->mVolume->m_uBlockSideLength - this->mVolume->m_uBlockSideLength*this->mVolume->m_uBlockSideLength); + return *(mCurrentVoxel + 1 + this->mVolume->m_uChunkSideLength - this->mVolume->m_uChunkSideLength*this->mVolume->m_uChunkSideLength); } return this->mVolume->getVoxel(this->mXPosInVolume+1,this->mYPosInVolume+1,this->mZPosInVolume-1,this->m_eWrapMode, this->m_tBorder); } @@ -538,7 +538,7 @@ namespace PolyVox { if((this->isCurrentPositionValid()) && CAN_GO_POS_X(this->mXPosInVolume) && CAN_GO_POS_Y(this->mYPosInVolume) ) { - return *(mCurrentVoxel + 1 + this->mVolume->m_uBlockSideLength); + return *(mCurrentVoxel + 1 + this->mVolume->m_uChunkSideLength); } return this->mVolume->getVoxel(this->mXPosInVolume+1,this->mYPosInVolume+1,this->mZPosInVolume,this->m_eWrapMode, this->m_tBorder); } @@ -548,7 +548,7 @@ namespace PolyVox { if((this->isCurrentPositionValid()) && CAN_GO_POS_X(this->mXPosInVolume) && CAN_GO_POS_Y(this->mYPosInVolume) && CAN_GO_POS_Z(this->mZPosInVolume) ) { - return *(mCurrentVoxel + 1 + this->mVolume->m_uBlockSideLength + this->mVolume->m_uBlockSideLength*this->mVolume->m_uBlockSideLength); + return *(mCurrentVoxel + 1 + this->mVolume->m_uChunkSideLength + this->mVolume->m_uChunkSideLength*this->mVolume->m_uChunkSideLength); } return this->mVolume->getVoxel(this->mXPosInVolume+1,this->mYPosInVolume+1,this->mZPosInVolume+1,this->m_eWrapMode, this->m_tBorder); } diff --git a/library/PolyVoxCore/include/PolyVoxCore/Pager.h b/library/PolyVoxCore/include/PolyVoxCore/Pager.h index f8edc620..39d016bb 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/Pager.h +++ b/library/PolyVoxCore/include/PolyVoxCore/Pager.h @@ -43,8 +43,8 @@ namespace PolyVox /// Destructor virtual ~Pager() {}; - virtual void pageIn(const Region& region, Chunk* pBlockData) = 0; - virtual void pageOut(const Region& region, Chunk* pBlockData) = 0; + virtual void pageIn(const Region& region, Chunk* pChunk) = 0; + virtual void pageOut(const Region& region, Chunk* pChunk) = 0; }; } From f3a18befad765ee4b518e1bc0a2755302d212246 Mon Sep 17 00:00:00 2001 From: David Williams Date: Sat, 20 Sep 2014 21:17:21 +0200 Subject: [PATCH 31/42] More renaming of 'block' to 'chunk' --- .../MarchingCubesSurfaceExtractor.inl | 1 - .../include/PolyVoxCore/PagedVolume.inl | 65 +++++++++---------- 2 files changed, 32 insertions(+), 34 deletions(-) diff --git a/library/PolyVoxCore/include/PolyVoxCore/MarchingCubesSurfaceExtractor.inl b/library/PolyVoxCore/include/PolyVoxCore/MarchingCubesSurfaceExtractor.inl index acd0f77b..4c2402cf 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/MarchingCubesSurfaceExtractor.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/MarchingCubesSurfaceExtractor.inl @@ -50,7 +50,6 @@ namespace PolyVox const uint32_t uArrayWidth = m_regSizeInVoxels.getUpperX() - m_regSizeInVoxels.getLowerX() + 1; const uint32_t uArrayHeight = m_regSizeInVoxels.getUpperY() - m_regSizeInVoxels.getLowerY() + 1; - const uint32_t arraySizes[2]= {uArrayWidth, uArrayHeight}; // Array dimensions //For edge indices Array2DInt32 m_pPreviousVertexIndicesX(uArrayWidth, uArrayHeight); diff --git a/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.inl index 48763014..8db258d7 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.inl @@ -31,7 +31,6 @@ namespace PolyVox //////////////////////////////////////////////////////////////////////////////// /// This constructor creates a volume with a fixed size which is specified as a parameter. By default this constructor will not enable paging but you can override this if desired. If you do wish to enable paging then you are required to provide the call back function (see the other PagedVolume constructor). /// \param regValid Specifies the minimum and maximum valid voxel positions. - /// \param pBlockCompressor An implementation of the Compressor interface which is used to compress blocks in memory. /// \param dataRequiredHandler The callback function which will be called when PolyVox tries to use data which is not currently in momory. /// \param dataOverflowHandler The callback function which will be called when PolyVox has too much data and needs to remove some from memory. /// \param bPagingEnabled Controls whether or not paging is enabled for this PagedVolume. @@ -215,15 +214,15 @@ namespace PolyVox { if(this->m_regValidRegion.containsPoint(Vector3DInt32(uXPos, uYPos, uZPos))) { - const int32_t blockX = uXPos >> m_uChunkSideLengthPower; - const int32_t blockY = uYPos >> m_uChunkSideLengthPower; - const int32_t blockZ = uZPos >> m_uChunkSideLengthPower; + const int32_t chunkX = uXPos >> m_uChunkSideLengthPower; + const int32_t chunkY = uYPos >> m_uChunkSideLengthPower; + const int32_t chunkZ = uZPos >> m_uChunkSideLengthPower; - const uint16_t xOffset = static_cast(uXPos - (blockX << m_uChunkSideLengthPower)); - const uint16_t yOffset = static_cast(uYPos - (blockY << m_uChunkSideLengthPower)); - const uint16_t zOffset = static_cast(uZPos - (blockZ << m_uChunkSideLengthPower)); + const uint16_t xOffset = static_cast(uXPos - (chunkX << m_uChunkSideLengthPower)); + const uint16_t yOffset = static_cast(uYPos - (chunkY << m_uChunkSideLengthPower)); + const uint16_t zOffset = static_cast(uZPos - (chunkZ << m_uChunkSideLengthPower)); - auto pChunk = getChunk(blockX, blockY, blockZ); + auto pChunk = getChunk(chunkX, chunkY, chunkZ); return pChunk->getVoxel(xOffset, yOffset, zOffset); } @@ -299,15 +298,15 @@ namespace PolyVox } } - const int32_t blockX = uXPos >> m_uChunkSideLengthPower; - const int32_t blockY = uYPos >> m_uChunkSideLengthPower; - const int32_t blockZ = uZPos >> m_uChunkSideLengthPower; + const int32_t chunkX = uXPos >> m_uChunkSideLengthPower; + const int32_t chunkY = uYPos >> m_uChunkSideLengthPower; + const int32_t chunkZ = uZPos >> m_uChunkSideLengthPower; - const uint16_t xOffset = static_cast(uXPos - (blockX << m_uChunkSideLengthPower)); - const uint16_t yOffset = static_cast(uYPos - (blockY << m_uChunkSideLengthPower)); - const uint16_t zOffset = static_cast(uZPos - (blockZ << m_uChunkSideLengthPower)); + const uint16_t xOffset = static_cast(uXPos - (chunkX << m_uChunkSideLengthPower)); + const uint16_t yOffset = static_cast(uYPos - (chunkY << m_uChunkSideLengthPower)); + const uint16_t zOffset = static_cast(uZPos - (chunkZ << m_uChunkSideLengthPower)); - auto pChunk = getChunk(blockX, blockY, blockZ); + auto pChunk = getChunk(chunkX, chunkY, chunkZ); pChunk->setVoxelAt(xOffset, yOffset, zOffset, tValue); } @@ -336,15 +335,15 @@ namespace PolyVox // PolyVox does not throw an exception when a voxel is out of range. Please see 'Error Handling' in the User Manual. POLYVOX_ASSERT(this->m_regValidRegion.containsPoint(Vector3DInt32(uXPos, uYPos, uZPos)), "Position is outside valid region"); - const int32_t blockX = uXPos >> m_uChunkSideLengthPower; - const int32_t blockY = uYPos >> m_uChunkSideLengthPower; - const int32_t blockZ = uZPos >> m_uChunkSideLengthPower; + const int32_t chunkX = uXPos >> m_uChunkSideLengthPower; + const int32_t chunkY = uYPos >> m_uChunkSideLengthPower; + const int32_t chunkZ = uZPos >> m_uChunkSideLengthPower; - const uint16_t xOffset = static_cast(uXPos - (blockX << m_uChunkSideLengthPower)); - const uint16_t yOffset = static_cast(uYPos - (blockY << m_uChunkSideLengthPower)); - const uint16_t zOffset = static_cast(uZPos - (blockZ << m_uChunkSideLengthPower)); + const uint16_t xOffset = static_cast(uXPos - (chunkX << m_uChunkSideLengthPower)); + const uint16_t yOffset = static_cast(uYPos - (chunkY << m_uChunkSideLengthPower)); + const uint16_t zOffset = static_cast(uZPos - (chunkZ << m_uChunkSideLengthPower)); - auto pChunk = getChunk(blockX, blockY, blockZ); + auto pChunk = getChunk(chunkX, chunkY, chunkZ); pChunk->setVoxelAt(xOffset, yOffset, zOffset, tValue); @@ -622,15 +621,15 @@ namespace PolyVox template void PagedVolume::purgeNullPtrsFromAllChunks(void) const { - for (auto blockIter = m_pAllChunks.begin(); blockIter != m_pAllChunks.end();) + for (auto chunkIter = m_pAllChunks.begin(); chunkIter != m_pAllChunks.end();) { - if (blockIter->second.expired()) + if (chunkIter->second.expired()) { - blockIter = m_pAllChunks.erase(blockIter); + chunkIter = m_pAllChunks.erase(chunkIter); } else { - blockIter++; + chunkIter++; } } } @@ -684,15 +683,15 @@ namespace PolyVox template VoxelType PagedVolume::getVoxelImpl(int32_t uXPos, int32_t uYPos, int32_t uZPos, WrapModeType, VoxelType /*tBorder*/) const { - const int32_t blockX = uXPos >> m_uChunkSideLengthPower; - const int32_t blockY = uYPos >> m_uChunkSideLengthPower; - const int32_t blockZ = uZPos >> m_uChunkSideLengthPower; + const int32_t chunkX = uXPos >> m_uChunkSideLengthPower; + const int32_t chunkY = uYPos >> m_uChunkSideLengthPower; + const int32_t chunkZ = uZPos >> m_uChunkSideLengthPower; - const uint16_t xOffset = static_cast(uXPos - (blockX << m_uChunkSideLengthPower)); - const uint16_t yOffset = static_cast(uYPos - (blockY << m_uChunkSideLengthPower)); - const uint16_t zOffset = static_cast(uZPos - (blockZ << m_uChunkSideLengthPower)); + const uint16_t xOffset = static_cast(uXPos - (chunkX << m_uChunkSideLengthPower)); + const uint16_t yOffset = static_cast(uYPos - (chunkY << m_uChunkSideLengthPower)); + const uint16_t zOffset = static_cast(uZPos - (chunkZ << m_uChunkSideLengthPower)); - auto pChunk = getChunk(blockX, blockY, blockZ); + auto pChunk = getChunk(chunkX, chunkY, chunkZ); return pChunk->getVoxel(xOffset, yOffset, zOffset); } } From 0ab7f27f0f5b55873482b4522f4821d133d48565 Mon Sep 17 00:00:00 2001 From: David Williams Date: Sat, 20 Sep 2014 21:27:26 +0200 Subject: [PATCH 32/42] More renaming of block to chunk. --- examples/Paging/main.cpp | 15 +-- .../PolyVoxCore/include/PolyVoxCore/Chunk.h | 8 +- .../PolyVoxCore/include/PolyVoxCore/Chunk.inl | 10 +- .../include/PolyVoxCore/FilePager.h | 14 +-- .../include/PolyVoxCore/PagedVolume.h | 32 +++---- .../include/PolyVoxCore/PagedVolume.inl | 96 +++++++++---------- .../PolyVoxCore/PagedVolumeSampler.inl | 24 ++--- .../include/PolyVoxCore/RawVolume.h | 2 +- 8 files changed, 95 insertions(+), 106 deletions(-) diff --git a/examples/Paging/main.cpp b/examples/Paging/main.cpp index b111a6bf..6d140e2f 100644 --- a/examples/Paging/main.cpp +++ b/examples/Paging/main.cpp @@ -92,10 +92,6 @@ public: virtual void pageIn(const PolyVox::Region& region, Chunk* pChunk) { - // FIXME - this isn't a great example... it's a shame we have to hard clode the block size and also create/destroy - // a compressor each time. These could at least be moved outside somewhere if we can't fix it in a better way... - //Chunk block(256); - Perlin perlin(2,2,1,234); for(int x = region.getLowerX(); x <= region.getUpperX(); x++) @@ -129,18 +125,13 @@ public: voxel.setDensity(MaterialDensityPair44::getMinDensity()); } - // Voxel position within a block always start from zero. So if a block represents region (4, 8, 12) to (11, 19, 15) - // then the valid block voxels are from (0, 0, 0) to (7, 11, 3). Hence we subtract the lower corner position of the - // region from the volume space position in order to get the block space position. + // Voxel position within a chunk always start from zero. So if a chunk represents region (4, 8, 12) to (11, 19, 15) + // then the valid chunk voxels are from (0, 0, 0) to (7, 11, 3). Hence we subtract the lower corner position of the + // region from the volume space position in order to get the chunk space position. pChunk->setVoxelAt(x - region.getLowerX(), y - region.getLowerY(), z - region.getLowerZ(), voxel); } } } - - // Now compress the computed data into the provided block. - //RLEBlockCompressor* compressor = new RLEBlockCompressor(); - //compressor->compress(&block, pChunk); - //delete compressor; } virtual void pageOut(const PolyVox::Region& region, Chunk* /*pChunk*/) diff --git a/library/PolyVoxCore/include/PolyVoxCore/Chunk.h b/library/PolyVoxCore/include/PolyVoxCore/Chunk.h index 69df02b3..1eeec066 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/Chunk.h +++ b/library/PolyVoxCore/include/PolyVoxCore/Chunk.h @@ -54,15 +54,13 @@ namespace PolyVox /// Private assignment operator to prevent accisdental copying Chunk& operator=(const Chunk& /*rhs*/) {}; - // This is updated by the PagedVolume and used to discard the least recently used blocks. + // This is updated by the PagedVolume and used to discard the least recently used chunks. uint32_t m_uChunkLastAccessed; - // This is so we can tell whether a uncompressed block has to be recompressed and whether - // a compressed block has to be paged back to disk, or whether they can just be discarded. + // This is so we can tell whether a uncompressed chunk has to be recompressed and whether + // a compressed chunk has to be paged back to disk, or whether they can just be discarded. bool m_bDataModified; - // Made this private for consistancy with CompressedBlock. - // Users shouldn't really need this for Chunk anyway. uint32_t calculateSizeInBytes(void); static uint32_t calculateSizeInBytes(uint32_t uSideLength); diff --git a/library/PolyVoxCore/include/PolyVoxCore/Chunk.inl b/library/PolyVoxCore/include/PolyVoxCore/Chunk.inl index 16440c7b..49d25aeb 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/Chunk.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/Chunk.inl @@ -43,10 +43,10 @@ namespace PolyVox const uint32_t uNoOfVoxels = m_uSideLength * m_uSideLength * m_uSideLength; m_tData = new VoxelType[uNoOfVoxels]; - // Pass the block to the Pager to give it a chance to initialise it with any data + // Pass the chunk to the Pager to give it a chance to initialise it with any data if (m_pPager) { - // From the coordinates of the block we deduce the coordinates of the contained voxels. + // From the coordinates of the chunk we deduce the coordinates of the contained voxels. Vector3DInt32 v3dLower = m_v3dChunkSpacePosition * static_cast(m_uSideLength); Vector3DInt32 v3dUpper = v3dLower + Vector3DInt32(m_uSideLength - 1, m_uSideLength - 1, m_uSideLength - 1); Region reg(v3dLower, v3dUpper); @@ -69,7 +69,7 @@ namespace PolyVox { if (m_pPager && m_bDataModified) { - // From the coordinates of the block we deduce the coordinates of the contained voxels. + // From the coordinates of the chunk we deduce the coordinates of the contained voxels. Vector3DInt32 v3dLower = m_v3dChunkSpacePosition * static_cast(m_uSideLength); Vector3DInt32 v3dUpper = v3dLower + Vector3DInt32(m_uSideLength - 1, m_uSideLength - 1, m_uSideLength - 1); @@ -101,7 +101,7 @@ namespace PolyVox POLYVOX_ASSERT(uXPos < m_uSideLength, "Supplied position is outside of the chunk"); POLYVOX_ASSERT(uYPos < m_uSideLength, "Supplied position is outside of the chunk"); POLYVOX_ASSERT(uZPos < m_uSideLength, "Supplied position is outside of the chunk"); - POLYVOX_ASSERT(m_tData, "No uncompressed data - block must be decompressed before accessing voxels."); + POLYVOX_ASSERT(m_tData, "No uncompressed data - chunk must be decompressed before accessing voxels."); return m_tData [ @@ -125,7 +125,7 @@ namespace PolyVox POLYVOX_ASSERT(uXPos < m_uSideLength, "Supplied position is outside of the chunk"); POLYVOX_ASSERT(uYPos < m_uSideLength, "Supplied position is outside of the chunk"); POLYVOX_ASSERT(uZPos < m_uSideLength, "Supplied position is outside of the chunk"); - POLYVOX_ASSERT(m_tData, "No uncompressed data - block must be decompressed before accessing voxels."); + POLYVOX_ASSERT(m_tData, "No uncompressed data - chunk must be decompressed before accessing voxels."); m_tData [ diff --git a/library/PolyVoxCore/include/PolyVoxCore/FilePager.h b/library/PolyVoxCore/include/PolyVoxCore/FilePager.h index 671023cc..26f20c21 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/FilePager.h +++ b/library/PolyVoxCore/include/PolyVoxCore/FilePager.h @@ -78,8 +78,8 @@ namespace PolyVox virtual void pageIn(const Region& region, Chunk* pChunk) { - POLYVOX_ASSERT(pChunk, "Attempting to page in NULL block"); - //POLYVOX_ASSERT(pChunk->hasUncompressedData() == false, "Block should not have uncompressed data"); + POLYVOX_ASSERT(pChunk, "Attempting to page in NULL chunk"); + POLYVOX_ASSERT(pChunk->getData() == false, "Chunk must have valid data"); std::stringstream ssFilename; ssFilename << m_strFolderName << "/" << m_strRandomPrefix << "-" @@ -109,7 +109,7 @@ namespace PolyVox if(ferror(pFile)) { - POLYVOX_THROW(std::runtime_error, "Error reading in block data, even though a file exists."); + POLYVOX_THROW(std::runtime_error, "Error reading in chunk data, even though a file exists."); } fclose(pFile); @@ -122,8 +122,8 @@ namespace PolyVox virtual void pageOut(const Region& region, Chunk* pChunk) { - POLYVOX_ASSERT(pChunk, "Attempting to page out NULL block"); - //POLYVOX_ASSERT(pChunk->hasUncompressedData() == false, "Block should not have uncompressed data"); + POLYVOX_ASSERT(pChunk, "Attempting to page out NULL chunk"); + POLYVOX_ASSERT(pChunk->getData() == false, "Chunk must have valid data"); POLYVOX_LOG_TRACE("Paging out data for " << region); @@ -140,7 +140,7 @@ namespace PolyVox FILE* pFile = fopen(filename.c_str(), "wb"); if(!pFile) { - POLYVOX_THROW(std::runtime_error, "Unable to open file to write out block data."); + POLYVOX_THROW(std::runtime_error, "Unable to open file to write out chunk data."); } //The file has been created, so add it to the list to delete on shutdown. @@ -150,7 +150,7 @@ namespace PolyVox if(ferror(pFile)) { - POLYVOX_THROW(std::runtime_error, "Error writing out block data."); + POLYVOX_THROW(std::runtime_error, "Error writing out chunk data."); } fclose(pFile); diff --git a/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.h b/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.h index de85418e..bc5ebb3e 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.h +++ b/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.h @@ -80,14 +80,14 @@ namespace PolyVox /// 1 byte per voxel will require 1GB of memory if stored in an uncompressed form. Natuarally our PagedVolume class is much more efficient /// than this and it is worth understanding (at least at a high level) the approach which is used. /// - /// Essentially, the PagedVolume class stores its data as a collection of blocks. Each of these block is much smaller than the whole volume, + /// Essentially, the PagedVolume class stores its data as a collection of chunks. Each of these chunk is much smaller than the whole volume, /// for example a typical size might be 32x32x32 voxels (though is is configurable by the user). In this case, a 256x512x1024 volume - /// would contain 8x16x32 = 4096 blocks. The data for each block is stored in a compressed form, which uses only a small amout of - /// memory but it is hard to modify the data. Therefore, before any given voxel can be modified, its corresponding block must be uncompressed. + /// would contain 8x16x32 = 4096 chunks. The data for each chunk is stored in a compressed form, which uses only a small amout of + /// memory but it is hard to modify the data. Therefore, before any given voxel can be modified, its corresponding chunk must be uncompressed. /// - /// The compression and decompression of block is a relatively slow process and so we aim to do this as rarely as possible. In order - /// to achive this, the volume class stores a cache of recently used blocks and their associated uncompressed data. Each time a voxel - /// is touched a timestamp is updated on the corresponding block. When the cache becomes full the block with the oldest timestamp is + /// The compression and decompression of chunk is a relatively slow process and so we aim to do this as rarely as possible. In order + /// to achive this, the volume class stores a cache of recently used chunks and their associated uncompressed data. Each time a voxel + /// is touched a timestamp is updated on the corresponding chunk. When the cache becomes full the chunk with the oldest timestamp is /// recompressed and moved out of the cache. /// /// Achieving high compression rates @@ -136,11 +136,11 @@ namespace PolyVox /// Cache-aware traversal /// --------------------- /// You might be suprised at just how many cache misses can occur when you traverse the volume in a naive manner. Consider a 1024x1024x1024 volume - /// with blocks of size 32x32x32. And imagine you iterate over this volume with a simple three-level for loop which iterates over x, the y, then z. + /// with chunks of size 32x32x32. And imagine you iterate over this volume with a simple three-level for loop which iterates over x, the y, then z. /// If you start at position (0,0,0) then ny the time you reach position (1023,0,0) you have touched 1024 voxels along one edge of the volume and - /// have pulled 32 blocks into the cache. By the time you reach (1023,1023,0) you have hit 1024x1024 voxels and pulled 32x32 blocks into the cache. - /// You are now ready to touch voxel (0,0,1) which is right nect to where you started, but unless your cache is at least 32x32 blocks large then this - /// initial block has already been cleared from the cache. + /// have pulled 32 chunks into the cache. By the time you reach (1023,1023,0) you have hit 1024x1024 voxels and pulled 32x32 chunks into the cache. + /// You are now ready to touch voxel (0,0,1) which is right nect to where you started, but unless your cache is at least 32x32 chunks large then this + /// initial chunk has already been cleared from the cache. /// /// Ensuring you have a large enough cache size can obviously help the above situation, but you might also consider iterating over the voxels in a /// different order. For example, if you replace your three-level loop with a six-level loop then you can first process all the voxels between (0,0,0) @@ -151,7 +151,7 @@ namespace PolyVox /// --------- /// The PagedVolume class does not make any guarentees about thread safety. You should ensure that all accesses are performed from the same thread. /// This is true even if you are only reading data from the volume, as concurrently reading from different threads can invalidate the contents - /// of the block cache (amoung other problems). + /// of the chunk cache (amoung other problems). //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// template class PagedVolume : public BaseVolume @@ -256,7 +256,7 @@ namespace PolyVox /// Gets a voxel at the position given by a 3D vector POLYVOX_DEPRECATED VoxelType getVoxelAt(const Vector3DInt32& v3dPos) const; - /// Sets the number of blocks for which uncompressed data is stored + /// Sets the number of chunks for which uncompressed data is stored void setMemoryUsageLimit(uint32_t uMemoryUsageInBytes); /// Sets the voxel at the position given by x,y,z coordinates void setVoxel(int32_t uXPos, int32_t uYPos, int32_t uZPos, VoxelType tValue, WrapMode eWrapMode = WrapModes::Validate); @@ -302,7 +302,7 @@ namespace PolyVox void purgeNullPtrsFromAllChunks(void) const; - // The block data + // The chunk data mutable WeakPtrChunkMap m_pAllChunks; mutable SharedPtrChunkMap m_pRecentlyUsedChunks; @@ -314,15 +314,15 @@ namespace PolyVox // The size of the volume Region m_regValidRegionInBlocks; - // The size of the blocks + // The size of the chunks uint16_t m_uChunkSideLength; uint8_t m_uChunkSideLengthPower; Pager* m_pPager; - // Enough to make sure a blocks and it's neighbours can be loaded, with a few to spare. + // Enough to make sure a chunks and it's neighbours can be loaded, with a few to spare. static const uint32_t uMinPracticalNoOfBlocks = 32; - // Should preent multi-gigabyte volumes with reasonable block sizes. + // Should preent multi-gigabyte volumes with reasonable chunk sizes. static const uint32_t uMaxPracticalNoOfBlocks = 32768; }; } diff --git a/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.inl index 8db258d7..cd8654c8 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.inl @@ -34,7 +34,7 @@ namespace PolyVox /// \param dataRequiredHandler The callback function which will be called when PolyVox tries to use data which is not currently in momory. /// \param dataOverflowHandler The callback function which will be called when PolyVox has too much data and needs to remove some from memory. /// \param bPagingEnabled Controls whether or not paging is enabled for this PagedVolume. - /// \param uChunkSideLength The size of the blocks making up the volume. Small blocks will compress/decompress faster, but there will also be more of them meaning voxel access could be slower. + /// \param uChunkSideLength The size of the chunks making up the volume. Small chunks will compress/decompress faster, but there will also be more of them meaning voxel access could be slower. //////////////////////////////////////////////////////////////////////////////// template PagedVolume::PagedVolume @@ -51,15 +51,15 @@ namespace PolyVox if (m_pPager) { // If the user is creating a vast (almost infinite) volume then we can bet they will be - // expecting a high memory usage and will want a fair number of blocks to play around with. + // expecting a high memory usage and will want a fair number of chunks to play around with. if (regValid == Region::MaxRegion) { m_uBlockCountLimit = 1024; } else { - // Otherwise we try to choose a block count to avoid too much thrashing, particularly when iterating - // over the whole volume. This means at least enough blocks to cover one edge of the volume, and ideally + // Otherwise we try to choose a chunk count to avoid too much thrashing, particularly when iterating + // over the whole volume. This means at least enough chunks to cover one edge of the volume, and ideally // enough for a whole face. Which face? Longest edge by shortest edge seems like a reasonable guess. uint32_t longestSide = (std::max)(regValid.getWidthInVoxels(), (std::max)(regValid.getHeightInVoxels(), regValid.getDepthInVoxels())); uint32_t shortestSide = (std::min)(regValid.getWidthInVoxels(), (std::min)(regValid.getHeightInVoxels(), regValid.getDepthInVoxels())); @@ -72,18 +72,18 @@ namespace PolyVox } else { - // If there is no pager provided then we set the block limit to the maximum - // value to ensure the system never attempts to page blocks out of memory. + // If there is no pager provided then we set the chunk limit to the maximum + // value to ensure the system never attempts to page chunks out of memory. m_uBlockCountLimit = (std::numeric_limits::max)(); } - // Make sure the calculated block limit is within practical bounds + // Make sure the calculated chunk limit is within practical bounds m_uBlockCountLimit = (std::max)(m_uBlockCountLimit, uMinPracticalNoOfBlocks); m_uBlockCountLimit = (std::min)(m_uBlockCountLimit, uMaxPracticalNoOfBlocks); uint32_t uChunkSizeInBytes = Chunk::calculateSizeInBytes(m_uChunkSideLength); POLYVOX_LOG_DEBUG("Memory usage limit for volume initially set to " << (m_uBlockCountLimit * uChunkSizeInBytes) / (1024 * 1024) - << "Mb (" << m_uBlockCountLimit << " blocks of " << uChunkSizeInBytes / 1024 << "Kb each)."); + << "Mb (" << m_uBlockCountLimit << " chunks of " << uChunkSizeInBytes / 1024 << "Kb each)."); initialise(); } @@ -243,34 +243,34 @@ namespace PolyVox } //////////////////////////////////////////////////////////////////////////////// - /// Increasing the size of the block cache will increase memory but may improve performance. + /// Increasing the size of the chunk cache will increase memory but may improve performance. /// You may want to set this to a large value (e.g. 1024) when you are first loading your /// volume data and then set it to a smaller value (e.g.64) for general processing. - /// \param uMaxNumberOfChunks The number of blocks for which uncompressed data can be cached. + /// \param uMaxNumberOfChunks The number of chunks for which uncompressed data can be cached. //////////////////////////////////////////////////////////////////////////////// template void PagedVolume::setMemoryUsageLimit(uint32_t uMemoryUsageInBytes) { POLYVOX_THROW_IF(!m_pPager, invalid_operation, "You cannot limit the memory usage of the volume because it was created without a pager attached."); - // Calculate the number of blocks based on the memory limit and the size of each block. + // Calculate the number of chunks based on the memory limit and the size of each chunk. uint32_t uChunkSizeInBytes = Chunk::calculateSizeInBytes(m_uChunkSideLength); m_uBlockCountLimit = uMemoryUsageInBytes / uChunkSizeInBytes; - // We need at least a few blocks available to avoid thrashing, and in pratice there will probably be hundreds. + // We need at least a few chunks available to avoid thrashing, and in pratice there will probably be hundreds. POLYVOX_LOG_WARNING_IF(m_uBlockCountLimit < uMinPracticalNoOfBlocks, "Requested memory usage limit of " << uMemoryUsageInBytes / (1024 * 1024) << "Mb is too low and cannot be adhered to."); m_uBlockCountLimit = (std::max)(m_uBlockCountLimit, uMinPracticalNoOfBlocks); m_uBlockCountLimit = (std::min)(m_uBlockCountLimit, uMaxPracticalNoOfBlocks); - // If the new limit is less than the number of blocks already loaded then the easiest solution is to flush and start loading again. + // If the new limit is less than the number of chunks already loaded then the easiest solution is to flush and start loading again. if (m_pRecentlyUsedChunks.size() > m_uBlockCountLimit) { flushAll(); } POLYVOX_LOG_DEBUG("Memory usage limit for volume now set to " << (m_uBlockCountLimit * uChunkSizeInBytes) / (1024 * 1024) - << "Mb (" << m_uBlockCountLimit << " blocks of " << uChunkSizeInBytes / 1024 << "Kb each)."); + << "Mb (" << m_uBlockCountLimit << " chunks of " << uChunkSizeInBytes / 1024 << "Kb each)."); } //////////////////////////////////////////////////////////////////////////////// @@ -370,7 +370,7 @@ namespace PolyVox template void PagedVolume::prefetch(Region regPrefetch) { - // Convert the start and end positions into block space coordinates + // Convert the start and end positions into chunk space coordinates Vector3DInt32 v3dStart; for(int i = 0; i < 3; i++) { @@ -383,13 +383,13 @@ namespace PolyVox v3dEnd.setElement(i, regPrefetch.getUpperCorner().getElement(i) >> m_uChunkSideLengthPower); } - // Ensure we don't page in more blocks than the volume can hold. + // Ensure we don't page in more chunks than the volume can hold. Region region(v3dStart, v3dEnd); uint32_t uNoOfBlocks = static_cast(region.getWidthInVoxels() * region.getHeightInVoxels() * region.getDepthInVoxels()); - POLYVOX_LOG_WARNING_IF(uNoOfBlocks > m_uBlockCountLimit, "Attempting to prefetch more than the maximum number of blocks."); + POLYVOX_LOG_WARNING_IF(uNoOfBlocks > m_uBlockCountLimit, "Attempting to prefetch more than the maximum number of chunks."); uNoOfBlocks = (std::min)(uNoOfBlocks, m_uBlockCountLimit); - // Loops over the specified positions and touch the corresponding blocks. + // Loops over the specified positions and touch the corresponding chunks. for(int32_t x = v3dStart.getX(); x <= v3dEnd.getX(); x++) { for(int32_t y = v3dStart.getY(); y <= v3dEnd.getY(); y++) @@ -410,16 +410,16 @@ namespace PolyVox { POLYVOX_LOG_WARNING_IF(!m_pPager, "Data discarded by flush operation as no pager is attached."); - // Clear this pointer so it doesn't hang on to any blocks. + // Clear this pointer so it doesn't hang on to any chunks. m_pLastAccessedChunk = nullptr; - // Erase all the most recently used blocks. + // Erase all the most recently used chunks. m_pRecentlyUsedChunks.clear(); - // Remove deleted blocks from the list of all loaded blocks. + // Remove deleted chunks from the list of all loaded chunks. purgeNullPtrsFromAllChunks(); - // If there are still some blocks left then this is a cause for concern. Perhaps samplers are holding on to them? + // If there are still some chunks left then this is a cause for concern. Perhaps samplers are holding on to them? POLYVOX_LOG_WARNING_IF(m_pAllChunks.size() > 0, "Blocks still exist after performing flushAll()! Perhaps you have samplers attached?"); } @@ -431,10 +431,10 @@ namespace PolyVox { POLYVOX_LOG_WARNING_IF(!m_pPager, "Data discarded by flush operation as no pager is attached."); - // Clear this pointer so it doesn't hang on to any blocks. + // Clear this pointer so it doesn't hang on to any chunks. m_pLastAccessedChunk = nullptr; - // Convert the start and end positions into block space coordinates + // Convert the start and end positions into chunk space coordinates Vector3DInt32 v3dStart; for(int i = 0; i < 3; i++) { @@ -447,7 +447,7 @@ namespace PolyVox v3dEnd.setElement(i, regFlush.getUpperCorner().getElement(i) >> m_uChunkSideLengthPower); } - // Loops over the specified positions and delete the corresponding blocks. + // Loops over the specified positions and delete the corresponding chunks. for(int32_t x = v3dStart.getX(); x <= v3dEnd.getX(); x++) { for(int32_t y = v3dStart.getY(); y <= v3dEnd.getY(); y++) @@ -459,7 +459,7 @@ namespace PolyVox } } - // We might now have so null pointers in the 'all blocks' list so clean them up. + // We might now have so null pointers in the 'all chunks' list so clean them up. purgeNullPtrsFromAllChunks(); } @@ -486,7 +486,7 @@ namespace PolyVox m_v3dLastAccessedChunkPos = Vector3DInt32(0,0,0); //There are no invalid positions, but initially the m_pLastAccessedChunk pointer will be null; m_pLastAccessedChunk = 0; - //Compute the block side length + //Compute the chunk side length m_uChunkSideLengthPower = logBase2(m_uChunkSideLength); m_regValidRegionInBlocks.setLowerX(this->m_regValidRegion.getLowerX() >> m_uChunkSideLengthPower); @@ -512,7 +512,7 @@ namespace PolyVox { Vector3DInt32 v3dBlockPos(uBlockX, uBlockY, uBlockZ); - //Check if we have the same block as last time, if so there's no need to even update + //Check if we have the same chunk as last time, if so there's no need to even update //the time stamp. If we updated it everytime then that would be every time we touched //a voxel, which would overflow a uint32_t and require us to use a uint64_t instead. //This check should also provide a significant speed boost as usually it is true. @@ -521,26 +521,26 @@ namespace PolyVox return m_pLastAccessedChunk; } - // The block was not the same as last time, but we can now hope it is in the set of most recently used blocks. + // The chunk was not the same as last time, but we can now hope it is in the set of most recently used chunks. std::shared_ptr< Chunk > pChunk = nullptr; typename SharedPtrChunkMap::iterator itChunk = m_pRecentlyUsedChunks.find(v3dBlockPos); - // Check whether the block was found. + // Check whether the chunk was found. if ((itChunk) != m_pRecentlyUsedChunks.end()) { - // The block was found so we can use it. + // The chunk was found so we can use it. pChunk = itChunk->second; - POLYVOX_ASSERT(pChunk, "Recent block list shold never contain a null pointer."); + POLYVOX_ASSERT(pChunk, "Recent chunk list shold never contain a null pointer."); } if (!pChunk) { - // Although it's not in our recently use blocks, there's some (slim) chance that it - // exists in the list of all loaded blocks, because a sampler may be holding on to it. + // Although it's not in our recently use chunks, there's some (slim) chance that it + // exists in the list of all loaded chunks, because a sampler may be holding on to it. typename WeakPtrChunkMap::iterator itWeakChunk = m_pAllChunks.find(v3dBlockPos); if (itWeakChunk != m_pAllChunks.end()) { - // We've found an entry in the 'all blocks' list, but it can be null. This happens if a sampler was the + // We've found an entry in the 'all chunks' list, but it can be null. This happens if a sampler was the // last thing to be keeping it alive and then the sampler let it go. In this case we remove it from the // list, and it will get added again soon when we page it in and fill it with valid data. if (itWeakChunk->second.expired()) @@ -549,28 +549,28 @@ namespace PolyVox } else { - // The block is valid. We know it's not in the recently used list (we checked earlier) so it should be added. + // The chunk is valid. We know it's not in the recently used list (we checked earlier) so it should be added. pChunk = std::shared_ptr< Chunk >(itWeakChunk->second); m_pRecentlyUsedChunks.insert(std::make_pair(v3dBlockPos, pChunk)); } } } - // If we still haven't found the block then it's time to create a new one and page it in from disk. + // If we still haven't found the chunk then it's time to create a new one and page it in from disk. if (!pChunk) { - // The block was not found so we will create a new one. + // The chunk was not found so we will create a new one. pChunk = std::make_shared< Chunk >(v3dBlockPos, m_uChunkSideLength, m_pPager); - // As we are loading a new block we should try to ensure we don't go over our target memory usage. + // As we are loading a new chunk we should try to ensure we don't go over our target memory usage. bool erasedBlock = false; - while (m_pRecentlyUsedChunks.size() + 1 > m_uBlockCountLimit) // +1 ready for new block we will add next. + while (m_pRecentlyUsedChunks.size() + 1 > m_uBlockCountLimit) // +1 ready for new chunk we will add next. { // This should never hit, because it should not have been possible for - // the user to limit the number of blocks if they did not provide a pager. - POLYVOX_ASSERT(m_pPager, "A valid pager is required to limit number of blocks"); + // the user to limit the number of chunks if they did not provide a pager. + POLYVOX_ASSERT(m_pPager, "A valid pager is required to limit number of chunks"); - // Find the least recently used block. Hopefully this isn't too slow. + // Find the least recently used chunk. Hopefully this isn't too slow. typename SharedPtrChunkMap::iterator itUnloadBlock = m_pRecentlyUsedChunks.begin(); for (typename SharedPtrChunkMap::iterator i = m_pRecentlyUsedChunks.begin(); i != m_pRecentlyUsedChunks.end(); i++) { @@ -580,19 +580,19 @@ namespace PolyVox } } - // Erase the least recently used block + // Erase the least recently used chunk m_pRecentlyUsedChunks.erase(itUnloadBlock); erasedBlock = true; } - // If we've deleted any blocks from the recently used list then this - // seems like a good place to purge the 'all blocks' list as well. + // If we've deleted any chunks from the recently used list then this + // seems like a good place to purge the 'all chunks' list as well. if (erasedBlock) { purgeNullPtrsFromAllChunks(); } - // Add our new block to the maps. + // Add our new chunk to the maps. m_pAllChunks.insert(std::make_pair(v3dBlockPos, pChunk)); m_pRecentlyUsedChunks.insert(std::make_pair(v3dBlockPos, pChunk)); } @@ -610,7 +610,7 @@ namespace PolyVox template uint32_t PagedVolume::calculateSizeInBytes(void) { - // Purge null blocks so we know that all blocks are used. + // Purge null chunks so we know that all chunks are used. purgeNullPtrsFromAllChunks(); // Note: We disregard the size of the other class members as they are likely to be very small compared to the size of the diff --git a/library/PolyVoxCore/include/PolyVoxCore/PagedVolumeSampler.inl b/library/PolyVoxCore/include/PolyVoxCore/PagedVolumeSampler.inl index bd7b8d7f..d325b01d 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/PagedVolumeSampler.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/PagedVolumeSampler.inl @@ -159,12 +159,12 @@ namespace PolyVox // Then we update the voxel pointer if((this->isCurrentPositionValid()) && bIsOldPositionValid && ((this->mXPosInVolume) % this->mVolume->m_uChunkSideLength != 0)) { - //No need to compute new block. + //No need to compute new chunk. ++mCurrentVoxel; } else { - //We've hit the block boundary. Just calling setPosition() is the easiest way to resolve this. + //We've hit the chunk boundary. Just calling setPosition() is the easiest way to resolve this. setPosition(this->mXPosInVolume, this->mYPosInVolume, this->mZPosInVolume); } } @@ -181,12 +181,12 @@ namespace PolyVox // Then we update the voxel pointer if((this->isCurrentPositionValid()) && bIsOldPositionValid && ((this->mYPosInVolume) % this->mVolume->m_uChunkSideLength != 0)) { - //No need to compute new block. + //No need to compute new chunk. mCurrentVoxel += this->mVolume->m_uChunkSideLength; } else { - //We've hit the block boundary. Just calling setPosition() is the easiest way to resolve this. + //We've hit the chunk boundary. Just calling setPosition() is the easiest way to resolve this. setPosition(this->mXPosInVolume, this->mYPosInVolume, this->mZPosInVolume); } } @@ -203,12 +203,12 @@ namespace PolyVox // Then we update the voxel pointer if((this->isCurrentPositionValid()) && bIsOldPositionValid && ((this->mZPosInVolume) % this->mVolume->m_uChunkSideLength != 0)) { - //No need to compute new block. + //No need to compute new chunk. mCurrentVoxel += this->mVolume->m_uChunkSideLength * this->mVolume->m_uChunkSideLength; } else { - //We've hit the block boundary. Just calling setPosition() is the easiest way to resolve this. + //We've hit the chunk boundary. Just calling setPosition() is the easiest way to resolve this. setPosition(this->mXPosInVolume, this->mYPosInVolume, this->mZPosInVolume); } } @@ -225,12 +225,12 @@ namespace PolyVox // Then we update the voxel pointer if((this->isCurrentPositionValid()) && bIsOldPositionValid && ((this->mXPosInVolume + 1) % this->mVolume->m_uChunkSideLength != 0)) { - //No need to compute new block. + //No need to compute new chunk. --mCurrentVoxel; } else { - //We've hit the block boundary. Just calling setPosition() is the easiest way to resolve this. + //We've hit the chunk boundary. Just calling setPosition() is the easiest way to resolve this. setPosition(this->mXPosInVolume, this->mYPosInVolume, this->mZPosInVolume); } } @@ -247,12 +247,12 @@ namespace PolyVox // Then we update the voxel pointer if((this->isCurrentPositionValid()) && bIsOldPositionValid && ((this->mYPosInVolume + 1) % this->mVolume->m_uChunkSideLength != 0)) { - //No need to compute new block. + //No need to compute new chunk. mCurrentVoxel -= this->mVolume->m_uChunkSideLength; } else { - //We've hit the block boundary. Just calling setPosition() is the easiest way to resolve this. + //We've hit the chunk boundary. Just calling setPosition() is the easiest way to resolve this. setPosition(this->mXPosInVolume, this->mYPosInVolume, this->mZPosInVolume); } } @@ -269,12 +269,12 @@ namespace PolyVox // Then we update the voxel pointer if((this->isCurrentPositionValid()) && bIsOldPositionValid && ((this->mZPosInVolume + 1) % this->mVolume->m_uChunkSideLength != 0)) { - //No need to compute new block. + //No need to compute new chunk. mCurrentVoxel -= this->mVolume->m_uChunkSideLength * this->mVolume->m_uChunkSideLength; } else { - //We've hit the block boundary. Just calling setPosition() is the easiest way to resolve this. + //We've hit the chunk boundary. Just calling setPosition() is the easiest way to resolve this. setPosition(this->mXPosInVolume, this->mYPosInVolume, this->mZPosInVolume); } } diff --git a/library/PolyVoxCore/include/PolyVoxCore/RawVolume.h b/library/PolyVoxCore/include/PolyVoxCore/RawVolume.h index 71f2be01..7c24e023 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/RawVolume.h +++ b/library/PolyVoxCore/include/PolyVoxCore/RawVolume.h @@ -162,7 +162,7 @@ namespace PolyVox VoxelType getVoxelImpl(int32_t uXPos, int32_t uYPos, int32_t uZPos, WrapModeType, VoxelType tBorder) const; VoxelType getVoxelImpl(int32_t uXPos, int32_t uYPos, int32_t uZPos, WrapModeType, VoxelType tBorder) const; - //The block data + //The voxel data VoxelType* m_pData; }; } From 71035029d1c51fb3eb1ae812ec6a9dc140433119 Mon Sep 17 00:00:00 2001 From: David Williams Date: Sat, 20 Sep 2014 23:27:28 +0200 Subject: [PATCH 33/42] More renaming blocks to chunks. --- examples/Paging/main.cpp | 1 - .../PolyVoxCore/CubicSurfaceExtractor.h | 2 +- .../include/PolyVoxCore/PagedVolume.h | 10 +- .../include/PolyVoxCore/PagedVolume.inl | 94 +++++++++---------- .../PolyVoxCore/PagedVolumeSampler.inl | 22 ++--- .../PolyVoxCore/PolyVoxForwardDeclarations.h | 5 - 6 files changed, 63 insertions(+), 71 deletions(-) diff --git a/examples/Paging/main.cpp b/examples/Paging/main.cpp index 6d140e2f..0b0d3de6 100644 --- a/examples/Paging/main.cpp +++ b/examples/Paging/main.cpp @@ -156,7 +156,6 @@ int main(int argc, char *argv[]) //createPerlinVolumeSlow(volData); std::cout << "Memory usage: " << (volData.calculateSizeInBytes()/1024.0/1024.0) << "MB" << std::endl; //std::cout << "Compression ratio: 1 to " << (1.0/(volData.calculateCompressionRatio())) << std::endl; - //volData.setBlockCacheSize(64); PolyVox::Region reg(Vector3DInt32(-255,0,0), Vector3DInt32(255,255,255)); std::cout << "Prefetching region: " << reg.getLowerCorner() << " -> " << reg.getUpperCorner() << std::endl; volData.prefetch(reg); diff --git a/library/PolyVoxCore/include/PolyVoxCore/CubicSurfaceExtractor.h b/library/PolyVoxCore/include/PolyVoxCore/CubicSurfaceExtractor.h index 409ed096..9e86ee1a 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/CubicSurfaceExtractor.h +++ b/library/PolyVoxCore/include/PolyVoxCore/CubicSurfaceExtractor.h @@ -206,7 +206,7 @@ namespace PolyVox /// /// \image html CubicSurfaceExtractor3.png /// - /// We could choose to add the quads to *both* regions, but this can cause confusion when one of the region is modified (causing the face to disappear or a new one to be created) as *both* regions need to have their mesh regenerated to correctly represent the new state of the volume data. Such pairs of coplanar quads can also cause problems with physics engines, and may prevent transparent block from rendering correctly. Therefore we choose to instead only add the quad to one of the the regions and we always choose the one with the greater coordinate value in the direction in which they differ. In the above example the regions differ by the 'x' component of their position, and so the quad is added to the region with the greater 'x' value (the one marked in blue). + /// We could choose to add the quads to *both* regions, but this can cause confusion when one of the region is modified (causing the face to disappear or a new one to be created) as *both* regions need to have their mesh regenerated to correctly represent the new state of the volume data. Such pairs of coplanar quads can also cause problems with physics engines, and may prevent transparent voxels from rendering correctly. Therefore we choose to instead only add the quad to one of the the regions and we always choose the one with the greater coordinate value in the direction in which they differ. In the above example the regions differ by the 'x' component of their position, and so the quad is added to the region with the greater 'x' value (the one marked in blue). /// /// **Note:** *This behaviour has changed recently (September 2012). Earlier versions of PolyVox tried to be smart about this problem by looking beyond the region which was being processed, but this complicated the code and didn't work very well. Ultimatly we decided to simply stick with the convention outlined above.* /// diff --git a/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.h b/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.h index bc5ebb3e..60852007 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.h +++ b/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.h @@ -298,7 +298,7 @@ namespace PolyVox VoxelType getVoxelImpl(int32_t uXPos, int32_t uYPos, int32_t uZPos, WrapModeType, VoxelType tBorder) const; VoxelType getVoxelImpl(int32_t uXPos, int32_t uYPos, int32_t uZPos, WrapModeType, VoxelType tBorder) const; - std::shared_ptr< Chunk > getChunk(int32_t uBlockX, int32_t uBlockY, int32_t uBlockZ) const; + std::shared_ptr< Chunk > getChunk(int32_t uChunkX, int32_t uChunkY, int32_t uChunkZ) const; void purgeNullPtrsFromAllChunks(void) const; @@ -309,10 +309,10 @@ namespace PolyVox mutable uint32_t m_uTimestamper; mutable Vector3DInt32 m_v3dLastAccessedChunkPos; mutable std::shared_ptr< Chunk > m_pLastAccessedChunk; - uint32_t m_uBlockCountLimit; + uint32_t m_uChunkCountLimit; // The size of the volume - Region m_regValidRegionInBlocks; + Region m_regValidRegionInChunks; // The size of the chunks uint16_t m_uChunkSideLength; @@ -321,9 +321,9 @@ namespace PolyVox Pager* m_pPager; // Enough to make sure a chunks and it's neighbours can be loaded, with a few to spare. - static const uint32_t uMinPracticalNoOfBlocks = 32; + static const uint32_t uMinPracticalNoOfChunks = 32; // Should preent multi-gigabyte volumes with reasonable chunk sizes. - static const uint32_t uMaxPracticalNoOfBlocks = 32768; + static const uint32_t uMaxPracticalNoOfChunks = 32768; }; } diff --git a/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.inl index cd8654c8..e307ceb1 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.inl @@ -54,7 +54,7 @@ namespace PolyVox // expecting a high memory usage and will want a fair number of chunks to play around with. if (regValid == Region::MaxRegion) { - m_uBlockCountLimit = 1024; + m_uChunkCountLimit = 1024; } else { @@ -67,23 +67,23 @@ namespace PolyVox longestSide /= m_uChunkSideLength; shortestSide /= m_uChunkSideLength; - m_uBlockCountLimit = longestSide * shortestSide; + m_uChunkCountLimit = longestSide * shortestSide; } } else { // If there is no pager provided then we set the chunk limit to the maximum // value to ensure the system never attempts to page chunks out of memory. - m_uBlockCountLimit = (std::numeric_limits::max)(); + m_uChunkCountLimit = (std::numeric_limits::max)(); } // Make sure the calculated chunk limit is within practical bounds - m_uBlockCountLimit = (std::max)(m_uBlockCountLimit, uMinPracticalNoOfBlocks); - m_uBlockCountLimit = (std::min)(m_uBlockCountLimit, uMaxPracticalNoOfBlocks); + m_uChunkCountLimit = (std::max)(m_uChunkCountLimit, uMinPracticalNoOfChunks); + m_uChunkCountLimit = (std::min)(m_uChunkCountLimit, uMaxPracticalNoOfChunks); uint32_t uChunkSizeInBytes = Chunk::calculateSizeInBytes(m_uChunkSideLength); - POLYVOX_LOG_DEBUG("Memory usage limit for volume initially set to " << (m_uBlockCountLimit * uChunkSizeInBytes) / (1024 * 1024) - << "Mb (" << m_uBlockCountLimit << " chunks of " << uChunkSizeInBytes / 1024 << "Kb each)."); + POLYVOX_LOG_DEBUG("Memory usage limit for volume initially set to " << (m_uChunkCountLimit * uChunkSizeInBytes) / (1024 * 1024) + << "Mb (" << m_uChunkCountLimit << " chunks of " << uChunkSizeInBytes / 1024 << "Kb each)."); initialise(); } @@ -255,22 +255,22 @@ namespace PolyVox // Calculate the number of chunks based on the memory limit and the size of each chunk. uint32_t uChunkSizeInBytes = Chunk::calculateSizeInBytes(m_uChunkSideLength); - m_uBlockCountLimit = uMemoryUsageInBytes / uChunkSizeInBytes; + m_uChunkCountLimit = uMemoryUsageInBytes / uChunkSizeInBytes; // We need at least a few chunks available to avoid thrashing, and in pratice there will probably be hundreds. - POLYVOX_LOG_WARNING_IF(m_uBlockCountLimit < uMinPracticalNoOfBlocks, "Requested memory usage limit of " + POLYVOX_LOG_WARNING_IF(m_uChunkCountLimit < uMinPracticalNoOfChunks, "Requested memory usage limit of " << uMemoryUsageInBytes / (1024 * 1024) << "Mb is too low and cannot be adhered to."); - m_uBlockCountLimit = (std::max)(m_uBlockCountLimit, uMinPracticalNoOfBlocks); - m_uBlockCountLimit = (std::min)(m_uBlockCountLimit, uMaxPracticalNoOfBlocks); + m_uChunkCountLimit = (std::max)(m_uChunkCountLimit, uMinPracticalNoOfChunks); + m_uChunkCountLimit = (std::min)(m_uChunkCountLimit, uMaxPracticalNoOfChunks); // If the new limit is less than the number of chunks already loaded then the easiest solution is to flush and start loading again. - if (m_pRecentlyUsedChunks.size() > m_uBlockCountLimit) + if (m_pRecentlyUsedChunks.size() > m_uChunkCountLimit) { flushAll(); } - POLYVOX_LOG_DEBUG("Memory usage limit for volume now set to " << (m_uBlockCountLimit * uChunkSizeInBytes) / (1024 * 1024) - << "Mb (" << m_uBlockCountLimit << " chunks of " << uChunkSizeInBytes / 1024 << "Kb each)."); + POLYVOX_LOG_DEBUG("Memory usage limit for volume now set to " << (m_uChunkCountLimit * uChunkSizeInBytes) / (1024 * 1024) + << "Mb (" << m_uChunkCountLimit << " chunks of " << uChunkSizeInBytes / 1024 << "Kb each)."); } //////////////////////////////////////////////////////////////////////////////// @@ -364,7 +364,7 @@ namespace PolyVox //////////////////////////////////////////////////////////////////////////////// - /// Note that if *NOTE - update docs - MaxNumberOfBlocksInMemory no longer exists* MaxNumberOfBlocksInMemory is not large enough to support the region this function will only load part of the region. In this case it is undefined which parts will actually be loaded. If all the voxels in the given region are already loaded, this function will not do anything. Other voxels might be unloaded to make space for the new voxels. + /// Note that if the memory usage limit is not large enough to support the region this function will only load part of the region. In this case it is undefined which parts will actually be loaded. If all the voxels in the given region are already loaded, this function will not do anything. Other voxels might be unloaded to make space for the new voxels. /// \param regPrefetch The Region of voxels to prefetch into memory. //////////////////////////////////////////////////////////////////////////////// template @@ -385,9 +385,9 @@ namespace PolyVox // Ensure we don't page in more chunks than the volume can hold. Region region(v3dStart, v3dEnd); - uint32_t uNoOfBlocks = static_cast(region.getWidthInVoxels() * region.getHeightInVoxels() * region.getDepthInVoxels()); - POLYVOX_LOG_WARNING_IF(uNoOfBlocks > m_uBlockCountLimit, "Attempting to prefetch more than the maximum number of chunks."); - uNoOfBlocks = (std::min)(uNoOfBlocks, m_uBlockCountLimit); + uint32_t uNoOfChunks = static_cast(region.getWidthInVoxels() * region.getHeightInVoxels() * region.getDepthInVoxels()); + POLYVOX_LOG_WARNING_IF(uNoOfChunks > m_uChunkCountLimit, "Attempting to prefetch more than the maximum number of chunks."); + uNoOfChunks = (std::min)(uNoOfChunks, m_uChunkCountLimit); // Loops over the specified positions and touch the corresponding chunks. for(int32_t x = v3dStart.getX(); x <= v3dEnd.getX(); x++) @@ -420,7 +420,7 @@ namespace PolyVox purgeNullPtrsFromAllChunks(); // If there are still some chunks left then this is a cause for concern. Perhaps samplers are holding on to them? - POLYVOX_LOG_WARNING_IF(m_pAllChunks.size() > 0, "Blocks still exist after performing flushAll()! Perhaps you have samplers attached?"); + POLYVOX_LOG_WARNING_IF(m_pAllChunks.size() > 0, "Chunks still exist after performing flushAll()! Perhaps you have samplers attached?"); } //////////////////////////////////////////////////////////////////////////////// @@ -472,31 +472,29 @@ namespace PolyVox //Validate parameters if(m_uChunkSideLength == 0) { - POLYVOX_THROW(std::invalid_argument, "Block side length cannot be zero."); + POLYVOX_THROW(std::invalid_argument, "Chunk side length cannot be zero."); } if(!isPowerOf2(m_uChunkSideLength)) { - POLYVOX_THROW(std::invalid_argument, "Block side length must be a power of two."); + POLYVOX_THROW(std::invalid_argument, "Chunk side length must be a power of two."); } m_uTimestamper = 0; - //m_uBlockCountLimit = 16; - //m_uMaxNumberOfBlocksInMemory = 1024; m_v3dLastAccessedChunkPos = Vector3DInt32(0,0,0); //There are no invalid positions, but initially the m_pLastAccessedChunk pointer will be null; m_pLastAccessedChunk = 0; //Compute the chunk side length m_uChunkSideLengthPower = logBase2(m_uChunkSideLength); - m_regValidRegionInBlocks.setLowerX(this->m_regValidRegion.getLowerX() >> m_uChunkSideLengthPower); - m_regValidRegionInBlocks.setLowerY(this->m_regValidRegion.getLowerY() >> m_uChunkSideLengthPower); - m_regValidRegionInBlocks.setLowerZ(this->m_regValidRegion.getLowerZ() >> m_uChunkSideLengthPower); - m_regValidRegionInBlocks.setUpperX(this->m_regValidRegion.getUpperX() >> m_uChunkSideLengthPower); - m_regValidRegionInBlocks.setUpperY(this->m_regValidRegion.getUpperY() >> m_uChunkSideLengthPower); - m_regValidRegionInBlocks.setUpperZ(this->m_regValidRegion.getUpperZ() >> m_uChunkSideLengthPower); + m_regValidRegionInChunks.setLowerX(this->m_regValidRegion.getLowerX() >> m_uChunkSideLengthPower); + m_regValidRegionInChunks.setLowerY(this->m_regValidRegion.getLowerY() >> m_uChunkSideLengthPower); + m_regValidRegionInChunks.setLowerZ(this->m_regValidRegion.getLowerZ() >> m_uChunkSideLengthPower); + m_regValidRegionInChunks.setUpperX(this->m_regValidRegion.getUpperX() >> m_uChunkSideLengthPower); + m_regValidRegionInChunks.setUpperY(this->m_regValidRegion.getUpperY() >> m_uChunkSideLengthPower); + m_regValidRegionInChunks.setUpperZ(this->m_regValidRegion.getUpperZ() >> m_uChunkSideLengthPower); - //setMaxNumberOfChunks(m_uBlockCountLimit); + //setMaxNumberOfChunks(m_uChunkCountLimit); //Clear the previous data m_pRecentlyUsedChunks.clear(); @@ -508,22 +506,22 @@ namespace PolyVox } template - std::shared_ptr< Chunk > PagedVolume::getChunk(int32_t uBlockX, int32_t uBlockY, int32_t uBlockZ) const + std::shared_ptr< Chunk > PagedVolume::getChunk(int32_t uChunkX, int32_t uChunkY, int32_t uChunkZ) const { - Vector3DInt32 v3dBlockPos(uBlockX, uBlockY, uBlockZ); + Vector3DInt32 v3dChunkPos(uChunkX, uChunkY, uChunkZ); //Check if we have the same chunk as last time, if so there's no need to even update //the time stamp. If we updated it everytime then that would be every time we touched //a voxel, which would overflow a uint32_t and require us to use a uint64_t instead. //This check should also provide a significant speed boost as usually it is true. - if((v3dBlockPos == m_v3dLastAccessedChunkPos) && (m_pLastAccessedChunk != 0)) + if((v3dChunkPos == m_v3dLastAccessedChunkPos) && (m_pLastAccessedChunk != 0)) { return m_pLastAccessedChunk; } // The chunk was not the same as last time, but we can now hope it is in the set of most recently used chunks. std::shared_ptr< Chunk > pChunk = nullptr; - typename SharedPtrChunkMap::iterator itChunk = m_pRecentlyUsedChunks.find(v3dBlockPos); + typename SharedPtrChunkMap::iterator itChunk = m_pRecentlyUsedChunks.find(v3dChunkPos); // Check whether the chunk was found. if ((itChunk) != m_pRecentlyUsedChunks.end()) @@ -537,7 +535,7 @@ namespace PolyVox { // Although it's not in our recently use chunks, there's some (slim) chance that it // exists in the list of all loaded chunks, because a sampler may be holding on to it. - typename WeakPtrChunkMap::iterator itWeakChunk = m_pAllChunks.find(v3dBlockPos); + typename WeakPtrChunkMap::iterator itWeakChunk = m_pAllChunks.find(v3dChunkPos); if (itWeakChunk != m_pAllChunks.end()) { // We've found an entry in the 'all chunks' list, but it can be null. This happens if a sampler was the @@ -551,7 +549,7 @@ namespace PolyVox { // The chunk is valid. We know it's not in the recently used list (we checked earlier) so it should be added. pChunk = std::shared_ptr< Chunk >(itWeakChunk->second); - m_pRecentlyUsedChunks.insert(std::make_pair(v3dBlockPos, pChunk)); + m_pRecentlyUsedChunks.insert(std::make_pair(v3dChunkPos, pChunk)); } } } @@ -560,46 +558,46 @@ namespace PolyVox if (!pChunk) { // The chunk was not found so we will create a new one. - pChunk = std::make_shared< Chunk >(v3dBlockPos, m_uChunkSideLength, m_pPager); + pChunk = std::make_shared< Chunk >(v3dChunkPos, m_uChunkSideLength, m_pPager); // As we are loading a new chunk we should try to ensure we don't go over our target memory usage. - bool erasedBlock = false; - while (m_pRecentlyUsedChunks.size() + 1 > m_uBlockCountLimit) // +1 ready for new chunk we will add next. + bool erasedChunk = false; + while (m_pRecentlyUsedChunks.size() + 1 > m_uChunkCountLimit) // +1 ready for new chunk we will add next. { // This should never hit, because it should not have been possible for // the user to limit the number of chunks if they did not provide a pager. POLYVOX_ASSERT(m_pPager, "A valid pager is required to limit number of chunks"); // Find the least recently used chunk. Hopefully this isn't too slow. - typename SharedPtrChunkMap::iterator itUnloadBlock = m_pRecentlyUsedChunks.begin(); + typename SharedPtrChunkMap::iterator itUnloadChunk = m_pRecentlyUsedChunks.begin(); for (typename SharedPtrChunkMap::iterator i = m_pRecentlyUsedChunks.begin(); i != m_pRecentlyUsedChunks.end(); i++) { - if (i->second->m_uChunkLastAccessed < itUnloadBlock->second->m_uChunkLastAccessed) + if (i->second->m_uChunkLastAccessed < itUnloadChunk->second->m_uChunkLastAccessed) { - itUnloadBlock = i; + itUnloadChunk = i; } } // Erase the least recently used chunk - m_pRecentlyUsedChunks.erase(itUnloadBlock); - erasedBlock = true; + m_pRecentlyUsedChunks.erase(itUnloadChunk); + erasedChunk = true; } // If we've deleted any chunks from the recently used list then this // seems like a good place to purge the 'all chunks' list as well. - if (erasedBlock) + if (erasedChunk) { purgeNullPtrsFromAllChunks(); } // Add our new chunk to the maps. - m_pAllChunks.insert(std::make_pair(v3dBlockPos, pChunk)); - m_pRecentlyUsedChunks.insert(std::make_pair(v3dBlockPos, pChunk)); + m_pAllChunks.insert(std::make_pair(v3dChunkPos, pChunk)); + m_pRecentlyUsedChunks.insert(std::make_pair(v3dChunkPos, pChunk)); } pChunk->m_uChunkLastAccessed = ++m_uTimestamper; m_pLastAccessedChunk = pChunk; - m_v3dLastAccessedChunkPos = v3dBlockPos; + m_v3dLastAccessedChunkPos = v3dChunkPos; return pChunk; } diff --git a/library/PolyVoxCore/include/PolyVoxCore/PagedVolumeSampler.inl b/library/PolyVoxCore/include/PolyVoxCore/PagedVolumeSampler.inl index d325b01d..90909cb7 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/PagedVolumeSampler.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/PagedVolumeSampler.inl @@ -107,21 +107,21 @@ namespace PolyVox // Then we update the voxel pointer if(this->isCurrentPositionValid()) { - const int32_t uXBlock = this->mXPosInVolume >> this->mVolume->m_uChunkSideLengthPower; - const int32_t uYBlock = this->mYPosInVolume >> this->mVolume->m_uChunkSideLengthPower; - const int32_t uZBlock = this->mZPosInVolume >> this->mVolume->m_uChunkSideLengthPower; + const int32_t uXChunk = this->mXPosInVolume >> this->mVolume->m_uChunkSideLengthPower; + const int32_t uYChunk = this->mYPosInVolume >> this->mVolume->m_uChunkSideLengthPower; + const int32_t uZChunk = this->mZPosInVolume >> this->mVolume->m_uChunkSideLengthPower; - const uint16_t uXPosInBlock = static_cast(this->mXPosInVolume - (uXBlock << this->mVolume->m_uChunkSideLengthPower)); - const uint16_t uYPosInBlock = static_cast(this->mYPosInVolume - (uYBlock << this->mVolume->m_uChunkSideLengthPower)); - const uint16_t uZPosInBlock = static_cast(this->mZPosInVolume - (uZBlock << this->mVolume->m_uChunkSideLengthPower)); + const uint16_t uXPosInChunk = static_cast(this->mXPosInVolume - (uXChunk << this->mVolume->m_uChunkSideLengthPower)); + const uint16_t uYPosInChunk = static_cast(this->mYPosInVolume - (uYChunk << this->mVolume->m_uChunkSideLengthPower)); + const uint16_t uZPosInChunk = static_cast(this->mZPosInVolume - (uZChunk << this->mVolume->m_uChunkSideLengthPower)); - const uint32_t uVoxelIndexInBlock = uXPosInBlock + - uYPosInBlock * this->mVolume->m_uChunkSideLength + - uZPosInBlock * this->mVolume->m_uChunkSideLength * this->mVolume->m_uChunkSideLength; + const uint32_t uVoxelIndexInChunk = uXPosInChunk + + uYPosInChunk * this->mVolume->m_uChunkSideLength + + uZPosInChunk * this->mVolume->m_uChunkSideLength * this->mVolume->m_uChunkSideLength; - auto pUncompressedCurrentBlock = this->mVolume->getChunk(uXBlock, uYBlock, uZBlock); + auto pCurrentChunk = this->mVolume->getChunk(uXChunk, uYChunk, uZChunk); - mCurrentVoxel = pUncompressedCurrentBlock->m_tData + uVoxelIndexInBlock; + mCurrentVoxel = pCurrentChunk->m_tData + uVoxelIndexInChunk; } else { diff --git a/library/PolyVoxCore/include/PolyVoxCore/PolyVoxForwardDeclarations.h b/library/PolyVoxCore/include/PolyVoxCore/PolyVoxForwardDeclarations.h index 449ff4a9..58aceae2 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/PolyVoxForwardDeclarations.h +++ b/library/PolyVoxCore/include/PolyVoxCore/PolyVoxForwardDeclarations.h @@ -62,11 +62,6 @@ namespace PolyVox typedef Array<3,int32_t> Array3DInt32; typedef Array<3,uint32_t> Array3DUint32;*/ - //////////////////////////////////////////////////////////////////////////////// - // BlockCompressor - //////////////////////////////////////////////////////////////////////////////// - template class BlockCompressor; - //////////////////////////////////////////////////////////////////////////////// // Compressor //////////////////////////////////////////////////////////////////////////////// From db9a74fdb4656fbe80dce56302f1881758b387ba Mon Sep 17 00:00:00 2001 From: David Williams Date: Sun, 21 Sep 2014 10:27:55 +0200 Subject: [PATCH 34/42] Moving Chunk to be a nested class of PagedVolume. --- examples/Paging/main.cpp | 4 +- library/PolyVoxCore/CMakeLists.txt | 3 +- .../PolyVoxCore/include/PolyVoxCore/Chunk.h | 77 ------------------- .../include/PolyVoxCore/FilePager.h | 8 +- .../include/PolyVoxCore/PagedVolume.h | 50 ++++++++++-- .../include/PolyVoxCore/PagedVolume.inl | 14 ++-- .../{Chunk.inl => PagedVolumeChunk.inl} | 20 ++--- .../PolyVoxCore/include/PolyVoxCore/Pager.h | 6 +- 8 files changed, 72 insertions(+), 110 deletions(-) delete mode 100644 library/PolyVoxCore/include/PolyVoxCore/Chunk.h rename library/PolyVoxCore/include/PolyVoxCore/{Chunk.inl => PagedVolumeChunk.inl} (86%) diff --git a/examples/Paging/main.cpp b/examples/Paging/main.cpp index 0b0d3de6..f0248c14 100644 --- a/examples/Paging/main.cpp +++ b/examples/Paging/main.cpp @@ -90,7 +90,7 @@ public: /// Destructor virtual ~PerlinNoisePager() {}; - virtual void pageIn(const PolyVox::Region& region, Chunk* pChunk) + virtual void pageIn(const PolyVox::Region& region, PagedVolume::Chunk* pChunk) { Perlin perlin(2,2,1,234); @@ -134,7 +134,7 @@ public: } } - virtual void pageOut(const PolyVox::Region& region, Chunk* /*pChunk*/) + virtual void pageOut(const PolyVox::Region& region, PagedVolume::Chunk* /*pChunk*/) { std::cout << "warning unloading region: " << region.getLowerCorner() << " -> " << region.getUpperCorner() << std::endl; } diff --git a/library/PolyVoxCore/CMakeLists.txt b/library/PolyVoxCore/CMakeLists.txt index 286f5e14..9704e593 100644 --- a/library/PolyVoxCore/CMakeLists.txt +++ b/library/PolyVoxCore/CMakeLists.txt @@ -38,8 +38,6 @@ SET(CORE_INC_FILES include/PolyVoxCore/BaseVolume.h include/PolyVoxCore/BaseVolume.inl include/PolyVoxCore/BaseVolumeSampler.inl - include/PolyVoxCore/Chunk.h - include/PolyVoxCore/Chunk.inl include/PolyVoxCore/CubicSurfaceExtractor.h include/PolyVoxCore/CubicSurfaceExtractor.inl include/PolyVoxCore/DefaultIsQuadNeeded.h @@ -62,6 +60,7 @@ SET(CORE_INC_FILES include/PolyVoxCore/Mesh.inl include/PolyVoxCore/PagedVolume.h include/PolyVoxCore/PagedVolume.inl + include/PolyVoxCore/PagedVolumeChunk.inl include/PolyVoxCore/PagedVolumeSampler.inl include/PolyVoxCore/Pager.h include/PolyVoxCore/PolyVoxForwardDeclarations.h diff --git a/library/PolyVoxCore/include/PolyVoxCore/Chunk.h b/library/PolyVoxCore/include/PolyVoxCore/Chunk.h deleted file mode 100644 index 1eeec066..00000000 --- a/library/PolyVoxCore/include/PolyVoxCore/Chunk.h +++ /dev/null @@ -1,77 +0,0 @@ -/******************************************************************************* -Copyright (c) 2005-2013 David Williams and Matthew Williams - -This software is provided 'as-is', without any express or implied -warranty. In no event will the authors be held liable for any damages -arising from the use of this software. - -Permission is granted to anyone to use this software for any purpose, -including commercial applications, and to alter it and redistribute it -freely, subject to the following restrictions: - - 1. The origin of this software must not be misrepresented; you must not - claim that you wrote the original software. If you use this software - in a product, an acknowledgment in the product documentation would be - appreciated but is not required. - - 2. Altered source versions must be plainly marked as such, and must not be - misrepresented as being the original software. - - 3. This notice may not be removed or altered from any source - distribution. -*******************************************************************************/ - -#ifndef __PolyVox_Chunk_H__ -#define __PolyVox_Chunk_H__ - -#include "PolyVoxCore/PolyVoxForwardDeclarations.h" -#include "PolyVoxCore/Vector.h" - -namespace PolyVox -{ - template - class Chunk - { - friend class PagedVolume; - - public: - Chunk(Vector3DInt32 v3dPosition, uint16_t uSideLength, Pager* pPager = nullptr); - ~Chunk(); - - VoxelType* getData(void) const; - uint32_t getDataSizeInBytes(void) const; - - VoxelType getVoxel(uint16_t uXPos, uint16_t uYPos, uint16_t uZPos) const; - VoxelType getVoxel(const Vector3DUint16& v3dPos) const; - - void setVoxelAt(uint16_t uXPos, uint16_t uYPos, uint16_t uZPos, VoxelType tValue); - void setVoxelAt(const Vector3DUint16& v3dPos, VoxelType tValue); - - private: - /// Private copy constructor to prevent accisdental copying - Chunk(const Chunk& /*rhs*/) {}; - - /// Private assignment operator to prevent accisdental copying - Chunk& operator=(const Chunk& /*rhs*/) {}; - - // This is updated by the PagedVolume and used to discard the least recently used chunks. - uint32_t m_uChunkLastAccessed; - - // This is so we can tell whether a uncompressed chunk has to be recompressed and whether - // a compressed chunk has to be paged back to disk, or whether they can just be discarded. - bool m_bDataModified; - - uint32_t calculateSizeInBytes(void); - static uint32_t calculateSizeInBytes(uint32_t uSideLength); - - VoxelType* m_tData; - uint16_t m_uSideLength; - uint8_t m_uSideLengthPower; - Pager* m_pPager; - Vector3DInt32 m_v3dChunkSpacePosition; - }; -} - -#include "PolyVoxCore/Chunk.inl" - -#endif diff --git a/library/PolyVoxCore/include/PolyVoxCore/FilePager.h b/library/PolyVoxCore/include/PolyVoxCore/FilePager.h index 26f20c21..269af29c 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/FilePager.h +++ b/library/PolyVoxCore/include/PolyVoxCore/FilePager.h @@ -76,10 +76,10 @@ namespace PolyVox m_vecCreatedFiles.clear(); } - virtual void pageIn(const Region& region, Chunk* pChunk) + virtual void pageIn(const Region& region, typename PagedVolume::Chunk* pChunk) { POLYVOX_ASSERT(pChunk, "Attempting to page in NULL chunk"); - POLYVOX_ASSERT(pChunk->getData() == false, "Chunk must have valid data"); + POLYVOX_ASSERT(pChunk->getData(), "Chunk must have valid data"); std::stringstream ssFilename; ssFilename << m_strFolderName << "/" << m_strRandomPrefix << "-" @@ -120,10 +120,10 @@ namespace PolyVox } } - virtual void pageOut(const Region& region, Chunk* pChunk) + virtual void pageOut(const Region& region, typename PagedVolume::Chunk* pChunk) { POLYVOX_ASSERT(pChunk, "Attempting to page out NULL chunk"); - POLYVOX_ASSERT(pChunk->getData() == false, "Chunk must have valid data"); + POLYVOX_ASSERT(pChunk->getData(), "Chunk must have valid data"); POLYVOX_LOG_TRACE("Paging out data for " << region); diff --git a/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.h b/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.h index 60852007..75acaaca 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.h +++ b/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.h @@ -27,7 +27,6 @@ freely, subject to the following restrictions: #include "PolyVoxCore/BaseVolume.h" #include "PolyVoxCore/Pager.h" #include "PolyVoxCore/Region.h" -#include "PolyVoxCore/Chunk.h" #include "PolyVoxCore/Vector.h" #include @@ -157,6 +156,46 @@ namespace PolyVox class PagedVolume : public BaseVolume { public: + + class Chunk + { + public: + Chunk(Vector3DInt32 v3dPosition, uint16_t uSideLength, Pager* pPager = nullptr); + ~Chunk(); + + VoxelType* getData(void) const; + uint32_t getDataSizeInBytes(void) const; + + VoxelType getVoxel(uint16_t uXPos, uint16_t uYPos, uint16_t uZPos) const; + VoxelType getVoxel(const Vector3DUint16& v3dPos) const; + + void setVoxelAt(uint16_t uXPos, uint16_t uYPos, uint16_t uZPos, VoxelType tValue); + void setVoxelAt(const Vector3DUint16& v3dPos, VoxelType tValue); + + public: + /// Private copy constructor to prevent accisdental copying + Chunk(const Chunk& /*rhs*/) {}; + + /// Private assignment operator to prevent accisdental copying + Chunk& operator=(const Chunk& /*rhs*/) {}; + + // This is updated by the PagedVolume and used to discard the least recently used chunks. + uint32_t m_uChunkLastAccessed; + + // This is so we can tell whether a uncompressed chunk has to be recompressed and whether + // a compressed chunk has to be paged back to disk, or whether they can just be discarded. + bool m_bDataModified; + + uint32_t calculateSizeInBytes(void); + static uint32_t calculateSizeInBytes(uint32_t uSideLength); + + VoxelType* m_tData; + uint16_t m_uSideLength; + uint8_t m_uSideLengthPower; + Pager* m_pPager; + Vector3DInt32 m_v3dChunkSpacePosition; + }; + //There seems to be some descrepency between Visual Studio and GCC about how the following class should be declared. //There is a work around (see also See http://goo.gl/qu1wn) given below which appears to work on VS2010 and GCC, but //which seems to cause internal compiler errors on VS2008 when building with the /Gm 'Enable Minimal Rebuild' compiler @@ -285,8 +324,8 @@ namespace PolyVox private: - typedef std::unordered_map > > SharedPtrChunkMap; - typedef std::unordered_map > > WeakPtrChunkMap; + typedef std::unordered_map > SharedPtrChunkMap; + typedef std::unordered_map > WeakPtrChunkMap; void initialise(); @@ -298,7 +337,7 @@ namespace PolyVox VoxelType getVoxelImpl(int32_t uXPos, int32_t uYPos, int32_t uZPos, WrapModeType, VoxelType tBorder) const; VoxelType getVoxelImpl(int32_t uXPos, int32_t uYPos, int32_t uZPos, WrapModeType, VoxelType tBorder) const; - std::shared_ptr< Chunk > getChunk(int32_t uChunkX, int32_t uChunkY, int32_t uChunkZ) const; + std::shared_ptr getChunk(int32_t uChunkX, int32_t uChunkY, int32_t uChunkZ) const; void purgeNullPtrsFromAllChunks(void) const; @@ -308,7 +347,7 @@ namespace PolyVox mutable uint32_t m_uTimestamper; mutable Vector3DInt32 m_v3dLastAccessedChunkPos; - mutable std::shared_ptr< Chunk > m_pLastAccessedChunk; + mutable std::shared_ptr m_pLastAccessedChunk; uint32_t m_uChunkCountLimit; // The size of the volume @@ -328,6 +367,7 @@ namespace PolyVox } #include "PolyVoxCore/PagedVolume.inl" +#include "PolyVoxCore/PagedVolumeChunk.inl" #include "PolyVoxCore/PagedVolumeSampler.inl" #endif //__PolyVox_PagedVolume_H__ diff --git a/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.inl index e307ceb1..ae5aeea3 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.inl @@ -81,7 +81,7 @@ namespace PolyVox m_uChunkCountLimit = (std::max)(m_uChunkCountLimit, uMinPracticalNoOfChunks); m_uChunkCountLimit = (std::min)(m_uChunkCountLimit, uMaxPracticalNoOfChunks); - uint32_t uChunkSizeInBytes = Chunk::calculateSizeInBytes(m_uChunkSideLength); + uint32_t uChunkSizeInBytes = PagedVolume::Chunk::calculateSizeInBytes(m_uChunkSideLength); POLYVOX_LOG_DEBUG("Memory usage limit for volume initially set to " << (m_uChunkCountLimit * uChunkSizeInBytes) / (1024 * 1024) << "Mb (" << m_uChunkCountLimit << " chunks of " << uChunkSizeInBytes / 1024 << "Kb each)."); @@ -254,7 +254,7 @@ namespace PolyVox POLYVOX_THROW_IF(!m_pPager, invalid_operation, "You cannot limit the memory usage of the volume because it was created without a pager attached."); // Calculate the number of chunks based on the memory limit and the size of each chunk. - uint32_t uChunkSizeInBytes = Chunk::calculateSizeInBytes(m_uChunkSideLength); + uint32_t uChunkSizeInBytes = PagedVolume::Chunk::calculateSizeInBytes(m_uChunkSideLength); m_uChunkCountLimit = uMemoryUsageInBytes / uChunkSizeInBytes; // We need at least a few chunks available to avoid thrashing, and in pratice there will probably be hundreds. @@ -506,7 +506,7 @@ namespace PolyVox } template - std::shared_ptr< Chunk > PagedVolume::getChunk(int32_t uChunkX, int32_t uChunkY, int32_t uChunkZ) const + std::shared_ptr::Chunk> PagedVolume::getChunk(int32_t uChunkX, int32_t uChunkY, int32_t uChunkZ) const { Vector3DInt32 v3dChunkPos(uChunkX, uChunkY, uChunkZ); @@ -520,7 +520,7 @@ namespace PolyVox } // The chunk was not the same as last time, but we can now hope it is in the set of most recently used chunks. - std::shared_ptr< Chunk > pChunk = nullptr; + std::shared_ptr::Chunk> pChunk = nullptr; typename SharedPtrChunkMap::iterator itChunk = m_pRecentlyUsedChunks.find(v3dChunkPos); // Check whether the chunk was found. @@ -548,7 +548,7 @@ namespace PolyVox else { // The chunk is valid. We know it's not in the recently used list (we checked earlier) so it should be added. - pChunk = std::shared_ptr< Chunk >(itWeakChunk->second); + pChunk = std::shared_ptr< PagedVolume::Chunk >(itWeakChunk->second); m_pRecentlyUsedChunks.insert(std::make_pair(v3dChunkPos, pChunk)); } } @@ -558,7 +558,7 @@ namespace PolyVox if (!pChunk) { // The chunk was not found so we will create a new one. - pChunk = std::make_shared< Chunk >(v3dChunkPos, m_uChunkSideLength, m_pPager); + pChunk = std::make_shared< PagedVolume::Chunk >(v3dChunkPos, m_uChunkSideLength, m_pPager); // As we are loading a new chunk we should try to ensure we don't go over our target memory usage. bool erasedChunk = false; @@ -613,7 +613,7 @@ namespace PolyVox // Note: We disregard the size of the other class members as they are likely to be very small compared to the size of the // allocated voxel data. This also keeps the reported size as a power of two, which makes other memory calculations easier. - return Chunk::calculateSizeInBytes(m_uChunkSideLength) * m_pAllChunks.size(); + return PagedVolume::Chunk::calculateSizeInBytes(m_uChunkSideLength) * m_pAllChunks.size(); } template diff --git a/library/PolyVoxCore/include/PolyVoxCore/Chunk.inl b/library/PolyVoxCore/include/PolyVoxCore/PagedVolumeChunk.inl similarity index 86% rename from library/PolyVoxCore/include/PolyVoxCore/Chunk.inl rename to library/PolyVoxCore/include/PolyVoxCore/PagedVolumeChunk.inl index 49d25aeb..05b713d2 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/Chunk.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/PagedVolumeChunk.inl @@ -26,7 +26,7 @@ freely, subject to the following restrictions: namespace PolyVox { template - Chunk::Chunk(Vector3DInt32 v3dPosition, uint16_t uSideLength, Pager* pPager) + PagedVolume::Chunk::Chunk(Vector3DInt32 v3dPosition, uint16_t uSideLength, Pager* pPager) :m_uChunkLastAccessed(0) ,m_bDataModified(true) ,m_tData(0) @@ -65,7 +65,7 @@ namespace PolyVox } template - Chunk::~Chunk() + PagedVolume::Chunk::~Chunk() { if (m_pPager && m_bDataModified) { @@ -82,19 +82,19 @@ namespace PolyVox } template - VoxelType* Chunk::getData(void) const + VoxelType* PagedVolume::Chunk::getData(void) const { return m_tData; } template - uint32_t Chunk::getDataSizeInBytes(void) const + uint32_t PagedVolume::Chunk::getDataSizeInBytes(void) const { return m_uSideLength * m_uSideLength * m_uSideLength * sizeof(VoxelType); } template - VoxelType Chunk::getVoxel(uint16_t uXPos, uint16_t uYPos, uint16_t uZPos) const + VoxelType PagedVolume::Chunk::getVoxel(uint16_t uXPos, uint16_t uYPos, uint16_t uZPos) const { // This code is not usually expected to be called by the user, with the exception of when implementing paging // of uncompressed data. It's a performance critical code path so we use asserts rather than exceptions. @@ -112,13 +112,13 @@ namespace PolyVox } template - VoxelType Chunk::getVoxel(const Vector3DUint16& v3dPos) const + VoxelType PagedVolume::Chunk::getVoxel(const Vector3DUint16& v3dPos) const { return getVoxel(v3dPos.getX(), v3dPos.getY(), v3dPos.getZ()); } template - void Chunk::setVoxelAt(uint16_t uXPos, uint16_t uYPos, uint16_t uZPos, VoxelType tValue) + void PagedVolume::Chunk::setVoxelAt(uint16_t uXPos, uint16_t uYPos, uint16_t uZPos, VoxelType tValue) { // This code is not usually expected to be called by the user, with the exception of when implementing paging // of uncompressed data. It's a performance critical code path so we use asserts rather than exceptions. @@ -138,20 +138,20 @@ namespace PolyVox } template - void Chunk::setVoxelAt(const Vector3DUint16& v3dPos, VoxelType tValue) + void PagedVolume::Chunk::setVoxelAt(const Vector3DUint16& v3dPos, VoxelType tValue) { setVoxelAt(v3dPos.getX(), v3dPos.getY(), v3dPos.getZ(), tValue); } template - uint32_t Chunk::calculateSizeInBytes(void) + uint32_t PagedVolume::Chunk::calculateSizeInBytes(void) { // Call through to the static version return calculateSizeInBytes(m_uSideLength); } template - uint32_t Chunk::calculateSizeInBytes(uint32_t uSideLength) + uint32_t PagedVolume::Chunk::calculateSizeInBytes(uint32_t uSideLength) { // Note: We disregard the size of the other class members as they are likely to be very small compared to the size of the // allocated voxel data. This also keeps the reported size as a power of two, which makes other memory calculations easier. diff --git a/library/PolyVoxCore/include/PolyVoxCore/Pager.h b/library/PolyVoxCore/include/PolyVoxCore/Pager.h index 39d016bb..0280883c 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/Pager.h +++ b/library/PolyVoxCore/include/PolyVoxCore/Pager.h @@ -24,7 +24,7 @@ freely, subject to the following restrictions: #ifndef __PolyVox_Pager_H__ #define __PolyVox_Pager_H__ -#include "PolyVoxCore/Chunk.h" +#include "PolyVoxCore/PagedVolume.h" #include "PolyVoxCore/Impl/TypeDef.h" #include @@ -43,8 +43,8 @@ namespace PolyVox /// Destructor virtual ~Pager() {}; - virtual void pageIn(const Region& region, Chunk* pChunk) = 0; - virtual void pageOut(const Region& region, Chunk* pChunk) = 0; + virtual void pageIn(const Region& region, typename PagedVolume::Chunk* pChunk) = 0; + virtual void pageOut(const Region& region, typename PagedVolume::Chunk* pChunk) = 0; }; } From d6a7b83698f3a4e24679bb4800bc12e92166cee7 Mon Sep 17 00:00:00 2001 From: David Williams Date: Sun, 21 Sep 2014 11:42:12 +0200 Subject: [PATCH 35/42] Moved Pager to be a nested class of PagedVolume. --- examples/Paging/main.cpp | 7 ++- library/PolyVoxCore/CMakeLists.txt | 1 - .../include/PolyVoxCore/FilePager.h | 6 +-- .../include/PolyVoxCore/PagedVolume.h | 28 +++++++--- .../include/PolyVoxCore/PagedVolume.inl | 2 +- .../include/PolyVoxCore/PagedVolumeChunk.inl | 2 +- .../PolyVoxCore/include/PolyVoxCore/Pager.h | 51 ------------------- 7 files changed, 30 insertions(+), 67 deletions(-) delete mode 100644 library/PolyVoxCore/include/PolyVoxCore/Pager.h diff --git a/examples/Paging/main.cpp b/examples/Paging/main.cpp index f0248c14..f70de28f 100644 --- a/examples/Paging/main.cpp +++ b/examples/Paging/main.cpp @@ -27,9 +27,8 @@ freely, subject to the following restrictions: #include "PolyVoxCore/MaterialDensityPair.h" #include "PolyVoxCore/CubicSurfaceExtractor.h" #include "PolyVoxCore/MarchingCubesSurfaceExtractor.h" -#include "PolyVoxCore/Pager.h" #include "PolyVoxCore/Mesh.h" -#include "PolyVoxCore/LargeVolume.h" +#include "PolyVoxCore/PagedVolume.h" #include @@ -78,12 +77,12 @@ void createSphereInVolume(LargeVolume& volData, Vector3DF /** * Generates data using Perlin noise. */ -class PerlinNoisePager : public PolyVox::Pager +class PerlinNoisePager : public PolyVox::PagedVolume::Pager { public: /// Constructor PerlinNoisePager() - :Pager() + :PagedVolume::Pager() { } diff --git a/library/PolyVoxCore/CMakeLists.txt b/library/PolyVoxCore/CMakeLists.txt index 9704e593..42d8c752 100644 --- a/library/PolyVoxCore/CMakeLists.txt +++ b/library/PolyVoxCore/CMakeLists.txt @@ -62,7 +62,6 @@ SET(CORE_INC_FILES include/PolyVoxCore/PagedVolume.inl include/PolyVoxCore/PagedVolumeChunk.inl include/PolyVoxCore/PagedVolumeSampler.inl - include/PolyVoxCore/Pager.h include/PolyVoxCore/PolyVoxForwardDeclarations.h include/PolyVoxCore/Picking.h include/PolyVoxCore/Picking.inl diff --git a/library/PolyVoxCore/include/PolyVoxCore/FilePager.h b/library/PolyVoxCore/include/PolyVoxCore/FilePager.h index 269af29c..ed3b7a9a 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/FilePager.h +++ b/library/PolyVoxCore/include/PolyVoxCore/FilePager.h @@ -26,7 +26,7 @@ freely, subject to the following restrictions: #include "PolyVoxCore/Impl/TypeDef.h" -#include "PolyVoxCore/Pager.h" +#include "PolyVoxCore/PagedVolume.h" #include "PolyVoxCore/Region.h" #include @@ -42,12 +42,12 @@ namespace PolyVox * Provides an interface for performing paging of data. */ template - class FilePager : public Pager + class FilePager : public PagedVolume::Pager { public: /// Constructor FilePager(const std::string& strFolderName = ".") - :Pager() + :PagedVolume::Pager() ,m_strFolderName(strFolderName) { // Add the trailing slash, assuming the user dind't already do it. diff --git a/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.h b/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.h index 75acaaca..7ee41ec4 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.h +++ b/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.h @@ -25,7 +25,6 @@ freely, subject to the following restrictions: #define __PolyVox_PagedVolume_H__ #include "PolyVoxCore/BaseVolume.h" -#include "PolyVoxCore/Pager.h" #include "PolyVoxCore/Region.h" #include "PolyVoxCore/Vector.h" @@ -156,11 +155,13 @@ namespace PolyVox class PagedVolume : public BaseVolume { public: + class Chunk; + class Pager; class Chunk { public: - Chunk(Vector3DInt32 v3dPosition, uint16_t uSideLength, Pager* pPager = nullptr); + Chunk(Vector3DInt32 v3dPosition, uint16_t uSideLength, typename PagedVolume::Pager* pPager = nullptr); ~Chunk(); VoxelType* getData(void) const; @@ -192,10 +193,25 @@ namespace PolyVox VoxelType* m_tData; uint16_t m_uSideLength; uint8_t m_uSideLengthPower; - Pager* m_pPager; + typename PagedVolume::Pager* m_pPager; Vector3DInt32 m_v3dChunkSpacePosition; }; + /** + * Provides an interface for performing paging of data. + */ + class Pager + { + public: + /// Constructor + Pager() {}; + /// Destructor + virtual ~Pager() {}; + + virtual void pageIn(const Region& region, typename PagedVolume::Chunk* pChunk) = 0; + virtual void pageOut(const Region& region, typename PagedVolume::Chunk* pChunk) = 0; + }; + //There seems to be some descrepency between Visual Studio and GCC about how the following class should be declared. //There is a work around (see also See http://goo.gl/qu1wn) given below which appears to work on VS2010 and GCC, but //which seems to cause internal compiler errors on VS2008 when building with the /Gm 'Enable Minimal Rebuild' compiler @@ -272,7 +288,7 @@ namespace PolyVox PagedVolume ( const Region& regValid, - Pager* pPager = nullptr, + typename PagedVolume::Pager* pPager = nullptr, uint16_t uChunkSideLength = 32 ); /// Destructor @@ -357,11 +373,11 @@ namespace PolyVox uint16_t m_uChunkSideLength; uint8_t m_uChunkSideLengthPower; - Pager* m_pPager; + typename PagedVolume::Pager* m_pPager; // Enough to make sure a chunks and it's neighbours can be loaded, with a few to spare. static const uint32_t uMinPracticalNoOfChunks = 32; - // Should preent multi-gigabyte volumes with reasonable chunk sizes. + // Should prevent multi-gigabyte volumes when chunk sizes are reasonable. static const uint32_t uMaxPracticalNoOfChunks = 32768; }; } diff --git a/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.inl index ae5aeea3..7027bf68 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.inl @@ -40,7 +40,7 @@ namespace PolyVox PagedVolume::PagedVolume ( const Region& regValid, - Pager* pPager, + typename PagedVolume::Pager* pPager, uint16_t uChunkSideLength ) :BaseVolume(regValid) diff --git a/library/PolyVoxCore/include/PolyVoxCore/PagedVolumeChunk.inl b/library/PolyVoxCore/include/PolyVoxCore/PagedVolumeChunk.inl index 05b713d2..7bb904a8 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/PagedVolumeChunk.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/PagedVolumeChunk.inl @@ -26,7 +26,7 @@ freely, subject to the following restrictions: namespace PolyVox { template - PagedVolume::Chunk::Chunk(Vector3DInt32 v3dPosition, uint16_t uSideLength, Pager* pPager) + PagedVolume::Chunk::Chunk(Vector3DInt32 v3dPosition, uint16_t uSideLength, typename PagedVolume::Pager* pPager) :m_uChunkLastAccessed(0) ,m_bDataModified(true) ,m_tData(0) diff --git a/library/PolyVoxCore/include/PolyVoxCore/Pager.h b/library/PolyVoxCore/include/PolyVoxCore/Pager.h deleted file mode 100644 index 0280883c..00000000 --- a/library/PolyVoxCore/include/PolyVoxCore/Pager.h +++ /dev/null @@ -1,51 +0,0 @@ -/******************************************************************************* -Copyright (c) 2005-2009 David Williams - -This software is provided 'as-is', without any express or implied -warranty. In no event will the authors be held liable for any damages -arising from the use of this software. - -Permission is granted to anyone to use this software for any purpose, -including commercial applications, and to alter it and redistribute it -freely, subject to the following restrictions: - - 1. The origin of this software must not be misrepresented; you must not - claim that you wrote the original software. If you use this software - in a product, an acknowledgment in the product documentation would be - appreciated but is not required. - - 2. Altered source versions must be plainly marked as such, and must not be - misrepresented as being the original software. - - 3. This notice may not be removed or altered from any source - distribution. -*******************************************************************************/ - -#ifndef __PolyVox_Pager_H__ -#define __PolyVox_Pager_H__ - -#include "PolyVoxCore/PagedVolume.h" -#include "PolyVoxCore/Impl/TypeDef.h" - -#include - -namespace PolyVox -{ - /** - * Provides an interface for performing paging of data. - */ - template - class Pager - { - public: - /// Constructor - Pager() {}; - /// Destructor - virtual ~Pager() {}; - - virtual void pageIn(const Region& region, typename PagedVolume::Chunk* pChunk) = 0; - virtual void pageOut(const Region& region, typename PagedVolume::Chunk* pChunk) = 0; - }; -} - -#endif //__PolyVox_Pager_H__ From 33ec5e1d79e1d5f27e920729dc4955c437d6ae5f Mon Sep 17 00:00:00 2001 From: David Williams Date: Sun, 21 Sep 2014 12:16:54 +0200 Subject: [PATCH 36/42] Added a couple of comments. --- library/PolyVoxCore/include/PolyVoxCore/PagedVolume.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.h b/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.h index 7ee41ec4..b7f03aa1 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.h +++ b/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.h @@ -155,7 +155,9 @@ namespace PolyVox class PagedVolume : public BaseVolume { public: + /// The PagedVolume stores it data as a set of Chunk instances which can be loaded and unloaded as memory requirements dictate. class Chunk; + /// The Pager class is responsible for the loading and unloading of Chunks, and can be overridden by the user. class Pager; class Chunk From ae0c5b3702caf52288e1716ab772aaf540d95a72 Mon Sep 17 00:00:00 2001 From: David Williams Date: Sun, 21 Sep 2014 14:47:46 +0200 Subject: [PATCH 37/42] Added comments and removed deprecated functions. --- .../include/PolyVoxCore/PagedVolume.h | 40 +++++---- .../include/PolyVoxCore/PagedVolume.inl | 82 +------------------ .../include/PolyVoxCore/PagedVolumeChunk.inl | 2 +- 3 files changed, 25 insertions(+), 99 deletions(-) diff --git a/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.h b/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.h index b7f03aa1..304909ca 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.h +++ b/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.h @@ -163,7 +163,7 @@ namespace PolyVox class Chunk { public: - Chunk(Vector3DInt32 v3dPosition, uint16_t uSideLength, typename PagedVolume::Pager* pPager = nullptr); + Chunk(Vector3DInt32 v3dPosition, uint16_t uSideLength, Pager* pPager = nullptr); ~Chunk(); VoxelType* getData(void) const; @@ -195,12 +195,14 @@ namespace PolyVox VoxelType* m_tData; uint16_t m_uSideLength; uint8_t m_uSideLengthPower; - typename PagedVolume::Pager* m_pPager; + Pager* m_pPager; Vector3DInt32 m_v3dChunkSpacePosition; }; /** - * Provides an interface for performing paging of data. + * Users can override this class and provide an instance of the derived class to the PagedVolume constructor. This derived class + * could then perform tasks such as compression and decompression of the data, and read/writing it to a file, database, network, + * or other storage as appropriate. See FilePager for a simple example of such a derived class. */ class Pager { @@ -210,8 +212,8 @@ namespace PolyVox /// Destructor virtual ~Pager() {}; - virtual void pageIn(const Region& region, typename PagedVolume::Chunk* pChunk) = 0; - virtual void pageOut(const Region& region, typename PagedVolume::Chunk* pChunk) = 0; + virtual void pageIn(const Region& region, Chunk* pChunk) = 0; + virtual void pageOut(const Region& region, Chunk* pChunk) = 0; }; //There seems to be some descrepency between Visual Studio and GCC about how the following class should be declared. @@ -290,7 +292,7 @@ namespace PolyVox PagedVolume ( const Region& regValid, - typename PagedVolume::Pager* pPager = nullptr, + Pager* pPager = nullptr, uint16_t uChunkSideLength = 32 ); /// Destructor @@ -308,21 +310,13 @@ namespace PolyVox /// Gets a voxel at the position given by a 3D vector VoxelType getVoxel(const Vector3DInt32& v3dPos, WrapMode eWrapMode = WrapModes::Validate, VoxelType tBorder = VoxelType()) const; - /// Gets a voxel at the position given by x,y,z coordinates - POLYVOX_DEPRECATED VoxelType getVoxelAt(int32_t uXPos, int32_t uYPos, int32_t uZPos) const; - /// Gets a voxel at the position given by a 3D vector - POLYVOX_DEPRECATED VoxelType getVoxelAt(const Vector3DInt32& v3dPos) const; - /// Sets the number of chunks for which uncompressed data is stored void setMemoryUsageLimit(uint32_t uMemoryUsageInBytes); /// Sets the voxel at the position given by x,y,z coordinates void setVoxel(int32_t uXPos, int32_t uYPos, int32_t uZPos, VoxelType tValue, WrapMode eWrapMode = WrapModes::Validate); /// Sets the voxel at the position given by a 3D vector void setVoxel(const Vector3DInt32& v3dPos, VoxelType tValue, WrapMode eWrapMode = WrapModes::Validate); - /// Sets the voxel at the position given by x,y,z coordinates - bool setVoxelAt(int32_t uXPos, int32_t uYPos, int32_t uZPos, VoxelType tValue); - /// Sets the voxel at the position given by a 3D vector - bool setVoxelAt(const Vector3DInt32& v3dPos, VoxelType tValue); + /// Tries to ensure that the voxels within the specified Region are loaded into memory. void prefetch(Region regPrefetch); /// Ensures that any voxels within the specified Region are removed from memory. @@ -359,8 +353,20 @@ namespace PolyVox void purgeNullPtrsFromAllChunks(void) const; - // The chunk data + // The chunk data is stored in the pair of maps below. This will often hold the same set of chunks but there are occasions + // when they can differ (more on that in a moment). The main store is the set of 'recently used chunks' which holds shared_ptr's + // to the chunk data. When memory is low chunks can be removed from this list and, assuming there are no more references to + // them, they will be deleted. However, it is also possible that a Sampler is pointing at a given chunk, and in this case the + // reference counting will ensure that the chunk survives until the sampler has finished with it. + // + // However, this leaves us open to a subtle bug. If a chunk is removed from the recently used set, continues to exist due to a + // sampler using it, and then the user tries to access it through the volume interface, then the volume will page the chunk + // back in (not knowing the sampler still has it). This would result in two chunks in memory with the same position is space. + // To avoid this, the 'all chunks' set tracks chunks with are used by the volume *and/or* the samplers. It holds weak pointers + // so does not cause chunks to persist. + typedef std::unordered_map > WeakPtrChunkMap; mutable WeakPtrChunkMap m_pAllChunks; + typedef std::unordered_map > SharedPtrChunkMap; mutable SharedPtrChunkMap m_pRecentlyUsedChunks; mutable uint32_t m_uTimestamper; @@ -375,7 +381,7 @@ namespace PolyVox uint16_t m_uChunkSideLength; uint8_t m_uChunkSideLengthPower; - typename PagedVolume::Pager* m_pPager; + Pager* m_pPager; // Enough to make sure a chunks and it's neighbours can be loaded, with a few to spare. static const uint32_t uMinPracticalNoOfChunks = 32; diff --git a/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.inl index 7027bf68..9919da74 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.inl @@ -40,7 +40,7 @@ namespace PolyVox PagedVolume::PagedVolume ( const Region& regValid, - typename PagedVolume::Pager* pPager, + Pager* pPager, uint16_t uChunkSideLength ) :BaseVolume(regValid) @@ -203,45 +203,6 @@ namespace PolyVox return getVoxel(v3dPos.getX(), v3dPos.getY(), v3dPos.getZ(), eWrapMode, tBorder); } - //////////////////////////////////////////////////////////////////////////////// - /// \param uXPos The \c x position of the voxel - /// \param uYPos The \c y position of the voxel - /// \param uZPos The \c z position of the voxel - /// \return The voxel value - //////////////////////////////////////////////////////////////////////////////// - template - VoxelType PagedVolume::getVoxelAt(int32_t uXPos, int32_t uYPos, int32_t uZPos) const - { - if(this->m_regValidRegion.containsPoint(Vector3DInt32(uXPos, uYPos, uZPos))) - { - const int32_t chunkX = uXPos >> m_uChunkSideLengthPower; - const int32_t chunkY = uYPos >> m_uChunkSideLengthPower; - const int32_t chunkZ = uZPos >> m_uChunkSideLengthPower; - - const uint16_t xOffset = static_cast(uXPos - (chunkX << m_uChunkSideLengthPower)); - const uint16_t yOffset = static_cast(uYPos - (chunkY << m_uChunkSideLengthPower)); - const uint16_t zOffset = static_cast(uZPos - (chunkZ << m_uChunkSideLengthPower)); - - auto pChunk = getChunk(chunkX, chunkY, chunkZ); - - return pChunk->getVoxel(xOffset, yOffset, zOffset); - } - else - { - return this->getBorderValue(); - } - } - - //////////////////////////////////////////////////////////////////////////////// - /// \param v3dPos The 3D position of the voxel - /// \return The voxel value - //////////////////////////////////////////////////////////////////////////////// - template - VoxelType PagedVolume::getVoxelAt(const Vector3DInt32& v3dPos) const - { - return getVoxelAt(v3dPos.getX(), v3dPos.getY(), v3dPos.getZ()); - } - //////////////////////////////////////////////////////////////////////////////// /// Increasing the size of the chunk cache will increase memory but may improve performance. /// You may want to set this to a large value (e.g. 1024) when you are first loading your @@ -322,47 +283,6 @@ namespace PolyVox setVoxel(v3dPos.getX(), v3dPos.getY(), v3dPos.getZ(), tValue, eWrapMode); } - //////////////////////////////////////////////////////////////////////////////// - /// \param uXPos the \c x position of the voxel - /// \param uYPos the \c y position of the voxel - /// \param uZPos the \c z position of the voxel - /// \param tValue the value to which the voxel will be set - /// \return whether the requested position is inside the volume - //////////////////////////////////////////////////////////////////////////////// - template - bool PagedVolume::setVoxelAt(int32_t uXPos, int32_t uYPos, int32_t uZPos, VoxelType tValue) - { - // PolyVox does not throw an exception when a voxel is out of range. Please see 'Error Handling' in the User Manual. - POLYVOX_ASSERT(this->m_regValidRegion.containsPoint(Vector3DInt32(uXPos, uYPos, uZPos)), "Position is outside valid region"); - - const int32_t chunkX = uXPos >> m_uChunkSideLengthPower; - const int32_t chunkY = uYPos >> m_uChunkSideLengthPower; - const int32_t chunkZ = uZPos >> m_uChunkSideLengthPower; - - const uint16_t xOffset = static_cast(uXPos - (chunkX << m_uChunkSideLengthPower)); - const uint16_t yOffset = static_cast(uYPos - (chunkY << m_uChunkSideLengthPower)); - const uint16_t zOffset = static_cast(uZPos - (chunkZ << m_uChunkSideLengthPower)); - - auto pChunk = getChunk(chunkX, chunkY, chunkZ); - - pChunk->setVoxelAt(xOffset, yOffset, zOffset, tValue); - - //Return true to indicate that we modified a voxel. - return true; - } - - //////////////////////////////////////////////////////////////////////////////// - /// \param v3dPos the 3D position of the voxel - /// \param tValue the value to which the voxel will be set - /// \return whether the requested position is inside the volume - //////////////////////////////////////////////////////////////////////////////// - template - bool PagedVolume::setVoxelAt(const Vector3DInt32& v3dPos, VoxelType tValue) - { - return setVoxelAt(v3dPos.getX(), v3dPos.getY(), v3dPos.getZ(), tValue); - } - - //////////////////////////////////////////////////////////////////////////////// /// Note that if the memory usage limit is not large enough to support the region this function will only load part of the region. In this case it is undefined which parts will actually be loaded. If all the voxels in the given region are already loaded, this function will not do anything. Other voxels might be unloaded to make space for the new voxels. /// \param regPrefetch The Region of voxels to prefetch into memory. diff --git a/library/PolyVoxCore/include/PolyVoxCore/PagedVolumeChunk.inl b/library/PolyVoxCore/include/PolyVoxCore/PagedVolumeChunk.inl index 7bb904a8..d7e249bd 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/PagedVolumeChunk.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/PagedVolumeChunk.inl @@ -26,7 +26,7 @@ freely, subject to the following restrictions: namespace PolyVox { template - PagedVolume::Chunk::Chunk(Vector3DInt32 v3dPosition, uint16_t uSideLength, typename PagedVolume::Pager* pPager) + PagedVolume::Chunk::Chunk(Vector3DInt32 v3dPosition, uint16_t uSideLength, Pager* pPager) :m_uChunkLastAccessed(0) ,m_bDataModified(true) ,m_tData(0) From ccb9f8d5641059bc7ebed9ce71c24455831faa91 Mon Sep 17 00:00:00 2001 From: David Williams Date: Sun, 21 Sep 2014 15:03:13 +0200 Subject: [PATCH 38/42] Updated documentation. --- .../include/PolyVoxCore/PagedVolume.h | 70 ++++--------------- 1 file changed, 13 insertions(+), 57 deletions(-) diff --git a/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.h b/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.h index 304909ca..d0ed5d51 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.h +++ b/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.h @@ -52,22 +52,18 @@ namespace PolyVox /// The following code snippet shows how to construct a volume and demonstrates basic usage: /// /// \code - /// PagedVolume volume(Region(Vector3DInt32(0,0,0), Vector3DInt32(63,127,255))); - /// volume.setVoxelAt(15, 90, 42, Material8(5)); - /// std::cout << "Voxel at (15, 90, 42) has value: " << volume.getVoxelAt(15, 90, 42).getMaterial() << std::endl; + /// PagedVolume volume(Region(Vector3DInt32(0,0,0), Vector3DInt32(63,127,255))); + /// volume.setVoxel(15, 90, 42, int(5)); + /// std::cout << "Voxel at (15, 90, 42) has value: " << volume.getVoxel(15, 90, 42) << std::endl; /// std::cout << "Width = " << volume.getWidth() << ", Height = " << volume.getHeight() << ", Depth = " << volume.getDepth() << std::endl; /// \endcode - /// - /// In this particular example each voxel in the PagedVolume is of type 'Material8', as specified by the template parameter. This is one of several - /// predefined voxel types, and it is also possible to define your own. The Material8 type simply holds an integer value where zero represents - /// empty space and any other value represents a solid material. /// /// The PagedVolume constructor takes a Region as a parameter. This specifies the valid range of voxels which can be held in the volume, so in this - /// particular case the valid voxel positions are (0,0,0) to (63, 127, 255). Attempts to access voxels outside this range will result is accessing the - /// border value (see getBorderValue() and setBorderValue()). PolyVox also has support for near infinite volumes which will be discussed later. + /// particular case the valid voxel positions are (0,0,0) to (63, 127, 255). The result of attempts to access voxels outside this range will result + /// are defined by the WrapMode). PolyVox also has support for near infinite volumes which will be discussed later. /// - /// Access to individual voxels is provided via the setVoxelAt() and getVoxelAt() member functions. Advanced users may also be interested in - /// the Sampler class for faster read-only access to a large number of voxels. + /// Access to individual voxels is provided via the setVoxel() and getVoxel() member functions. Advanced users may also be interested in + /// the Sampler nested class for faster read-only access to a large number of voxels. /// /// Lastly the example prints out some properties of the PagedVolume. Note that the dimentsions getWidth(), getHeight(), and getDepth() are inclusive, such /// that the width is 64 when the range of valid x coordinates goes from 0 to 63. @@ -80,70 +76,30 @@ namespace PolyVox /// /// Essentially, the PagedVolume class stores its data as a collection of chunks. Each of these chunk is much smaller than the whole volume, /// for example a typical size might be 32x32x32 voxels (though is is configurable by the user). In this case, a 256x512x1024 volume - /// would contain 8x16x32 = 4096 chunks. The data for each chunk is stored in a compressed form, which uses only a small amout of - /// memory but it is hard to modify the data. Therefore, before any given voxel can be modified, its corresponding chunk must be uncompressed. - /// - /// The compression and decompression of chunk is a relatively slow process and so we aim to do this as rarely as possible. In order - /// to achive this, the volume class stores a cache of recently used chunks and their associated uncompressed data. Each time a voxel - /// is touched a timestamp is updated on the corresponding chunk. When the cache becomes full the chunk with the oldest timestamp is - /// recompressed and moved out of the cache. - /// - /// Achieving high compression rates - /// -------------------------------- - /// The compression rates which can be achieved can vary significantly depending the nature of the data you are storing, but you can - /// encourage high compression rates by making your data as homogenous as possible. If you are simply storing a material with each - /// voxel then this will probably happen naturally. Games such as Minecraft which use this approach will typically involve large areas - /// of the same material which will compress down well. - /// - /// However, if you are storing density values then you may want to take some care. The advantage of storing smoothly changing values - /// is that you can get smooth surfaces extracted, but storing smoothly changing values inside or outside objects (rather than just - /// on the boundary) does not benefit the surface and is very hard to compress effectively. You may wish to apply some thresholding to - /// your density values to reduce this problem (this threasholding should only be applied to voxels who don't contribute to the surface). - /// - /// Paging large volumes - /// -------------------- - /// The compression scheme described previously will typically allow you to load several billion voxels into a few hundred megabytes of memory, - /// though as explained the exact compression rate is highly dependant on your data. If you have more data than this then PolyVox provides a - /// mechanism by which parts of the volume can be paged out of memory by calling user supplied callback functions. This mechanism allows a + /// would contain 8x16x32 = 4096 chunks. Typically these chunks do not need to all be in memory all the time, and the Pager class can + /// be used to control how they are loaded and unloaded. This mechanism allows a /// potentially unlimited amount of data to be loaded, provided the user is able to take responsibility for storing any data which PolyVox /// cannot fit in memory, and then returning it back to PolyVox on demand. For example, the user might choose to temporarily store this data /// on disk or stream it to a remote database. /// - /// You can construct such a PagedVolume as follows: - /// - /// \code - /// void myDataRequiredHandler(const ConstVolumeProxy& volume, const PolyVox::Region& reg) - /// { - /// //This function is being called because part of the data is missing from memory and needs to be supplied. The parameter - /// //'volume' provides access to the volume data, and the parameter 'reg' indicates which region of the volume you need fill. - /// } - /// - /// void myDataOverflowHandler(const ConstVolumeProxy& vol, const PolyVox::Region& reg) - /// { - /// //This function is being called because part of the data is about to be removed from memory. The parameter 'volume' - /// //provides access to the volume data, and the parameter 'reg' indicates which region of the volume you need to store. - /// } - /// - /// PagedVolumevolData(&myDataRequiredHandler, &myDataOverflowHandler); - /// \endcode - /// /// Essentially you are providing an extension to the PagedVolume class - a way for data to be stored once PolyVox has run out of memory for it. Note /// that you don't actually have to do anything with the data - you could simply decide that once it gets removed from memory it doesn't matter - /// anymore. But you still need to be ready to then provide something to PolyVox (even if it's just default data) in the event that it is requested. + /// anymore. /// /// Cache-aware traversal /// --------------------- + /// *NOTE: This needs updating for PagedVolume rather than the old LargeVolume* /// You might be suprised at just how many cache misses can occur when you traverse the volume in a naive manner. Consider a 1024x1024x1024 volume /// with chunks of size 32x32x32. And imagine you iterate over this volume with a simple three-level for loop which iterates over x, the y, then z. /// If you start at position (0,0,0) then ny the time you reach position (1023,0,0) you have touched 1024 voxels along one edge of the volume and /// have pulled 32 chunks into the cache. By the time you reach (1023,1023,0) you have hit 1024x1024 voxels and pulled 32x32 chunks into the cache. - /// You are now ready to touch voxel (0,0,1) which is right nect to where you started, but unless your cache is at least 32x32 chunks large then this + /// You are now ready to touch voxel (0,0,1) which is right next to where you started, but unless your cache is at least 32x32 chunks large then this /// initial chunk has already been cleared from the cache. /// /// Ensuring you have a large enough cache size can obviously help the above situation, but you might also consider iterating over the voxels in a /// different order. For example, if you replace your three-level loop with a six-level loop then you can first process all the voxels between (0,0,0) /// and (31,31,31), then process all the voxels between (32,0,0) and (63,0,0), and so forth. Using this approach you will have no cache misses even - /// is your cache sise is only one. Of course the logic is more complex, but writing code in such a cache-aware manner may be beneficial in some situations. + /// is your cache size is only one. Of course the logic is more complex, but writing code in such a cache-aware manner may be beneficial in some situations. /// /// Threading /// --------- From 3a56a70f38f9fab5262b9ae9ca98fc195a39ece8 Mon Sep 17 00:00:00 2001 From: David Williams Date: Sun, 21 Sep 2014 15:07:54 +0200 Subject: [PATCH 39/42] Put deprecated functions back in - lets remove them in the future after more thought. --- .../include/PolyVoxCore/PagedVolume.h | 10 ++- .../include/PolyVoxCore/PagedVolume.inl | 80 +++++++++++++++++++ 2 files changed, 89 insertions(+), 1 deletion(-) diff --git a/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.h b/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.h index d0ed5d51..b014c9a3 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.h +++ b/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.h @@ -266,13 +266,21 @@ namespace PolyVox /// Gets a voxel at the position given by a 3D vector VoxelType getVoxel(const Vector3DInt32& v3dPos, WrapMode eWrapMode = WrapModes::Validate, VoxelType tBorder = VoxelType()) const; + /// Gets a voxel at the position given by x,y,z coordinates + POLYVOX_DEPRECATED VoxelType getVoxelAt(int32_t uXPos, int32_t uYPos, int32_t uZPos) const; + /// Gets a voxel at the position given by a 3D vector + POLYVOX_DEPRECATED VoxelType getVoxelAt(const Vector3DInt32& v3dPos) const; + /// Sets the number of chunks for which uncompressed data is stored void setMemoryUsageLimit(uint32_t uMemoryUsageInBytes); /// Sets the voxel at the position given by x,y,z coordinates void setVoxel(int32_t uXPos, int32_t uYPos, int32_t uZPos, VoxelType tValue, WrapMode eWrapMode = WrapModes::Validate); /// Sets the voxel at the position given by a 3D vector void setVoxel(const Vector3DInt32& v3dPos, VoxelType tValue, WrapMode eWrapMode = WrapModes::Validate); - + /// Sets the voxel at the position given by x,y,z coordinates + bool setVoxelAt(int32_t uXPos, int32_t uYPos, int32_t uZPos, VoxelType tValue); + /// Sets the voxel at the position given by a 3D vector + bool setVoxelAt(const Vector3DInt32& v3dPos, VoxelType tValue); /// Tries to ensure that the voxels within the specified Region are loaded into memory. void prefetch(Region regPrefetch); /// Ensures that any voxels within the specified Region are removed from memory. diff --git a/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.inl index 9919da74..28babf13 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.inl @@ -203,6 +203,45 @@ namespace PolyVox return getVoxel(v3dPos.getX(), v3dPos.getY(), v3dPos.getZ(), eWrapMode, tBorder); } + //////////////////////////////////////////////////////////////////////////////// + /// \param uXPos The \c x position of the voxel + /// \param uYPos The \c y position of the voxel + /// \param uZPos The \c z position of the voxel + /// \return The voxel value + //////////////////////////////////////////////////////////////////////////////// + template + VoxelType PagedVolume::getVoxelAt(int32_t uXPos, int32_t uYPos, int32_t uZPos) const + { + if(this->m_regValidRegion.containsPoint(Vector3DInt32(uXPos, uYPos, uZPos))) + { + const int32_t chunkX = uXPos >> m_uChunkSideLengthPower; + const int32_t chunkY = uYPos >> m_uChunkSideLengthPower; + const int32_t chunkZ = uZPos >> m_uChunkSideLengthPower; + + const uint16_t xOffset = static_cast(uXPos - (chunkX << m_uChunkSideLengthPower)); + const uint16_t yOffset = static_cast(uYPos - (chunkY << m_uChunkSideLengthPower)); + const uint16_t zOffset = static_cast(uZPos - (chunkZ << m_uChunkSideLengthPower)); + + auto pChunk = getChunk(chunkX, chunkY, chunkZ); + + return pChunk->getVoxel(xOffset, yOffset, zOffset); + } + else + { + return this->getBorderValue(); + } + } + + //////////////////////////////////////////////////////////////////////////////// + /// \param v3dPos The 3D position of the voxel + /// \return The voxel value + //////////////////////////////////////////////////////////////////////////////// + template + VoxelType PagedVolume::getVoxelAt(const Vector3DInt32& v3dPos) const + { + return getVoxelAt(v3dPos.getX(), v3dPos.getY(), v3dPos.getZ()); + } + //////////////////////////////////////////////////////////////////////////////// /// Increasing the size of the chunk cache will increase memory but may improve performance. /// You may want to set this to a large value (e.g. 1024) when you are first loading your @@ -283,6 +322,47 @@ namespace PolyVox setVoxel(v3dPos.getX(), v3dPos.getY(), v3dPos.getZ(), tValue, eWrapMode); } + //////////////////////////////////////////////////////////////////////////////// + /// \param uXPos the \c x position of the voxel + /// \param uYPos the \c y position of the voxel + /// \param uZPos the \c z position of the voxel + /// \param tValue the value to which the voxel will be set + /// \return whether the requested position is inside the volume + //////////////////////////////////////////////////////////////////////////////// + template + bool PagedVolume::setVoxelAt(int32_t uXPos, int32_t uYPos, int32_t uZPos, VoxelType tValue) + { + // PolyVox does not throw an exception when a voxel is out of range. Please see 'Error Handling' in the User Manual. + POLYVOX_ASSERT(this->m_regValidRegion.containsPoint(Vector3DInt32(uXPos, uYPos, uZPos)), "Position is outside valid region"); + + const int32_t chunkX = uXPos >> m_uChunkSideLengthPower; + const int32_t chunkY = uYPos >> m_uChunkSideLengthPower; + const int32_t chunkZ = uZPos >> m_uChunkSideLengthPower; + + const uint16_t xOffset = static_cast(uXPos - (chunkX << m_uChunkSideLengthPower)); + const uint16_t yOffset = static_cast(uYPos - (chunkY << m_uChunkSideLengthPower)); + const uint16_t zOffset = static_cast(uZPos - (chunkZ << m_uChunkSideLengthPower)); + + auto pChunk = getChunk(chunkX, chunkY, chunkZ); + + pChunk->setVoxelAt(xOffset, yOffset, zOffset, tValue); + + //Return true to indicate that we modified a voxel. + return true; + } + + //////////////////////////////////////////////////////////////////////////////// + /// \param v3dPos the 3D position of the voxel + /// \param tValue the value to which the voxel will be set + /// \return whether the requested position is inside the volume + //////////////////////////////////////////////////////////////////////////////// + template + bool PagedVolume::setVoxelAt(const Vector3DInt32& v3dPos, VoxelType tValue) + { + return setVoxelAt(v3dPos.getX(), v3dPos.getY(), v3dPos.getZ(), tValue); + } + + //////////////////////////////////////////////////////////////////////////////// /// Note that if the memory usage limit is not large enough to support the region this function will only load part of the region. In this case it is undefined which parts will actually be loaded. If all the voxels in the given region are already loaded, this function will not do anything. Other voxels might be unloaded to make space for the new voxels. /// \param regPrefetch The Region of voxels to prefetch into memory. From f95cc6bfcac0284ddad573883196405b841b96df Mon Sep 17 00:00:00 2001 From: David Williams Date: Sun, 21 Sep 2014 17:50:35 +0200 Subject: [PATCH 40/42] Added comments and warnings. --- documentation/Threading.rst | 4 +++- .../PolyVoxCore/include/PolyVoxCore/LargeVolume.h | 2 ++ .../PolyVoxCore/include/PolyVoxCore/PagedVolume.h | 13 +++++++++---- .../PolyVoxCore/include/PolyVoxCore/SimpleVolume.h | 2 ++ 4 files changed, 16 insertions(+), 5 deletions(-) diff --git a/documentation/Threading.rst b/documentation/Threading.rst index c2d8991a..fc4d2245 100644 --- a/documentation/Threading.rst +++ b/documentation/Threading.rst @@ -27,8 +27,10 @@ C++ does provide the 'volatile' keyword which can be used to ensure a variable i Lastly, note that PolyVox volumes are templatised which means the voxel type might be something other than a simple int. However we don't think this actually makes a difference given that so few guarantees are made anyway, and it should still be safe to perform multiple concurrent reads for more complex types. -LargeVolume +PagedVolume ----------- +NOTE: The info below is based on LargeVolume, which PagedVolume has replaced. It is likely that the same limitations apply but this has not been well tested. We do intend to improve the thread safty of PagedVolume in the future. + The LargeVolume provides even less thread safety than the RawVolume, in that even concurrent read operations can cause problems. The reason for this is the more complex memory management which is performed behind the scenes, and which allows pieces of volume data to be moved around and deleted. For example, a read of a single voxel may mean that the block of data associated with that voxel has to be paged in to memory, which in turn may mean that another block of data has to be paged out of memory. If second thread was halfway through reading a voxel in this second block of data then a problem will occur. In the future we may do a more comprehensive analysis of thread safety in the LargeVolume, but for now you should assume that any multithreaded access can cause problems. diff --git a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h index ff60e49f..a018e67c 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h +++ b/library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h @@ -1,6 +1,8 @@ #ifndef __PolyVox_LargeVolume_H__ #define __PolyVox_LargeVolume_H__ +#pragma message("WARNING - The LargeVolume class has been replaced by PagedVolume. Please use that instead.") + #include "PagedVolume.h" #include "PolyVoxForwardDeclarations.h" diff --git a/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.h b/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.h index b014c9a3..848486e2 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.h +++ b/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.h @@ -118,6 +118,8 @@ namespace PolyVox class Chunk { + friend class PagedVolume; + public: Chunk(Vector3DInt32 v3dPosition, uint16_t uSideLength, Pager* pPager = nullptr); ~Chunk(); @@ -131,7 +133,7 @@ namespace PolyVox void setVoxelAt(uint16_t uXPos, uint16_t uYPos, uint16_t uZPos, VoxelType tValue); void setVoxelAt(const Vector3DUint16& v3dPos, VoxelType tValue); - public: + private: /// Private copy constructor to prevent accisdental copying Chunk(const Chunk& /*rhs*/) {}; @@ -152,6 +154,8 @@ namespace PolyVox uint16_t m_uSideLength; uint8_t m_uSideLengthPower; Pager* m_pPager; + + // Note: Do we really need to store this position here as well as in the block maps? Vector3DInt32 m_v3dChunkSpacePosition; }; @@ -300,9 +304,7 @@ namespace PolyVox private: - typedef std::unordered_map > SharedPtrChunkMap; - typedef std::unordered_map > WeakPtrChunkMap; - + // FIXME - We can probably ove this into the constructor void initialise(); // A trick to implement specialization of template member functions in template classes. See http://stackoverflow.com/a/4951057 @@ -328,6 +330,9 @@ namespace PolyVox // back in (not knowing the sampler still has it). This would result in two chunks in memory with the same position is space. // To avoid this, the 'all chunks' set tracks chunks with are used by the volume *and/or* the samplers. It holds weak pointers // so does not cause chunks to persist. + // + // TODO: Do we really need maps here? It means we are storing the chunk positions in the map, but they are also stored in the + // chunks themselves (so they can be passed to the pager from the chunks destructor). Can we use a set here? What is a better approach? typedef std::unordered_map > WeakPtrChunkMap; mutable WeakPtrChunkMap m_pAllChunks; typedef std::unordered_map > SharedPtrChunkMap; diff --git a/library/PolyVoxCore/include/PolyVoxCore/SimpleVolume.h b/library/PolyVoxCore/include/PolyVoxCore/SimpleVolume.h index 9967d874..84ecd46b 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/SimpleVolume.h +++ b/library/PolyVoxCore/include/PolyVoxCore/SimpleVolume.h @@ -1,6 +1,8 @@ #ifndef __PolyVox_SimpleVolume_H__ #define __PolyVox_SimpleVolume_H__ +#pragma message("WARNING - The LargeVolume class has been replaced by PagedVolume. Please use that instead.") + #include "PagedVolume.h" #include "PolyVoxForwardDeclarations.h" From db2e62d2a8ed9c29ca4b46c6e5eaf5e253abdce9 Mon Sep 17 00:00:00 2001 From: David Williams Date: Sun, 21 Sep 2014 17:57:42 +0200 Subject: [PATCH 41/42] Replaced LargeVolume and SimpleVolume with PagedVolume in tests and examples. --- examples/Basic/main.cpp | 6 +++--- examples/DecodeOnGPU/main.cpp | 6 +++--- examples/OpenGL/Shapes.cpp | 4 ++-- examples/OpenGL/Shapes.h | 2 +- examples/OpenGL/main.cpp | 3 +-- examples/Paging/main.cpp | 4 ++-- examples/SmoothLOD/main.cpp | 14 +++++++------- .../PolyVoxCore/AmbientOcclusionCalculator.h | 4 ++-- .../include/PolyVoxCore/BaseVolume.h | 2 +- .../include/PolyVoxCore/BaseVolume.inl | 2 +- .../include/PolyVoxCore/SimpleVolume.h | 2 +- tests/TestAmbientOcclusionGenerator.cpp | 4 ++-- tests/TestCubicSurfaceExtractor.cpp | 16 ++++++++-------- tests/TestPicking.cpp | 4 ++-- tests/TestRaycast.cpp | 6 +++--- tests/TestSurfaceExtractor.cpp | 17 ++++++++--------- tests/testvolume.cpp | 6 +++--- 17 files changed, 50 insertions(+), 52 deletions(-) diff --git a/examples/Basic/main.cpp b/examples/Basic/main.cpp index b697b189..a3fdbbd5 100644 --- a/examples/Basic/main.cpp +++ b/examples/Basic/main.cpp @@ -26,14 +26,14 @@ freely, subject to the following restrictions: #include "PolyVoxCore/CubicSurfaceExtractor.h" #include "PolyVoxCore/MarchingCubesSurfaceExtractor.h" #include "PolyVoxCore/Mesh.h" -#include "PolyVoxCore/SimpleVolume.h" +#include "PolyVoxCore/PagedVolume.h" #include //Use the PolyVox namespace using namespace PolyVox; -void createSphereInVolume(SimpleVolume& volData, float fRadius) +void createSphereInVolume(PagedVolume& volData, float fRadius) { //This vector hold the position of the center of the volume Vector3DFloat v3dVolCenter(volData.getWidth() / 2, volData.getHeight() / 2, volData.getDepth() / 2); @@ -74,7 +74,7 @@ int main(int argc, char *argv[]) openGLWidget.show(); //Create an empty volume and then place a sphere in it - SimpleVolume volData(PolyVox::Region(Vector3DInt32(0,0,0), Vector3DInt32(63, 63, 63))); + PagedVolume volData(PolyVox::Region(Vector3DInt32(0, 0, 0), Vector3DInt32(63, 63, 63))); createSphereInVolume(volData, 30); // Extract the surface for the specified region of the volume. Uncomment the line for the kind of surface extraction you want to see. diff --git a/examples/DecodeOnGPU/main.cpp b/examples/DecodeOnGPU/main.cpp index 34c86bef..19ef97fd 100644 --- a/examples/DecodeOnGPU/main.cpp +++ b/examples/DecodeOnGPU/main.cpp @@ -26,14 +26,14 @@ freely, subject to the following restrictions: #include "PolyVoxCore/CubicSurfaceExtractor.h" #include "PolyVoxCore/MarchingCubesSurfaceExtractor.h" #include "PolyVoxCore/Mesh.h" -#include "PolyVoxCore/SimpleVolume.h" +#include "PolyVoxCore/PagedVolume.h" #include //Use the PolyVox namespace using namespace PolyVox; -void createSphereInVolume(SimpleVolume& volData, float fRadius) +void createSphereInVolume(PagedVolume& volData, float fRadius) { //This vector hold the position of the center of the volume Vector3DFloat v3dVolCenter(volData.getWidth() / 2, volData.getHeight() / 2, volData.getDepth() / 2); @@ -145,7 +145,7 @@ int main(int argc, char *argv[]) openGLWidget.setShader(shader); //Create an empty volume and then place a sphere in it - SimpleVolume volData(PolyVox::Region(Vector3DInt32(0,0,0), Vector3DInt32(63, 63, 63))); + PagedVolume volData(PolyVox::Region(Vector3DInt32(0, 0, 0), Vector3DInt32(63, 63, 63))); createSphereInVolume(volData, 30); // Extract the surface for the specified region of the volume. Uncomment the line for the kind of surface extraction you want to see. diff --git a/examples/OpenGL/Shapes.cpp b/examples/OpenGL/Shapes.cpp index 7a66b988..419359d0 100644 --- a/examples/OpenGL/Shapes.cpp +++ b/examples/OpenGL/Shapes.cpp @@ -27,7 +27,7 @@ freely, subject to the following restrictions: using namespace PolyVox; -void createSphereInVolume(LargeVolume& volData, float fRadius, uint8_t uValue) +void createSphereInVolume(PagedVolume& volData, float fRadius, uint8_t uValue) { //This vector hold the position of the center of the volume Vector3DInt32 v3dVolCenter = (volData.getEnclosingRegion().getUpperCorner() - volData.getEnclosingRegion().getLowerCorner()) / static_cast(2); @@ -55,7 +55,7 @@ void createSphereInVolume(LargeVolume& volData, float fRa } } -void createCubeInVolume(LargeVolume& volData, Vector3DInt32 lowerCorner, Vector3DInt32 upperCorner, uint8_t uValue) +void createCubeInVolume(PagedVolume& volData, Vector3DInt32 lowerCorner, Vector3DInt32 upperCorner, uint8_t uValue) { uint8_t maxDen = MaterialDensityPair88::getMaxDensity(); uint8_t minDen = MaterialDensityPair88::getMinDensity(); diff --git a/examples/OpenGL/Shapes.h b/examples/OpenGL/Shapes.h index 712558fd..25f2acfc 100644 --- a/examples/OpenGL/Shapes.h +++ b/examples/OpenGL/Shapes.h @@ -24,7 +24,7 @@ freely, subject to the following restrictions: #ifndef __OpenGLExample_Shapes_H__ #define __OpenGLExample_Shapes_H__ -#include "PolyVoxCore/LargeVolume.h" +#include "PolyVoxCore/PagedVolume.h" #include "PolyVoxCore/MaterialDensityPair.h" void createSphereInVolume(PolyVox::LargeVolume& volData, float fRadius, uint8_t uValue); diff --git a/examples/OpenGL/main.cpp b/examples/OpenGL/main.cpp index 9a36fac8..edd7cdbc 100644 --- a/examples/OpenGL/main.cpp +++ b/examples/OpenGL/main.cpp @@ -24,7 +24,6 @@ freely, subject to the following restrictions: #include "PolyVoxCore/FilePager.h" #include "PolyVoxCore/MarchingCubesSurfaceExtractor.h" #include "PolyVoxCore/MaterialDensityPair.h" -#include "PolyVoxCore/LargeVolume.h" #include "PolyVoxCore/LowPassFilter.h" #include "PolyVoxCore/RawVolume.h" #include "PolyVoxCore/Mesh.h" @@ -51,7 +50,7 @@ const int32_t g_uVolumeSideLength = 128; int main(int argc, char *argv[]) { FilePager* pager = new FilePager("."); - LargeVolume volData(PolyVox::Region(Vector3DInt32(0, 0, 0), Vector3DInt32(g_uVolumeSideLength - 1, g_uVolumeSideLength - 1, g_uVolumeSideLength - 1)), pager); + PagedVolume volData(PolyVox::Region(Vector3DInt32(0, 0, 0), Vector3DInt32(g_uVolumeSideLength - 1, g_uVolumeSideLength - 1, g_uVolumeSideLength - 1)), pager); //Make our volume contain a sphere in the center. int32_t minPos = 0; diff --git a/examples/Paging/main.cpp b/examples/Paging/main.cpp index f70de28f..95c93972 100644 --- a/examples/Paging/main.cpp +++ b/examples/Paging/main.cpp @@ -35,7 +35,7 @@ freely, subject to the following restrictions: //Use the PolyVox namespace using namespace PolyVox; -void createSphereInVolume(LargeVolume& volData, Vector3DFloat v3dVolCenter, float fRadius) +void createSphereInVolume(PagedVolume& volData, Vector3DFloat v3dVolCenter, float fRadius) { //This vector hold the position of the center of the volume //Vector3DFloat v3dVolCenter(volData.getWidth() / 2, volData.getHeight() / 2, volData.getDepth() / 2); @@ -147,7 +147,7 @@ int main(int argc, char *argv[]) openGLWidget.show(); PerlinNoisePager* pager = new PerlinNoisePager(); - LargeVolume volData(PolyVox::Region::MaxRegion, pager, 64); + PagedVolume volData(PolyVox::Region::MaxRegion, pager, 64); volData.setMemoryUsageLimit(8 * 1024 * 1024); // 8Mb //createSphereInVolume(volData, 30); diff --git a/examples/SmoothLOD/main.cpp b/examples/SmoothLOD/main.cpp index 0054c5ce..978f9935 100644 --- a/examples/SmoothLOD/main.cpp +++ b/examples/SmoothLOD/main.cpp @@ -27,7 +27,7 @@ freely, subject to the following restrictions: #include "PolyVoxCore/MarchingCubesSurfaceExtractor.h" #include "PolyVoxCore/Mesh.h" #include "PolyVoxCore/RawVolume.h" -#include "PolyVoxCore/SimpleVolume.h" +#include "PolyVoxCore/PagedVolume.h" #include "PolyVoxCore/VolumeResampler.h" #include @@ -35,7 +35,7 @@ freely, subject to the following restrictions: //Use the PolyVox namespace using namespace PolyVox; -void createSphereInVolume(SimpleVolume& volData, float fRadius) +void createSphereInVolume(PagedVolume& volData, float fRadius) { //This vector hold the position of the center of the volume Vector3DFloat v3dVolCenter(volData.getWidth() / 2, volData.getHeight() / 2, volData.getDepth() / 2); @@ -76,17 +76,17 @@ int main(int argc, char *argv[]) openGLWidget.show(); //Create an empty volume and then place a sphere in it - SimpleVolume volData(PolyVox::Region(Vector3DInt32(0,0,0), Vector3DInt32(63, 63, 63))); + PagedVolume volData(PolyVox::Region(Vector3DInt32(0, 0, 0), Vector3DInt32(63, 63, 63))); createSphereInVolume(volData, 28); //Smooth the data - should reimplement this using LowPassFilter - //smoothRegion(volData, volData.getEnclosingRegion()); - //smoothRegion(volData, volData.getEnclosingRegion()); - //smoothRegion(volData, volData.getEnclosingRegion()); + //smoothRegion(volData, volData.getEnclosingRegion()); + //smoothRegion(volData, volData.getEnclosingRegion()); + //smoothRegion(volData, volData.getEnclosingRegion()); RawVolume volDataLowLOD(PolyVox::Region(Vector3DInt32(0,0,0), Vector3DInt32(15, 31, 31))); - VolumeResampler< SimpleVolume, RawVolume > volumeResampler(&volData, PolyVox::Region(Vector3DInt32(0,0,0), Vector3DInt32(31, 63, 63)), &volDataLowLOD, volDataLowLOD.getEnclosingRegion()); + VolumeResampler< PagedVolume, RawVolume > volumeResampler(&volData, PolyVox::Region(Vector3DInt32(0, 0, 0), Vector3DInt32(31, 63, 63)), &volDataLowLOD, volDataLowLOD.getEnclosingRegion()); volumeResampler.execute(); //Extract the surface diff --git a/library/PolyVoxCore/include/PolyVoxCore/AmbientOcclusionCalculator.h b/library/PolyVoxCore/include/PolyVoxCore/AmbientOcclusionCalculator.h index 696d21e1..70a5d89d 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/AmbientOcclusionCalculator.h +++ b/library/PolyVoxCore/include/PolyVoxCore/AmbientOcclusionCalculator.h @@ -33,7 +33,7 @@ freely, subject to the following restrictions: //These two should not be here! #include "PolyVoxCore/Material.h" -#include "PolyVoxCore/SimpleVolume.h" +#include "PolyVoxCore/PagedVolume.h" #include @@ -53,7 +53,7 @@ namespace PolyVox { } - bool operator()(const SimpleVolume::Sampler& sampler) + bool operator()(const PagedVolume::Sampler& sampler) { uint8_t sample = sampler.getVoxel(); bool func = mIsVoxelTransparentCallback(sample); diff --git a/library/PolyVoxCore/include/PolyVoxCore/BaseVolume.h b/library/PolyVoxCore/include/PolyVoxCore/BaseVolume.h index 3f5f1c0f..84d46c9b 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/BaseVolume.h +++ b/library/PolyVoxCore/include/PolyVoxCore/BaseVolume.h @@ -32,7 +32,7 @@ freely, subject to the following restrictions: namespace PolyVox { /// The BaseVolume class provides common functionality and an interface for other volume classes to implement. You should not try to create an instance of this - /// class directly. Instead you should use RawVolume, SimpleVolume, or PagedVolume. + /// class directly. Instead you should use RawVolume or PagedVolume. //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// /// More details to come... //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/library/PolyVoxCore/include/PolyVoxCore/BaseVolume.inl b/library/PolyVoxCore/include/PolyVoxCore/BaseVolume.inl index 7669a4ca..aeefc305 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/BaseVolume.inl +++ b/library/PolyVoxCore/include/PolyVoxCore/BaseVolume.inl @@ -26,7 +26,7 @@ namespace PolyVox //////////////////////////////////////////////////////////////////////////////// /// This is protected because you should never create a BaseVolume directly, you should instead use one of the derived classes. /// - /// \sa RawVolume, SimpleVolume, PagedVolume + /// \sa RawVolume, PagedVolume //////////////////////////////////////////////////////////////////////////////// template BaseVolume::BaseVolume(const Region& regValid) diff --git a/library/PolyVoxCore/include/PolyVoxCore/SimpleVolume.h b/library/PolyVoxCore/include/PolyVoxCore/SimpleVolume.h index 84ecd46b..a97c7be3 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/SimpleVolume.h +++ b/library/PolyVoxCore/include/PolyVoxCore/SimpleVolume.h @@ -1,7 +1,7 @@ #ifndef __PolyVox_SimpleVolume_H__ #define __PolyVox_SimpleVolume_H__ -#pragma message("WARNING - The LargeVolume class has been replaced by PagedVolume. Please use that instead.") +#pragma message("WARNING - The SimpleVolume class has been replaced by PagedVolume. Please use that instead.") #include "PagedVolume.h" #include "PolyVoxForwardDeclarations.h" diff --git a/tests/TestAmbientOcclusionGenerator.cpp b/tests/TestAmbientOcclusionGenerator.cpp index cb077ec2..97ab7993 100644 --- a/tests/TestAmbientOcclusionGenerator.cpp +++ b/tests/TestAmbientOcclusionGenerator.cpp @@ -24,7 +24,7 @@ freely, subject to the following restrictions: #include "TestAmbientOcclusionGenerator.h" #include "PolyVoxCore/AmbientOcclusionCalculator.h" -#include "PolyVoxCore/SimpleVolume.h" +#include "PolyVoxCore/PagedVolume.h" #include @@ -49,7 +49,7 @@ void TestAmbientOcclusionGenerator::testExecute() const int32_t g_uVolumeSideLength = 64; //Create empty volume - SimpleVolume volData(Region(Vector3DInt32(0,0,0), Vector3DInt32(g_uVolumeSideLength-1, g_uVolumeSideLength-1, g_uVolumeSideLength-1))); + PagedVolume volData(Region(Vector3DInt32(0, 0, 0), Vector3DInt32(g_uVolumeSideLength - 1, g_uVolumeSideLength - 1, g_uVolumeSideLength - 1))); //Create two solid walls at opposite sides of the volume for (int32_t z = 0; z < g_uVolumeSideLength; z++) diff --git a/tests/TestCubicSurfaceExtractor.cpp b/tests/TestCubicSurfaceExtractor.cpp index 906a2e80..74182fda 100644 --- a/tests/TestCubicSurfaceExtractor.cpp +++ b/tests/TestCubicSurfaceExtractor.cpp @@ -27,7 +27,7 @@ freely, subject to the following restrictions: #include "PolyVoxCore/Material.h" #include "PolyVoxCore/MaterialDensityPair.h" #include "PolyVoxCore/RawVolume.h" -#include "PolyVoxCore/SimpleVolume.h" +#include "PolyVoxCore/PagedVolume.h" #include "PolyVoxCore/CubicSurfaceExtractor.h" #include @@ -128,26 +128,26 @@ VolumeType* createAndFillVolumeRealistic(int32_t iVolumeSideLength) void TestCubicSurfaceExtractor::testBehaviour() { // Test with default mesh and contoller types. - auto uint8Vol = createAndFillVolumeWithNoise< SimpleVolume >(32, 0, 2); + auto uint8Vol = createAndFillVolumeWithNoise< PagedVolume >(32, 0, 2); auto uint8Mesh = extractCubicMesh(uint8Vol, uint8Vol->getEnclosingRegion()); QCOMPARE(uint8Mesh.getNoOfVertices(), uint32_t(57687)); QCOMPARE(uint8Mesh.getNoOfIndices(), uint32_t(216234)); // Test with default mesh type but user-provided controller. - auto int8Vol = createAndFillVolumeWithNoise< SimpleVolume >(32, 0, 2); + auto int8Vol = createAndFillVolumeWithNoise< PagedVolume >(32, 0, 2); auto int8Mesh = extractCubicMesh(int8Vol, int8Vol->getEnclosingRegion(), CustomIsQuadNeeded()); QCOMPARE(int8Mesh.getNoOfVertices(), uint32_t(29027)); QCOMPARE(int8Mesh.getNoOfIndices(), uint32_t(178356)); // Test with default controller but user-provided mesh. - auto uint32Vol = createAndFillVolumeWithNoise< SimpleVolume >(32, 0, 2); + auto uint32Vol = createAndFillVolumeWithNoise< PagedVolume >(32, 0, 2); Mesh< CubicVertex< uint32_t >, uint16_t > uint32Mesh; extractCubicMeshCustom(uint32Vol, uint32Vol->getEnclosingRegion(), &uint32Mesh); QCOMPARE(uint32Mesh.getNoOfVertices(), uint16_t(57687)); QCOMPARE(uint32Mesh.getNoOfIndices(), uint32_t(216234)); // Test with both mesh and controller being provided by the user. - auto int32Vol = createAndFillVolumeWithNoise< SimpleVolume >(32, 0, 2); + auto int32Vol = createAndFillVolumeWithNoise< PagedVolume >(32, 0, 2); Mesh< CubicVertex< int32_t >, uint16_t > int32Mesh; extractCubicMeshCustom(int32Vol, int32Vol->getEnclosingRegion(), &int32Mesh, CustomIsQuadNeeded()); QCOMPARE(int32Mesh.getNoOfVertices(), uint16_t(29027)); @@ -156,7 +156,7 @@ void TestCubicSurfaceExtractor::testBehaviour() void TestCubicSurfaceExtractor::testEmptyVolumePerformance() { - auto emptyVol = createAndFillVolumeWithNoise< SimpleVolume >(128, 0, 0); + auto emptyVol = createAndFillVolumeWithNoise< PagedVolume >(128, 0, 0); Mesh< CubicVertex< uint32_t >, uint16_t > emptyMesh; QBENCHMARK{ extractCubicMeshCustom(emptyVol, Region(32, 32, 32, 63, 63, 63), &emptyMesh); } QCOMPARE(emptyMesh.getNoOfVertices(), uint16_t(0)); @@ -164,7 +164,7 @@ void TestCubicSurfaceExtractor::testEmptyVolumePerformance() void TestCubicSurfaceExtractor::testRealisticVolumePerformance() { - auto realisticVol = createAndFillVolumeRealistic< SimpleVolume >(128); + auto realisticVol = createAndFillVolumeRealistic< PagedVolume >(128); Mesh< CubicVertex< uint32_t >, uint16_t > realisticMesh; QBENCHMARK{ extractCubicMeshCustom(realisticVol, Region(32, 32, 32, 63, 63, 63), &realisticMesh); } QCOMPARE(realisticMesh.getNoOfVertices(), uint16_t(2176)); @@ -172,7 +172,7 @@ void TestCubicSurfaceExtractor::testRealisticVolumePerformance() void TestCubicSurfaceExtractor::testNoiseVolumePerformance() { - auto noiseVol = createAndFillVolumeWithNoise< SimpleVolume >(128, 0, 2); + auto noiseVol = createAndFillVolumeWithNoise< PagedVolume >(128, 0, 2); Mesh< CubicVertex< uint32_t >, uint16_t > noiseMesh; QBENCHMARK{ extractCubicMeshCustom(noiseVol, Region(32, 32, 32, 63, 63, 63), &noiseMesh); } QCOMPARE(noiseMesh.getNoOfVertices(), uint16_t(57729)); diff --git a/tests/TestPicking.cpp b/tests/TestPicking.cpp index 6271d004..fa255b55 100644 --- a/tests/TestPicking.cpp +++ b/tests/TestPicking.cpp @@ -24,7 +24,7 @@ freely, subject to the following restrictions: #include "TestPicking.h" #include "PolyVoxCore/Picking.h" -#include "PolyVoxCore/SimpleVolume.h" +#include "PolyVoxCore/PagedVolume.h" #include @@ -34,7 +34,7 @@ void TestPicking::testExecute() { const int32_t uVolumeSideLength = 32; - SimpleVolume volData(Region(Vector3DInt32(0,0,0), Vector3DInt32(uVolumeSideLength-1, uVolumeSideLength-1, uVolumeSideLength-1))); + PagedVolume volData(Region(Vector3DInt32(0, 0, 0), Vector3DInt32(uVolumeSideLength - 1, uVolumeSideLength - 1, uVolumeSideLength - 1))); for (int32_t z = 0; z < uVolumeSideLength; z++) { for (int32_t y = 0; y < uVolumeSideLength; y++) diff --git a/tests/TestRaycast.cpp b/tests/TestRaycast.cpp index 364bff04..a7b148fa 100644 --- a/tests/TestRaycast.cpp +++ b/tests/TestRaycast.cpp @@ -25,7 +25,7 @@ freely, subject to the following restrictions: #include "PolyVoxCore/Density.h" #include "PolyVoxCore/Raycast.h" -#include "PolyVoxCore/SimpleVolume.h" +#include "PolyVoxCore/PagedVolume.h" #include "PolyVoxCore/Impl/RandomUnitVectors.h" @@ -47,7 +47,7 @@ public: { } - bool operator()(const SimpleVolume::Sampler& sampler) + bool operator()(const PagedVolume::Sampler& sampler) { m_uVoxelsTouched++; @@ -73,7 +73,7 @@ void TestRaycast::testExecute() const int32_t uVolumeSideLength = 32; //Create a hollow volume, with solid sides on x and y but with open ends in z. - SimpleVolume volData(Region(Vector3DInt32(0,0,0), Vector3DInt32(uVolumeSideLength-1, uVolumeSideLength-1, uVolumeSideLength-1))); + PagedVolume volData(Region(Vector3DInt32(0, 0, 0), Vector3DInt32(uVolumeSideLength - 1, uVolumeSideLength - 1, uVolumeSideLength - 1))); for (int32_t z = 0; z < uVolumeSideLength; z++) { for (int32_t y = 0; y < uVolumeSideLength; y++) diff --git a/tests/TestSurfaceExtractor.cpp b/tests/TestSurfaceExtractor.cpp index 599281bb..9bf96a6c 100644 --- a/tests/TestSurfaceExtractor.cpp +++ b/tests/TestSurfaceExtractor.cpp @@ -27,8 +27,7 @@ freely, subject to the following restrictions: #include "PolyVoxCore/FilePager.h" #include "PolyVoxCore/MaterialDensityPair.h" #include "PolyVoxCore/RawVolume.h" -#include "PolyVoxCore/SimpleVolume.h" -#include "PolyVoxCore/LargeVolume.h" +#include "PolyVoxCore/PagedVolume.h" #include "PolyVoxCore/MarchingCubesSurfaceExtractor.h" #include @@ -173,7 +172,7 @@ void TestSurfaceExtractor::testBehaviour() // Of course, the use of a custom controller will also make a significant diference, but this probably does need investigating further in the future. // This basic test just uses the default controller and automatically generates a mesh of the appropriate type. - auto uintVol = createAndFillVolume< LargeVolume >(); + auto uintVol = createAndFillVolume< PagedVolume >(); auto uintMesh = extractMarchingCubesMesh(uintVol, uintVol->getEnclosingRegion()); QCOMPARE(uintMesh.getNoOfVertices(), uint32_t(12096)); // Verifies size of mesh and that we have 32-bit indices QCOMPARE(uintMesh.getNoOfIndices(), uint32_t(35157)); // Verifies size of mesh @@ -181,7 +180,7 @@ void TestSurfaceExtractor::testBehaviour() QCOMPARE(uintMesh.getVertex(100).data, uint8_t(1)); // Not really meaningful for a primative type // This test makes use of a custom controller - auto floatVol = createAndFillVolume< LargeVolume >(); + auto floatVol = createAndFillVolume< PagedVolume >(); CustomMarchingCubesController floatCustomController; auto floatMesh = extractMarchingCubesMesh(floatVol, floatVol->getEnclosingRegion(), floatCustomController); QCOMPARE(floatMesh.getNoOfVertices(), uint32_t(16113)); // Verifies size of mesh and that we have 32-bit indices @@ -191,7 +190,7 @@ void TestSurfaceExtractor::testBehaviour() // This test makes use of a user provided mesh. It uses the default controller, but we have to explicitly provide this because C++ won't let us // use a default for the second-to-last parameter but noot use a default for the last parameter. - auto intVol = createAndFillVolume< LargeVolume >(); + auto intVol = createAndFillVolume< PagedVolume >(); Mesh< MarchingCubesVertex< int8_t >, uint16_t > intMesh; extractMarchingCubesMeshCustom(intVol, intVol->getEnclosingRegion(), &intMesh); QCOMPARE(intMesh.getNoOfVertices(), uint16_t(11718)); // Verifies size of mesh and that we have 16-bit indices @@ -200,7 +199,7 @@ void TestSurfaceExtractor::testBehaviour() QCOMPARE(intMesh.getVertex(100).data, int8_t(1)); // Not really meaningful for a primative type // This test makes use of a user-provided mesh and also a custom controller. - auto doubleVol = createAndFillVolume< LargeVolume >(); + auto doubleVol = createAndFillVolume< PagedVolume >(); CustomMarchingCubesController doubleCustomController; Mesh< MarchingCubesVertex< double >, uint16_t > doubleMesh; extractMarchingCubesMeshCustom(doubleVol, doubleVol->getEnclosingRegion(), &doubleMesh, doubleCustomController); @@ -210,7 +209,7 @@ void TestSurfaceExtractor::testBehaviour() QCOMPARE(doubleMesh.getVertex(100).data, double(1.0f)); // Not really meaningful for a primative type // This test ensures the extractor works on a non-primitive voxel type. - auto materialVol = createAndFillVolume< LargeVolume >(); + auto materialVol = createAndFillVolume< PagedVolume >(); auto materialMesh = extractMarchingCubesMesh(materialVol, materialVol->getEnclosingRegion()); QCOMPARE(materialMesh.getNoOfVertices(), uint32_t(12096)); // Verifies size of mesh and that we have 32-bit indices QCOMPARE(materialMesh.getNoOfIndices(), uint32_t(35157)); // Verifies size of mesh @@ -220,7 +219,7 @@ void TestSurfaceExtractor::testBehaviour() void TestSurfaceExtractor::testEmptyVolumePerformance() { - auto emptyVol = createAndFillVolumeWithNoise< LargeVolume >(128, -2.0f, -1.0f); + auto emptyVol = createAndFillVolumeWithNoise< PagedVolume >(128, -2.0f, -1.0f); Mesh< MarchingCubesVertex< float >, uint16_t > emptyMesh; QBENCHMARK{ extractMarchingCubesMeshCustom(emptyVol, Region(32, 32, 32, 63, 63, 63), &emptyMesh); } QCOMPARE(emptyMesh.getNoOfVertices(), uint16_t(0)); @@ -228,7 +227,7 @@ void TestSurfaceExtractor::testEmptyVolumePerformance() void TestSurfaceExtractor::testNoiseVolumePerformance() { - auto noiseVol = createAndFillVolumeWithNoise< LargeVolume >(128, -1.0f, 1.0f); + auto noiseVol = createAndFillVolumeWithNoise< PagedVolume >(128, -1.0f, 1.0f); Mesh< MarchingCubesVertex< float >, uint16_t > noiseMesh; QBENCHMARK{ extractMarchingCubesMeshCustom(noiseVol, Region(32, 32, 32, 63, 63, 63), &noiseMesh); } QCOMPARE(noiseMesh.getNoOfVertices(), uint16_t(48967)); diff --git a/tests/testvolume.cpp b/tests/testvolume.cpp index a8f8202b..3eafceb2 100644 --- a/tests/testvolume.cpp +++ b/tests/testvolume.cpp @@ -110,7 +110,7 @@ int32_t testSamplersWithWrappingForwards(VolumeType* volume, int lowXOffset, int xSampler = ySampler; for(int x = volume->getEnclosingRegion().getLowerX() + lowXOffset; x <= volume->getEnclosingRegion().getUpperX() + highXOffset; x++) { - xSampler.setPosition(x, y, z); // HACK - Accessing a volume through multiple samplers currently breaks the LargeVolume. + xSampler.setPosition(x, y, z); // HACK - Accessing a volume through multiple samplers currently breaks the PagedVolume. result = cantorTupleFunction(result, xSampler.peekVoxel1nx1ny1nz()); result = cantorTupleFunction(result, xSampler.peekVoxel0px1ny1nz()); @@ -223,7 +223,7 @@ int32_t testSamplersWithWrappingBackwards(VolumeType* volume, int lowXOffset, in xSampler = ySampler; for(int x = volume->getEnclosingRegion().getUpperX() + highXOffset; x >= volume->getEnclosingRegion().getLowerX() + lowXOffset; x--) { - xSampler.setPosition(x, y, z); // HACK - Accessing a volume through multiple samplers currently breaks the LargeVolume. + xSampler.setPosition(x, y, z); // HACK - Accessing a volume through multiple samplers currently breaks the PagedVolume. result = cantorTupleFunction(result, xSampler.peekVoxel1nx1ny1nz()); result = cantorTupleFunction(result, xSampler.peekVoxel0px1ny1nz()); @@ -273,7 +273,7 @@ TestVolume::TestVolume() //Create the volumes m_pRawVolume = new RawVolume(region); - m_pPagedVolume = new LargeVolume(region, m_pFilePager, 32); + m_pPagedVolume = new PagedVolume(region, m_pFilePager, 32); m_pPagedVolume->setMemoryUsageLimit(1 * 1024 * 1024); From 75df6621bd144801af4eacef4ca55152bfdd55db Mon Sep 17 00:00:00 2001 From: David Williams Date: Sun, 21 Sep 2014 23:07:14 +0200 Subject: [PATCH 42/42] GCC fixes --- .../include/PolyVoxCore/PagedVolume.h | 4 +- tests/TestSurfaceExtractor.cpp | 456 +++++++++--------- 2 files changed, 230 insertions(+), 230 deletions(-) diff --git a/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.h b/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.h index 848486e2..adf27243 100644 --- a/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.h +++ b/library/PolyVoxCore/include/PolyVoxCore/PagedVolume.h @@ -353,9 +353,9 @@ namespace PolyVox Pager* m_pPager; // Enough to make sure a chunks and it's neighbours can be loaded, with a few to spare. - static const uint32_t uMinPracticalNoOfChunks = 32; + const uint32_t uMinPracticalNoOfChunks = 32; // Should prevent multi-gigabyte volumes when chunk sizes are reasonable. - static const uint32_t uMaxPracticalNoOfChunks = 32768; + const uint32_t uMaxPracticalNoOfChunks = 32768; }; } diff --git a/tests/TestSurfaceExtractor.cpp b/tests/TestSurfaceExtractor.cpp index 9bf96a6c..a3a70085 100644 --- a/tests/TestSurfaceExtractor.cpp +++ b/tests/TestSurfaceExtractor.cpp @@ -1,236 +1,236 @@ -/******************************************************************************* -Copyright (c) 2010 Matt Williams - -This software is provided 'as-is', without any express or implied -warranty. In no event will the authors be held liable for any damages -arising from the use of this software. - -Permission is granted to anyone to use this software for any purpose, -including commercial applications, and to alter it and redistribute it -freely, subject to the following restrictions: - - 1. The origin of this software must not be misrepresented; you must not - claim that you wrote the original software. If you use this software - in a product, an acknowledgment in the product documentation would be - appreciated but is not required. - - 2. Altered source versions must be plainly marked as such, and must not be - misrepresented as being the original software. - - 3. This notice may not be removed or altered from any source - distribution. -*******************************************************************************/ - -#include "TestSurfaceExtractor.h" - -#include "PolyVoxCore/Density.h" -#include "PolyVoxCore/FilePager.h" -#include "PolyVoxCore/MaterialDensityPair.h" -#include "PolyVoxCore/RawVolume.h" -#include "PolyVoxCore/PagedVolume.h" -#include "PolyVoxCore/MarchingCubesSurfaceExtractor.h" - -#include - -using namespace PolyVox; - -// Test our ability to modify the behaviour of the MarchingCubesSurfaceExtractor. This simple example only modifies -// the threshold (and actually this can be achieved by passing a parameter to the constructor of the -// DefaultMarchingCubesController) but you could implement custom behaviour in the other members -// if you wanted too. Actually, it's not clear if this ability is really useful because I can't think -// what you'd modify apart from the threshold but the ability is at least available. Also, the -// DefaultMarchingCubesController is templatised whereas this exmple shows that controllers don't -// have to be. -class CustomMarchingCubesController -{ -public: - typedef float DensityType; - typedef float MaterialType; - - float convertToDensity(float voxel) - { - return voxel; - } - - float convertToMaterial(float /*voxel*/) - { - return 1.0f; - } - - float blendMaterials(float /*a*/, float /*b*/, float /*weight*/) - { - return 1.0f; - } - - float getThreshold(void) - { - return 50.0f; - } -}; - -// These 'writeDensityValueToVoxel' functions provide a unified interface for writting densities to primative and class voxel types. -// They are conceptually the inverse of the 'convertToDensity' function used by the MarchingCubesSurfaceExtractor. They probably shouldn't be part -// of PolyVox, but they might be usful to other tests so we cold move them into a 'Tests.h' or something in the future. -template -void writeDensityValueToVoxel(int valueToWrite, VoxelType& voxel) -{ - voxel = valueToWrite; -} - -template<> -void writeDensityValueToVoxel(int valueToWrite, MaterialDensityPair88& voxel) -{ - voxel.setDensity(valueToWrite); -} - -template -void writeMaterialValueToVoxel(int /*valueToWrite*/, VoxelType& /*voxel*/) -{ - //Most types don't have a material - return; -} - -template<> -void writeMaterialValueToVoxel(int valueToWrite, MaterialDensityPair88& voxel) -{ - voxel.setMaterial(valueToWrite); -} - -template -VolumeType* createAndFillVolume(void) -{ - const int32_t uVolumeSideLength = 64; +/******************************************************************************* +Copyright (c) 2010 Matt Williams - FilePager* pager = new FilePager("."); - - //Create empty volume - VolumeType* volData = new VolumeType(Region(Vector3DInt32(0, 0, 0), Vector3DInt32(uVolumeSideLength - 1, uVolumeSideLength - 1, uVolumeSideLength - 1)), pager); - - // Fill - for (int32_t z = 0; z < uVolumeSideLength; z++) - { - for (int32_t y = 0; y < uVolumeSideLength; y++) - { - for (int32_t x = 0; x < uVolumeSideLength; x++) - { - // Create a density field which changes throughout the volume. It's - // zero in the lower corner and increasing as the coordinates increase. - VolumeType::VoxelType voxelValue; - writeDensityValueToVoxel(x + y + z, voxelValue); - writeMaterialValueToVoxel(z > uVolumeSideLength / 2 ? 42 : 79, voxelValue); - volData->setVoxelAt(x, y, z, voxelValue); - } - } - } - - return volData; -} - -// From http://stackoverflow.com/a/5289624 +This software is provided 'as-is', without any express or implied +warranty. In no event will the authors be held liable for any damages +arising from the use of this software. + +Permission is granted to anyone to use this software for any purpose, +including commercial applications, and to alter it and redistribute it +freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + + 3. This notice may not be removed or altered from any source + distribution. +*******************************************************************************/ + +#include "TestSurfaceExtractor.h" + +#include "PolyVoxCore/Density.h" +#include "PolyVoxCore/FilePager.h" +#include "PolyVoxCore/MaterialDensityPair.h" +#include "PolyVoxCore/RawVolume.h" +#include "PolyVoxCore/PagedVolume.h" +#include "PolyVoxCore/MarchingCubesSurfaceExtractor.h" + +#include + +using namespace PolyVox; + +// Test our ability to modify the behaviour of the MarchingCubesSurfaceExtractor. This simple example only modifies +// the threshold (and actually this can be achieved by passing a parameter to the constructor of the +// DefaultMarchingCubesController) but you could implement custom behaviour in the other members +// if you wanted too. Actually, it's not clear if this ability is really useful because I can't think +// what you'd modify apart from the threshold but the ability is at least available. Also, the +// DefaultMarchingCubesController is templatised whereas this exmple shows that controllers don't +// have to be. +class CustomMarchingCubesController +{ +public: + typedef float DensityType; + typedef float MaterialType; + + float convertToDensity(float voxel) + { + return voxel; + } + + float convertToMaterial(float /*voxel*/) + { + return 1.0f; + } + + float blendMaterials(float /*a*/, float /*b*/, float /*weight*/) + { + return 1.0f; + } + + float getThreshold(void) + { + return 50.0f; + } +}; + +// These 'writeDensityValueToVoxel' functions provide a unified interface for writting densities to primative and class voxel types. +// They are conceptually the inverse of the 'convertToDensity' function used by the MarchingCubesSurfaceExtractor. They probably shouldn't be part +// of PolyVox, but they might be usful to other tests so we cold move them into a 'Tests.h' or something in the future. +template +void writeDensityValueToVoxel(int valueToWrite, VoxelType& voxel) +{ + voxel = valueToWrite; +} + +template<> +void writeDensityValueToVoxel(int valueToWrite, MaterialDensityPair88& voxel) +{ + voxel.setDensity(valueToWrite); +} + +template +void writeMaterialValueToVoxel(int /*valueToWrite*/, VoxelType& /*voxel*/) +{ + //Most types don't have a material + return; +} + +template<> +void writeMaterialValueToVoxel(int valueToWrite, MaterialDensityPair88& voxel) +{ + voxel.setMaterial(valueToWrite); +} + +template +VolumeType* createAndFillVolume(void) +{ + const int32_t uVolumeSideLength = 64; + + FilePager* pager = new FilePager("."); + + //Create empty volume + VolumeType* volData = new VolumeType(Region(Vector3DInt32(0, 0, 0), Vector3DInt32(uVolumeSideLength - 1, uVolumeSideLength - 1, uVolumeSideLength - 1)), pager); + + // Fill + for (int32_t z = 0; z < uVolumeSideLength; z++) + { + for (int32_t y = 0; y < uVolumeSideLength; y++) + { + for (int32_t x = 0; x < uVolumeSideLength; x++) + { + // Create a density field which changes throughout the volume. It's + // zero in the lower corner and increasing as the coordinates increase. + typename VolumeType::VoxelType voxelValue; + writeDensityValueToVoxel(x + y + z, voxelValue); + writeMaterialValueToVoxel(z > uVolumeSideLength / 2 ? 42 : 79, voxelValue); + volData->setVoxelAt(x, y, z, voxelValue); + } + } + } + + return volData; +} + +// From http://stackoverflow.com/a/5289624 float randomFloat(float a, float b) { float random = ((float)rand()) / (float)RAND_MAX; float diff = b - a; float r = random * diff; return a + r; -} - -template -VolumeType* createAndFillVolumeWithNoise(int32_t iVolumeSideLength, float minValue, float maxValue) +} + +template +VolumeType* createAndFillVolumeWithNoise(int32_t iVolumeSideLength, float minValue, float maxValue) { - FilePager* pager = new FilePager("."); - - //Create empty volume - VolumeType* volData = new VolumeType(Region(Vector3DInt32(0, 0, 0), Vector3DInt32(iVolumeSideLength - 1, iVolumeSideLength - 1, iVolumeSideLength - 1)), pager); - - // Seed generator for consistency between runs. - srand(12345); - - // Fill - for (int32_t z = 0; z < iVolumeSideLength; z++) - { - for (int32_t y = 0; y < iVolumeSideLength; y++) - { - for (int32_t x = 0; x < iVolumeSideLength; x++) - { - float voxelValue = randomFloat(minValue, maxValue); - volData->setVoxelAt(x, y, z, voxelValue); - } - } - } - - return volData; -} - -void TestSurfaceExtractor::testBehaviour() -{ - // These tests apply the Marching Cubes surface extractor to volumes of various voxel types. In addition we sometimes make use of custom controllers - // and user-provided meshes to make sure these various combinations work as expected. - // - // It is also noted that the number of indices and vertices is varying quite significantly based on the voxel type. This seems unexpected, but could - // be explained if some overflow is occuring when writing data into the volume, causing volumes of different voxel types to have different distributions. - // Of course, the use of a custom controller will also make a significant diference, but this probably does need investigating further in the future. - - // This basic test just uses the default controller and automatically generates a mesh of the appropriate type. - auto uintVol = createAndFillVolume< PagedVolume >(); - auto uintMesh = extractMarchingCubesMesh(uintVol, uintVol->getEnclosingRegion()); - QCOMPARE(uintMesh.getNoOfVertices(), uint32_t(12096)); // Verifies size of mesh and that we have 32-bit indices - QCOMPARE(uintMesh.getNoOfIndices(), uint32_t(35157)); // Verifies size of mesh - QCOMPARE(uintMesh.getIndex(100), uint32_t(44)); // Verifies that we have 32-bit indices - QCOMPARE(uintMesh.getVertex(100).data, uint8_t(1)); // Not really meaningful for a primative type - - // This test makes use of a custom controller - auto floatVol = createAndFillVolume< PagedVolume >(); - CustomMarchingCubesController floatCustomController; - auto floatMesh = extractMarchingCubesMesh(floatVol, floatVol->getEnclosingRegion(), floatCustomController); - QCOMPARE(floatMesh.getNoOfVertices(), uint32_t(16113)); // Verifies size of mesh and that we have 32-bit indices - QCOMPARE(floatMesh.getNoOfIndices(), uint32_t(22053)); // Verifies size of mesh - QCOMPARE(floatMesh.getIndex(100), uint32_t(26)); // Verifies that we have 32-bit indices - QCOMPARE(floatMesh.getVertex(100).data, float(1.0f)); // Not really meaningful for a primative type - - // This test makes use of a user provided mesh. It uses the default controller, but we have to explicitly provide this because C++ won't let us - // use a default for the second-to-last parameter but noot use a default for the last parameter. - auto intVol = createAndFillVolume< PagedVolume >(); - Mesh< MarchingCubesVertex< int8_t >, uint16_t > intMesh; - extractMarchingCubesMeshCustom(intVol, intVol->getEnclosingRegion(), &intMesh); - QCOMPARE(intMesh.getNoOfVertices(), uint16_t(11718)); // Verifies size of mesh and that we have 16-bit indices - QCOMPARE(intMesh.getNoOfIndices(), uint32_t(34041)); // Verifies size of mesh - QCOMPARE(intMesh.getIndex(100), uint16_t(29)); // Verifies that we have 16-bit indices - QCOMPARE(intMesh.getVertex(100).data, int8_t(1)); // Not really meaningful for a primative type - - // This test makes use of a user-provided mesh and also a custom controller. - auto doubleVol = createAndFillVolume< PagedVolume >(); - CustomMarchingCubesController doubleCustomController; - Mesh< MarchingCubesVertex< double >, uint16_t > doubleMesh; - extractMarchingCubesMeshCustom(doubleVol, doubleVol->getEnclosingRegion(), &doubleMesh, doubleCustomController); - QCOMPARE(doubleMesh.getNoOfVertices(), uint16_t(16113)); // Verifies size of mesh and that we have 32-bit indices - QCOMPARE(doubleMesh.getNoOfIndices(), uint32_t(22053)); // Verifies size of mesh - QCOMPARE(doubleMesh.getIndex(100), uint16_t(26)); // Verifies that we have 32-bit indices - QCOMPARE(doubleMesh.getVertex(100).data, double(1.0f)); // Not really meaningful for a primative type - - // This test ensures the extractor works on a non-primitive voxel type. - auto materialVol = createAndFillVolume< PagedVolume >(); - auto materialMesh = extractMarchingCubesMesh(materialVol, materialVol->getEnclosingRegion()); - QCOMPARE(materialMesh.getNoOfVertices(), uint32_t(12096)); // Verifies size of mesh and that we have 32-bit indices - QCOMPARE(materialMesh.getNoOfIndices(), uint32_t(35157)); // Verifies size of mesh - QCOMPARE(materialMesh.getIndex(100), uint32_t(44)); // Verifies that we have 32-bit indices - QCOMPARE(materialMesh.getVertex(100).data.getMaterial(), uint16_t(79)); // Verify the data attached to the vertex -} - -void TestSurfaceExtractor::testEmptyVolumePerformance() -{ - auto emptyVol = createAndFillVolumeWithNoise< PagedVolume >(128, -2.0f, -1.0f); - Mesh< MarchingCubesVertex< float >, uint16_t > emptyMesh; - QBENCHMARK{ extractMarchingCubesMeshCustom(emptyVol, Region(32, 32, 32, 63, 63, 63), &emptyMesh); } - QCOMPARE(emptyMesh.getNoOfVertices(), uint16_t(0)); -} - -void TestSurfaceExtractor::testNoiseVolumePerformance() -{ - auto noiseVol = createAndFillVolumeWithNoise< PagedVolume >(128, -1.0f, 1.0f); - Mesh< MarchingCubesVertex< float >, uint16_t > noiseMesh; - QBENCHMARK{ extractMarchingCubesMeshCustom(noiseVol, Region(32, 32, 32, 63, 63, 63), &noiseMesh); } - QCOMPARE(noiseMesh.getNoOfVertices(), uint16_t(48967)); -} - -QTEST_MAIN(TestSurfaceExtractor) + FilePager* pager = new FilePager("."); + + //Create empty volume + VolumeType* volData = new VolumeType(Region(Vector3DInt32(0, 0, 0), Vector3DInt32(iVolumeSideLength - 1, iVolumeSideLength - 1, iVolumeSideLength - 1)), pager); + + // Seed generator for consistency between runs. + srand(12345); + + // Fill + for (int32_t z = 0; z < iVolumeSideLength; z++) + { + for (int32_t y = 0; y < iVolumeSideLength; y++) + { + for (int32_t x = 0; x < iVolumeSideLength; x++) + { + float voxelValue = randomFloat(minValue, maxValue); + volData->setVoxelAt(x, y, z, voxelValue); + } + } + } + + return volData; +} + +void TestSurfaceExtractor::testBehaviour() +{ + // These tests apply the Marching Cubes surface extractor to volumes of various voxel types. In addition we sometimes make use of custom controllers + // and user-provided meshes to make sure these various combinations work as expected. + // + // It is also noted that the number of indices and vertices is varying quite significantly based on the voxel type. This seems unexpected, but could + // be explained if some overflow is occuring when writing data into the volume, causing volumes of different voxel types to have different distributions. + // Of course, the use of a custom controller will also make a significant diference, but this probably does need investigating further in the future. + + // This basic test just uses the default controller and automatically generates a mesh of the appropriate type. + auto uintVol = createAndFillVolume< PagedVolume >(); + auto uintMesh = extractMarchingCubesMesh(uintVol, uintVol->getEnclosingRegion()); + QCOMPARE(uintMesh.getNoOfVertices(), uint32_t(12096)); // Verifies size of mesh and that we have 32-bit indices + QCOMPARE(uintMesh.getNoOfIndices(), uint32_t(35157)); // Verifies size of mesh + QCOMPARE(uintMesh.getIndex(100), uint32_t(44)); // Verifies that we have 32-bit indices + QCOMPARE(uintMesh.getVertex(100).data, uint8_t(1)); // Not really meaningful for a primative type + + // This test makes use of a custom controller + auto floatVol = createAndFillVolume< PagedVolume >(); + CustomMarchingCubesController floatCustomController; + auto floatMesh = extractMarchingCubesMesh(floatVol, floatVol->getEnclosingRegion(), floatCustomController); + QCOMPARE(floatMesh.getNoOfVertices(), uint32_t(16113)); // Verifies size of mesh and that we have 32-bit indices + QCOMPARE(floatMesh.getNoOfIndices(), uint32_t(22053)); // Verifies size of mesh + QCOMPARE(floatMesh.getIndex(100), uint32_t(26)); // Verifies that we have 32-bit indices + QCOMPARE(floatMesh.getVertex(100).data, float(1.0f)); // Not really meaningful for a primative type + + // This test makes use of a user provided mesh. It uses the default controller, but we have to explicitly provide this because C++ won't let us + // use a default for the second-to-last parameter but noot use a default for the last parameter. + auto intVol = createAndFillVolume< PagedVolume >(); + Mesh< MarchingCubesVertex< int8_t >, uint16_t > intMesh; + extractMarchingCubesMeshCustom(intVol, intVol->getEnclosingRegion(), &intMesh); + QCOMPARE(intMesh.getNoOfVertices(), uint16_t(11718)); // Verifies size of mesh and that we have 16-bit indices + QCOMPARE(intMesh.getNoOfIndices(), uint32_t(34041)); // Verifies size of mesh + QCOMPARE(intMesh.getIndex(100), uint16_t(29)); // Verifies that we have 16-bit indices + QCOMPARE(intMesh.getVertex(100).data, int8_t(1)); // Not really meaningful for a primative type + + // This test makes use of a user-provided mesh and also a custom controller. + auto doubleVol = createAndFillVolume< PagedVolume >(); + CustomMarchingCubesController doubleCustomController; + Mesh< MarchingCubesVertex< double >, uint16_t > doubleMesh; + extractMarchingCubesMeshCustom(doubleVol, doubleVol->getEnclosingRegion(), &doubleMesh, doubleCustomController); + QCOMPARE(doubleMesh.getNoOfVertices(), uint16_t(16113)); // Verifies size of mesh and that we have 32-bit indices + QCOMPARE(doubleMesh.getNoOfIndices(), uint32_t(22053)); // Verifies size of mesh + QCOMPARE(doubleMesh.getIndex(100), uint16_t(26)); // Verifies that we have 32-bit indices + QCOMPARE(doubleMesh.getVertex(100).data, double(1.0f)); // Not really meaningful for a primative type + + // This test ensures the extractor works on a non-primitive voxel type. + auto materialVol = createAndFillVolume< PagedVolume >(); + auto materialMesh = extractMarchingCubesMesh(materialVol, materialVol->getEnclosingRegion()); + QCOMPARE(materialMesh.getNoOfVertices(), uint32_t(12096)); // Verifies size of mesh and that we have 32-bit indices + QCOMPARE(materialMesh.getNoOfIndices(), uint32_t(35157)); // Verifies size of mesh + QCOMPARE(materialMesh.getIndex(100), uint32_t(44)); // Verifies that we have 32-bit indices + QCOMPARE(materialMesh.getVertex(100).data.getMaterial(), uint16_t(79)); // Verify the data attached to the vertex +} + +void TestSurfaceExtractor::testEmptyVolumePerformance() +{ + auto emptyVol = createAndFillVolumeWithNoise< PagedVolume >(128, -2.0f, -1.0f); + Mesh< MarchingCubesVertex< float >, uint16_t > emptyMesh; + QBENCHMARK{ extractMarchingCubesMeshCustom(emptyVol, Region(32, 32, 32, 63, 63, 63), &emptyMesh); } + QCOMPARE(emptyMesh.getNoOfVertices(), uint16_t(0)); +} + +void TestSurfaceExtractor::testNoiseVolumePerformance() +{ + auto noiseVol = createAndFillVolumeWithNoise< PagedVolume >(128, -1.0f, 1.0f); + Mesh< MarchingCubesVertex< float >, uint16_t > noiseMesh; + QBENCHMARK{ extractMarchingCubesMeshCustom(noiseVol, Region(32, 32, 32, 63, 63, 63), &noiseMesh); } + QCOMPARE(noiseMesh.getNoOfVertices(), uint16_t(48967)); +} + +QTEST_MAIN(TestSurfaceExtractor)