Merge branch 'feature/paging' into develop

CHANGELOG.txt

@@ -10,9 +10,13 @@ The getVoxelAt() and setVoxelAt() functions have been deprecated and replaced by
 
 LargeVolume
 -----------
+Behaviour and interface of the LargeVolume has changed significantly and code which uses it will certainly need to be updated. Plese see the LargeVolume API docs for full details of how it now works.
+
 It is now possible to provide custom compressors for the data which is stored in a LargeVolume. Two compressor implementation are provided with PolyVox - RLECompressor which is suitable for cubic-style terrain, and MinizCompressor which uses the 'miniz' zlib implementation for smooth terrain or general purpose use. Users can provide their own implementation of the compressor interface if they wish.
 
-Note that the setCompressionEnabled() functionality has been removed and a compressor must always be provided when constructing the volume. The ability to disable compression was of questionable benefit and was complicating the logic in the code. In practice it is still possible to mostly disable compression by setting the maximum number of uncompressed blocks to a high value by calling setMaxNumberOfUncompressedBlocks().
+Note that the setCompressionEnabled() functionality has been removed and a compressor must always be provided when constructing the volume. The ability to disable compression was of questionable benefit and was complicating the logic in the code.
+
+The LargeVolume also supports custom paging code which can be supplied by providing a subclass of Pager and implementing the relevant methods. This replaces the dataRequiredHandler() and dataOverflowHandler() functions. 
 
 These changes regarding compression and paging have also affected the LargeVolume constructors(s). Please see the API docs to see how they look now.
 

examples/Basic/main.cpp

@@ -92,4 +92,4 @@ int main(int argc, char *argv[])
 
 	//Run the message pump.
 	return app.exec();
-}
+}

examples/OpenGL/main.cpp

@@ -25,6 +25,7 @@ freely, subject to the following restrictions:
 #include "PolyVoxCore/LargeVolume.h"
 #include "PolyVoxCore/LowPassFilter.h"
 #include "PolyVoxCore/RawVolume.h"
+#include "PolyVoxCore/RLECompressor.h"
 #include "PolyVoxCore/SurfaceMesh.h"
 #include "PolyVoxCore/Impl/Utility.h"
 
@@ -48,7 +49,8 @@ using namespace std;
 
 int main(int argc, char *argv[])
 {
-	LargeVolume<MaterialDensityPair44> volData(PolyVox::Region(Vector3DInt32(0,0,0), Vector3DInt32(g_uVolumeSideLength-1, g_uVolumeSideLength-1, g_uVolumeSideLength-1)));
+	RLECompressor<MaterialDensityPair44, uint16_t>* compressor = new RLECompressor<MaterialDensityPair44, uint16_t>();
+	LargeVolume<MaterialDensityPair44> volData(PolyVox::Region(Vector3DInt32(0,0,0), Vector3DInt32(g_uVolumeSideLength-1, g_uVolumeSideLength-1, g_uVolumeSideLength-1)), compressor, 0);
 
 	//Make our volume contain a sphere in the center.
 	int32_t minPos = 0;

examples/Paging/main.cpp

@@ -27,6 +27,8 @@ freely, subject to the following restrictions:
 #include "PolyVoxCore/MaterialDensityPair.h"
 #include "PolyVoxCore/CubicSurfaceExtractorWithNormals.h"
 #include "PolyVoxCore/MarchingCubesSurfaceExtractor.h"
+#include "PolyVoxCore/Pager.h"
+#include "PolyVoxCore/RLECompressor.h"
 #include "PolyVoxCore/SurfaceMesh.h"
 #include "PolyVoxCore/LargeVolume.h"
 
@@ -35,129 +37,6 @@ freely, subject to the following restrictions:
 //Use the PolyVox namespace
 using namespace PolyVox;
 
-void createPerlinVolumeSlow(LargeVolume<MaterialDensityPair44>& volData)
-{
-	Perlin perlin(2,8,1,234);
-
-	for(int z = 1; z < 256-1; z++)
-	{
-		std::cout << z << std::endl;
-		for(int y = 1; y < 256-1; y++)
-		{
-			for(int x = 1; x < 256-1; x++)
-			{							
-				float perlinVal = perlin.Get3D(x /static_cast<float>(256-1), (y) / static_cast<float>(256-1), z / static_cast<float>(256-1));
-
-				perlinVal += 1.0f;
-				perlinVal *= 0.5f;
-				perlinVal *= MaterialDensityPair44::getMaxDensity();
-
-				MaterialDensityPair44 voxel;
-
-				voxel.setMaterial(245);
-				voxel.setDensity(perlinVal);
-
-				/*if(perlinVal < 0.0f)
-				{
-					voxel.setMaterial(245);
-					voxel.setDensity(MaterialDensityPair44::getMaxDensity());
-				}
-				else
-				{
-					voxel.setMaterial(0);
-					voxel.setDensity(MaterialDensityPair44::getMinDensity());
-				}*/
-
-				volData.setVoxelAt(x, y, z, voxel);
-			}
-		}
-	}
-}
-
-/*void createPerlinVolumeFast(LargeVolume<MaterialDensityPair44>& volData)
-{
-	Perlin perlin(2,8,1,234);
-
-	for(int blockZ = 0; blockZ < volData.m_uDepthInBlocks; blockZ++)
-	{		
-		std::cout << blockZ << std::endl;
-		for(int blockY = 0; blockY < volData.m_uHeightInBlocks; blockY++)
-		{
-			for(int blockX = 0; blockX < volData.m_uWidthInBlocks; blockX++)
-			{
-				for(int offsetz = 0; offsetz < volData.m_uBlockSideLength; offsetz++)
-				{
-					for(int offsety = 0; offsety < volData.m_uBlockSideLength; offsety++)
-					{
-						for(int offsetx = 0; offsetx < volData.m_uBlockSideLength; offsetx++) 
-						{							
-							int x = blockX * volData.m_uBlockSideLength + offsetx;
-							int y = blockY * volData.m_uBlockSideLength + offsety;
-							int z = blockZ * volData.m_uBlockSideLength + offsetz;
-
-							if((x == 0) || (x == volData.getWidth()-1)) continue;
-							if((y == 0) || (y == volData.getHeight()-1)) continue;
-							if((z == 0) || (z == volData.getDepth()-1)) continue;
-
-							float perlinVal = perlin.Get3D(x /static_cast<float>(volData.getWidth()-1), (y) / static_cast<float>(volData.getHeight()-1), z / static_cast<float>(volData.getDepth()-1));
-
-							MaterialDensityPair44 voxel;
-							if(perlinVal < 0.0f)
-							{
-								voxel.setMaterial(245);
-								voxel.setDensity(MaterialDensityPair44::getMaxDensity());
-							}
-							else
-							{
-								voxel.setMaterial(0);
-								voxel.setDensity(MaterialDensityPair44::getMinDensity());
-							}
-
-							volData.setVoxelAt(x, y, z, voxel);
-						}
-					}
-				}
-			}
-		}			
-	}
-}*/
-
-void createPerlinTerrain(LargeVolume<MaterialDensityPair44>& volData)
-{
-	Perlin perlin(2,2,1,234);
-
-	for(int x = 1; x < 255-1; x++)
-	{
-		if(x%(255/100) == 0) {
-			std::cout << "." << std::flush;
-		}
-		for(int y = 1; y < 255-1; y++)
-		{
-			float perlinVal = perlin.Get(x / static_cast<float>(255-1), y / static_cast<float>(255-1));
-			perlinVal += 1.0f;
-			perlinVal *= 0.5f;
-			perlinVal *= 255;
-			for(int z = 1; z < 255-1; z++)
-			{							
-				MaterialDensityPair44 voxel;
-				if(z < perlinVal)
-				{
-					voxel.setMaterial(245);
-					voxel.setDensity(MaterialDensityPair44::getMaxDensity());
-				}
-				else
-				{
-					voxel.setMaterial(0);
-					voxel.setDensity(MaterialDensityPair44::getMinDensity());
-				}
-
-				volData.setVoxelAt(x, y, z, voxel);
-			}
-		}
-	}
-	std::cout << std::endl;
-}
-
 void createSphereInVolume(LargeVolume<MaterialDensityPair44>& volData, Vector3DFloat v3dVolCenter, float fRadius)
 {
 	//This vector hold the position of the center of the volume
@@ -197,51 +76,72 @@ void createSphereInVolume(LargeVolume<MaterialDensityPair44>& volData, Vector3DF
 	}
 }
 
-void load(const ConstVolumeProxy<MaterialDensityPair44>& volume, const PolyVox::Region& reg)
+/**
+ * Generates data using Perlin noise.
+ */
+class PerlinNoisePager : public PolyVox::Pager<MaterialDensityPair44>
 {
-	Perlin perlin(2,2,1,234);
+public:
-
+	/// Constructor
-	for(int x = reg.getLowerX(); x <= reg.getUpperX(); x++)
+	PerlinNoisePager()
+		:Pager()
 	{
-		for(int y = reg.getLowerY(); y <= reg.getUpperY(); y++)
+	}
+
+	/// Destructor
+	virtual ~PerlinNoisePager() {};
+
+	virtual void pageIn(const Region& region, Block<MaterialDensityPair44>* pBlockData)
+	{
+		pBlockData->createUncompressedData();
+
+		Perlin perlin(2,2,1,234);
+
+		for(int x = region.getLowerX(); x <= region.getUpperX(); x++)
 		{
-			float perlinVal = perlin.Get(x / static_cast<float>(255-1), y / static_cast<float>(255-1));
+			for(int y = region.getLowerY(); y <= region.getUpperY(); y++)
-			perlinVal += 1.0f;
-			perlinVal *= 0.5f;
-			perlinVal *= 255;
-			for(int z = reg.getLowerZ(); z <= reg.getUpperZ(); z++)
 			{
-				MaterialDensityPair44 voxel;
+				float perlinVal = perlin.Get(x / static_cast<float>(255-1), y / static_cast<float>(255-1));
-				if(z < perlinVal)
+				perlinVal += 1.0f;
+				perlinVal *= 0.5f;
+				perlinVal *= 255;
+				for(int z = region.getLowerZ(); z <= region.getUpperZ(); z++)
 				{
-					const int xpos = 50;
+					MaterialDensityPair44 voxel;
-					const int zpos = 100;
+					if(z < perlinVal)
-					if((x-xpos)*(x-xpos) + (z-zpos)*(z-zpos) < 200) {
+					{
-						// tunnel
+						const int xpos = 50;
+						const int zpos = 100;
+						if((x-xpos)*(x-xpos) + (z-zpos)*(z-zpos) < 200) {
+							// tunnel
+							voxel.setMaterial(0);
+							voxel.setDensity(MaterialDensityPair44::getMinDensity());
+						} else {
+							// solid
+							voxel.setMaterial(245);
+							voxel.setDensity(MaterialDensityPair44::getMaxDensity());
+						}
+					}
+					else
+					{
 						voxel.setMaterial(0);
 						voxel.setDensity(MaterialDensityPair44::getMinDensity());
-					} else {
-						// solid
-						voxel.setMaterial(245);
-						voxel.setDensity(MaterialDensityPair44::getMaxDensity());
 					}
-				}
-				else
-				{
-					voxel.setMaterial(0);
-					voxel.setDensity(MaterialDensityPair44::getMinDensity());
-				}
 
-				volume.setVoxelAt(x, y, z, voxel);
+					// Voxel position within a block always start from zero. So if a block represents region (4, 8, 12) to (11, 19, 15)
+					// then the valid block voxels are from (0, 0, 0) to (7, 11, 3). Hence we subtract the lower corner position of the
+					// region from the volume space position in order to get the block space position.
+					pBlockData->setVoxelAt(x - region.getLowerX(), y - region.getLowerY(), z - region.getLowerZ(), voxel);
+				}
 			}
 		}
 	}
-}
 
-void unload(const ConstVolumeProxy<MaterialDensityPair44>& /*vol*/, const PolyVox::Region& reg)
+	virtual void pageOut(const Region& region, Block<MaterialDensityPair44>* pBlockData)
-{
+	{
-	std::cout << "warning unloading region: " << reg.getLowerCorner() << " -> " << reg.getUpperCorner() << std::endl;
+		std::cout << "warning unloading region: " << region.getLowerCorner() << " -> " << region.getUpperCorner() << std::endl;
-}
+	}
+};
 
 int main(int argc, char *argv[])
 {
@@ -250,18 +150,12 @@ int main(int argc, char *argv[])
 	OpenGLWidget openGLWidget(0);
 	openGLWidget.show();
 
-	//If these lines don't compile, please try commenting them out and using the two lines after
+	RLECompressor<MaterialDensityPair44, uint16_t>* compressor = new RLECompressor<MaterialDensityPair44, uint16_t>();
-	//(you will need Boost for this). If you have to do this then please let us know in the forums as
+	PerlinNoisePager* pager = new PerlinNoisePager();
-	//we rely on community feedback to keep the Boost version running.
+	LargeVolume<MaterialDensityPair44> volData(Region::MaxRegion, compressor, pager, 256);
-	LargeVolume<MaterialDensityPair44> volData(&load, &unload, 256);
-	//LargeVolume<MaterialDensityPair44> volData(polyvox_bind(&load, polyvox_placeholder_1, polyvox_placeholder_2),
-	//	polyvox_bind(&unload, polyvox_placeholder_1, polyvox_placeholder_2), 256);
 	volData.setMaxNumberOfBlocksInMemory(4096);
 	volData.setMaxNumberOfUncompressedBlocks(64);
 
-	//volData.dataRequiredHandler = &load;
-	//volData.dataOverflowHandler = &unload;
-
 	//volData.setMaxNumberOfUncompressedBlocks(4096);
 	//createSphereInVolume(volData, 30);
 	//createPerlinTerrain(volData);

library/PolyVoxCore/CMakeLists.txt

@@ -49,7 +49,6 @@ SET(CORE_INC_FILES
 	include/PolyVoxCore/BaseVolume.inl
 	include/PolyVoxCore/BaseVolumeSampler.inl
 	include/PolyVoxCore/Compressor.h
-	include/PolyVoxCore/ConstVolumeProxy.h
 	include/PolyVoxCore/CubicSurfaceExtractor.h
 	include/PolyVoxCore/CubicSurfaceExtractor.inl
 	include/PolyVoxCore/CubicSurfaceExtractorWithNormals.h
@@ -57,6 +56,7 @@ SET(CORE_INC_FILES
 	include/PolyVoxCore/DefaultIsQuadNeeded.h
 	include/PolyVoxCore/DefaultMarchingCubesController.h
 	include/PolyVoxCore/Density.h
+	include/PolyVoxCore/FilePager.h
 	include/PolyVoxCore/GradientEstimators.h
 	include/PolyVoxCore/GradientEstimators.inl
 	include/PolyVoxCore/Interpolation.h
@@ -72,6 +72,7 @@ SET(CORE_INC_FILES
 	include/PolyVoxCore/Material.h
 	include/PolyVoxCore/MaterialDensityPair.h
 	include/PolyVoxCore/MinizCompressor.h
+	include/PolyVoxCore/Pager.h
 	include/PolyVoxCore/PolyVoxForwardDeclarations.h
 	include/PolyVoxCore/Picking.h
 	include/PolyVoxCore/Picking.inl

library/PolyVoxCore/include/PolyVoxCore/Array.inl

@@ -73,10 +73,7 @@ namespace PolyVox
 	template <uint32_t noOfDims, typename ElementType>
 	SubArray<noOfDims-1, ElementType> Array<noOfDims, ElementType>::operator[](uint32_t uIndex)
 	{
-		if(uIndex >= m_pDimensions[0])
+		POLYVOX_THROW_IF(uIndex >= m_pDimensions[0], std::out_of_range, "Array index out of range");
-		{
-			POLYVOX_THROW(std::out_of_range, "Array index out of range");
-		}
 
 		return
 			SubArray<noOfDims-1, ElementType>(&m_pElements[uIndex*m_pOffsets[0]],
@@ -95,10 +92,7 @@ namespace PolyVox
 	template <uint32_t noOfDims, typename ElementType>
 	const SubArray<noOfDims-1, ElementType> Array<noOfDims, ElementType>::operator[](uint32_t uIndex) const
 	{
-		if(uIndex >= m_pDimensions[0])
+		POLYVOX_THROW_IF(uIndex >= m_pDimensions[0], std::out_of_range, "Array index out of range");
-		{
-			POLYVOX_THROW(std::out_of_range, "Array index out of range");
-		}
 
 		return
 			SubArray<noOfDims-1, ElementType>(&m_pElements[uIndex*m_pOffsets[0]],
@@ -147,10 +141,7 @@ namespace PolyVox
 		m_uNoOfElements = 1;
 		for (uint32_t i = 0; i<noOfDims; i++)
 		{
-			if(pDimensions[i] == 0)
+			POLYVOX_THROW_IF(pDimensions[i] == 0, std::out_of_range, "Invalid array dimension");
-			{
-				POLYVOX_THROW(std::out_of_range, "Invalid array dimension");
-			}
 
 			m_uNoOfElements *= pDimensions[i];
 			m_pDimensions[i] = pDimensions[i];
@@ -197,10 +188,7 @@ namespace PolyVox
 	template <uint32_t noOfDims, typename ElementType>
 	uint32_t Array<noOfDims, ElementType>::getDimension(uint32_t uDimension)
 	{
-		if(uDimension >= noOfDims)
+		POLYVOX_THROW_IF(uDimension >= noOfDims, std::out_of_range, "Array dimension out of range");
-		{
-			POLYVOX_THROW(std::out_of_range, "Array dimension out of range");
-		}
 
 		return m_pDimensions[uDimension];
 	}
@@ -266,10 +254,7 @@ namespace PolyVox
 	template <typename ElementType>
 	ElementType& Array<1, ElementType>::operator[] (uint32_t uIndex)
 	{
-		if(uIndex >= m_pDimensions[0])
+		POLYVOX_THROW_IF(uIndex >= m_pDimensions[0], std::out_of_range, "Array index out of range");
-		{
-			POLYVOX_THROW(std::out_of_range, "Array index out of range");
-		}
 
 		return m_pElements[uIndex];
 	}
@@ -277,10 +262,7 @@ namespace PolyVox
 	template <typename ElementType>
 	const ElementType& Array<1, ElementType>::operator[] (uint32_t uIndex) const
 	{
-		if(uIndex >= m_pDimensions[0])
+		POLYVOX_THROW_IF(uIndex >= m_pDimensions[0], std::out_of_range, "Array index out of range");
-		{
-			POLYVOX_THROW(std::out_of_range, "Array index out of range");
-		}
 
 		return m_pElements[uIndex];
 	}

library/PolyVoxCore/include/PolyVoxCore/ConstVolumeProxy.h

@@ -1,83 +0,0 @@
-/*******************************************************************************
-Copyright (c) 2005-2009 David Williams
-
-This software is provided 'as-is', without any express or implied
-warranty. In no event will the authors be held liable for any damages
-arising from the use of this software.
-
-Permission is granted to anyone to use this software for any purpose,
-including commercial applications, and to alter it and redistribute it
-freely, subject to the following restrictions:
-
-    1. The origin of this software must not be misrepresented; you must not
-    claim that you wrote the original software. If you use this software
-    in a product, an acknowledgment in the product documentation would be
-    appreciated but is not required.
-
-    2. Altered source versions must be plainly marked as such, and must not be
-    misrepresented as being the original software.
-
-    3. This notice may not be removed or altered from any source
-    distribution. 	
-*******************************************************************************/
-
-#ifndef __PolyVox_ConstVolumeProxy_H__
-#define __PolyVox_ConstVolumeProxy_H__
-
-#include "PolyVoxCore/Region.h"
-#include "PolyVoxCore/Vector.h"
-
-namespace PolyVox
-{
-	template <typename VoxelType>
-	class ConstVolumeProxy
-	{
-		//LargeVolume is a friend so it can call the constructor.
-		friend class LargeVolume<VoxelType>;
-	public:
-		VoxelType getVoxelAt(int32_t uXPos, int32_t uYPos, int32_t uZPos) const
-		{
-			// PolyVox does not throw an exception when a voxel is out of range. Please see 'Error Handling' in the User Manual.
-			POLYVOX_ASSERT(m_regValid.containsPoint(Vector3DInt32(uXPos, uYPos, uZPos)), "Position is outside valid region");
-			return m_pVolume.getVoxel<WrapModes::AssumeValid>(uXPos, uYPos, uZPos);
-		}
-
-		VoxelType getVoxelAt(const Vector3DInt32& v3dPos) const
-		{
-			// PolyVox does not throw an exception when a voxel is out of range. Please see 'Error Handling' in the User Manual.
-			POLYVOX_ASSERT(m_regValid.containsPoint(v3dPos), "Position is outside valid region");
-			return getVoxelAt<WrapModes::AssumeValid>(v3dPos.getX(), v3dPos.getY(), v3dPos.getZ());
-		}
-
-		void setVoxelAt(int32_t uXPos, int32_t uYPos, int32_t uZPos, VoxelType tValue) const
-		{
-			// PolyVox does not throw an exception when a voxel is out of range. Please see 'Error Handling' in the User Manual.
-			POLYVOX_ASSERT(m_regValid.containsPoint(Vector3DInt32(uXPos, uYPos, uZPos)), "Position is outside valid region");
-			m_pVolume.setVoxelAtConst(uXPos, uYPos, uZPos, tValue);
-		}
-
-		void setVoxelAt(const Vector3DInt32& v3dPos, VoxelType tValue) const
-		{
-			// PolyVox does not throw an exception when a voxel is out of range. Please see 'Error Handling' in the User Manual.
-			POLYVOX_ASSERT(m_regValid.containsPoint(v3dPos), "Position is outside valid region");
-			setVoxelAt(v3dPos.getX(), v3dPos.getY(), v3dPos.getZ(), tValue);
-		}
-	private:
-		//Private constructor, so client code can't abuse this class.
-		ConstVolumeProxy(const LargeVolume<VoxelType>& pVolume, const Region& regValid)
-			:m_pVolume(pVolume)
-			,m_regValid(regValid)
-		{
-		}
-
-		//Private assignment operator, so client code can't abuse this class.
-		ConstVolumeProxy& operator=(const ConstVolumeProxy& rhs)
-		{
-		}
-
-		const LargeVolume<VoxelType>& m_pVolume;
-		const Region& m_regValid;
-	};
-}
-
-#endif //__PolyVox_ConstVolumeProxy_H__

library/PolyVoxCore/include/PolyVoxCore/FilePager.h

@@ -0,0 +1,132 @@
+/*******************************************************************************
+Copyright (c) 2005-2009 David Williams
+
+This software is provided 'as-is', without any express or implied
+warranty. In no event will the authors be held liable for any damages
+arising from the use of this software.
+
+Permission is granted to anyone to use this software for any purpose,
+including commercial applications, and to alter it and redistribute it
+freely, subject to the following restrictions:
+
+    1. The origin of this software must not be misrepresented; you must not
+    claim that you wrote the original software. If you use this software
+    in a product, an acknowledgment in the product documentation would be
+    appreciated but is not required.
+
+    2. Altered source versions must be plainly marked as such, and must not be
+    misrepresented as being the original software.
+
+    3. This notice may not be removed or altered from any source
+    distribution. 	
+*******************************************************************************/
+
+#ifndef __PolyVox_FilePager_H__
+#define __PolyVox_FilePager_H__
+
+#include "PolyVoxCore/Impl/TypeDef.h"
+
+#include "PolyVoxCore/Pager.h"
+
+#include <fstream>
+#include <stdexcept>
+#include <string>
+
+namespace PolyVox
+{
+	/**
+	 * Provides an interface for performing paging of data.
+	 */
+	template <typename VoxelType>
+	class FilePager : public Pager<typename VoxelType>
+	{
+	public:
+		/// Constructor
+		FilePager(const std::string& strFolderName)
+			:Pager()
+			,m_strFolderName(strFolderName)
+		{
+		}
+
+		/// Destructor
+		virtual ~FilePager() {};
+
+		virtual void pageIn(const Region& region, Block<VoxelType>* pBlockData)
+		{
+			POLYVOX_ASSERT(pBlockData, "Attempting to page in NULL block");
+			POLYVOX_ASSERT(pBlockData->hasUncompressedData() == false, "Block should not have uncompressed data");
+
+			std::stringstream ss;
+			ss << region.getLowerX() << "_" << region.getLowerY() << "_" << region.getLowerZ() << "_"
+				 << region.getUpperX() << "_" << region.getUpperY() << "_" << region.getUpperZ();
+
+			std::string filename = m_strFolderName + ss.str();
+
+			// FIXME - This should be replaced by C++ style IO, but currently this causes problems with
+			// the gameplay-cubiquity integration. See: https://github.com/blackberry/GamePlay/issues/919
+
+			FILE* pFile = fopen(filename.c_str(), "rb");
+			if(pFile)
+			{
+				logTrace() << "Paging in data for " << region;
+
+				fseek(pFile, 0L, SEEK_END);
+				size_t fileSizeInBytes = ftell(pFile);
+				fseek(pFile, 0L, SEEK_SET);
+				
+				uint8_t* buffer = new uint8_t[fileSizeInBytes];
+				fread(buffer, sizeof(uint8_t), fileSizeInBytes, pFile);
+				pBlockData->setCompressedData(buffer, fileSizeInBytes);
+				delete[] buffer;
+
+				if(ferror(pFile))
+				{
+					POLYVOX_THROW(std::runtime_error, "Error reading in block data, even though a file exists.");
+				}
+
+				fclose(pFile);
+			}
+			else
+			{
+				logTrace() << "No data found for " << region << " during paging in.";
+			}
+		}
+
+		virtual void pageOut(const Region& region, Block<VoxelType>* pBlockData)
+		{
+			POLYVOX_ASSERT(pBlockData, "Attempting to page out NULL block");
+			POLYVOX_ASSERT(pBlockData->hasUncompressedData() == false, "Block should not have uncompressed data");
+
+			logTrace() << "Paging out data for " << region;
+
+			std::stringstream ss;
+			ss << region.getLowerX() << "_" << region.getLowerY() << "_" << region.getLowerZ() << "_"
+				 << region.getUpperX() << "_" << region.getUpperY() << "_" << region.getUpperZ();
+
+			std::string filename = m_strFolderName + ss.str();
+
+			// FIXME - This should be replaced by C++ style IO, but currently this causes problems with
+			// the gameplay-cubiquity integration. See: https://github.com/blackberry/GamePlay/issues/919
+
+			FILE* pFile = fopen(filename.c_str(), "wb");
+			if(!pFile)
+			{
+				POLYVOX_THROW(std::runtime_error, "Unable to open file to write out block data.");
+			}
+
+			fwrite(pBlockData->getCompressedData(), sizeof(uint8_t), pBlockData->getCompressedDataLength(), pFile);
+
+			if(ferror(pFile))
+			{
+				POLYVOX_THROW(std::runtime_error, "Error writing out block data.");
+			}
+
+			fclose(pFile);
+		}
+
+	protected:
+		std::string m_strFolderName;
+	};
+}
+
+#endif //__PolyVox_FilePager_H__

library/PolyVoxCore/include/PolyVoxCore/Impl/Block.h

@@ -37,41 +37,36 @@ namespace PolyVox
 	template <typename VoxelType>
 	class Block
 	{
-		template <typename LengthType>
-		struct RunlengthEntry
-		{
-			LengthType length;
-			VoxelType value;
-
-			//We can parametise the length on anything up to uint32_t.
-			//This lets us experiment with the optimal size in the future.
-			static uint32_t maxRunlength(void) {return (std::numeric_limits<LengthType>::max)();}
-		};
-
 	public:
-		Block(uint16_t uSideLength = 0);
+		Block(uint16_t uSideLength, Compressor* pCompressor);
 
+		const uint8_t* const getCompressedData(void) const;
+		const uint32_t getCompressedDataLength(void) const;
 		uint16_t getSideLength(void) const;
 		VoxelType getVoxel(uint16_t uXPos, uint16_t uYPos, uint16_t uZPos) const;
 		VoxelType getVoxel(const Vector3DUint16& v3dPos) const;
 
+		bool hasUncompressedData(void) const;
+
+		void setCompressedData(const uint8_t* const data, uint32_t dataLength);
 		void setVoxelAt(uint16_t uXPos, uint16_t uYPos, uint16_t uZPos, VoxelType tValue);
 		void setVoxelAt(const Vector3DUint16& v3dPos, VoxelType tValue);
 
-		void initialise(uint16_t uSideLength);
+		void createUncompressedData(void);
+		void destroyUncompressedData(void);
+
 		uint32_t calculateSizeInBytes(void);
 
 	public:
-		void compress(Compressor* pCompressor);
+		Compressor* m_pCompressor;
-		void uncompress(Compressor* pCompressor);
-
 		uint8_t* m_pCompressedData;
 		uint32_t m_uCompressedDataLength;
 		VoxelType* m_tUncompressedData;
 		uint16_t m_uSideLength;
 		uint8_t m_uSideLengthPower;	
-		bool m_bIsCompressed;
 		bool m_bIsUncompressedDataModified;
+
+		uint32_t timestamp;
 	};
 }
 

library/PolyVoxCore/include/PolyVoxCore/Impl/Block.inl

@@ -36,19 +36,56 @@ freely, subject to the following restrictions:
 namespace PolyVox
 {
 	template <typename VoxelType>
-	Block<VoxelType>::Block(uint16_t uSideLength)
+	Block<VoxelType>::Block(uint16_t uSideLength, Compressor* pCompressor)
-		:m_pCompressedData(0)
+		:m_pCompressor(pCompressor)
+		,m_pCompressedData(0)
 		,m_uCompressedDataLength(0)
 		,m_tUncompressedData(0)
 		,m_uSideLength(0)
 		,m_uSideLengthPower(0)
-		,m_bIsCompressed(false)
 		,m_bIsUncompressedDataModified(true)
 	{
-		if(uSideLength != 0)
+		if(uSideLength == 0)
 		{
-			initialise(uSideLength);
+			POLYVOX_THROW(std::invalid_argument, "Block side length cannot be zero.");
 		}
+
+		if(!isPowerOf2(uSideLength))
+		{
+			POLYVOX_THROW(std::invalid_argument, "Block side length must be a power of two.");
+		}
+
+		if(pCompressor == 0)
+		{
+			POLYVOX_THROW(std::invalid_argument, "Block must be provided with a valid compressor.");
+		}
+
+		//Compute the side length		
+		m_uSideLength = uSideLength;
+		m_uSideLengthPower = logBase2(uSideLength);
+
+		//Temporarily create the block data. This is just so we can compress it an discard it.
+		// FIXME - this is a temporary solution.
+		const uint32_t uNoOfVoxels = m_uSideLength * m_uSideLength * m_uSideLength;
+		m_tUncompressedData = new VoxelType[uNoOfVoxels];		
+		std::fill(m_tUncompressedData, m_tUncompressedData + uNoOfVoxels, VoxelType());
+		m_bIsUncompressedDataModified = true;
+
+		destroyUncompressedData();
+	}
+
+	template <typename VoxelType>
+	const uint8_t* const Block<VoxelType>::getCompressedData(void) const
+	{
+		POLYVOX_ASSERT(m_pCompressedData, "Compressed data is NULL");
+		return m_pCompressedData;
+	}
+	
+	template <typename VoxelType>
+	const uint32_t Block<VoxelType>::getCompressedDataLength(void) const
+	{
+		POLYVOX_ASSERT(m_pCompressedData, "Compressed data is NULL");
+		return m_uCompressedDataLength;
 	}
 
 	template <typename VoxelType>
@@ -64,6 +101,7 @@ namespace PolyVox
 		POLYVOX_ASSERT(uXPos < m_uSideLength, "Supplied position is outside of the block");
 		POLYVOX_ASSERT(uYPos < m_uSideLength, "Supplied position is outside of the block");
 		POLYVOX_ASSERT(uZPos < m_uSideLength, "Supplied position is outside of the block");
+		POLYVOX_ASSERT(hasUncompressedData(), "The block must have uncompressed data to call getVoxel()");
 		POLYVOX_ASSERT(m_tUncompressedData, "No uncompressed data - block must be decompressed before accessing voxels.");
 
 		return m_tUncompressedData
@@ -80,6 +118,25 @@ namespace PolyVox
 		return getVoxel(v3dPos.getX(), v3dPos.getY(), v3dPos.getZ());
 	}
 
+	template <typename VoxelType>
+	bool Block<VoxelType>::hasUncompressedData(void) const
+	{
+		return m_tUncompressedData != 0;
+	}
+
+	template <typename VoxelType>
+	void Block<VoxelType>::setCompressedData(const uint8_t* const data, uint32_t dataLength)
+	{
+		POLYVOX_ASSERT(m_pCompressedData, "Compressed data is NULL");
+		POLYVOX_ASSERT(m_pCompressedData != data, "Attempting to copy data onto itself");
+
+		delete[] m_pCompressedData;
+
+		m_uCompressedDataLength = dataLength;
+		m_pCompressedData = new uint8_t[dataLength];
+		memcpy(m_pCompressedData, data, dataLength);
+	}
+
 	template <typename VoxelType>
 	void Block<VoxelType>::setVoxelAt(uint16_t uXPos, uint16_t uYPos, uint16_t uZPos, VoxelType tValue)
 	{
@@ -87,6 +144,7 @@ namespace PolyVox
 		POLYVOX_ASSERT(uXPos < m_uSideLength, "Supplied position is outside of the block");
 		POLYVOX_ASSERT(uYPos < m_uSideLength, "Supplied position is outside of the block");
 		POLYVOX_ASSERT(uZPos < m_uSideLength, "Supplied position is outside of the block");
+		POLYVOX_ASSERT(hasUncompressedData(), "The block must have uncompressed data to call setVoxelAt()");
 		POLYVOX_ASSERT(m_tUncompressedData, "No uncompressed data - block must be decompressed before accessing voxels.");
 
 		m_tUncompressedData
@@ -106,46 +164,11 @@ namespace PolyVox
 	}
 
 	template <typename VoxelType>
-	void Block<VoxelType>::initialise(uint16_t uSideLength)
+	void Block<VoxelType>::destroyUncompressedData()
 	{
-		//Release mode validation
+		if(!hasUncompressedData())
-		if(!isPowerOf2(uSideLength))
-		{
-			POLYVOX_THROW(std::invalid_argument, "Block side length must be a power of two.");
-		}
-
-		//Compute the side length		
-		m_uSideLength = uSideLength;
-		m_uSideLengthPower = logBase2(uSideLength);
-
-		//Create the block data
-		m_tUncompressedData = new VoxelType[m_uSideLength * m_uSideLength * m_uSideLength];
-
-		//Clear it (should we bother?)
-		const uint32_t uNoOfVoxels = m_uSideLength * m_uSideLength * m_uSideLength;
-		std::fill(m_tUncompressedData, m_tUncompressedData + uNoOfVoxels, VoxelType());
-		m_bIsUncompressedDataModified = true;
-	}
-
-	template <typename VoxelType>
-	uint32_t Block<VoxelType>::calculateSizeInBytes(void)
-	{
-		//FIXME - This function is incomplete.
-		uint32_t uSizeInBytes = sizeof(Block<VoxelType>);
-		return  uSizeInBytes;
-	}
-
-	template <typename VoxelType>
-	void Block<VoxelType>::compress(Compressor* pCompressor)
-	{
-		if(m_bIsCompressed)
 		{ 
-			POLYVOX_THROW(invalid_operation, "Attempted to compress block which is already flagged as compressed.");
+			POLYVOX_THROW(invalid_operation, "No uncompressed data to compress.");
-		}
-
-		if(!pCompressor)
-		{
-			POLYVOX_THROW(std::invalid_argument, "A valid compressor must be provided");
 		}
 
 		POLYVOX_ASSERT(m_tUncompressedData != 0, "No uncompressed data is present.");
@@ -169,7 +192,7 @@ namespace PolyVox
 
 			try
 			{
-				uCompressedLength = pCompressor->compress(pSrcData, uSrcLength, pDstData, uDstLength);
+				uCompressedLength = m_pCompressor->compress(pSrcData, uSrcLength, pDstData, uDstLength);
 
 				// Create new compressed data and copy across
 				m_pCompressedData = new uint8_t[uCompressedLength];
@@ -181,7 +204,9 @@ namespace PolyVox
 				// It is possible for the compression to fail. A common cause for this would be if the destination
 				// buffer is not big enough. So now we try again using a buffer that is definitely big enough.
 				// Note that ideally we will choose our earlier buffer size so that this almost never happens.
-				uint32_t uMaxCompressedSize = pCompressor->getMaxCompressedSize(uSrcLength);
+				logWarning() << "The compressor failed to compress the block, proabaly due to the buffer being too small.";
+				logWarning() << "The compression will be tried again with a larger buffer";
+				uint32_t uMaxCompressedSize = m_pCompressor->getMaxCompressedSize(uSrcLength);
 				uint8_t* buffer = new uint8_t[ uMaxCompressedSize ];
 
 				pDstData = reinterpret_cast<void*>( buffer );				
@@ -189,7 +214,7 @@ namespace PolyVox
 
 				try
 				{		
-					uCompressedLength = pCompressor->compress(pSrcData, uSrcLength, pDstData, uDstLength);
+					uCompressedLength = m_pCompressor->compress(pSrcData, uSrcLength, pDstData, uDstLength);
 
 					// Create new compressed data and copy across
 					m_pCompressedData = new uint8_t[uCompressedLength];
@@ -211,20 +236,14 @@ namespace PolyVox
 		//Flag the uncompressed data as no longer being used.
 		delete[] m_tUncompressedData;
 		m_tUncompressedData = 0;
-		m_bIsCompressed = true;
 	}
 
 	template <typename VoxelType>
-	void Block<VoxelType>::uncompress(Compressor* pCompressor)
+	void Block<VoxelType>::createUncompressedData()
 	{
-		if(!m_bIsCompressed)
+		if(hasUncompressedData())
 		{
-			POLYVOX_THROW(invalid_operation, "Attempted to uncompress block which is not flagged as compressed.");
+			POLYVOX_THROW(invalid_operation, "Uncompressed data already exists.");
-		}
-
-		if(!pCompressor)
-		{
-			POLYVOX_THROW(std::invalid_argument, "A valid compressor must be provided");
 		}
 
 		POLYVOX_ASSERT(m_tUncompressedData == 0, "Uncompressed data already exists.");
@@ -238,13 +257,20 @@ namespace PolyVox
 
 		//MinizCompressor compressor;
 		//RLECompressor<VoxelType, uint16_t> compressor;
-		uint32_t uUncompressedLength = pCompressor->decompress(pSrcData, uSrcLength, pDstData, uDstLength);
+		uint32_t uUncompressedLength = m_pCompressor->decompress(pSrcData, uSrcLength, pDstData, uDstLength);
 
 		POLYVOX_ASSERT(uUncompressedLength == m_uSideLength * m_uSideLength * m_uSideLength * sizeof(VoxelType), "Destination length has changed.");
 
 		//m_tUncompressedData = reinterpret_cast<VoxelType*>(uncompressedResult.ptr);
 
-		m_bIsCompressed = false;
 		m_bIsUncompressedDataModified = false;
 	}
+
+	template <typename VoxelType>
+	uint32_t Block<VoxelType>::calculateSizeInBytes(void)
+	{
+		//FIXME - This function is incomplete.
+		uint32_t uSizeInBytes = sizeof(Block<VoxelType>);
+		return  uSizeInBytes;
+	}
 }

library/PolyVoxCore/include/PolyVoxCore/Impl/ErrorHandling.h

@@ -255,15 +255,36 @@ namespace PolyVox
  * ...
  */
 #ifdef POLYVOX_THROW_ENABLED
-	#define POLYVOX_THROW(type, message) \
-		PolyVox::logError() << (message); \
-		throw type((message))
 
-	// Some fast functions (getVoxel(), etc) use exceptions for error handling but don't want the overhead of logging.
+	#define POLYVOX_THROW_IF(condition, type, message) \
-	// This overhead is present even if no exception is thrown, probably because the presence of the logging code prevents
+		/* We use the do...while(0) construct in our macros (for reasons see here: http://stackoverflow.com/a/154138) \
-	// some inlining. Therefore we provide this macro which doesn't log for such specialised circumstances.
+		   but Visual Studio gives unhelpful 'conditional expression is constant' warnings. The recommended solution \
-	#define POLYVOX_THROW_DONT_LOG(type, message) \
+		   (http://stackoverflow.com/a/1946485) is to disable these warnings. */ \
-		throw type((message))
+		POLYVOX_MSC_WARNING_PUSH \
+		POLYVOX_DISABLE_MSC_WARNING(4127) \
+		do \
+		{ \
+			if ((condition)) \
+			{ \
+				PolyVox::logError() << (message); \
+				throw type((message)); \
+			} \
+		} while(0) \
+		POLYVOX_MSC_WARNING_POP
+
+	#define POLYVOX_THROW(type, message) \
+		/* We use the do...while(0) construct in our macros (for reasons see here: http://stackoverflow.com/a/154138) \
+		   but Visual Studio gives unhelpful 'conditional expression is constant' warnings. The recommended solution \
+		   (http://stackoverflow.com/a/1946485) is to disable these warnings. */ \
+		POLYVOX_MSC_WARNING_PUSH \
+		POLYVOX_DISABLE_MSC_WARNING(4127) \
+		do \
+		{ \
+			PolyVox::logError() << (message); \
+			throw type((message)); \
+		} while(0) \
+		POLYVOX_MSC_WARNING_POP
+
 #else
 	namespace PolyVox
 	{
@@ -273,17 +294,37 @@ namespace PolyVox
 		void setThrowHandler(ThrowHandler newHandler);
 	}
 
-	#define POLYVOX_THROW(type, message) \
+	#define POLYVOX_THROW_IF(condition, type, message) \
-		PolyVox::logError() << (message); \
+		/* We use the do...while(0) construct in our macros (for reasons see here: http://stackoverflow.com/a/154138) \
-		type except = (type)((message)); \
+		   but Visual Studio gives unhelpful 'conditional expression is constant' warnings. The recommended solution \
-		getThrowHandler()((except), __FILE__, __LINE__)
+		   (http://stackoverflow.com/a/1946485) is to disable these warnings. */ \
+		POLYVOX_MSC_WARNING_PUSH \
+		POLYVOX_DISABLE_MSC_WARNING(4127) \
+		do \
+		{ \
+			if ((condition)) \
+			{ \
+				PolyVox::logError() << (message); \
+				type except = (type)((message)); \
+				getThrowHandler()((except), __FILE__, __LINE__); \
+			} \
+		} while(0) \
+		POLYVOX_MSC_WARNING_POP
+
+	#define POLYVOX_THROW(type, message) \
+		/* We use the do...while(0) construct in our macros (for reasons see here: http://stackoverflow.com/a/154138) \
+		   but Visual Studio gives unhelpful 'conditional expression is constant' warnings. The recommended solution \
+		   (http://stackoverflow.com/a/1946485) is to disable these warnings. */ \
+		POLYVOX_MSC_WARNING_PUSH \
+		POLYVOX_DISABLE_MSC_WARNING(4127) \
+		do \
+		{ \
+			PolyVox::logError() << (message); \
+			type except = (type)((message)); \
+			getThrowHandler()((except), __FILE__, __LINE__); \
+		} while(0) \
+		POLYVOX_MSC_WARNING_POP
 
-	// Some fast functions (getVoxel(), etc) use exceptions for error handling but don't want the overhead of logging.
-	// This overhead is present even if no exception is thrown, probably because the presence of the logging code prevents
-	// some inlining. Therefore we provide this macro which doesn't log for such specialised circumstances.
-	#define POLYVOX_THROW_DONT_LOG(type, message) \
-		type except = (type)((message)); \
-		getThrowHandler()((except), __FILE__, __LINE__)
 #endif
 
 namespace PolyVox

library/PolyVoxCore/include/PolyVoxCore/LargeVolume.h

@@ -27,6 +27,7 @@ freely, subject to the following restrictions:
 #include "PolyVoxCore/BaseVolume.h"
 #include "Impl/Block.h"
 #include "PolyVoxCore/Compressor.h"
+#include "PolyVoxCore/Pager.h"
 #include "PolyVoxCore/Region.h"
 #include "PolyVoxCore/Vector.h"
 
@@ -41,8 +42,6 @@ freely, subject to the following restrictions:
 
 namespace PolyVox
 {
-	template <typename VoxelType> class ConstVolumeProxy;
-
 	/// The LargeVolume class provides a memory efficient method of storing voxel data while also allowing fast access and modification.
 	////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 	/// A LargeVolume is essentially a 3D array in which each element (or <i>voxel</i>) is identified by a three dimensional (x,y,z) coordinate.
@@ -227,39 +226,21 @@ namespace PolyVox
 			VoxelType* mCurrentVoxel;
 		};
 
-		// Make the ConstVolumeProxy a friend
-		friend class ConstVolumeProxy<VoxelType>;
-
-		struct LoadedBlock
-		{
-		public:
-			LoadedBlock(uint16_t uSideLength = 0)
-				:block(uSideLength)
-				,timestamp(0)
-			{
-			}
-
-			Block<VoxelType> block;
-			uint32_t timestamp;
-		};
 		#endif
 
-	public:		
+	public:
-		/// Constructor for creating a very large paging volume.
+		/// Constructor for creating a fixed size volume.
 		LargeVolume
 		(
-			polyvox_function<void(const ConstVolumeProxy<VoxelType>&, const Region&)> dataRequiredHandler,
+			const Region& regValid,
-			polyvox_function<void(const ConstVolumeProxy<VoxelType>&, const Region&)> dataOverflowHandler,
+			uint16_t uBlockSideLength = 32			
-			uint16_t uBlockSideLength = 32
 		);
 		/// Constructor for creating a fixed size volume.
 		LargeVolume
 		(
 			const Region& regValid,
-			Compressor* pCompressor = 0,
+			Compressor* pCompressor,	
-			polyvox_function<void(const ConstVolumeProxy<VoxelType>&, const Region&)> dataRequiredHandler = 0,
+			Pager<VoxelType>* pPager ,	
-			polyvox_function<void(const ConstVolumeProxy<VoxelType>&, const Region&)> dataOverflowHandler = 0,
-			bool bPagingEnabled = false,
 			uint16_t uBlockSideLength = 32
 		);
 		/// Destructor
@@ -332,7 +313,7 @@ namespace PolyVox
 				return false;
 			}
 		};
-		void initialise(const Region& regValidRegion, uint16_t uBlockSideLength);
+		void initialise();
 
 		// A trick to implement specialization of template member functions in template classes. See http://stackoverflow.com/a/4951057
 		template <WrapMode eWrapMode>
@@ -341,47 +322,38 @@ namespace PolyVox
 		VoxelType getVoxelImpl(int32_t uXPos, int32_t uYPos, int32_t uZPos, WrapModeType<WrapModes::Clamp>, VoxelType tBorder) const;
 		VoxelType getVoxelImpl(int32_t uXPos, int32_t uYPos, int32_t uZPos, WrapModeType<WrapModes::Border>, VoxelType tBorder) const;
 		VoxelType getVoxelImpl(int32_t uXPos, int32_t uYPos, int32_t uZPos, WrapModeType<WrapModes::AssumeValid>, VoxelType tBorder) const;
-
-		/// gets called when a new region is allocated and needs to be filled
-		/// NOTE: accessing ANY voxels outside this region during the process of this function
-		/// is absolutely unsafe
-		polyvox_function<void(const ConstVolumeProxy<VoxelType>&, const Region&)> m_funcDataRequiredHandler;
-		/// gets called when a Region needs to be stored by the user, because LargeVolume will erase it right after
-		/// this function returns
-		/// NOTE: accessing ANY voxels outside this region during the process of this function
-		/// is absolutely unsafe
-		polyvox_function<void(const ConstVolumeProxy<VoxelType>&, const Region&)> m_funcDataOverflowHandler;
 	
 		Block<VoxelType>* getUncompressedBlock(int32_t uBlockX, int32_t uBlockY, int32_t uBlockZ) const;
-		void eraseBlock(typename std::map<Vector3DInt32, LoadedBlock, BlockPositionCompare>::iterator itBlock) const;
+		void eraseBlock(typename std::map<Vector3DInt32, Block<VoxelType>, BlockPositionCompare>::iterator itBlock) const;
-		/// this function can be called by m_funcDataRequiredHandler without causing any weird effects
-		bool setVoxelAtConst(int32_t uXPos, int32_t uYPos, int32_t uZPos, VoxelType tValue) const;
 
-		//The block data
+		// The block data
-		mutable std::map<Vector3DInt32, LoadedBlock, BlockPositionCompare> m_pBlocks;
+		mutable std::map<Vector3DInt32, Block<VoxelType>, BlockPositionCompare> m_pBlocks;
 
-		//The cache of uncompressed blocks. The uncompressed block data and the timestamps are stored here rather
+		// The cache of uncompressed blocks. The uncompressed block data and the timestamps are stored here rather
-		//than in the Block class. This is so that in the future each VolumeIterator might to maintain its own cache
+		// than in the Block class. This is so that in the future each VolumeIterator might to maintain its own cache
-		//of blocks. However, this could mean the same block data is uncompressed and modified in more than one
+		// of blocks. However, this could mean the same block data is uncompressed and modified in more than one
-		//location in memory... could be messy with threading.
+		// location in memory... could be messy with threading.
-		mutable std::vector< LoadedBlock* > m_vecUncompressedBlockCache;
+		mutable std::vector< Block<VoxelType>* > m_vecBlocksWithUncompressedData;
 		mutable uint32_t m_uTimestamper;
 		mutable Vector3DInt32 m_v3dLastAccessedBlockPos;
 		mutable Block<VoxelType>* m_pLastAccessedBlock;
 		uint32_t m_uMaxNumberOfUncompressedBlocks;
 		uint32_t m_uMaxNumberOfBlocksInMemory;
 
-		//The size of the volume
+		// The size of the volume
 		Region m_regValidRegionInBlocks;
 
-		//The size of the blocks
+		// The size of the blocks
 		uint16_t m_uBlockSideLength;
 		uint8_t m_uBlockSideLengthPower;
 
-		//The compressor used by the Blocks to compress their data if required.
+		// The compressor used by the Blocks to compress their data if required.
 		Compressor* m_pCompressor;
+		Pager<VoxelType>* m_pPager;
 
-		bool m_bPagingEnabled;
+		// Whether we created the compressor or whether it was provided
+		// by the user. This controls whether we delete it on destruction.
+		bool m_bIsOurCompressor;
 	};
 }
 

library/PolyVoxCore/include/PolyVoxCore/LargeVolume.inl

@@ -23,8 +23,7 @@ freely, subject to the following restrictions:
 
 #include "PolyVoxCore/Impl/ErrorHandling.h"
 
-//Included here rather than in the .h because it refers to LargeVolume (avoids forward declaration)
+#include "PolyVoxCore/MinizCompressor.h"
-#include "PolyVoxCore/ConstVolumeProxy.h"
 
 namespace PolyVox
 {
@@ -37,17 +36,19 @@ namespace PolyVox
 	template <typename VoxelType>
 	LargeVolume<VoxelType>::LargeVolume
 	(
-		polyvox_function<void(const ConstVolumeProxy<VoxelType>&, const Region&)> dataRequiredHandler,
+		const Region& regValid,
-		polyvox_function<void(const ConstVolumeProxy<VoxelType>&, const Region&)> dataOverflowHandler,
+		uint16_t uBlockSideLength	
-		uint16_t uBlockSideLength
 	)
-	:BaseVolume<VoxelType>(Region::MaxRegion)
+	:BaseVolume<VoxelType>(regValid)
 	{
-		m_funcDataRequiredHandler = dataRequiredHandler;
+		m_uBlockSideLength = uBlockSideLength;
-		m_funcDataOverflowHandler = dataOverflowHandler;
+
-		m_bPagingEnabled = true;
+		m_pCompressor = new MinizCompressor();
-		//Create a volume of the right size.
+		m_bIsOurCompressor = true;
-		initialise(Region::MaxRegion,uBlockSideLength);
+
+		m_pPager = 0;
+
+		initialise();
 	}
 
 	////////////////////////////////////////////////////////////////////////////////
@@ -64,20 +65,20 @@ namespace PolyVox
 	(
 		const Region& regValid,
 		Compressor* pCompressor,
-		polyvox_function<void(const ConstVolumeProxy<VoxelType>&, const Region&)> dataRequiredHandler,
+		Pager<VoxelType>* pPager,
-		polyvox_function<void(const ConstVolumeProxy<VoxelType>&, const Region&)> dataOverflowHandler,
-		bool bPagingEnabled,
 		uint16_t uBlockSideLength
 	)
 	:BaseVolume<VoxelType>(regValid)
-	,m_pCompressor(pCompressor)
 	{
-		m_funcDataRequiredHandler = dataRequiredHandler;
-		m_funcDataOverflowHandler = dataOverflowHandler;
-		m_bPagingEnabled = bPagingEnabled;
 
-		//Create a volume of the right size.
+		m_uBlockSideLength = uBlockSideLength;
-		initialise(regValid,uBlockSideLength);
+
+		m_pCompressor = pCompressor;
+		m_bIsOurCompressor = false;
+
+		m_pPager = pPager;
+
+		initialise();
 	}
 
 	////////////////////////////////////////////////////////////////////////////////
@@ -100,6 +101,12 @@ namespace PolyVox
 	LargeVolume<VoxelType>::~LargeVolume()
 	{
 		flushAll();
+
+		// Only delete the compressor if it was created by us (in the constructor), not by the user.
+		if(m_bIsOurCompressor)
+		{
+			delete m_pCompressor;
+		}
 	}
 
 	////////////////////////////////////////////////////////////////////////////////
@@ -386,7 +393,7 @@ namespace PolyVox
 				for(int32_t z = v3dStart.getZ(); z <= v3dEnd.getZ(); z++)
 				{
 					Vector3DInt32 pos(x,y,z);
-					typename std::map<Vector3DInt32, LoadedBlock, BlockPositionCompare>::iterator itBlock = m_pBlocks.find(pos);
+					typename std::map<Vector3DInt32, Block<VoxelType>, BlockPositionCompare>::iterator itBlock = m_pBlocks.find(pos);
 					
 					if(itBlock != m_pBlocks.end())
 					{
@@ -417,7 +424,7 @@ namespace PolyVox
 	template <typename VoxelType>
 	void LargeVolume<VoxelType>::flushAll()
 	{
-		typename std::map<Vector3DInt32, LoadedBlock, BlockPositionCompare>::iterator i;
+		typename std::map<Vector3DInt32, Block<VoxelType>, BlockPositionCompare>::iterator i;
 		//Replaced the for loop here as the call to
 		//eraseBlock was invalidating the iterator.
 		while(m_pBlocks.size() > 0)
@@ -451,7 +458,7 @@ namespace PolyVox
 				for(int32_t z = v3dStart.getZ(); z <= v3dEnd.getZ(); z++)
 				{
 					Vector3DInt32 pos(x,y,z);
-					typename std::map<Vector3DInt32, LoadedBlock, BlockPositionCompare>::iterator itBlock = m_pBlocks.find(pos);
+					typename std::map<Vector3DInt32, Block<VoxelType>, BlockPositionCompare>::iterator itBlock = m_pBlocks.find(pos);
 					if(itBlock == m_pBlocks.end())
 					{
 						// not loaded, not unloading
@@ -474,44 +481,42 @@ namespace PolyVox
 	template <typename VoxelType>
 	void LargeVolume<VoxelType>::clearBlockCache(void)
 	{
-		for(uint32_t ct = 0; ct < m_vecUncompressedBlockCache.size(); ct++)
+		for(uint32_t ct = 0; ct < m_vecBlocksWithUncompressedData.size(); ct++)
 		{
-			m_vecUncompressedBlockCache[ct]->block.compress(m_pCompressor);
+			m_vecBlocksWithUncompressedData[ct]->destroyUncompressedData();
 		}
-		m_vecUncompressedBlockCache.clear();
+		m_vecBlocksWithUncompressedData.clear();
 	}
 
 	////////////////////////////////////////////////////////////////////////////////
 	/// This function should probably be made internal...
 	////////////////////////////////////////////////////////////////////////////////
 	template <typename VoxelType>
-	void LargeVolume<VoxelType>::initialise(const Region& regValidRegion, uint16_t uBlockSideLength)
+	void LargeVolume<VoxelType>::initialise()
 	{		
 		//Validate parameters
-		if(uBlockSideLength == 0)
+		if(m_uBlockSideLength == 0)
 		{
 			POLYVOX_THROW(std::invalid_argument, "Block side length cannot be zero.");
 		}
-		if(!isPowerOf2(uBlockSideLength))
+
+		if(!isPowerOf2(m_uBlockSideLength))
 		{
 			POLYVOX_THROW(std::invalid_argument, "Block side length must be a power of two.");
 		}
+
 		if(!m_pCompressor)
 		{
-			POLYVOX_THROW(std::invalid_argument, "You must provide a compressor for the LargeVolume to use.");
+			POLYVOX_THROW(std::invalid_argument, "You must provide a valid compressor for the LargeVolume to use.");
 		}
 
 		m_uTimestamper = 0;
 		m_uMaxNumberOfUncompressedBlocks = 16;
-		m_uBlockSideLength = uBlockSideLength;
 		m_uMaxNumberOfBlocksInMemory = 1024;
 		m_v3dLastAccessedBlockPos = Vector3DInt32(0,0,0); //There are no invalid positions, but initially the m_pLastAccessedBlock pointer will be null;
 		m_pLastAccessedBlock = 0;
 
-		this->m_regValidRegion = regValidRegion;
-
 		//Compute the block side length
-		m_uBlockSideLength = uBlockSideLength;
 		m_uBlockSideLengthPower = logBase2(m_uBlockSideLength);
 
 		m_regValidRegionInBlocks.setLowerX(this->m_regValidRegion.getLowerX() >> m_uBlockSideLengthPower);
@@ -536,63 +541,40 @@ namespace PolyVox
 	}
 
 	template <typename VoxelType>
-	void LargeVolume<VoxelType>::eraseBlock(typename std::map<Vector3DInt32, LoadedBlock, BlockPositionCompare>::iterator itBlock) const
+	void LargeVolume<VoxelType>::eraseBlock(typename std::map<Vector3DInt32, Block<VoxelType>, BlockPositionCompare>::iterator itBlock) const
 	{
-		if(m_funcDataOverflowHandler)
+		if(itBlock->second.hasUncompressedData())
+		{
+			itBlock->second.destroyUncompressedData();
+		}
+
+		if(m_pPager)
 		{
 			Vector3DInt32 v3dPos = itBlock->first;
 			Vector3DInt32 v3dLower(v3dPos.getX() << m_uBlockSideLengthPower, v3dPos.getY() << m_uBlockSideLengthPower, v3dPos.getZ() << m_uBlockSideLengthPower);
 			Vector3DInt32 v3dUpper = v3dLower + Vector3DInt32(m_uBlockSideLength-1, m_uBlockSideLength-1, m_uBlockSideLength-1);
 
 			Region reg(v3dLower, v3dUpper);
-			ConstVolumeProxy<VoxelType> ConstVolumeProxy(*this, reg);
 
-			m_funcDataOverflowHandler(ConstVolumeProxy, reg);
+			m_pPager->pageOut(reg, &(itBlock->second));
 		}
-		if(m_pCompressor)
+		
+		for(uint32_t ct = 0; ct < m_vecBlocksWithUncompressedData.size(); ct++)
 		{
-			for(uint32_t ct = 0; ct < m_vecUncompressedBlockCache.size(); ct++)
+			// find the block in the uncompressed cache
+			if(m_vecBlocksWithUncompressedData[ct] == &(itBlock->second))
 			{
-				// find the block in the uncompressed cache
+				// put last object in cache here
-				if(m_vecUncompressedBlockCache[ct] == &(itBlock->second))
+				m_vecBlocksWithUncompressedData[ct] = m_vecBlocksWithUncompressedData.back();
-				{
+				// decrease cache size by one since last element is now in here twice
-					// TODO: compression is unneccessary? or will not compressing this cause a memleak?
+				m_vecBlocksWithUncompressedData.resize(m_vecBlocksWithUncompressedData.size()-1);
-					itBlock->second.block.compress(m_pCompressor);
+				break;
-					// put last object in cache here
-					m_vecUncompressedBlockCache[ct] = m_vecUncompressedBlockCache.back();
-					// decrease cache size by one since last element is now in here twice
-					m_vecUncompressedBlockCache.resize(m_vecUncompressedBlockCache.size()-1);
-					break;
-				}
 			}
 		}
+
 		m_pBlocks.erase(itBlock);
 	}
 
-	template <typename VoxelType>
-	bool LargeVolume<VoxelType>::setVoxelAtConst(int32_t uXPos, int32_t uYPos, int32_t uZPos, VoxelType tValue) const
-	{
-		//We don't have any range checks in this function because it
-		//is a private function only called by the ConstVolumeProxy. The
-		//ConstVolumeProxy takes care of ensuring the range is appropriate.
-
-		const int32_t blockX = uXPos >> m_uBlockSideLengthPower;
-		const int32_t blockY = uYPos >> m_uBlockSideLengthPower;
-		const int32_t blockZ = uZPos >> m_uBlockSideLengthPower;
-
-		const uint16_t xOffset = uXPos - (blockX << m_uBlockSideLengthPower);
-		const uint16_t yOffset = uYPos - (blockY << m_uBlockSideLengthPower);
-		const uint16_t zOffset = uZPos - (blockZ << m_uBlockSideLengthPower);
-
-		Block<VoxelType>* pUncompressedBlock = getUncompressedBlock(blockX, blockY, blockZ);
-
-		pUncompressedBlock->setVoxelAt(xOffset,yOffset,zOffset, tValue);
-
-		//Return true to indicate that we modified a voxel.
-		return true;
-	}
-
-
 	template <typename VoxelType>
 	Block<VoxelType>* LargeVolume<VoxelType>::getUncompressedBlock(int32_t uBlockX, int32_t uBlockY, int32_t uBlockZ) const
 	{
@@ -604,25 +586,25 @@ namespace PolyVox
 		//This check should also provide a significant speed boost as usually it is true.
 		if((v3dBlockPos == m_v3dLastAccessedBlockPos) && (m_pLastAccessedBlock != 0))
 		{
-			POLYVOX_ASSERT(m_pLastAccessedBlock->m_tUncompressedData, "Block has no uncompressed data");
+			POLYVOX_ASSERT(m_pLastAccessedBlock->hasUncompressedData(), "Last accessed block has no uncompressed data.");
 			return m_pLastAccessedBlock;
 		}		
 
-		typename std::map<Vector3DInt32, LoadedBlock, BlockPositionCompare>::iterator itBlock = m_pBlocks.find(v3dBlockPos);
+		typename std::map<Vector3DInt32, Block<VoxelType>, BlockPositionCompare>::iterator itBlock = m_pBlocks.find(v3dBlockPos);
 		// check whether the block is already loaded
 		if(itBlock == m_pBlocks.end())
 		{
 			//The block is not in the map, so we will have to create a new block and add it.
 			//Before we do so, we might want to dump some existing data to make space. We 
 			//Only do this if paging is enabled.
-			if(m_bPagingEnabled)
+			if(m_pPager)
 			{
 				// check wether another block needs to be unloaded before this one can be loaded
 				if(m_pBlocks.size() == m_uMaxNumberOfBlocksInMemory)
 				{
 					// find the least recently used block
-					typename std::map<Vector3DInt32, LoadedBlock, BlockPositionCompare>::iterator i;
+					typename std::map<Vector3DInt32, Block<VoxelType>, BlockPositionCompare>::iterator i;
-					typename std::map<Vector3DInt32, LoadedBlock, BlockPositionCompare>::iterator itUnloadBlock = m_pBlocks.begin();
+					typename std::map<Vector3DInt32, Block<VoxelType>, BlockPositionCompare>::iterator itUnloadBlock = m_pBlocks.begin();
 					for(i = m_pBlocks.begin(); i != m_pBlocks.end(); i++)
 					{
 						if(i->second.timestamp < itUnloadBlock->second.timestamp)
@@ -635,20 +617,17 @@ namespace PolyVox
 			}
 			
 			// create the new block
-			LoadedBlock newBlock(m_uBlockSideLength);
+			Block<VoxelType> newBlock(m_uBlockSideLength,  m_pCompressor);
 
-			// Blocks start out compressed - should we change this?
+			auto retVal = m_pBlocks.insert(std::make_pair(v3dBlockPos, newBlock));
-			// Or maybe we should just 'seed' them with compressed data,
+			itBlock = retVal.first;
-			// rather than creating an empty block and then compressing?
+			POLYVOX_ASSERT(retVal.second == true, "Element was not supposed to exist!");
-			newBlock.block.compress(m_pCompressor);
-
-			itBlock = m_pBlocks.insert(std::make_pair(v3dBlockPos, newBlock)).first;
 
 			//We have created the new block. If paging is enabled it should be used to
 			//fill in the required data. Otherwise it is just left in the default state.
-			if(m_bPagingEnabled)
+			if(m_pPager)
 			{
-				if(m_funcDataRequiredHandler)
+				//if(m_funcDataRequiredHandler)
 				{
 					// "load" will actually call setVoxel, which will in turn call this function again but the block will be found
 					// so this if(itBlock == m_pBlocks.end()) never is entered		
@@ -656,26 +635,25 @@ namespace PolyVox
 					Vector3DInt32 v3dLower(v3dBlockPos.getX() << m_uBlockSideLengthPower, v3dBlockPos.getY() << m_uBlockSideLengthPower, v3dBlockPos.getZ() << m_uBlockSideLengthPower);
 					Vector3DInt32 v3dUpper = v3dLower + Vector3DInt32(m_uBlockSideLength-1, m_uBlockSideLength-1, m_uBlockSideLength-1);
 					Region reg(v3dLower, v3dUpper);
-					ConstVolumeProxy<VoxelType> ConstVolumeProxy(*this, reg);
+
-					m_funcDataRequiredHandler(ConstVolumeProxy, reg);
+					m_pPager->pageIn(reg, &(itBlock->second));
 				}
 			}
 		}		
 
 		//Get the block and mark that we accessed it
-		LoadedBlock& loadedBlock = itBlock->second;
+		Block<VoxelType>& block = itBlock->second;
-		loadedBlock.timestamp = ++m_uTimestamper;
+		block.timestamp = ++m_uTimestamper;
 		m_v3dLastAccessedBlockPos = v3dBlockPos;
-		m_pLastAccessedBlock = &(loadedBlock.block);
+		m_pLastAccessedBlock = &block;
 
-		if(loadedBlock.block.m_bIsCompressed == false)
+		if(block.hasUncompressedData())
 		{ 			
-			POLYVOX_ASSERT(m_pLastAccessedBlock->m_tUncompressedData, "Block has no uncompressed data");
 			return m_pLastAccessedBlock;
 		}
 
 		//If we are allowed to compress then check whether we need to
-		if((m_pCompressor) && (m_vecUncompressedBlockCache.size() == m_uMaxNumberOfUncompressedBlocks))
+		if(m_vecBlocksWithUncompressedData.size() == m_uMaxNumberOfUncompressedBlocks)
 		{
 			int32_t leastRecentlyUsedBlockIndex = -1;
 			uint32_t uLeastRecentTimestamp = (std::numeric_limits<uint32_t>::max)();
@@ -683,30 +661,30 @@ namespace PolyVox
 			//Currently we find the oldest block by iterating over the whole array. Of course we could store the blocks sorted by
 			//timestamp (set, priority_queue, etc) but then we'll need to move them around as the timestamp changes. Can come back 
 			//to this if it proves to be a bottleneck (compraed to the cost of actually doing the compression/decompression).
-			for(uint32_t ct = 0; ct < m_vecUncompressedBlockCache.size(); ct++)
+			for(uint32_t ct = 0; ct < m_vecBlocksWithUncompressedData.size(); ct++)
 			{
-				if(m_vecUncompressedBlockCache[ct]->timestamp < uLeastRecentTimestamp)
+				if(m_vecBlocksWithUncompressedData[ct]->timestamp < uLeastRecentTimestamp)
 				{
-					uLeastRecentTimestamp = m_vecUncompressedBlockCache[ct]->timestamp;
+					uLeastRecentTimestamp = m_vecBlocksWithUncompressedData[ct]->timestamp;
 					leastRecentlyUsedBlockIndex = ct;
 				}
 			}
 			
 			//Compress the least recently used block.
-			m_vecUncompressedBlockCache[leastRecentlyUsedBlockIndex]->block.compress(m_pCompressor);
+			m_vecBlocksWithUncompressedData[leastRecentlyUsedBlockIndex]->destroyUncompressedData();
 
 			//We don't actually remove any elements from this vector, we
 			//simply change the pointer to point at the new uncompressed bloack.			
-			m_vecUncompressedBlockCache[leastRecentlyUsedBlockIndex] = &loadedBlock;
+			m_vecBlocksWithUncompressedData[leastRecentlyUsedBlockIndex] = &block;
 		}
 		else
 		{
-			m_vecUncompressedBlockCache.push_back(&loadedBlock);
+			m_vecBlocksWithUncompressedData.push_back(&block);
 		}
 		
-		loadedBlock.block.uncompress(m_pCompressor);
+		block.createUncompressedData();
 
-		m_pLastAccessedBlock = &(loadedBlock.block);
+		m_pLastAccessedBlock = &(block);
 		POLYVOX_ASSERT(m_pLastAccessedBlock->m_tUncompressedData, "Block has no uncompressed data");
 		return m_pLastAccessedBlock;
 	}
@@ -731,16 +709,16 @@ namespace PolyVox
 		uint32_t uSizeInBytes = sizeof(LargeVolume);
 
 		//Memory used by the blocks
-		typename std::map<Vector3DInt32, LoadedBlock, BlockPositionCompare>::iterator i;
+		typename std::map<Vector3DInt32, Block<VoxelType>, BlockPositionCompare>::iterator i;
 		for(i = m_pBlocks.begin(); i != m_pBlocks.end(); i++)
 		{
 			//Inaccurate - account for rest of loaded block.
-			uSizeInBytes += i->second.block.calculateSizeInBytes();
+			uSizeInBytes += i->second.calculateSizeInBytes();
 		}
 
 		//Memory used by the block cache.
-		uSizeInBytes += m_vecUncompressedBlockCache.capacity() * sizeof(LoadedBlock);
+		uSizeInBytes += m_vecBlocksWithUncompressedData.capacity() * sizeof(Block<VoxelType>);
-		uSizeInBytes += m_vecUncompressedBlockCache.size() * m_uBlockSideLength * m_uBlockSideLength * m_uBlockSideLength * sizeof(VoxelType);
+		uSizeInBytes += m_vecBlocksWithUncompressedData.size() * m_uBlockSideLength * m_uBlockSideLength * m_uBlockSideLength * sizeof(VoxelType);
 
 		return uSizeInBytes;
 	}

library/PolyVoxCore/include/PolyVoxCore/Pager.h

@@ -0,0 +1,49 @@
+/*******************************************************************************
+Copyright (c) 2005-2009 David Williams
+
+This software is provided 'as-is', without any express or implied
+warranty. In no event will the authors be held liable for any damages
+arising from the use of this software.
+
+Permission is granted to anyone to use this software for any purpose,
+including commercial applications, and to alter it and redistribute it
+freely, subject to the following restrictions:
+
+    1. The origin of this software must not be misrepresented; you must not
+    claim that you wrote the original software. If you use this software
+    in a product, an acknowledgment in the product documentation would be
+    appreciated but is not required.
+
+    2. Altered source versions must be plainly marked as such, and must not be
+    misrepresented as being the original software.
+
+    3. This notice may not be removed or altered from any source
+    distribution. 	
+*******************************************************************************/
+
+#ifndef __PolyVox_Pager_H__
+#define __PolyVox_Pager_H__
+
+#include "PolyVoxCore/Impl/Block.h"
+#include "PolyVoxCore/Impl/TypeDef.h"
+
+namespace PolyVox
+{
+	/**
+	 * Provides an interface for performing paging of data.
+	 */
+	template <typename VoxelType>
+	class Pager
+	{
+	public:
+		/// Constructor
+		Pager() {};
+		/// Destructor
+		virtual ~Pager() {};
+
+		virtual void pageIn(const Region& region, Block<VoxelType>* pBlockData) = 0;
+		virtual void pageOut(const Region& region, Block<VoxelType>* pBlockData) = 0;
+	};
+}
+
+#endif //__PolyVox_Pager_H__

library/PolyVoxCore/include/PolyVoxCore/Region.h

@@ -202,6 +202,10 @@ namespace PolyVox
 		int32_t m_iUpperZ;
 	};
 
+	// Non-member overloaded operators. 
+	/// Stream insertion operator.
+    std::ostream& operator<<(std::ostream& os, const Region& region);
+
 	// Functions to be inlined to to be in the header rather than the .cpp.
 	// 'inline' keyword is used for the definition rather than the declaration.
 	// See also http://www.parashift.com/c++-faq-lite/inline-functions.html

library/PolyVoxCore/source/Region.cpp

@@ -487,4 +487,17 @@ namespace PolyVox
 	{
 		shrink(v3dAmount.getX(), v3dAmount.getY(), v3dAmount.getZ());
 	}
-}
+
+	/**
+     * Enables the Region to be used intuitively with output streams such as cout.
+     * \param os The output stream to write to.
+     * \param region The Region to write to the stream.
+     * \return A reference to the output stream to allow chaining.
+     */
+	std::ostream& operator<<(std::ostream& os, const Region& region)
+    {
+		os << "(" << region.getLowerX() << "," << region.getLowerY() << "," << region.getLowerZ() <<
+			") to (" << region.getUpperX() << "," << region.getUpperY() << "," << region.getUpperZ() << ")";
+        return os;
+    }	
+}

tests/testvolume.cpp

@@ -277,7 +277,7 @@ TestVolume::TestVolume()
 	//Create the volumes
 	m_pRawVolume = new RawVolume<int32_t>(region);
 	m_pSimpleVolume = new SimpleVolume<int32_t>(region);
-	m_pLargeVolume = new LargeVolume<int32_t>(region, m_pCompressor);
+	m_pLargeVolume = new LargeVolume<int32_t>(region, m_pCompressor, 0, 32);
 
 	// LargeVolume currently fails a test if compression is enabled. It
 	// may be related to accessing the data through more than one sampler?