From a2fe1944af01faad847ca9bb9e021cd5014c21b5 Mon Sep 17 00:00:00 2001 From: David Williams Date: Thu, 9 Apr 2015 23:44:25 +0200 Subject: [PATCH 01/12] Initial work on replacing std::unordered_map with a specialized hash table for looking up chunks based on their 3D position. --- include/PolyVox/PagedVolume.h | 2 ++ include/PolyVox/PagedVolume.inl | 57 ++++++++++++++++++++++++++++++--- 2 files changed, 55 insertions(+), 4 deletions(-) diff --git a/include/PolyVox/PagedVolume.h b/include/PolyVox/PagedVolume.h index a3eb271d..38db965b 100644 --- a/include/PolyVox/PagedVolume.h +++ b/include/PolyVox/PagedVolume.h @@ -310,6 +310,8 @@ namespace PolyVox // The size of the volume //Region m_regValidRegionInChunks; + mutable std::unique_ptr< Chunk > m_arrayChunks[16384]; + // The size of the chunks uint16_t m_uChunkSideLength; uint8_t m_uChunkSideLengthPower; diff --git a/include/PolyVox/PagedVolume.inl b/include/PolyVox/PagedVolume.inl index 3afc3d1e..124fbb16 100644 --- a/include/PolyVox/PagedVolume.inl +++ b/include/PolyVox/PagedVolume.inl @@ -273,7 +273,7 @@ namespace PolyVox Vector3DInt32 v3dChunkPos(uChunkX, uChunkY, uChunkZ); // The chunk was not the same as last time, but we can now hope it is in the set of most recently used chunks. - Chunk* pChunk = nullptr; + /*Chunk* pChunk = nullptr; auto itChunk = m_mapChunks.find(v3dChunkPos); // Check whether the chunk was found. @@ -283,6 +283,45 @@ namespace PolyVox pChunk = itChunk->second.get(); POLYVOX_ASSERT(pChunk, "Recent chunk list shold never contain a null pointer."); pChunk->m_uChunkLastAccessed = ++m_uTimestamper; + }*/ + + Chunk* pChunk = nullptr; + const int32_t startIndex = (((uChunkX & 0xFF)) | ((uChunkY & 0x0F) << 4) | ((uChunkZ & 0x0F) << 8) << 2 ); + for (uint32_t ct = startIndex; ct < 16384; ct++) + { + if (m_arrayChunks[ct]) + { + Vector3DInt32& entryPos = m_arrayChunks[ct]->m_v3dChunkSpacePosition; + if (entryPos.getX() == uChunkX && entryPos.getY() == uChunkY && entryPos.getZ() == uChunkZ) + { + pChunk = m_arrayChunks[ct].get(); + break; + } + } + } + + if (pChunk == nullptr) + { + for (uint32_t ct = 0; ct < startIndex; ct++) + { + if (m_arrayChunks[ct]) + { + Vector3DInt32& entryPos = m_arrayChunks[ct]->m_v3dChunkSpacePosition; + if (entryPos.getX() == uChunkX && entryPos.getY() == uChunkY && entryPos.getZ() == uChunkZ) + { + pChunk = m_arrayChunks[ct].get(); + break; + } + } + } + } + + // Check whether the chunk was found. + if (pChunk) + { + // The chunk was found so we can use it. + POLYVOX_ASSERT(pChunk, "Recent chunk list shold never contain a null pointer."); + pChunk->m_uChunkLastAccessed = ++m_uTimestamper; } // If we still haven't found the chunk then it's time to create a new one and page it in from disk. @@ -291,10 +330,18 @@ namespace PolyVox // The chunk was not found so we will create a new one. pChunk = new PagedVolume::Chunk(v3dChunkPos, m_uChunkSideLength, m_pPager); pChunk->m_uChunkLastAccessed = ++m_uTimestamper; // Important, as we may soon delete the oldest chunk - m_mapChunks.insert(std::make_pair(v3dChunkPos, std::unique_ptr(pChunk))); + //m_mapChunks.insert(std::make_pair(v3dChunkPos, std::unique_ptr(pChunk))); + for (uint32_t ct = startIndex; ct < 16384; ct++) + { + if (m_arrayChunks[ct] == nullptr) + { + m_arrayChunks[ct] = std::move(std::unique_ptr< Chunk >(pChunk)); + break; + } + } // As we are loading a new chunk we should try to ensure we don't go over our target memory usage. - while (m_mapChunks.size() > m_uChunkCountLimit) + /*while (m_mapChunks.size() > m_uChunkCountLimit) { // Find the least recently used chunk. Hopefully this isn't too slow. auto itUnloadChunk = m_mapChunks.begin(); @@ -308,7 +355,9 @@ namespace PolyVox // Erase the least recently used chunk m_mapChunks.erase(itUnloadChunk); - } + }*/ + + } m_pLastAccessedChunk = pChunk; From f35581506c2bbbe9fce8c294aaf30d24b6a8088a Mon Sep 17 00:00:00 2001 From: David Williams Date: Sun, 12 Apr 2015 09:42:15 +0200 Subject: [PATCH 02/12] Minor optimization - only creating vector if we are going to use it. --- include/PolyVox/PagedVolume.inl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/include/PolyVox/PagedVolume.inl b/include/PolyVox/PagedVolume.inl index 124fbb16..9bcb3f10 100644 --- a/include/PolyVox/PagedVolume.inl +++ b/include/PolyVox/PagedVolume.inl @@ -270,8 +270,6 @@ namespace PolyVox template typename PagedVolume::Chunk* PagedVolume::getChunk(int32_t uChunkX, int32_t uChunkY, int32_t uChunkZ) const { - Vector3DInt32 v3dChunkPos(uChunkX, uChunkY, uChunkZ); - // The chunk was not the same as last time, but we can now hope it is in the set of most recently used chunks. /*Chunk* pChunk = nullptr; auto itChunk = m_mapChunks.find(v3dChunkPos); @@ -328,6 +326,7 @@ namespace PolyVox if (!pChunk) { // The chunk was not found so we will create a new one. + Vector3DInt32 v3dChunkPos(uChunkX, uChunkY, uChunkZ); pChunk = new PagedVolume::Chunk(v3dChunkPos, m_uChunkSideLength, m_pPager); pChunk->m_uChunkLastAccessed = ++m_uTimestamper; // Important, as we may soon delete the oldest chunk //m_mapChunks.insert(std::make_pair(v3dChunkPos, std::unique_ptr(pChunk))); From c4cccf904321d9ea77e62f183376754a6694472c Mon Sep 17 00:00:00 2001 From: David Williams Date: Sun, 12 Apr 2015 09:55:30 +0200 Subject: [PATCH 03/12] Replaced double for loop with cleaner do-while loop. --- include/PolyVox/PagedVolume.inl | 44 ++++++++------------------------- 1 file changed, 10 insertions(+), 34 deletions(-) diff --git a/include/PolyVox/PagedVolume.inl b/include/PolyVox/PagedVolume.inl index 9bcb3f10..ab6af57c 100644 --- a/include/PolyVox/PagedVolume.inl +++ b/include/PolyVox/PagedVolume.inl @@ -271,48 +271,24 @@ namespace PolyVox typename PagedVolume::Chunk* PagedVolume::getChunk(int32_t uChunkX, int32_t uChunkY, int32_t uChunkZ) const { // The chunk was not the same as last time, but we can now hope it is in the set of most recently used chunks. - /*Chunk* pChunk = nullptr; - auto itChunk = m_mapChunks.find(v3dChunkPos); - - // Check whether the chunk was found. - if ((itChunk) != m_mapChunks.end()) - { - // The chunk was found so we can use it. - pChunk = itChunk->second.get(); - POLYVOX_ASSERT(pChunk, "Recent chunk list shold never contain a null pointer."); - pChunk->m_uChunkLastAccessed = ++m_uTimestamper; - }*/ - Chunk* pChunk = nullptr; - const int32_t startIndex = (((uChunkX & 0xFF)) | ((uChunkY & 0x0F) << 4) | ((uChunkZ & 0x0F) << 8) << 2 ); - for (uint32_t ct = startIndex; ct < 16384; ct++) + const int32_t iStartIndex = (((uChunkX & 0xFF)) | ((uChunkY & 0x0F) << 4) | ((uChunkZ & 0x0F) << 8) << 2 ); + int32_t iIndex = iStartIndex; + do { - if (m_arrayChunks[ct]) + if (m_arrayChunks[iIndex]) { - Vector3DInt32& entryPos = m_arrayChunks[ct]->m_v3dChunkSpacePosition; + Vector3DInt32& entryPos = m_arrayChunks[iIndex]->m_v3dChunkSpacePosition; if (entryPos.getX() == uChunkX && entryPos.getY() == uChunkY && entryPos.getZ() == uChunkZ) { - pChunk = m_arrayChunks[ct].get(); + pChunk = m_arrayChunks[iIndex].get(); break; } } - } - if (pChunk == nullptr) - { - for (uint32_t ct = 0; ct < startIndex; ct++) - { - if (m_arrayChunks[ct]) - { - Vector3DInt32& entryPos = m_arrayChunks[ct]->m_v3dChunkSpacePosition; - if (entryPos.getX() == uChunkX && entryPos.getY() == uChunkY && entryPos.getZ() == uChunkZ) - { - pChunk = m_arrayChunks[ct].get(); - break; - } - } - } - } + iIndex++; + iIndex %= 16384; + } while (iIndex != iStartIndex); // Check whether the chunk was found. if (pChunk) @@ -331,7 +307,7 @@ namespace PolyVox pChunk->m_uChunkLastAccessed = ++m_uTimestamper; // Important, as we may soon delete the oldest chunk //m_mapChunks.insert(std::make_pair(v3dChunkPos, std::unique_ptr(pChunk))); - for (uint32_t ct = startIndex; ct < 16384; ct++) + for (uint32_t ct = iStartIndex; ct < 16384; ct++) { if (m_arrayChunks[ct] == nullptr) { From 99390580dd043cf771cfe1179221ea19584e21f0 Mon Sep 17 00:00:00 2001 From: David Williams Date: Sun, 12 Apr 2015 10:35:12 +0200 Subject: [PATCH 04/12] Replaced number with constant. --- include/PolyVox/PagedVolume.h | 13 +++++++++---- include/PolyVox/PagedVolume.inl | 6 +++--- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/include/PolyVox/PagedVolume.h b/include/PolyVox/PagedVolume.h index 38db965b..5dc5bf8e 100644 --- a/include/PolyVox/PagedVolume.h +++ b/include/PolyVox/PagedVolume.h @@ -307,10 +307,15 @@ namespace PolyVox uint32_t m_uChunkCountLimit = 0; - // The size of the volume - //Region m_regValidRegionInChunks; - - mutable std::unique_ptr< Chunk > m_arrayChunks[16384]; + // Chunks are stored in the following array which is used as a hash-table. Conventional wisdom is that such a hash-table + // should not be more than half full to avoid conflicts, and a practical chunk size seems to be 64^3. With this configuration + // there can be up to 32768*64^3 = 8 gigavoxels (with each voxel perhaps being many bytes). This should effectively make use + // of even high end machines. Of course, the user can choose to limit the memory usage in which case much less of the chunk + // array will actually be used. None-the-less, we have chosen to use a fixed size array (rather than a vector) as it appear to + // be slightly faster (probably due to the extra pointer indirection in a vector?) and the actual size of this array should + // just be 1Mb or so. + static const uint32_t uChunkArraySize = 65536; + mutable std::unique_ptr< Chunk > m_arrayChunks[uChunkArraySize]; // The size of the chunks uint16_t m_uChunkSideLength; diff --git a/include/PolyVox/PagedVolume.inl b/include/PolyVox/PagedVolume.inl index ab6af57c..8de87ce3 100644 --- a/include/PolyVox/PagedVolume.inl +++ b/include/PolyVox/PagedVolume.inl @@ -58,7 +58,7 @@ namespace PolyVox // Enforce sensible limits on the number of chunks. const uint32_t uMinPracticalNoOfChunks = 32; // Enough to make sure a chunks and it's neighbours can be loaded, with a few to spare. - const uint32_t uMaxPracticalNoOfChunks = 32768; // Should prevent multi-gigabyte volumes when chunk sizes are reasonable. + const uint32_t uMaxPracticalNoOfChunks = uChunkArraySize / 2; // A hash table should only become half-full to avoid too many clashes. POLYVOX_LOG_WARNING_IF(m_uChunkCountLimit < uMinPracticalNoOfChunks, "Requested memory usage limit of " << uTargetMemoryUsageInBytes / (1024 * 1024) << "Mb is too low and cannot be adhered to."); m_uChunkCountLimit = (std::max)(m_uChunkCountLimit, uMinPracticalNoOfChunks); @@ -287,7 +287,7 @@ namespace PolyVox } iIndex++; - iIndex %= 16384; + iIndex %= uChunkArraySize; } while (iIndex != iStartIndex); // Check whether the chunk was found. @@ -307,7 +307,7 @@ namespace PolyVox pChunk->m_uChunkLastAccessed = ++m_uTimestamper; // Important, as we may soon delete the oldest chunk //m_mapChunks.insert(std::make_pair(v3dChunkPos, std::unique_ptr(pChunk))); - for (uint32_t ct = iStartIndex; ct < 16384; ct++) + for (uint32_t ct = iStartIndex; ct < uChunkArraySize; ct++) { if (m_arrayChunks[ct] == nullptr) { From af70096fcccfb76f5617cbf296a0b50f5ac747e4 Mon Sep 17 00:00:00 2001 From: David Williams Date: Sun, 12 Apr 2015 16:46:43 +0200 Subject: [PATCH 05/12] Tidying and adding comments. --- include/PolyVox/PagedVolume.h | 2 +- include/PolyVox/PagedVolume.inl | 23 ++++++++++++++++++----- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/include/PolyVox/PagedVolume.h b/include/PolyVox/PagedVolume.h index 5dc5bf8e..826f3b29 100644 --- a/include/PolyVox/PagedVolume.h +++ b/include/PolyVox/PagedVolume.h @@ -311,7 +311,7 @@ namespace PolyVox // should not be more than half full to avoid conflicts, and a practical chunk size seems to be 64^3. With this configuration // there can be up to 32768*64^3 = 8 gigavoxels (with each voxel perhaps being many bytes). This should effectively make use // of even high end machines. Of course, the user can choose to limit the memory usage in which case much less of the chunk - // array will actually be used. None-the-less, we have chosen to use a fixed size array (rather than a vector) as it appear to + // array will actually be used. None-the-less, we have chosen to use a fixed size array (rather than a vector) as it appears to // be slightly faster (probably due to the extra pointer indirection in a vector?) and the actual size of this array should // just be 1Mb or so. static const uint32_t uChunkArraySize = 65536; diff --git a/include/PolyVox/PagedVolume.inl b/include/PolyVox/PagedVolume.inl index 8de87ce3..0221e38f 100644 --- a/include/PolyVox/PagedVolume.inl +++ b/include/PolyVox/PagedVolume.inl @@ -270,10 +270,23 @@ namespace PolyVox template typename PagedVolume::Chunk* PagedVolume::getChunk(int32_t uChunkX, int32_t uChunkY, int32_t uChunkZ) const { - // The chunk was not the same as last time, but we can now hope it is in the set of most recently used chunks. Chunk* pChunk = nullptr; - const int32_t iStartIndex = (((uChunkX & 0xFF)) | ((uChunkY & 0x0F) << 4) | ((uChunkZ & 0x0F) << 8) << 2 ); - int32_t iIndex = iStartIndex; + + // We generate a 16-bit hash here and assume this matches the range available in the chunk + // array. The assert here is just to make sure we take care if change this in the future. + static_assert(uChunkArraySize == 65536, "Chunk array size has changed, check if the hash calculation needs updating."); + // Extract the lower five bits from each position component. + const uint32_t uChunkXLowerBits = static_cast(uChunkX & 0x1F); + const uint32_t uChunkYLowerBits = static_cast(uChunkY & 0x1F); + const uint32_t uChunkZLowerBits = static_cast(uChunkZ & 0x1F); + // Combine then to form a 15-bit hash of the position. Also shift by one to spread the values out in the whole 16-bit space. + const uint32_t iPosisionHash = (((uChunkXLowerBits)) | ((uChunkYLowerBits) << 5) | ((uChunkZLowerBits) << 10) << 1); + + // Starting at the position indicated by the hash, and then search through the whole array looking for a chunk with the correct + // position. In most cases we expect to find it in the first place we look. Note that this algorithm is slow in the case that + // the chunk is not found because the whole array has to be searched, but in this case we are going to have to page the data in + // from an external source which is likely to be slow anyway. + uint32_t iIndex = iPosisionHash; do { if (m_arrayChunks[iIndex]) @@ -288,7 +301,7 @@ namespace PolyVox iIndex++; iIndex %= uChunkArraySize; - } while (iIndex != iStartIndex); + } while (iIndex != iPosisionHash); // Keep searching until we get back to our start position. // Check whether the chunk was found. if (pChunk) @@ -307,7 +320,7 @@ namespace PolyVox pChunk->m_uChunkLastAccessed = ++m_uTimestamper; // Important, as we may soon delete the oldest chunk //m_mapChunks.insert(std::make_pair(v3dChunkPos, std::unique_ptr(pChunk))); - for (uint32_t ct = iStartIndex; ct < uChunkArraySize; ct++) + for (uint32_t ct = iPosisionHash; ct < uChunkArraySize; ct++) { if (m_arrayChunks[ct] == nullptr) { From 64be18cd149952880b22e35eb35112b23705ba37 Mon Sep 17 00:00:00 2001 From: David Williams Date: Sun, 12 Apr 2015 20:55:49 +0200 Subject: [PATCH 06/12] Tidied up loop for inserting chunk into array. --- include/PolyVox/PagedVolume.inl | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/include/PolyVox/PagedVolume.inl b/include/PolyVox/PagedVolume.inl index 0221e38f..7a0a1483 100644 --- a/include/PolyVox/PagedVolume.inl +++ b/include/PolyVox/PagedVolume.inl @@ -318,16 +318,39 @@ namespace PolyVox Vector3DInt32 v3dChunkPos(uChunkX, uChunkY, uChunkZ); pChunk = new PagedVolume::Chunk(v3dChunkPos, m_uChunkSideLength, m_pPager); pChunk->m_uChunkLastAccessed = ++m_uTimestamper; // Important, as we may soon delete the oldest chunk + + // Store the chunk at the appropriate place in out chunk array. Ideally this place is + // given by the hash, otherwise we do a linear search for the next available location + // We always expect to find a free place because we aim to keep the array only half full. + uint32_t iIndex = iPosisionHash; + bool bInsertedSucessfully = false; + do + { + if (m_arrayChunks[iIndex] == nullptr) + { + m_arrayChunks[iIndex] = std::move(std::unique_ptr< Chunk >(pChunk)); + bInsertedSucessfully = true; + break; + } + + iIndex++; + iIndex %= uChunkArraySize; + } while (iIndex != iPosisionHash); // Keep searching until we get back to our start position. + + // This should never really happen unless we are failing to keep our number of active chunks + // significantly under the target amount. Perhaps if chunks are 'pinned' for threading purposes? + //POLYVOX_THROW_IF(!bInsertedSucessfully, std::logic_error, "No space in chunk array for new chunk."); + //m_mapChunks.insert(std::make_pair(v3dChunkPos, std::unique_ptr(pChunk))); - for (uint32_t ct = iPosisionHash; ct < uChunkArraySize; ct++) + /*for (uint32_t ct = iPosisionHash; ct < uChunkArraySize; ct++) { if (m_arrayChunks[ct] == nullptr) { m_arrayChunks[ct] = std::move(std::unique_ptr< Chunk >(pChunk)); break; } - } + }*/ // As we are loading a new chunk we should try to ensure we don't go over our target memory usage. /*while (m_mapChunks.size() > m_uChunkCountLimit) { From 8757f1e53ef47515d00e00d8680386ce411a67c7 Mon Sep 17 00:00:00 2001 From: David Williams Date: Mon, 13 Apr 2015 21:17:19 +0200 Subject: [PATCH 07/12] Removed unneeded assert. --- include/PolyVox/PagedVolume.inl | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/include/PolyVox/PagedVolume.inl b/include/PolyVox/PagedVolume.inl index 7a0a1483..edeb6f0f 100644 --- a/include/PolyVox/PagedVolume.inl +++ b/include/PolyVox/PagedVolume.inl @@ -295,6 +295,7 @@ namespace PolyVox if (entryPos.getX() == uChunkX && entryPos.getY() == uChunkY && entryPos.getZ() == uChunkZ) { pChunk = m_arrayChunks[iIndex].get(); + pChunk->m_uChunkLastAccessed = ++m_uTimestamper; break; } } @@ -303,14 +304,6 @@ namespace PolyVox iIndex %= uChunkArraySize; } while (iIndex != iPosisionHash); // Keep searching until we get back to our start position. - // Check whether the chunk was found. - if (pChunk) - { - // The chunk was found so we can use it. - POLYVOX_ASSERT(pChunk, "Recent chunk list shold never contain a null pointer."); - pChunk->m_uChunkLastAccessed = ++m_uTimestamper; - } - // If we still haven't found the chunk then it's time to create a new one and page it in from disk. if (!pChunk) { From 37c35a08db814da592498a6c348b7bac1dbf08eb Mon Sep 17 00:00:00 2001 From: David Williams Date: Mon, 13 Apr 2015 21:30:59 +0200 Subject: [PATCH 08/12] Added code to ensure the number of chunks doesn't go over our target limit. --- include/PolyVox/PagedVolume.inl | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/include/PolyVox/PagedVolume.inl b/include/PolyVox/PagedVolume.inl index edeb6f0f..d70242c7 100644 --- a/include/PolyVox/PagedVolume.inl +++ b/include/PolyVox/PagedVolume.inl @@ -334,6 +334,28 @@ namespace PolyVox // significantly under the target amount. Perhaps if chunks are 'pinned' for threading purposes? //POLYVOX_THROW_IF(!bInsertedSucessfully, std::logic_error, "No space in chunk array for new chunk."); + // As we have added a chunk we may have exceeded our target chunk limit. Search through the array to + // determine how many chunks we have, as well as finding the oldest timestamp. Note that this is potentially + // wasteful and we may instead wish to track how many chunks we have and/or delete a chunk at random (or + // just check e.g. 10 and delete the oldest of those) but we'll see if this is a bottleneck first. Paging + // the data in is probably more expensive. + uint32_t uChunkCount = 0; + uint32_t uOldestChunkIndex = std::numeric_limits::max(); + for (uint32_t uIndex = 0; uIndex < uChunkArraySize; uIndex++) + { + if (m_arrayChunks[uIndex]) + { + uChunkCount++; + uOldestChunkIndex = std::min(uOldestChunkIndex, m_arrayChunks[uIndex]->m_uChunkLastAccessed); + } + } + + // Check if we have too many chunks, and delete the oldest if so. + if (uChunkCount > m_uChunkCountLimit) + { + m_arrayChunks[uOldestChunkIndex] = nullptr; + } + //m_mapChunks.insert(std::make_pair(v3dChunkPos, std::unique_ptr(pChunk))); /*for (uint32_t ct = iPosisionHash; ct < uChunkArraySize; ct++) From 143c9fd08d95932d6908bdbdd6bf37281e55a725 Mon Sep 17 00:00:00 2001 From: David Williams Date: Mon, 13 Apr 2015 21:34:59 +0200 Subject: [PATCH 09/12] Made test 10x longer. --- tests/testvolume.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/testvolume.cpp b/tests/testvolume.cpp index a35cdaa6..08747e68 100644 --- a/tests/testvolume.cpp +++ b/tests/testvolume.cpp @@ -225,7 +225,7 @@ int32_t testDirectRandomAccess(const VolumeType* volume) std::mt19937 rng; int32_t result = 0; - for (uint32_t ct = 0; ct < 1000000; ct++) + for (uint32_t ct = 0; ct < 10000000; ct++) { uint32_t rand = rng(); @@ -490,7 +490,7 @@ void TestVolume::testRawVolumeDirectRandomAccess() { result = testDirectRandomAccess(m_pRawVolume); } - QCOMPARE(result, static_cast(267192737)); + QCOMPARE(result, static_cast(171835633)); } void TestVolume::testPagedVolumeDirectRandomAccess() @@ -500,7 +500,7 @@ void TestVolume::testPagedVolumeDirectRandomAccess() { result = testDirectRandomAccess(m_pPagedVolumeHighMem); } - QCOMPARE(result, static_cast(267192737)); + QCOMPARE(result, static_cast(171835633)); } int32_t TestVolume::testPagedVolumeChunkAccess(uint16_t localityMask) From f7c196277389b8e75e6b91c24c1e4e8c8ec5ad87 Mon Sep 17 00:00:00 2001 From: David Williams Date: Mon, 13 Apr 2015 23:32:23 +0200 Subject: [PATCH 10/12] Removed commented-out code. --- include/PolyVox/PagedVolume.inl | 30 ------------------------------ 1 file changed, 30 deletions(-) diff --git a/include/PolyVox/PagedVolume.inl b/include/PolyVox/PagedVolume.inl index d70242c7..736c76e0 100644 --- a/include/PolyVox/PagedVolume.inl +++ b/include/PolyVox/PagedVolume.inl @@ -355,39 +355,9 @@ namespace PolyVox { m_arrayChunks[uOldestChunkIndex] = nullptr; } - - //m_mapChunks.insert(std::make_pair(v3dChunkPos, std::unique_ptr(pChunk))); - - /*for (uint32_t ct = iPosisionHash; ct < uChunkArraySize; ct++) - { - if (m_arrayChunks[ct] == nullptr) - { - m_arrayChunks[ct] = std::move(std::unique_ptr< Chunk >(pChunk)); - break; - } - }*/ - // As we are loading a new chunk we should try to ensure we don't go over our target memory usage. - /*while (m_mapChunks.size() > m_uChunkCountLimit) - { - // Find the least recently used chunk. Hopefully this isn't too slow. - auto itUnloadChunk = m_mapChunks.begin(); - for (auto i = m_mapChunks.begin(); i != m_mapChunks.end(); i++) - { - if (i->second->m_uChunkLastAccessed < itUnloadChunk->second->m_uChunkLastAccessed) - { - itUnloadChunk = i; - } - } - - // Erase the least recently used chunk - m_mapChunks.erase(itUnloadChunk); - }*/ - - } m_pLastAccessedChunk = pChunk; - //m_v3dLastAccessedChunkPos = v3dChunkPos; m_v3dLastAccessedChunkX = uChunkX; m_v3dLastAccessedChunkY = uChunkY; m_v3dLastAccessedChunkZ = uChunkZ; From 1e0e8a8c167b6c786bd36cd616e6868319e7d476 Mon Sep 17 00:00:00 2001 From: David Williams Date: Mon, 13 Apr 2015 23:48:33 +0200 Subject: [PATCH 11/12] Fixed calculation of volume size in bytes. --- include/PolyVox/PagedVolume.inl | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/include/PolyVox/PagedVolume.inl b/include/PolyVox/PagedVolume.inl index 736c76e0..93e4e51f 100644 --- a/include/PolyVox/PagedVolume.inl +++ b/include/PolyVox/PagedVolume.inl @@ -371,9 +371,18 @@ namespace PolyVox template uint32_t PagedVolume::calculateSizeInBytes(void) { + uint32_t uChunkCount = 0; + for (uint32_t uIndex = 0; uIndex < uChunkArraySize; uIndex++) + { + if (m_arrayChunks[uIndex]) + { + uChunkCount++; + } + } + // Note: We disregard the size of the other class members as they are likely to be very small compared to the size of the // allocated voxel data. This also keeps the reported size as a power of two, which makes other memory calculations easier. - return PagedVolume::Chunk::calculateSizeInBytes(m_uChunkSideLength) * m_mapChunks.size(); + return PagedVolume::Chunk::calculateSizeInBytes(m_uChunkSideLength) * uChunkCount; } } From 12fdeb8e525ae922e717a0fb8eafbfaccd6f5cab Mon Sep 17 00:00:00 2001 From: David Williams Date: Mon, 13 Apr 2015 23:51:18 +0200 Subject: [PATCH 12/12] Removed old chunk map. Removed flush(Region) function as it's a bit trickier to implement with the new hash table, and it's not clear that we need it. --- examples/Paging/main.cpp | 6 +++--- include/PolyVox/PagedVolume.h | 4 +--- include/PolyVox/PagedVolume.inl | 9 ++++++--- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/examples/Paging/main.cpp b/examples/Paging/main.cpp index 3cb5415c..5b4bb191 100644 --- a/examples/Paging/main.cpp +++ b/examples/Paging/main.cpp @@ -164,9 +164,9 @@ protected: std::cout << "Memory usage: " << (volData.calculateSizeInBytes() / 1024.0 / 1024.0) << "MB" << std::endl; //std::cout << "Compression ratio: 1 to " << (1.0/(volData.calculateCompressionRatio())) << std::endl; PolyVox::Region reg2(Vector3DInt32(0, 0, 0), Vector3DInt32(255, 255, 255)); - std::cout << "Flushing region: " << reg2.getLowerCorner() << " -> " << reg2.getUpperCorner() << std::endl; - volData.flush(reg2); - std::cout << "Memory usage: " << (volData.calculateSizeInBytes() / 1024.0 / 1024.0) << "MB" << std::endl; + //std::cout << "Flushing region: " << reg2.getLowerCorner() << " -> " << reg2.getUpperCorner() << std::endl; + //volData.flush(reg2); + //std::cout << "Memory usage: " << (volData.calculateSizeInBytes() / 1024.0 / 1024.0) << "MB" << std::endl; //std::cout << "Compression ratio: 1 to " << (1.0/(volData.calculateCompressionRatio())) << std::endl; std::cout << "Flushing entire volume" << std::endl; volData.flushAll(); diff --git a/include/PolyVox/PagedVolume.h b/include/PolyVox/PagedVolume.h index 826f3b29..6c28a633 100644 --- a/include/PolyVox/PagedVolume.h +++ b/include/PolyVox/PagedVolume.h @@ -274,7 +274,7 @@ namespace PolyVox /// Tries to ensure that the voxels within the specified Region are loaded into memory. void prefetch(Region regPrefetch); /// Ensures that any voxels within the specified Region are removed from memory. - void flush(Region regFlush); + //void flush(Region regFlush); /// Removes all voxels from memory void flushAll(); @@ -301,8 +301,6 @@ namespace PolyVox mutable int32_t m_v3dLastAccessedChunkZ = 0; mutable Chunk* m_pLastAccessedChunk = nullptr; - mutable std::unordered_map > m_mapChunks; - mutable uint32_t m_uTimestamper = 0; uint32_t m_uChunkCountLimit = 0; diff --git a/include/PolyVox/PagedVolume.inl b/include/PolyVox/PagedVolume.inl index 93e4e51f..7c650e43 100644 --- a/include/PolyVox/PagedVolume.inl +++ b/include/PolyVox/PagedVolume.inl @@ -220,13 +220,16 @@ namespace PolyVox m_pLastAccessedChunk = nullptr; // Erase all the most recently used chunks. - m_mapChunks.clear(); + for (uint32_t uIndex = 0; uIndex < uChunkArraySize; uIndex++) + { + m_arrayChunks[uIndex] = nullptr; + } } //////////////////////////////////////////////////////////////////////////////// /// Removes all voxels in the specified Region from memory, and calls dataOverflowHandler() to ensure the application has a chance to store the data. It is possible that there are no voxels loaded in the Region, in which case the function will have no effect. //////////////////////////////////////////////////////////////////////////////// - template + /*template void PagedVolume::flush(Region regFlush) { // Clear this pointer in case the chunk it points at is flushed. @@ -256,7 +259,7 @@ namespace PolyVox } } } - } + }*/ template bool PagedVolume::canReuseLastAccessedChunk(int32_t iChunkX, int32_t iChunkY, int32_t iChunkZ) const