Merge branch 'master' into feature-small-buffers
diff --git a/include/vk_mem_alloc.h b/include/vk_mem_alloc.h
index 47f79db..71f8990 100644
--- a/include/vk_mem_alloc.h
+++ b/include/vk_mem_alloc.h
@@ -7262,6 +7262,8 @@
#endif
};
+struct VmaBlockVector;
+
/*
Represents a single block of device memory (`VkDeviceMemory`) with all the
data about its regions (aka suballocations, #VmaAllocation), assigned and free.
@@ -7285,6 +7287,7 @@
// Always call after construction.
void Init(
VmaAllocator hAllocator,
+ VmaBlockVector* parentBlockVector,
VmaPool hParentPool,
uint32_t newMemoryTypeIndex,
VkDeviceMemory newMemory,
@@ -7294,6 +7297,7 @@
// Always call before destruction.
void Destroy(VmaAllocator allocator);
+ VmaBlockVector* GetParentBlockVector() const { return m_ParentBlockVector; }
VmaPool GetParentPool() const { return m_hParentPool; }
VkDeviceMemory GetDeviceMemory() const { return m_hMemory; }
uint32_t GetMemoryTypeIndex() const { return m_MemoryTypeIndex; }
@@ -7326,10 +7330,11 @@
const void* pNext);
private:
- VmaPool m_hParentPool; // VK_NULL_HANDLE if not belongs to custom pool.
- uint32_t m_MemoryTypeIndex;
- uint32_t m_Id;
- VkDeviceMemory m_hMemory;
+ VmaBlockVector* m_ParentBlockVector = VMA_NULL;
+ VmaPool m_hParentPool = VK_NULL_HANDLE; // VK_NULL_HANDLE if not belongs to custom pool.
+ uint32_t m_MemoryTypeIndex = UINT32_MAX;
+ uint32_t m_Id = 0;
+ VkDeviceMemory m_hMemory = VK_NULL_HANDLE;
/*
Protects access to m_hMemory so it's not used by multiple threads simultaneously, e.g. vkMapMemory, vkBindBufferMemory.
@@ -7337,8 +7342,8 @@
Allocations, deallocations, any change in m_pMetadata is protected by parent's VmaBlockVector::m_Mutex.
*/
VMA_MUTEX m_Mutex;
- uint32_t m_MapCount;
- void* m_pMappedData;
+ uint32_t m_MapCount = 0;
+ void* m_pMappedData = VMA_NULL;
};
struct VmaDefragmentationMove
@@ -8235,6 +8240,7 @@
// Default pools.
VmaBlockVector* m_pBlockVectors[VK_MAX_MEMORY_TYPES];
+ VmaBlockVector* m_pSmallBufferBlockVectors[VK_MAX_MEMORY_TYPES];
typedef VmaIntrusiveLinkedList<VmaDedicatedAllocationListItemTraits> DedicatedAllocationLinkedList;
DedicatedAllocationLinkedList m_DedicatedAllocations[VK_MAX_MEMORY_TYPES];
@@ -12656,18 +12662,13 @@
////////////////////////////////////////////////////////////////////////////////
// class VmaDeviceMemoryBlock
-VmaDeviceMemoryBlock::VmaDeviceMemoryBlock(VmaAllocator hAllocator) :
- m_pMetadata(VMA_NULL),
- m_MemoryTypeIndex(UINT32_MAX),
- m_Id(0),
- m_hMemory(VK_NULL_HANDLE),
- m_MapCount(0),
- m_pMappedData(VMA_NULL)
+VmaDeviceMemoryBlock::VmaDeviceMemoryBlock(VmaAllocator hAllocator)
{
}
void VmaDeviceMemoryBlock::Init(
VmaAllocator hAllocator,
+ VmaBlockVector* parentBlockVector,
VmaPool hParentPool,
uint32_t newMemoryTypeIndex,
VkDeviceMemory newMemory,
@@ -12675,8 +12676,10 @@
uint32_t id,
uint32_t algorithm)
{
+ VMA_ASSERT(parentBlockVector != VMA_NULL);
VMA_ASSERT(m_hMemory == VK_NULL_HANDLE);
+ m_ParentBlockVector = parentBlockVector;
m_hParentPool = hParentPool;
m_MemoryTypeIndex = newMemoryTypeIndex;
m_Id = id;
@@ -13485,6 +13488,8 @@
void VmaBlockVector::Free(
const VmaAllocation hAllocation)
{
+ VMA_ASSERT(hAllocation->GetBlock()->GetParentBlockVector() == this);
+
VmaDeviceMemoryBlock* pBlockToDelete = VMA_NULL;
bool budgetExceeded = false;
@@ -13717,6 +13722,7 @@
VmaDeviceMemoryBlock* const pBlock = vma_new(m_hAllocator, VmaDeviceMemoryBlock)(m_hAllocator);
pBlock->Init(
m_hAllocator,
+ this, // parentBlockVector
m_hParentPool,
m_MemoryTypeIndex,
mem,
@@ -16146,6 +16152,7 @@
memset(&m_MemProps, 0, sizeof(m_MemProps));
memset(&m_pBlockVectors, 0, sizeof(m_pBlockVectors));
+ memset(&m_pSmallBufferBlockVectors, 0, sizeof(m_pSmallBufferBlockVectors));
memset(&m_VulkanFunctions, 0, sizeof(m_VulkanFunctions));
#if VMA_EXTERNAL_MEMORY
@@ -16216,6 +16223,20 @@
0.5f, // priority (0.5 is the default per Vulkan spec)
GetMemoryTypeMinAlignment(memTypeIndex), // minAllocationAlignment
VMA_NULL); // // pMemoryAllocateNext
+ m_pSmallBufferBlockVectors[memTypeIndex] = vma_new(this, VmaBlockVector)(
+ this,
+ VK_NULL_HANDLE, // hParentPool
+ memTypeIndex,
+ preferredBlockSize,
+ 0,
+ SIZE_MAX,
+ 1, // bufferImageGranularity forced to 1 !!!
+ pCreateInfo->frameInUseCount,
+ false, // explicitBlockSize
+ false, // linearAlgorithm
+ 0.5f, // priority (0.5 is the default per Vulkan spec)
+ GetMemoryTypeMinAlignment(memTypeIndex), // minAllocationAlignment
+ VMA_NULL); // // pMemoryAllocateNext
// No need to call m_pBlockVectors[memTypeIndex][blockVectorTypeIndex]->CreateMinBlocks here,
// becase minBlockCount is 0.
}
@@ -16279,6 +16300,7 @@
VMA_ASSERT(0 && "Unfreed dedicated allocations found.");
}
+ vma_delete(this, m_pSmallBufferBlockVectors[memTypeIndex]);
vma_delete(this, m_pBlockVectors[memTypeIndex]);
}
}
@@ -16534,7 +16556,8 @@
finalCreateInfo.flags |= VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT;
}
- VmaBlockVector* const blockVector = m_pBlockVectors[memTypeIndex];
+ bool isSmallBuffer = dedicatedBuffer != VK_NULL_HANDLE && size <= 4096; // TODO
+ VmaBlockVector* const blockVector = isSmallBuffer ? m_pSmallBufferBlockVectors[memTypeIndex] : m_pBlockVectors[memTypeIndex];
VMA_ASSERT(blockVector);
const VkDeviceSize preferredBlockSize = blockVector->GetPreferredBlockSize();
@@ -17071,8 +17094,7 @@
}
else
{
- const uint32_t memTypeIndex = allocation->GetMemoryTypeIndex();
- pBlockVector = m_pBlockVectors[memTypeIndex];
+ pBlockVector = allocation->GetBlock()->GetParentBlockVector();
}
pBlockVector->Free(allocation);
}
@@ -17108,6 +17130,10 @@
VmaBlockVector* const pBlockVector = m_pBlockVectors[memTypeIndex];
VMA_ASSERT(pBlockVector);
pBlockVector->AddStats(pStats);
+
+ VmaBlockVector* const pSmallBufferBlockVector = m_pSmallBufferBlockVectors[memTypeIndex];
+ VMA_ASSERT(pSmallBufferBlockVector);
+ pSmallBufferBlockVector->AddStats(pStats);
}
// Process custom pools.
@@ -18105,6 +18131,7 @@
json.EndObject();
}
+ // Default pools
{
bool allocationsStarted = false;
for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
@@ -18131,6 +18158,33 @@
}
}
+ // Small buffer pools
+ {
+ bool allocationsStarted = false;
+ for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
+ {
+ if(m_pSmallBufferBlockVectors[memTypeIndex]->IsEmpty() == false)
+ {
+ if(allocationsStarted == false)
+ {
+ allocationsStarted = true;
+ json.WriteString("SmallBufferPools");
+ json.BeginObject();
+ }
+
+ json.BeginString("Type ");
+ json.ContinueString(memTypeIndex);
+ json.EndString();
+
+ m_pSmallBufferBlockVectors[memTypeIndex]->PrintDetailedMap(json);
+ }
+ }
+ if(allocationsStarted)
+ {
+ json.EndObject();
+ }
+ }
+
// Custom pools
{
VmaMutexLockRead lock(m_PoolsMutex, m_UseMutex);
