Optimise buddy allocator to not search too high (#606)

The buddy allocator doesn't need to look at sizes above the current
highest size. This commit tracks the highest block that is stored in the
buddy allocator.

src/snmalloc/backend_helpers/buddy.h

@@ -16,6 +16,8 @@ namespace snmalloc
   class Buddy
   {
     std::array<RBTree<Rep>, MAX_SIZE_BITS - MIN_SIZE_BITS> trees;
+    // All RBtrees at or above this index should be empty.
+    size_t empty_at_or_above = 0;
 
     size_t to_index(size_t size)
     {
@@ -37,6 +39,16 @@ namespace snmalloc
       UNUSED(addr, size);
     }
 
+    void invariant()
+    {
+#ifndef NDEBUG
+      for (size_t i = empty_at_or_above; i < trees.size(); i++)
+      {
+        SNMALLOC_ASSERT(trees[i].is_empty());
+      }
+#endif
+    }
+
   public:
     constexpr Buddy() = default;
     /**
@@ -52,6 +64,7 @@ namespace snmalloc
     typename Rep::Contents add_block(typename Rep::Contents addr, size_t size)
     {
       auto idx = to_index(size);
+      empty_at_or_above = bits::max(empty_at_or_above, idx + 1);
 
       validate_block(addr, size);
 
@@ -74,8 +87,13 @@ namespace snmalloc
           size *= 2;
           addr = Rep::align_down(addr, size);
           if (size == bits::one_at_bit(MAX_SIZE_BITS))
+          {
+            // Invariant should be checked on all non-tail return paths.
+            // Holds trivially here with current design.
+            invariant();
             // Too big for this buddy allocator.
             return addr;
+          }
           return add_block(addr, size);
         }
 
@@ -87,6 +105,7 @@ namespace snmalloc
         trees[idx].find(path, addr);
       }
       trees[idx].insert_path(path, addr);
+      invariant();
       return Rep::null;
     }
 
@@ -97,7 +116,10 @@ namespace snmalloc
      */
     typename Rep::Contents remove_block(size_t size)
     {
+      invariant();
       auto idx = to_index(size);
+      if (idx >= empty_at_or_above)
+        return Rep::null;
 
       auto addr = trees[idx].remove_min();
       if (addr != Rep::null)
@@ -112,7 +134,11 @@ namespace snmalloc
 
       auto bigger = remove_block(size * 2);
       if (bigger == Rep::null)
+      {
+        empty_at_or_above = idx;
+        invariant();
         return Rep::null;
+      }
 
       auto second = Rep::offset(bigger, size);
 

src/snmalloc/ds_core/redblacktree.h

@@ -730,9 +730,14 @@ namespace snmalloc
       invariant();
     }
 
+    bool is_empty()
+    {
+      return get_root().is_null();
+    }
+
     K remove_min()
     {
-      if (get_root().is_null())
+      if (is_empty())
         return Rep::null;
 
       auto path = get_root_path();
@@ -748,7 +753,7 @@ namespace snmalloc
 
     bool remove_elem(K value)
     {
-      if (get_root().is_null())
+      if (is_empty())
         return false;
 
       auto path = get_root_path();