snmalloc/src/mem/chunkmap.h

#pragma once

using namespace std;

#include "../ds/address.h"
#include "largealloc.h"
#include "mediumslab.h"
#include "pagemap.h"
#include "slab.h"

namespace snmalloc
{
  enum ChunkMapSuperslabKind : uint8_t
  {
    CMNotOurs = 0,
    CMSuperslab = 1,
    CMMediumslab = 2,

    /*
     * Values 3 (inclusive) through SUPERSLAB_BITS (exclusive) are as yet
     * unused.
     *
     * Values SUPERSLAB_BITS (inclusive) through 64 (exclusive, as it would
     * represent the entire address space) are used for log2(size) at the
     * heads of large allocations.  See SuperslabMap::set_large_size.
     */
    CMLargeMin = SUPERSLAB_BITS,
    CMLargeMax = 63,

    /*
     * Values 64 (inclusive) through 64 + SUPERSLAB_BITS (exclusive) are unused
     */

    /*
     * Values 64 + SUPERSLAB_BITS (inclusive) through 128 (exclusive) are used
     * for entries within a large allocation.  A value of x at pagemap entry p
     * indicates that there are at least 2^(x-64) (inclusive) and at most
     * 2^(x+1-64) (exclusive) page map entries between p and the start of the
     * allocation.  See ChunkMap::set_large_size and external_address's
     * handling of large reallocation redirections.
     */
    CMLargeRangeMin = 64 + SUPERSLAB_BITS,
    CMLargeRangeMax = 127,

    /*
     * Values 128 (inclusive) through 255 (inclusive) are as yet unused.
     */

  };

  /*
   * Ensure that ChunkMapSuperslabKind values are actually disjoint, i.e.,
   * that large allocations don't land on CMMediumslab.
   */
  static_assert(
    SUPERSLAB_BITS > CMMediumslab, "Large allocations may be too small");

#ifndef SNMALLOC_MAX_FLATPAGEMAP_SIZE
/*
 * Unless otherwise specified, use a flat pagemap for the chunkmap (1 byte per
 * Superslab-sized and -aligned region of the address space) if either of the
 * following hold:
 *
 *   - the platform supports LazyCommit and the flat structure would occupy 256
 *     MiB or less.  256 MiB is more than adequate for 32-bit architectures and
 *     is the size of the flat pagemap for a 48-bit AS with the default chunk
 *     size or the USE_LARGE_CHUNKS chunksize (that is, configurations other
 *     than USE_SMALL_CHUNKS).
 *
 *   - the platform does not support LazyCommit but the flat structure would
 *     occupy less than PAGEMAP_NODE_SIZE (i.e., the backing store for an
 *     internal tree node in the non-flat pagemap).
 */
#  define SNMALLOC_MAX_FLATPAGEMAP_SIZE \
    (pal_supports<LazyCommit, Pal> ? 256ULL * 1024 * 1024 : PAGEMAP_NODE_SIZE)
#endif
  static constexpr bool CHUNKMAP_USE_FLATPAGEMAP =
    SNMALLOC_MAX_FLATPAGEMAP_SIZE >=
    sizeof(FlatPagemap<SUPERSLAB_BITS, uint8_t>);

  using ChunkmapPagemap = std::conditional_t<
    CHUNKMAP_USE_FLATPAGEMAP,
    FlatPagemap<SUPERSLAB_BITS, uint8_t>,
    Pagemap<SUPERSLAB_BITS, uint8_t, 0, DefaultPrimAlloc>>;

  struct ForChunkmap
  {};
  using GlobalChunkmap = GlobalPagemapTemplate<ChunkmapPagemap, ForChunkmap>;

  /**
   * Optionally exported function that accesses the global chunkmap pagemap
   * provided by a shared library.
   */
  extern "C" void*
  snmalloc_chunkmap_global_get(snmalloc::PagemapConfig const**);

  /**
   * Class that defines an interface to the pagemap.  This is provided to
   * `Allocator` as a template argument and so can be replaced by a compatible
   * implementation (for example, to move pagemap updates to a different
   * protection domain).
   */
  template<typename PagemapProvider = GlobalChunkmap>
  struct DefaultChunkMap
  {
    /**
     * Get the pagemap entry corresponding to a specific address.
     *
     * Despite the type, the return value is an enum ChunkMapSuperslabKind
     * or one of the reserved values described therewith.
     */
    static uint8_t get(address_t p)
    {
      return PagemapProvider::pagemap().get(p);
    }

    /**
     * Get the pagemap entry corresponding to a specific address.
     */
    static uint8_t get(void* p)
    {
      return get(address_cast(p));
    }

    /**
     * Set a pagemap entry indicating that there is a superslab at the
     * specified index.
     */
    static void set_slab(Superslab* slab)
    {
      set(slab, static_cast<size_t>(CMSuperslab));
    }
    /**
     * Add a pagemap entry indicating that a medium slab has been allocated.
     */
    static void set_slab(Mediumslab* slab)
    {
      set(slab, static_cast<size_t>(CMMediumslab));
    }
    /**
     * Remove an entry from the pagemap corresponding to a superslab.
     */
    static void clear_slab(Superslab* slab)
    {
      SNMALLOC_ASSERT(get(slab) == CMSuperslab);
      set(slab, static_cast<size_t>(CMNotOurs));
    }
    /**
     * Remove an entry corresponding to a medium slab.
     */
    static void clear_slab(Mediumslab* slab)
    {
      SNMALLOC_ASSERT(get(slab) == CMMediumslab);
      set(slab, static_cast<size_t>(CMNotOurs));
    }
    /**
     * Update the pagemap to reflect a large allocation, of `size` bytes from
     * address `p`.
     */
    static void set_large_size(void* p, size_t size)
    {
      size_t size_bits = bits::next_pow2_bits(size);
      set(p, static_cast<uint8_t>(size_bits));
      // Set redirect slide
      auto ss = address_cast(p) + SUPERSLAB_SIZE;
      for (size_t i = 0; i < size_bits - SUPERSLAB_BITS; i++)
      {
        size_t run = bits::one_at_bit(i);
        PagemapProvider::pagemap().set_range(
          ss, static_cast<uint8_t>(CMLargeRangeMin + i), run);
        ss = ss + SUPERSLAB_SIZE * run;
      }
    }
    /**
     * Update the pagemap to remove a large allocation, of `size` bytes from
     * address `p`.
     */
    static void clear_large_size(void* vp, size_t size)
    {
      auto p = address_cast(vp);
      size_t rounded_size = bits::next_pow2(size);
      SNMALLOC_ASSERT(get(p) == bits::next_pow2_bits(size));
      auto count = rounded_size >> SUPERSLAB_BITS;
      PagemapProvider::pagemap().set_range(p, CMNotOurs, count);
    }

  private:
    /**
     * Helper function to set a pagemap entry.  This is not part of the public
     * interface and exists to make it easy to reuse the code in the public
     * methods in other pagemap adaptors.
     */
    static void set(void* p, uint8_t x)
    {
      PagemapProvider::pagemap().set(address_cast(p), x);
    }
  };

#ifndef SNMALLOC_DEFAULT_CHUNKMAP
#  define SNMALLOC_DEFAULT_CHUNKMAP snmalloc::DefaultChunkMap<>
#endif

} // namespace snmalloc