jemalloc_internal.h

Go to the documentation of this file.

#ifndef JEMALLOC_INTERNAL_H
#define JEMALLOC_INTERNAL_H
 
#include "jemalloc_internal_defs.h"
#include "jemalloc_internal_decls.h"
 
#ifdef JEMALLOC_UTRACE
#include <sys/ktrace.h>
#endif
 
#define JEMALLOC_NO_DEMANGLE
#ifdef JEMALLOC_JET
#  define JEMALLOC_N(n) jet_##n
#  include "public_namespace.h"
#  define JEMALLOC_NO_RENAME
#  include "../jemalloc.h"
#  undef JEMALLOC_NO_RENAME
#else
#  define JEMALLOC_N(n) je_##n
#  include "../jemalloc.h"
#endif
#include "private_namespace.h"
 
static const bool config_debug =
#ifdef JEMALLOC_DEBUG
    true
#else
    false
#endif
    ;
static const bool have_dss =
#ifdef JEMALLOC_DSS
    true
#else
    false
#endif
    ;
static const bool config_fill =
#ifdef JEMALLOC_FILL
    true
#else
    false
#endif
    ;
static const bool config_lazy_lock =
#ifdef JEMALLOC_LAZY_LOCK
    true
#else
    false
#endif
    ;
static const char * const config_malloc_conf = JEMALLOC_CONFIG_MALLOC_CONF;
static const bool config_prof =
#ifdef JEMALLOC_PROF
    true
#else
    false
#endif
    ;
static const bool config_prof_libgcc =
#ifdef JEMALLOC_PROF_LIBGCC
    true
#else
    false
#endif
    ;
static const bool config_prof_libunwind =
#ifdef JEMALLOC_PROF_LIBUNWIND
    true
#else
    false
#endif
    ;
static const bool maps_coalesce =
#ifdef JEMALLOC_MAPS_COALESCE
    true
#else
    false
#endif
    ;
static const bool config_munmap =
#ifdef JEMALLOC_MUNMAP
    true
#else
    false
#endif
    ;
static const bool config_stats =
#ifdef JEMALLOC_STATS
    true
#else
    false
#endif
    ;
static const bool config_tcache =
#ifdef JEMALLOC_TCACHE
    true
#else
    false
#endif
    ;
static const bool config_thp =
#ifdef JEMALLOC_THP
    true
#else
    false
#endif
    ;
static const bool config_tls =
#ifdef JEMALLOC_TLS
    true
#else
    false
#endif
    ;
static const bool config_utrace =
#ifdef JEMALLOC_UTRACE
    true
#else
    false
#endif
    ;
static const bool config_valgrind =
#ifdef JEMALLOC_VALGRIND
    true
#else
    false
#endif
    ;
static const bool config_xmalloc =
#ifdef JEMALLOC_XMALLOC
    true
#else
    false
#endif
    ;
static const bool config_ivsalloc =
#ifdef JEMALLOC_IVSALLOC
    true
#else
    false
#endif
    ;
static const bool config_cache_oblivious =
#ifdef JEMALLOC_CACHE_OBLIVIOUS
    true
#else
    false
#endif
    ;
 
#ifdef JEMALLOC_ATOMIC9
#include <machine/atomic.h>
#endif
 
#if (defined(JEMALLOC_OSATOMIC) || defined(JEMALLOC_OSSPIN))
#include <libkern/OSAtomic.h>
#endif
 
#ifdef JEMALLOC_ZONE
#include <mach/mach_error.h>
#include <mach/mach_init.h>
#include <mach/vm_map.h>
#endif
 
#include "ph.h"
#ifndef __PGI
#define RB_COMPACT
#endif
#include "rb.h"
#include "qr.h"
#include "ql.h"
 
/*
 * jemalloc can conceptually be broken into components (arena, tcache, etc.),
 * but there are circular dependencies that cannot be broken without
 * substantial performance degradation.  In order to reduce the effect on
 * visual code flow, read the header files in multiple passes, with one of the
 * following cpp variables defined during each pass:
 *
 *   JEMALLOC_H_TYPES   : Preprocessor-defined constants and psuedo-opaque data
 *                        types.
 *   JEMALLOC_H_STRUCTS : Data structures.
 *   JEMALLOC_H_EXTERNS : Extern data declarations and function prototypes.
 *   JEMALLOC_H_INLINES : Inline functions.
 */
/******************************************************************************/
#define JEMALLOC_H_TYPES
 
#include "jemalloc_internal_macros.h"
 
/* Page size index type. */
typedef unsigned pszind_t;
 
/* Size class index type. */
typedef unsigned szind_t;
 
/*
 * Flags bits:
 *
 * a: arena
 * t: tcache
 * 0: unused
 * z: zero
 * n: alignment
 *
 * aaaaaaaa aaaatttt tttttttt 0znnnnnn
 */
#define MALLOCX_ARENA_MASK  ((int)~0xfffff)
#define MALLOCX_ARENA_MAX   0xffe
#define MALLOCX_TCACHE_MASK ((int)~0xfff000ffU)
#define MALLOCX_TCACHE_MAX  0xffd
#define MALLOCX_LG_ALIGN_MASK   ((int)0x3f)
/* Use MALLOCX_ALIGN_GET() if alignment may not be specified in flags. */
#define MALLOCX_ALIGN_GET_SPECIFIED(flags)              \
    (ZU(1) << (flags & MALLOCX_LG_ALIGN_MASK))
#define MALLOCX_ALIGN_GET(flags)                    \
    (MALLOCX_ALIGN_GET_SPECIFIED(flags) & (SIZE_T_MAX-1))
#define MALLOCX_ZERO_GET(flags)                     \
    ((bool)(flags & MALLOCX_ZERO))
 
#define MALLOCX_TCACHE_GET(flags)                   \
    (((unsigned)((flags & MALLOCX_TCACHE_MASK) >> 8)) - 2)
#define MALLOCX_ARENA_GET(flags)                    \
    (((unsigned)(((unsigned)flags) >> 20)) - 1)
 
/* Smallest size class to support. */
#define TINY_MIN        (1U << LG_TINY_MIN)
 
/*
 * Minimum allocation alignment is 2^LG_QUANTUM bytes (ignoring tiny size
 * classes).
 */
#ifndef LG_QUANTUM
#  if (defined(__i386__) || defined(_M_IX86))
#    define LG_QUANTUM      4
#  endif
#  ifdef __ia64__
#    define LG_QUANTUM      4
#  endif
#  ifdef __alpha__
#    define LG_QUANTUM      4
#  endif
#  if (defined(__sparc64__) || defined(__sparcv9) || defined(__sparc_v9__))
#    define LG_QUANTUM      4
#  endif
#  if (defined(__amd64__) || defined(__x86_64__) || defined(_M_X64))
#    define LG_QUANTUM      4
#  endif
#  ifdef __arm__
#    define LG_QUANTUM      3
#  endif
#  ifdef __aarch64__
#    define LG_QUANTUM      4
#  endif
#  ifdef __hppa__
#    define LG_QUANTUM      4
#  endif
#  ifdef __m68k__
#    define LG_QUANTUM          3
#  endif
#  ifdef __mips__
#    define LG_QUANTUM      3
#  endif
#  ifdef __or1k__
#    define LG_QUANTUM      3
#  endif
#  ifdef __powerpc__
#    define LG_QUANTUM      4
#  endif
#  if defined(__riscv) || defined(__riscv__)
#    define LG_QUANTUM      4
#  endif
#  ifdef __s390__
#    define LG_QUANTUM      4
#  endif
#  ifdef __SH4__
#    define LG_QUANTUM      4
#  endif
#  ifdef __tile__
#    define LG_QUANTUM      4
#  endif
#  ifdef __le32__
#    define LG_QUANTUM      4
#  endif
#  ifndef LG_QUANTUM
#    error "Unknown minimum alignment for architecture; specify via "
     "--with-lg-quantum"
#  endif
#endif
 
#define QUANTUM         ((size_t)(1U << LG_QUANTUM))
#define QUANTUM_MASK        (QUANTUM - 1)
 
/* Return the smallest quantum multiple that is >= a. */
#define QUANTUM_CEILING(a)                      \
    (((a) + QUANTUM_MASK) & ~QUANTUM_MASK)
 
#define LONG            ((size_t)(1U << LG_SIZEOF_LONG))
#define LONG_MASK       (LONG - 1)
 
/* Return the smallest long multiple that is >= a. */
#define LONG_CEILING(a)                         \
    (((a) + LONG_MASK) & ~LONG_MASK)
 
#define SIZEOF_PTR      (1U << LG_SIZEOF_PTR)
#define PTR_MASK        (SIZEOF_PTR - 1)
 
/* Return the smallest (void *) multiple that is >= a. */
#define PTR_CEILING(a)                          \
    (((a) + PTR_MASK) & ~PTR_MASK)
 
/*
 * Maximum size of L1 cache line.  This is used to avoid cache line aliasing.
 * In addition, this controls the spacing of cacheline-spaced size classes.
 *
 * CACHELINE cannot be based on LG_CACHELINE because __declspec(align()) can
 * only handle raw constants.
 */
#define LG_CACHELINE        6
#define CACHELINE       64
#define CACHELINE_MASK      (CACHELINE - 1)
 
/* Return the smallest cacheline multiple that is >= s. */
#define CACHELINE_CEILING(s)                        \
    (((s) + CACHELINE_MASK) & ~CACHELINE_MASK)
 
/* Page size.  LG_PAGE is determined by the configure script. */
#ifdef PAGE_MASK
#  undef PAGE_MASK
#endif
#define PAGE        ((size_t)(1U << LG_PAGE))
#define PAGE_MASK   ((size_t)(PAGE - 1))
 
/* Return the page base address for the page containing address a. */
#define PAGE_ADDR2BASE(a)                       \
    ((void *)((uintptr_t)(a) & ~PAGE_MASK))
 
/* Return the smallest pagesize multiple that is >= s. */
#define PAGE_CEILING(s)                         \
    (((s) + PAGE_MASK) & ~PAGE_MASK)
 
/* Return the nearest aligned address at or below a. */
#define ALIGNMENT_ADDR2BASE(a, alignment)               \
    ((void *)((uintptr_t)(a) & ((~(alignment)) + 1)))
 
/* Return the offset between a and the nearest aligned address at or below a. */
#define ALIGNMENT_ADDR2OFFSET(a, alignment)             \
    ((size_t)((uintptr_t)(a) & (alignment - 1)))
 
/* Return the smallest alignment multiple that is >= s. */
#define ALIGNMENT_CEILING(s, alignment)                 \
    (((s) + (alignment - 1)) & ((~(alignment)) + 1))
 
/* Declare a variable-length array. */
#if __STDC_VERSION__ < 199901L
#  ifdef _MSC_VER
#    include <malloc.h>
#    define alloca _alloca
#  else
#    include <stdlib.h>
#  endif
#  define VARIABLE_ARRAY(type, name, count) \
    type *name = alloca(sizeof(type) * (count))
#else
#  define VARIABLE_ARRAY(type, name, count) type name[(count)]
#endif
 
#include "nstime.h"
#include "valgrind.h"
#include "util.h"
#include "atomic.h"
#include "spin.h"
#include "prng.h"
#include "ticker.h"
#include "ckh.h"
#include "size_classes.h"
#include "smoothstep.h"
#include "stats.h"
#include "ctl.h"
#include "witness.h"
#include "mutex.h"
#include "tsd.h"
#include "mb.h"
#include "extent.h"
#include "arena.h"
#include "bitmap.h"
#include "base.h"
#include "rtree.h"
#include "pages.h"
#include "chunk.h"
#include "huge.h"
#include "tcache.h"
#include "hash.h"
#include "quarantine.h"
#include "prof.h"
 
#undef JEMALLOC_H_TYPES
/******************************************************************************/
#define JEMALLOC_H_STRUCTS
 
#include "nstime.h"
#include "valgrind.h"
#include "util.h"
#include "atomic.h"
#include "spin.h"
#include "prng.h"
#include "ticker.h"
#include "ckh.h"
#include "size_classes.h"
#include "smoothstep.h"
#include "stats.h"
#include "ctl.h"
#include "witness.h"
#include "mutex.h"
#include "mb.h"
#include "bitmap.h"
#define JEMALLOC_ARENA_STRUCTS_A
#include "arena.h"
#undef JEMALLOC_ARENA_STRUCTS_A
#include "extent.h"
#define JEMALLOC_ARENA_STRUCTS_B
#include "arena.h"
#undef JEMALLOC_ARENA_STRUCTS_B
#include "base.h"
#include "rtree.h"
#include "pages.h"
#include "chunk.h"
#include "huge.h"
#include "tcache.h"
#include "hash.h"
#include "quarantine.h"
#include "prof.h"
 
#include "tsd.h"
 
#undef JEMALLOC_H_STRUCTS
/******************************************************************************/
#define JEMALLOC_H_EXTERNS
 
extern bool opt_abort;
extern const char   *opt_junk;
extern bool opt_junk_alloc;
extern bool opt_junk_free;
extern size_t   opt_quarantine;
extern bool opt_redzone;
extern bool opt_utrace;
extern bool opt_xmalloc;
extern bool opt_zero;
extern unsigned opt_narenas;
 
extern bool in_valgrind;
 
/* Number of CPUs. */
extern unsigned ncpus;
 
/* Number of arenas used for automatic multiplexing of threads and arenas. */
extern unsigned narenas_auto;
 
/*
 * Arenas that are used to service external requests.  Not all elements of the
 * arenas array are necessarily used; arenas are created lazily as needed.
 */
extern arena_t  **arenas;
 
/*
 * pind2sz_tab encodes the same information as could be computed by
 * pind2sz_compute().
 */
extern size_t const pind2sz_tab[NPSIZES];
/*
 * index2size_tab encodes the same information as could be computed (at
 * unacceptable cost in some code paths) by index2size_compute().
 */
extern size_t const index2size_tab[NSIZES];
/*
 * size2index_tab is a compact lookup table that rounds request sizes up to
 * size classes.  In order to reduce cache footprint, the table is compressed,
 * and all accesses are via size2index().
 */
extern uint8_t const    size2index_tab[];
 
arena_t *a0get(void);
void    *a0malloc(size_t size);
void    a0dalloc(void *ptr);
void    *bootstrap_malloc(size_t size);
void    *bootstrap_calloc(size_t num, size_t size);
void    bootstrap_free(void *ptr);
unsigned    narenas_total_get(void);
arena_t *arena_init(tsdn_t *tsdn, unsigned ind);
arena_tdata_t   *arena_tdata_get_hard(tsd_t *tsd, unsigned ind);
arena_t *arena_choose_hard(tsd_t *tsd, bool internal);
void    arena_migrate(tsd_t *tsd, unsigned oldind, unsigned newind);
void    thread_allocated_cleanup(tsd_t *tsd);
void    thread_deallocated_cleanup(tsd_t *tsd);
void    iarena_cleanup(tsd_t *tsd);
void    arena_cleanup(tsd_t *tsd);
void    arenas_tdata_cleanup(tsd_t *tsd);
void    narenas_tdata_cleanup(tsd_t *tsd);
void    arenas_tdata_bypass_cleanup(tsd_t *tsd);
void    jemalloc_prefork(void);
void    jemalloc_postfork_parent(void);
void    jemalloc_postfork_child(void);
 
#include "nstime.h"
#include "valgrind.h"
#include "util.h"
#include "atomic.h"
#include "spin.h"
#include "prng.h"
#include "ticker.h"
#include "ckh.h"
#include "size_classes.h"
#include "smoothstep.h"
#include "stats.h"
#include "ctl.h"
#include "witness.h"
#include "mutex.h"
#include "mb.h"
#include "bitmap.h"
#include "extent.h"
#include "arena.h"
#include "base.h"
#include "rtree.h"
#include "pages.h"
#include "chunk.h"
#include "huge.h"
#include "tcache.h"
#include "hash.h"
#include "quarantine.h"
#include "prof.h"
#include "tsd.h"
 
#undef JEMALLOC_H_EXTERNS
/******************************************************************************/
#define JEMALLOC_H_INLINES
 
#include "nstime.h"
#include "valgrind.h"
#include "util.h"
#include "atomic.h"
#include "spin.h"
#include "prng.h"
#include "ticker.h"
#include "ckh.h"
#include "size_classes.h"
#include "smoothstep.h"
#include "stats.h"
#include "ctl.h"
#include "tsd.h"
#include "witness.h"
#include "mutex.h"
#include "mb.h"
#include "extent.h"
#include "base.h"
#include "rtree.h"
#include "pages.h"
#include "chunk.h"
#include "huge.h"
 
#ifndef JEMALLOC_ENABLE_INLINE
pszind_t    psz2ind(size_t psz);
size_t  pind2sz_compute(pszind_t pind);
size_t  pind2sz_lookup(pszind_t pind);
size_t  pind2sz(pszind_t pind);
size_t  psz2u(size_t psz);
szind_t size2index_compute(size_t size);
szind_t size2index_lookup(size_t size);
szind_t size2index(size_t size);
size_t  index2size_compute(szind_t index);
size_t  index2size_lookup(szind_t index);
size_t  index2size(szind_t index);
size_t  s2u_compute(size_t size);
size_t  s2u_lookup(size_t size);
size_t  s2u(size_t size);
size_t  sa2u(size_t size, size_t alignment);
arena_t *arena_choose_impl(tsd_t *tsd, arena_t *arena, bool internal);
arena_t *arena_choose(tsd_t *tsd, arena_t *arena);
arena_t *arena_ichoose(tsd_t *tsd, arena_t *arena);
arena_tdata_t   *arena_tdata_get(tsd_t *tsd, unsigned ind,
    bool refresh_if_missing);
arena_t *arena_get(tsdn_t *tsdn, unsigned ind, bool init_if_missing);
ticker_t    *decay_ticker_get(tsd_t *tsd, unsigned ind);
#endif
 
 
#include "bitmap.h"
/*
 * Include portions of arena.h interleaved with tcache.h in order to resolve
 * circular dependencies.
 */
#define JEMALLOC_ARENA_INLINE_A
#include "arena.h"
#undef JEMALLOC_ARENA_INLINE_A
#include "tcache.h"
#define JEMALLOC_ARENA_INLINE_B
#include "arena.h"
#undef JEMALLOC_ARENA_INLINE_B
#include "hash.h"
#include "quarantine.h"
 
#endif /* JEMALLOC_INTERNAL_H */