jemalloc_internal.h File Reference

#include "jemalloc_internal_defs.h"
#include "jemalloc_internal_decls.h"
#include "../jemalloc.h"
#include "private_namespace.h"
#include "ph.h"
#include "rb.h"
#include "qr.h"
#include "ql.h"
#include "jemalloc_internal_macros.h"
#include <stdlib.h>
#include "nstime.h"
#include "valgrind.h"
#include "util.h"
#include "atomic.h"
#include "spin.h"
#include "prng.h"
#include "ticker.h"
#include "ckh.h"
#include "size_classes.h"
#include "smoothstep.h"
#include "stats.h"
#include "ctl.h"
#include "witness.h"
#include "mutex.h"
#include "tsd.h"
#include "mb.h"
#include "extent.h"
#include "arena.h"
#include "bitmap.h"
#include "base.h"
#include "rtree.h"
#include "pages.h"
#include "chunk.h"
#include "huge.h"
#include "tcache.h"
#include "hash.h"
#include "quarantine.h"
#include "prof.h"

Go to the source code of this file.

Macros
#define	JEMALLOC_NO_DEMANGLE
#define	JEMALLOC_N(n)   je_##n
#define	RB_COMPACT
#define	JEMALLOC_H_TYPES
#define	MALLOCX_ARENA_MASK   ((int)~0xfffff)
#define	MALLOCX_ARENA_MAX   0xffe
#define	MALLOCX_TCACHE_MASK   ((int)~0xfff000ffU)
#define	MALLOCX_TCACHE_MAX   0xffd
#define	MALLOCX_LG_ALIGN_MASK   ((int)0x3f)
#define	MALLOCX_ALIGN_GET_SPECIFIED(flags)    (ZU(1) << (flags & MALLOCX_LG_ALIGN_MASK))
#define	MALLOCX_ALIGN_GET(flags)    (MALLOCX_ALIGN_GET_SPECIFIED(flags) & (SIZE_T_MAX-1))
#define	MALLOCX_ZERO_GET(flags)    ((bool)(flags & MALLOCX_ZERO))
#define	MALLOCX_TCACHE_GET(flags)    (((unsigned)((flags & MALLOCX_TCACHE_MASK) >> 8)) - 2)
#define	MALLOCX_ARENA_GET(flags)    (((unsigned)(((unsigned)flags) >> 20)) - 1)
#define	TINY_MIN   (1U << LG_TINY_MIN)
#define	QUANTUM   ((size_t)(1U << LG_QUANTUM))
#define	QUANTUM_MASK   (QUANTUM - 1)
#define	QUANTUM_CEILING(a)    (((a) + QUANTUM_MASK) & ~QUANTUM_MASK)
#define	LONG   ((size_t)(1U << LG_SIZEOF_LONG))
#define	LONG_MASK   (LONG - 1)
#define	LONG_CEILING(a)    (((a) + LONG_MASK) & ~LONG_MASK)
#define	SIZEOF_PTR   (1U << LG_SIZEOF_PTR)
#define	PTR_MASK   (SIZEOF_PTR - 1)
#define	PTR_CEILING(a)    (((a) + PTR_MASK) & ~PTR_MASK)
#define	LG_CACHELINE   6
#define	CACHELINE   64
#define	CACHELINE_MASK   (CACHELINE - 1)
#define	CACHELINE_CEILING(s)    (((s) + CACHELINE_MASK) & ~CACHELINE_MASK)
#define	PAGE   ((size_t)(1U << LG_PAGE))
#define	PAGE_MASK   ((size_t)(PAGE - 1))
#define	PAGE_ADDR2BASE(a)    ((void *)((uintptr_t)(a) & ~PAGE_MASK))
#define	PAGE_CEILING(s)    (((s) + PAGE_MASK) & ~PAGE_MASK)
#define	ALIGNMENT_ADDR2BASE(a, alignment)    ((void *)((uintptr_t)(a) & ((~(alignment)) + 1)))
#define	ALIGNMENT_ADDR2OFFSET(a, alignment)    ((size_t)((uintptr_t)(a) & (alignment - 1)))
#define	ALIGNMENT_CEILING(s, alignment)    (((s) + (alignment - 1)) & ((~(alignment)) + 1))
#define	VARIABLE_ARRAY(type, name, count)    type *name = alloca(sizeof(type) * (count))
#define	JEMALLOC_H_STRUCTS
#define	JEMALLOC_ARENA_STRUCTS_A
#define	JEMALLOC_ARENA_STRUCTS_B
#define	JEMALLOC_H_EXTERNS
#define	JEMALLOC_H_INLINES
#define	JEMALLOC_ARENA_INLINE_A
#define	JEMALLOC_ARENA_INLINE_B

Typedefs
typedef unsigned	pszind_t
typedef unsigned	szind_t

Functions
arena_t *	a0get (void)
void *	a0malloc (size_t size)
void	a0dalloc (void *ptr)
void *	bootstrap_malloc (size_t size)
void *	bootstrap_calloc (size_t num, size_t size)
void	bootstrap_free (void *ptr)
unsigned	narenas_total_get (void)
arena_t *	arena_init (tsdn_t *tsdn, unsigned ind)
arena_tdata_t *	arena_tdata_get_hard (tsd_t *tsd, unsigned ind)
arena_t *	arena_choose_hard (tsd_t *tsd, bool internal)
void	arena_migrate (tsd_t *tsd, unsigned oldind, unsigned newind)
void	thread_allocated_cleanup (tsd_t *tsd)
void	thread_deallocated_cleanup (tsd_t *tsd)
void	iarena_cleanup (tsd_t *tsd)
void	arena_cleanup (tsd_t *tsd)
void	arenas_tdata_cleanup (tsd_t *tsd)
void	narenas_tdata_cleanup (tsd_t *tsd)
void	arenas_tdata_bypass_cleanup (tsd_t *tsd)
void	jemalloc_prefork (void)
void	jemalloc_postfork_parent (void)
void	jemalloc_postfork_child (void)
pszind_t	psz2ind (size_t psz)
size_t	pind2sz_compute (pszind_t pind)
size_t	pind2sz_lookup (pszind_t pind)
size_t	pind2sz (pszind_t pind)
size_t	psz2u (size_t psz)
szind_t	size2index_compute (size_t size)
szind_t	size2index_lookup (size_t size)
szind_t	size2index (size_t size)
size_t	index2size_compute (szind_t index)
size_t	index2size_lookup (szind_t index)
size_t	index2size (szind_t index)
size_t	s2u_compute (size_t size)
size_t	s2u_lookup (size_t size)
size_t	s2u (size_t size)
size_t	sa2u (size_t size, size_t alignment)
arena_t *	arena_choose_impl (tsd_t *tsd, arena_t *arena, bool internal)
arena_t *	arena_choose (tsd_t *tsd, arena_t *arena)
arena_t *	arena_ichoose (tsd_t *tsd, arena_t *arena)
arena_tdata_t *	arena_tdata_get (tsd_t *tsd, unsigned ind, bool refresh_if_missing)
arena_t *	arena_get (tsdn_t *tsdn, unsigned ind, bool init_if_missing)
ticker_t *	decay_ticker_get (tsd_t *tsd, unsigned ind)

Variables
static const bool	config_debug
static const bool	have_dss
static const bool	config_fill
static const bool	config_lazy_lock
static const char *const	config_malloc_conf = JEMALLOC_CONFIG_MALLOC_CONF
static const bool	config_prof
static const bool	config_prof_libgcc
static const bool	config_prof_libunwind
static const bool	maps_coalesce
static const bool	config_munmap
static const bool	config_stats
static const bool	config_tcache
static const bool	config_thp
static const bool	config_tls
static const bool	config_utrace
static const bool	config_valgrind
static const bool	config_xmalloc
static const bool	config_ivsalloc
static const bool	config_cache_oblivious
bool	opt_abort
const char *	opt_junk
bool	opt_junk_alloc
bool	opt_junk_free
size_t	opt_quarantine
bool	opt_redzone
bool	opt_utrace
bool	opt_xmalloc
bool	opt_zero
unsigned	opt_narenas
bool	in_valgrind
unsigned	ncpus
unsigned	narenas_auto
arena_t **	arenas
size_t const	pind2sz_tab [NPSIZES]
size_t const	index2size_tab [NSIZES]
uint8_t const	size2index_tab []

Macro Definition Documentation

ALIGNMENT_ADDR2BASE

#define ALIGNMENT_ADDR2BASE	(		a,
			alignment
	)		((void *)((uintptr_t)(a) & ((~(alignment)) + 1)))

Definition at line 344 of file jemalloc_internal.h.

ALIGNMENT_ADDR2OFFSET

#define ALIGNMENT_ADDR2OFFSET	(		a,
			alignment
	)		((size_t)((uintptr_t)(a) & (alignment - 1)))

Definition at line 348 of file jemalloc_internal.h.

ALIGNMENT_CEILING

#define ALIGNMENT_CEILING	(		s,
			alignment
	)		(((s) + (alignment - 1)) & ((~(alignment)) + 1))

Definition at line 352 of file jemalloc_internal.h.

CACHELINE

#define CACHELINE   64

Definition at line 321 of file jemalloc_internal.h.

CACHELINE_CEILING

#define CACHELINE_CEILING

(

s

)

(((s) + CACHELINE_MASK) & ~CACHELINE_MASK)

Definition at line 325 of file jemalloc_internal.h.

CACHELINE_MASK

#define CACHELINE_MASK   (CACHELINE - 1)

Definition at line 322 of file jemalloc_internal.h.

JEMALLOC_ARENA_INLINE_A

#define JEMALLOC_ARENA_INLINE_A

Definition at line 592 of file jemalloc_internal.h.

JEMALLOC_ARENA_INLINE_B

#define JEMALLOC_ARENA_INLINE_B

Definition at line 596 of file jemalloc_internal.h.

JEMALLOC_ARENA_STRUCTS_A

#define JEMALLOC_ARENA_STRUCTS_A

Definition at line 418 of file jemalloc_internal.h.

JEMALLOC_ARENA_STRUCTS_B

#define JEMALLOC_ARENA_STRUCTS_B

Definition at line 422 of file jemalloc_internal.h.

JEMALLOC_H_EXTERNS

#define JEMALLOC_H_EXTERNS

Definition at line 439 of file jemalloc_internal.h.

JEMALLOC_H_INLINES

#define JEMALLOC_H_INLINES

Definition at line 536 of file jemalloc_internal.h.

JEMALLOC_H_STRUCTS

#define JEMALLOC_H_STRUCTS

Definition at line 400 of file jemalloc_internal.h.

JEMALLOC_H_TYPES

#define JEMALLOC_H_TYPES

Definition at line 188 of file jemalloc_internal.h.

JEMALLOC_N

#define JEMALLOC_N

(

n

)

je_##n

Definition at line 19 of file jemalloc_internal.h.

JEMALLOC_NO_DEMANGLE

#define JEMALLOC_NO_DEMANGLE

Definition at line 11 of file jemalloc_internal.h.

LG_CACHELINE

#define LG_CACHELINE   6

Definition at line 320 of file jemalloc_internal.h.

LONG

#define LONG   ((size_t)(1U << LG_SIZEOF_LONG))

Definition at line 299 of file jemalloc_internal.h.

LONG_CEILING

#define LONG_CEILING

(

a

)

(((a) + LONG_MASK) & ~LONG_MASK)

Definition at line 303 of file jemalloc_internal.h.

LONG_MASK

#define LONG_MASK   (LONG - 1)

Definition at line 300 of file jemalloc_internal.h.

MALLOCX_ALIGN_GET

#define MALLOCX_ALIGN_GET

(

flags

)

(MALLOCX_ALIGN_GET_SPECIFIED(flags) & (SIZE_T_MAX-1))

Definition at line 217 of file jemalloc_internal.h.

MALLOCX_ALIGN_GET_SPECIFIED

#define MALLOCX_ALIGN_GET_SPECIFIED

(

flags

)

(ZU(1) << (flags & MALLOCX_LG_ALIGN_MASK))

Definition at line 215 of file jemalloc_internal.h.

MALLOCX_ARENA_GET

#define MALLOCX_ARENA_GET

(

flags

)

(((unsigned)(((unsigned)flags) >> 20)) - 1)

Definition at line 224 of file jemalloc_internal.h.

MALLOCX_ARENA_MASK

#define MALLOCX_ARENA_MASK   ((int)~0xfffff)

Definition at line 209 of file jemalloc_internal.h.

MALLOCX_ARENA_MAX

#define MALLOCX_ARENA_MAX   0xffe

Definition at line 210 of file jemalloc_internal.h.

MALLOCX_LG_ALIGN_MASK

#define MALLOCX_LG_ALIGN_MASK   ((int)0x3f)

Definition at line 213 of file jemalloc_internal.h.

MALLOCX_TCACHE_GET

#define MALLOCX_TCACHE_GET

(

flags

)

(((unsigned)((flags & MALLOCX_TCACHE_MASK) >> 8)) - 2)

Definition at line 222 of file jemalloc_internal.h.

MALLOCX_TCACHE_MASK

#define MALLOCX_TCACHE_MASK   ((int)~0xfff000ffU)

Definition at line 211 of file jemalloc_internal.h.

MALLOCX_TCACHE_MAX

#define MALLOCX_TCACHE_MAX   0xffd

Definition at line 212 of file jemalloc_internal.h.

MALLOCX_ZERO_GET

#define MALLOCX_ZERO_GET

(

flags

)

((bool)(flags & MALLOCX_ZERO))

Definition at line 219 of file jemalloc_internal.h.

PAGE

#define PAGE   ((size_t)(1U << LG_PAGE))

Definition at line 332 of file jemalloc_internal.h.

PAGE_ADDR2BASE

#define PAGE_ADDR2BASE

(

a

)

((void *)((uintptr_t)(a) & ~PAGE_MASK))

Definition at line 336 of file jemalloc_internal.h.

PAGE_CEILING

#define PAGE_CEILING

(

s

)

(((s) + PAGE_MASK) & ~PAGE_MASK)

Definition at line 340 of file jemalloc_internal.h.

PAGE_MASK

#define PAGE_MASK   ((size_t)(PAGE - 1))

Definition at line 333 of file jemalloc_internal.h.

PTR_CEILING

#define PTR_CEILING

(

a

)

(((a) + PTR_MASK) & ~PTR_MASK)

Definition at line 310 of file jemalloc_internal.h.

PTR_MASK

#define PTR_MASK   (SIZEOF_PTR - 1)

Definition at line 307 of file jemalloc_internal.h.

QUANTUM

#define QUANTUM   ((size_t)(1U << LG_QUANTUM))

Definition at line 292 of file jemalloc_internal.h.

QUANTUM_CEILING

#define QUANTUM_CEILING

(

a

)

(((a) + QUANTUM_MASK) & ~QUANTUM_MASK)

Definition at line 296 of file jemalloc_internal.h.

QUANTUM_MASK

#define QUANTUM_MASK   (QUANTUM - 1)

Definition at line 293 of file jemalloc_internal.h.

RB_COMPACT

#define RB_COMPACT

Definition at line 168 of file jemalloc_internal.h.

SIZEOF_PTR

#define SIZEOF_PTR   (1U << LG_SIZEOF_PTR)

Definition at line 306 of file jemalloc_internal.h.

TINY_MIN

#define TINY_MIN   (1U << LG_TINY_MIN)

Definition at line 228 of file jemalloc_internal.h.

VARIABLE_ARRAY

#define VARIABLE_ARRAY	(		type,
			name,
			count
	)		type *name = alloca(sizeof(type) * (count))

Definition at line 363 of file jemalloc_internal.h.

Typedef Documentation

pszind_t

typedef unsigned pszind_t

Definition at line 193 of file jemalloc_internal.h.

szind_t

typedef unsigned szind_t

Definition at line 196 of file jemalloc_internal.h.

Function Documentation

a0dalloc()

void a0dalloc

(

void *

ptr

)

a0get()

arena_t* a0get

(

void

)

a0malloc()

void* a0malloc

(

size_t

size

)

arena_choose()

arena_t* arena_choose	(	tsd_t *	tsd,
		arena_t *	arena
	)

arena_choose_hard()

arena_t* arena_choose_hard	(	tsd_t *	tsd,
		bool	internal
	)

arena_choose_impl()

arena_t* arena_choose_impl	(	tsd_t *	tsd,
		arena_t *	arena,
		bool	internal
	)

arena_cleanup()

void arena_cleanup

(

tsd_t *

tsd

)

arena_get()

arena_t* arena_get	(	tsdn_t *	tsdn,
		unsigned	ind,
		bool	init_if_missing
	)

arena_ichoose()

arena_t* arena_ichoose	(	tsd_t *	tsd,
		arena_t *	arena
	)

arena_init()

arena_t* arena_init	(	tsdn_t *	tsdn,
		unsigned	ind
	)

arena_migrate()

void arena_migrate	(	tsd_t *	tsd,
		unsigned	oldind,
		unsigned	newind
	)

arena_tdata_get()

arena_tdata_t* arena_tdata_get	(	tsd_t *	tsd,
		unsigned	ind,
		bool	refresh_if_missing
	)

arena_tdata_get_hard()

arena_tdata_t* arena_tdata_get_hard	(	tsd_t *	tsd,
		unsigned	ind
	)

arenas_tdata_bypass_cleanup()

void arenas_tdata_bypass_cleanup

(

tsd_t *

tsd

)

arenas_tdata_cleanup()

void arenas_tdata_cleanup

(

tsd_t *

tsd

)

bootstrap_calloc()

void* bootstrap_calloc	(	size_t	num,
		size_t	size
	)

bootstrap_free()

void bootstrap_free

(

void *

ptr

)

bootstrap_malloc()

void* bootstrap_malloc

(

size_t

size

)

decay_ticker_get()

ticker_t* decay_ticker_get	(	tsd_t *	tsd,
		unsigned	ind
	)

iarena_cleanup()

void iarena_cleanup

(

tsd_t *

tsd

)

index2size()

size_t index2size

(

szind_t

index

)

index2size_compute()

size_t index2size_compute

(

szind_t

index

)

index2size_lookup()

size_t index2size_lookup

(

szind_t

index

)

jemalloc_postfork_child()

void jemalloc_postfork_child

(

void

)

jemalloc_postfork_parent()

void jemalloc_postfork_parent

(

void

)

jemalloc_prefork()

void jemalloc_prefork

(

void

)

narenas_tdata_cleanup()

void narenas_tdata_cleanup

(

tsd_t *

tsd

)

narenas_total_get()

unsigned narenas_total_get

(

void

)

pind2sz()

size_t pind2sz

(

pszind_t

pind

)

pind2sz_compute()

size_t pind2sz_compute

(

pszind_t

pind

)

pind2sz_lookup()

size_t pind2sz_lookup

(

pszind_t

pind

)

psz2ind()

pszind_t psz2ind

(

size_t

psz

)

psz2u()

size_t psz2u

(

size_t

psz

)

s2u()

size_t s2u

(

size_t

size

)

s2u_compute()

size_t s2u_compute

(

size_t

size

)

s2u_lookup()

size_t s2u_lookup

(

size_t

size

)

sa2u()

size_t sa2u	(	size_t	size,
		size_t	alignment
	)

size2index()

szind_t size2index

(

size_t

size

)

size2index_compute()

szind_t size2index_compute

(

size_t

size

)

size2index_lookup()

szind_t size2index_lookup

(

size_t

size

)

thread_allocated_cleanup()

void thread_allocated_cleanup

(

tsd_t *

tsd

)

thread_deallocated_cleanup()

void thread_deallocated_cleanup

(

tsd_t *

tsd

)

Variable Documentation

arenas

arena_t** arenas

extern

Referenced by jemalloc_get_bins(), jemalloc_get_chunks(), and jemalloc_print_narenas().

config_cache_oblivious

const bool config_cache_oblivious

static

Initial value:

=
 
    true

Definition at line 144 of file jemalloc_internal.h.

config_debug

const bool config_debug

static

Initial value:

=
 
 
 
    false

Definition at line 24 of file jemalloc_internal.h.

config_fill

const bool config_fill

static

Initial value:

=
 
    true

Definition at line 38 of file jemalloc_internal.h.

config_ivsalloc

const bool config_ivsalloc

static

Initial value:

=
 
 
 
    false

Definition at line 137 of file jemalloc_internal.h.

config_lazy_lock

const bool config_lazy_lock

static

Initial value:

=
 
 
 
    false

Definition at line 45 of file jemalloc_internal.h.

config_malloc_conf

const char* const config_malloc_conf = JEMALLOC_CONFIG_MALLOC_CONF

static

Definition at line 52 of file jemalloc_internal.h.

config_munmap

const bool config_munmap

static

Initial value:

=
 
 
 
    false

Definition at line 81 of file jemalloc_internal.h.

config_prof

const bool config_prof

static

Initial value:

=
 
 
 
    false

Definition at line 53 of file jemalloc_internal.h.

config_prof_libgcc

const bool config_prof_libgcc

static

Initial value:

=
 
 
 
    false

Definition at line 60 of file jemalloc_internal.h.

config_prof_libunwind

const bool config_prof_libunwind

static

Initial value:

=
 
 
 
    false

Definition at line 67 of file jemalloc_internal.h.

config_stats

const bool config_stats

static

Initial value:

=
 
    true

Definition at line 88 of file jemalloc_internal.h.

config_tcache

const bool config_tcache

static

Initial value:

=
 
    true

Definition at line 95 of file jemalloc_internal.h.

config_thp

const bool config_thp

static

Initial value:

=
 
    true

Definition at line 102 of file jemalloc_internal.h.

config_tls

const bool config_tls

static

Initial value:

=
 
    true

Definition at line 109 of file jemalloc_internal.h.

config_utrace

const bool config_utrace

static

Initial value:

=
 
 
 
    false

Definition at line 116 of file jemalloc_internal.h.

config_valgrind

const bool config_valgrind

static

Initial value:

=
 
 
 
    false

Definition at line 123 of file jemalloc_internal.h.

config_xmalloc

const bool config_xmalloc

static

Initial value:

=
 
 
 
    false

Definition at line 130 of file jemalloc_internal.h.

have_dss

const bool have_dss

static

Initial value:

=
 
    true

Definition at line 31 of file jemalloc_internal.h.

in_valgrind

bool in_valgrind

extern

index2size_tab

size_t const index2size_tab[NSIZES]

extern

maps_coalesce

const bool maps_coalesce

static

Initial value:

=
 
    true

Definition at line 74 of file jemalloc_internal.h.

narenas_auto

unsigned narenas_auto

extern

ncpus

unsigned ncpus

extern

Referenced by uv_cpu_info().

opt_abort

bool opt_abort

extern

opt_junk

const char* opt_junk

extern

opt_junk_alloc

bool opt_junk_alloc

extern

opt_junk_free

bool opt_junk_free

extern

opt_narenas

unsigned opt_narenas

extern

opt_quarantine

size_t opt_quarantine

extern

opt_redzone

bool opt_redzone

extern

opt_utrace

bool opt_utrace

extern

opt_xmalloc

bool opt_xmalloc

extern

opt_zero

bool opt_zero

extern

pind2sz_tab

size_t const pind2sz_tab[NPSIZES]

extern

size2index_tab

uint8_t const size2index_tab[]

extern