Rizin
unix-like reverse engineering framework and cli tools
outqueue.c File Reference

Output queue handling in multithreaded coding. More...

#include "outqueue.h"

Go to the source code of this file.

Macros

#define BUF_SIZE_MAX   (UINT64_MAX / LZMA_THREADS_MAX / 2 / 2)
 

Functions

static lzma_ret get_options (uint64_t *bufs_alloc_size, uint32_t *bufs_count, uint64_t buf_size_max, uint32_t threads)
 
uint64_t lzma_outq_memusage (uint64_t buf_size_max, uint32_t threads)
 Calculate the memory usage of an output queue. More...
 
lzma_ret lzma_outq_init (lzma_outq *outq, const lzma_allocator *allocator, uint64_t buf_size_max, uint32_t threads)
 Initialize an output queue. More...
 
void lzma_outq_end (lzma_outq *outq, const lzma_allocator *allocator)
 Free the memory associated with the output queue. More...
 
lzma_outbuflzma_outq_get_buf (lzma_outq *outq)
 Get a new buffer. More...
 
bool lzma_outq_is_readable (const lzma_outq *outq)
 Test if there is data ready to be read. More...
 
lzma_ret lzma_outq_read (lzma_outq *restrict outq, uint8_t *restrict out, size_t *restrict out_pos, size_t out_size, lzma_vli *restrict unpadded_size, lzma_vli *restrict uncompressed_size)
 Read finished data. More...
 

Detailed Description

Output queue handling in multithreaded coding.

Definition in file outqueue.c.

Macro Definition Documentation

◆ BUF_SIZE_MAX

#define BUF_SIZE_MAX   (UINT64_MAX / LZMA_THREADS_MAX / 2 / 2)

This is to ease integer overflow checking: We may allocate up to 2 * LZMA_THREADS_MAX buffers and we need some extra memory for other data structures (that's the second /2).

Definition at line 19 of file outqueue.c.

Function Documentation

◆ get_options()

static lzma_ret get_options ( uint64_t bufs_alloc_size,
uint32_t bufs_count,
uint64_t  buf_size_max,
uint32_t  threads 
)
static

Definition at line 23 of file outqueue.c.

25 {
26  if (threads > LZMA_THREADS_MAX || buf_size_max > BUF_SIZE_MAX)
27  return LZMA_OPTIONS_ERROR;
28 
29  // The number of buffers is twice the number of threads.
30  // This wastes RAM but keeps the threads busy when buffers
31  // finish out of order.
32  //
33  // NOTE: If this is changed, update BUF_SIZE_MAX too.
34  *bufs_count = threads * 2;
35  *bufs_alloc_size = *bufs_count * buf_size_max;
36 
37  return LZMA_OK;
38 }
#define BUF_SIZE_MAX
Definition: outqueue.c:19
#define LZMA_THREADS_MAX
Definition: common.h:56
static uv_thread_t * threads
Definition: threadpool.c:38
@ LZMA_OPTIONS_ERROR
Invalid or unsupported options.
Definition: base.h:160
@ LZMA_OK
Operation completed successfully.
Definition: base.h:58

References BUF_SIZE_MAX, LZMA_OK, LZMA_OPTIONS_ERROR, LZMA_THREADS_MAX, and threads.

Referenced by lzma_outq_init(), and lzma_outq_memusage().

◆ lzma_outq_end()

void lzma_outq_end ( lzma_outq outq,
const lzma_allocator allocator 
)

Free the memory associated with the output queue.

Definition at line 101 of file outqueue.c.

102 {
103  lzma_free(outq->bufs, allocator);
104  outq->bufs = NULL;
105 
106  lzma_free(outq->bufs_mem, allocator);
107  outq->bufs_mem = NULL;
108 
109  return;
110 }
const lzma_allocator * allocator
Definition: block.h:377
#define NULL
Definition: cris-opc.c:27
lzma_outbuf * bufs
Array of buffers that are used cyclically.
Definition: outqueue.h:39
uint8_t * bufs_mem
Memory allocated for all the buffers.
Definition: outqueue.h:42
void lzma_free(void *ptr, const lzma_allocator *allocator)
Frees memory.
Definition: common.c:78

References allocator, lzma_outq::bufs, lzma_outq::bufs_mem, lzma_free(), and NULL.

Referenced by lzma_outq_init(), and stream_encoder_mt_end().

◆ lzma_outq_get_buf()

lzma_outbuf* lzma_outq_get_buf ( lzma_outq outq)

Get a new buffer.

lzma_outq_has_buf() must be used to check that there is a buffer available before calling lzma_outq_get_buf().

Definition at line 114 of file outqueue.c.

115 {
116  // Caller must have checked it with lzma_outq_has_buf().
117  assert(outq->bufs_used < outq->bufs_allocated);
118 
119  // Initialize the new buffer.
120  lzma_outbuf *buf = &outq->bufs[outq->bufs_pos];
121  buf->buf = outq->bufs_mem + outq->bufs_pos * outq->buf_size_max;
122  buf->size = 0;
123  buf->finished = false;
124 
125  // Update the queue state.
126  if (++outq->bufs_pos == outq->bufs_allocated)
127  outq->bufs_pos = 0;
128 
129  ++outq->bufs_used;
130 
131  return buf;
132 }
voidpf void * buf
Definition: ioapi.h:138
assert(limit<=UINT32_MAX/2)
Output buffer for a single thread.
Definition: outqueue.h:17
uint8_t * buf
Pointer to the output buffer of lzma_outq.buf_size_max bytes.
Definition: outqueue.h:19
size_t buf_size_max
Amount of buffer space available in each buffer.
Definition: outqueue.h:45
uint32_t bufs_allocated
Number of buffers allocated.
Definition: outqueue.h:48
uint32_t bufs_pos
Definition: outqueue.h:52
uint32_t bufs_used
Number of buffers in use.
Definition: outqueue.h:55

References assert(), lzma_outbuf::buf, lzma_outq::buf_size_max, lzma_outq::bufs, lzma_outq::bufs_allocated, lzma_outq::bufs_mem, lzma_outq::bufs_pos, and lzma_outq::bufs_used.

Referenced by get_thread().

◆ lzma_outq_init()

lzma_ret lzma_outq_init ( lzma_outq outq,
const lzma_allocator allocator,
uint64_t  buf_size_max,
uint32_t  threads 
)

Initialize an output queue.

Parameters
outqPointer to an output queue. Before calling this function the first time, *outq should have been zeroed with memzero() so that this function knows that there are no previous allocations to free.
allocatorPointer to allocator or NULL
buf_size_maxMaximum amount of data that a single buffer in the queue may need to store.
threadsNumber of buffers that may be in use concurrently. Note that more than this number of buffers will actually get allocated to improve performance when buffers finish out of order.
Returns
- LZMA_OK
  • LZMA_MEM_ERROR

Definition at line 57 of file outqueue.c.

59 {
60  uint64_t bufs_alloc_size;
61  uint32_t bufs_count;
62 
63  // Set bufs_count and bufs_alloc_size.
64  return_if_error(get_options(&bufs_alloc_size, &bufs_count,
65  buf_size_max, threads));
66 
67  // Allocate memory if needed.
68  if (outq->buf_size_max != buf_size_max
69  || outq->bufs_allocated != bufs_count) {
70  lzma_outq_end(outq, allocator);
71 
72 #if SIZE_MAX < UINT64_MAX
73  if (bufs_alloc_size > SIZE_MAX)
74  return LZMA_MEM_ERROR;
75 #endif
76 
77  outq->bufs = lzma_alloc(bufs_count * sizeof(lzma_outbuf),
78  allocator);
79  outq->bufs_mem = lzma_alloc((size_t)(bufs_alloc_size),
80  allocator);
81 
82  if (outq->bufs == NULL || outq->bufs_mem == NULL) {
83  lzma_outq_end(outq, allocator);
84  return LZMA_MEM_ERROR;
85  }
86  }
87 
88  // Initialize the rest of the main structure. Initialization of
89  // outq->bufs[] is done when they are actually needed.
90  outq->buf_size_max = (size_t)(buf_size_max);
91  outq->bufs_allocated = bufs_count;
92  outq->bufs_pos = 0;
93  outq->bufs_used = 0;
94  outq->read_pos = 0;
95 
96  return LZMA_OK;
97 }
void lzma_outq_end(lzma_outq *outq, const lzma_allocator *allocator)
Free the memory associated with the output queue.
Definition: outqueue.c:101
static lzma_ret get_options(uint64_t *bufs_alloc_size, uint32_t *bufs_count, uint64_t buf_size_max, uint32_t threads)
Definition: outqueue.c:23
int size_t
Definition: sftypes.h:40
unsigned int uint32_t
Definition: sftypes.h:29
unsigned long uint64_t
Definition: sftypes.h:28
#define SIZE_MAX
size_t read_pos
Position in the buffer in lzma_outq_read()
Definition: outqueue.h:58
#define return_if_error(expr)
Return if expression doesn't evaluate to LZMA_OK.
Definition: common.h:278
void * lzma_alloc(size_t size, const lzma_allocator *allocator) lzma_attribute((__malloc__)) lzma_attr_alloc_size(1)
Allocates memory.
@ LZMA_MEM_ERROR
Cannot allocate memory.
Definition: base.h:128

References allocator, lzma_outq::buf_size_max, lzma_outq::bufs, lzma_outq::bufs_allocated, lzma_outq::bufs_mem, lzma_outq::bufs_pos, lzma_outq::bufs_used, get_options(), lzma_alloc(), LZMA_MEM_ERROR, LZMA_OK, lzma_outq_end(), NULL, lzma_outq::read_pos, return_if_error, SIZE_MAX, and threads.

Referenced by stream_encoder_mt_init().

◆ lzma_outq_is_readable()

bool lzma_outq_is_readable ( const lzma_outq outq)

Test if there is data ready to be read.

Call to this function must be protected with the same mutex that is used to protect lzma_outbuf.finished.

Definition at line 136 of file outqueue.c.

137 {
138  uint32_t i = outq->bufs_pos - outq->bufs_used;
139  if (outq->bufs_pos < outq->bufs_used)
140  i += outq->bufs_allocated;
141 
142  return outq->bufs[i].finished;
143 }
lzma_index ** i
Definition: index.h:629
bool finished
Definition: outqueue.h:32

References lzma_outq::bufs, lzma_outq::bufs_allocated, lzma_outq::bufs_pos, lzma_outq::bufs_used, lzma_outbuf::finished, and i.

Referenced by wait_for_work().

◆ lzma_outq_memusage()

uint64_t lzma_outq_memusage ( uint64_t  buf_size_max,
uint32_t  threads 
)

Calculate the memory usage of an output queue.

Returns
Approximate memory usage in bytes or UINT64_MAX on error.

Definition at line 42 of file outqueue.c.

43 {
44  uint64_t bufs_alloc_size;
45  uint32_t bufs_count;
46 
47  if (get_options(&bufs_alloc_size, &bufs_count, buf_size_max, threads)
48  != LZMA_OK)
49  return UINT64_MAX;
50 
51  return sizeof(lzma_outq) + bufs_count * sizeof(lzma_outbuf)
52  + bufs_alloc_size;
53 }
#define UINT64_MAX

References get_options(), LZMA_OK, threads, and UINT64_MAX.

Referenced by LZMA_API().

◆ lzma_outq_read()

lzma_ret lzma_outq_read ( lzma_outq *restrict  outq,
uint8_t *restrict  out,
size_t *restrict  out_pos,
size_t  out_size,
lzma_vli *restrict  unpadded_size,
lzma_vli *restrict  uncompressed_size 
)

Read finished data.

Parameters
outqPointer to an output queue
outBeginning of the output buffer
out_posThe next byte will be written to out[*out_pos].
out_sizeSize of the out buffer; the first byte into which no data is written to is out[out_size].
unpadded_sizeUnpadded Size from the Block encoder
uncompressed_sizeUncompressed Size from the Block encoder
Returns
- LZMA: All OK. Either no data was available or the buffer being read didn't become empty yet.
  • LZMA_STREAM_END: The buffer being read was finished. *unpadded_size and *uncompressed_size were set.
Note
This reads lzma_outbuf.finished variables and thus call to this function needs to be protected with a mutex.

Definition at line 147 of file outqueue.c.

151 {
152  // There must be at least one buffer from which to read.
153  if (outq->bufs_used == 0)
154  return LZMA_OK;
155 
156  // Get the buffer.
157  uint32_t i = outq->bufs_pos - outq->bufs_used;
158  if (outq->bufs_pos < outq->bufs_used)
159  i += outq->bufs_allocated;
160 
161  lzma_outbuf *buf = &outq->bufs[i];
162 
163  // If it isn't finished yet, we cannot read from it.
164  if (!buf->finished)
165  return LZMA_OK;
166 
167  // Copy from the buffer to output.
168  lzma_bufcpy(buf->buf, &outq->read_pos, buf->size,
169  out, out_pos, out_size);
170 
171  // Return if we didn't get all the data from the buffer.
172  if (outq->read_pos < buf->size)
173  return LZMA_OK;
174 
175  // The buffer was finished. Tell the caller its size information.
176  *unpadded_size = buf->unpadded_size;
177  *uncompressed_size = buf->uncompressed_size;
178 
179  // Free this buffer for further use.
180  --outq->bufs_used;
181  outq->read_pos = 0;
182 
183  return LZMA_STREAM_END;
184 }
const lzma_allocator lzma_vli unpadded_size
Definition: index.h:345
const lzma_allocator const uint8_t size_t uint8_t size_t * out_pos
Definition: block.h:528
const lzma_allocator const uint8_t size_t uint8_t * out
Definition: block.h:528
uint64_t uncompressed_size
Definition: list.c:106
@ LZMA_STREAM_END
End of stream was reached.
Definition: base.h:63
size_t lzma_bufcpy(const uint8_t *restrict in, size_t *restrict in_pos, size_t in_size, uint8_t *restrict out, size_t *restrict out_pos, size_t out_size)
Definition: common.c:94

References i, lzma_bufcpy(), LZMA_OK, LZMA_STREAM_END, out, out_pos, uncompressed_size, and unpadded_size.

Referenced by stream_encode_mt().