outqueue.c File Reference

Output queue handling in multithreaded coding. More...

#include "outqueue.h"

Go to the source code of this file.

Macros
#define	BUF_SIZE_MAX   (UINT64_MAX / LZMA_THREADS_MAX / 2 / 2)

Functions
static lzma_ret	get_options (uint64_t *bufs_alloc_size, uint32_t *bufs_count, uint64_t buf_size_max, uint32_t threads)
uint64_t	lzma_outq_memusage (uint64_t buf_size_max, uint32_t threads)
	Calculate the memory usage of an output queue. More...
lzma_ret	lzma_outq_init (lzma_outq *outq, const lzma_allocator *allocator, uint64_t buf_size_max, uint32_t threads)
	Initialize an output queue. More...
void	lzma_outq_end (lzma_outq *outq, const lzma_allocator *allocator)
	Free the memory associated with the output queue. More...
lzma_outbuf *	lzma_outq_get_buf (lzma_outq *outq)
	Get a new buffer. More...
bool	lzma_outq_is_readable (const lzma_outq *outq)
	Test if there is data ready to be read. More...
lzma_ret	lzma_outq_read (lzma_outq *restrict outq, uint8_t *restrict out, size_t *restrict out_pos, size_t out_size, lzma_vli *restrict unpadded_size, lzma_vli *restrict uncompressed_size)
	Read finished data. More...

Detailed Description

Output queue handling in multithreaded coding.

Definition in file outqueue.c.

Macro Definition Documentation

BUF_SIZE_MAX

#define BUF_SIZE_MAX   (UINT64_MAX / LZMA_THREADS_MAX / 2 / 2)

This is to ease integer overflow checking: We may allocate up to 2 * LZMA_THREADS_MAX buffers and we need some extra memory for other data structures (that's the second /2).

Definition at line 19 of file outqueue.c.

Function Documentation

get_options()

static lzma_ret get_options ( uint64_t *  bufs_alloc_size, uint32_t *  bufs_count, uint64_t  buf_size_max, uint32_t  threads  )

static

Definition at line 23 of file outqueue.c.

{
    if (threads > LZMA_THREADS_MAX || buf_size_max > BUF_SIZE_MAX)
        return LZMA_OPTIONS_ERROR;
 
    // The number of buffers is twice the number of threads.
    // This wastes RAM but keeps the threads busy when buffers
    // finish out of order.
    //
    // NOTE: If this is changed, update BUF_SIZE_MAX too.
    *bufs_count = threads * 2;
    *bufs_alloc_size = *bufs_count * buf_size_max;
 
    return LZMA_OK;
}

References BUF_SIZE_MAX, LZMA_OK, LZMA_OPTIONS_ERROR, LZMA_THREADS_MAX, and threads.

Referenced by lzma_outq_init(), and lzma_outq_memusage().

lzma_outq_end()

void lzma_outq_end	(	lzma_outq *	outq,
		const lzma_allocator *	allocator
	)

Free the memory associated with the output queue.

Definition at line 101 of file outqueue.c.

{
    lzma_free(outq->bufs, allocator);
    outq->bufs = NULL;
 
    lzma_free(outq->bufs_mem, allocator);
    outq->bufs_mem = NULL;
 
    return;
}

References allocator, lzma_outq::bufs, lzma_outq::bufs_mem, lzma_free(), and NULL.

Referenced by lzma_outq_init(), and stream_encoder_mt_end().

lzma_outq_get_buf()

lzma_outbuf* lzma_outq_get_buf

(

lzma_outq *

outq

)

Get a new buffer.

lzma_outq_has_buf() must be used to check that there is a buffer available before calling lzma_outq_get_buf().

Definition at line 114 of file outqueue.c.

{
    // Caller must have checked it with lzma_outq_has_buf().
    assert(outq->bufs_used < outq->bufs_allocated);
 
    // Initialize the new buffer.
    lzma_outbuf *buf = &outq->bufs[outq->bufs_pos];
    buf->buf = outq->bufs_mem + outq->bufs_pos * outq->buf_size_max;
    buf->size = 0;
    buf->finished = false;
 
    // Update the queue state.
    if (++outq->bufs_pos == outq->bufs_allocated)
        outq->bufs_pos = 0;
 
    ++outq->bufs_used;
 
    return buf;
}

References assert(), lzma_outbuf::buf, lzma_outq::buf_size_max, lzma_outq::bufs, lzma_outq::bufs_allocated, lzma_outq::bufs_mem, lzma_outq::bufs_pos, and lzma_outq::bufs_used.

Referenced by get_thread().

lzma_outq_init()

lzma_ret lzma_outq_init	(	lzma_outq *	outq,
		const lzma_allocator *	allocator,
		uint64_t	buf_size_max,
		uint32_t	threads
	)

Initialize an output queue.

Parameters

outq	Pointer to an output queue. Before calling this function the first time, *outq should have been zeroed with memzero() so that this function knows that there are no previous allocations to free.
allocator	Pointer to allocator or NULL
buf_size_max	Maximum amount of data that a single buffer in the queue may need to store.
threads	Number of buffers that may be in use concurrently. Note that more than this number of buffers will actually get allocated to improve performance when buffers finish out of order.

Returns

- LZMA_OK

• LZMA_MEM_ERROR

Definition at line 57 of file outqueue.c.

{
    uint64_t bufs_alloc_size;
    uint32_t bufs_count;
 
    // Set bufs_count and bufs_alloc_size.
    return_if_error(get_options(&bufs_alloc_size, &bufs_count,
            buf_size_max, threads));
 
    // Allocate memory if needed.
    if (outq->buf_size_max != buf_size_max
            || outq->bufs_allocated != bufs_count) {
        lzma_outq_end(outq, allocator);
 
#if SIZE_MAX < UINT64_MAX
        if (bufs_alloc_size > SIZE_MAX)
            return LZMA_MEM_ERROR;
#endif
 
        outq->bufs = lzma_alloc(bufs_count * sizeof(lzma_outbuf),
                allocator);
        outq->bufs_mem = lzma_alloc((size_t)(bufs_alloc_size),
                allocator);
 
        if (outq->bufs == NULL || outq->bufs_mem == NULL) {
            lzma_outq_end(outq, allocator);
            return LZMA_MEM_ERROR;
        }
    }
 
    // Initialize the rest of the main structure. Initialization of
    // outq->bufs[] is done when they are actually needed.
    outq->buf_size_max = (size_t)(buf_size_max);
    outq->bufs_allocated = bufs_count;
    outq->bufs_pos = 0;
    outq->bufs_used = 0;
    outq->read_pos = 0;
 
    return LZMA_OK;
}

References allocator, lzma_outq::buf_size_max, lzma_outq::bufs, lzma_outq::bufs_allocated, lzma_outq::bufs_mem, lzma_outq::bufs_pos, lzma_outq::bufs_used, get_options(), lzma_alloc(), LZMA_MEM_ERROR, LZMA_OK, lzma_outq_end(), NULL, lzma_outq::read_pos, return_if_error, SIZE_MAX, and threads.

Referenced by stream_encoder_mt_init().

lzma_outq_is_readable()

bool lzma_outq_is_readable

(

const lzma_outq *

outq

)

Test if there is data ready to be read.

Call to this function must be protected with the same mutex that is used to protect lzma_outbuf.finished.

Definition at line 136 of file outqueue.c.

{
    uint32_t i = outq->bufs_pos - outq->bufs_used;
    if (outq->bufs_pos < outq->bufs_used)
        i += outq->bufs_allocated;
 
    return outq->bufs[i].finished;
}

References lzma_outq::bufs, lzma_outq::bufs_allocated, lzma_outq::bufs_pos, lzma_outq::bufs_used, lzma_outbuf::finished, and i.

Referenced by wait_for_work().

lzma_outq_memusage()

uint64_t lzma_outq_memusage	(	uint64_t	buf_size_max,
		uint32_t	threads
	)

Calculate the memory usage of an output queue.

Returns

Approximate memory usage in bytes or UINT64_MAX on error.

Definition at line 42 of file outqueue.c.

{
    uint64_t bufs_alloc_size;
    uint32_t bufs_count;
 
    if (get_options(&bufs_alloc_size, &bufs_count, buf_size_max, threads)
            != LZMA_OK)
        return UINT64_MAX;
 
    return sizeof(lzma_outq) + bufs_count * sizeof(lzma_outbuf)
            + bufs_alloc_size;
}

References get_options(), LZMA_OK, threads, and UINT64_MAX.

Referenced by LZMA_API().

lzma_outq_read()

lzma_ret lzma_outq_read	(	lzma_outq *restrict	outq,
		uint8_t *restrict	out,
		size_t *restrict	out_pos,
		size_t	out_size,
		lzma_vli *restrict	unpadded_size,
		lzma_vli *restrict	uncompressed_size
	)

Read finished data.

Parameters

outq	Pointer to an output queue
out	Beginning of the output buffer
out_pos	The next byte will be written to out[*out_pos].
out_size	Size of the out buffer; the first byte into which no data is written to is out[out_size].
unpadded_size	Unpadded Size from the Block encoder
uncompressed_size	Uncompressed Size from the Block encoder

Returns

- LZMA: All OK. Either no data was available or the buffer being read didn't become empty yet.

• LZMA_STREAM_END: The buffer being read was finished. *unpadded_size and *uncompressed_size were set.

Note

This reads lzma_outbuf.finished variables and thus call to this function needs to be protected with a mutex.

Definition at line 147 of file outqueue.c.

{
    // There must be at least one buffer from which to read.
    if (outq->bufs_used == 0)
        return LZMA_OK;
 
    // Get the buffer.
    uint32_t i = outq->bufs_pos - outq->bufs_used;
    if (outq->bufs_pos < outq->bufs_used)
        i += outq->bufs_allocated;
 
    lzma_outbuf *buf = &outq->bufs[i];
 
    // If it isn't finished yet, we cannot read from it.
    if (!buf->finished)
        return LZMA_OK;
 
    // Copy from the buffer to output.
    lzma_bufcpy(buf->buf, &outq->read_pos, buf->size,
            out, out_pos, out_size);
 
    // Return if we didn't get all the data from the buffer.
    if (outq->read_pos < buf->size)
        return LZMA_OK;
 
    // The buffer was finished. Tell the caller its size information.
    *unpadded_size = buf->unpadded_size;
    *uncompressed_size = buf->uncompressed_size;
 
    // Free this buffer for further use.
    --outq->bufs_used;
    outq->read_pos = 0;
 
    return LZMA_STREAM_END;
}

References i, lzma_bufcpy(), LZMA_OK, LZMA_STREAM_END, out, out_pos, uncompressed_size, and unpadded_size.

Referenced by stream_encode_mt().