coder.c File Reference

Compresses or uncompresses a file. More...

#include "private.h"

Go to the source code of this file.

Enumerations
enum	coder_init_ret { CODER_INIT_NORMAL , CODER_INIT_PASSTHRU , CODER_INIT_ERROR }
	Return value type for coder_init(). More...

Functions
void	coder_set_check (lzma_check new_check)
	Set the integrity check type used when compressing. More...
static void	forget_filter_chain (void)
void	coder_set_preset (uint32_t new_preset)
	Set preset number. More...
void	coder_set_extreme (void)
	Enable extreme mode. More...
void	coder_add_filter (lzma_vli id, void *options)
	Add a filter to the custom filter chain. More...
static void	lzma_attribute ((__noreturn__))
void	coder_set_compression_settings (void)
static enum coder_init_ret	coder_init (file_pair *pair)
static void	split_block (uint64_t *block_remaining, uint64_t *next_block_remaining, size_t *list_pos)
static bool	coder_write_output (file_pair *pair)
static bool	coder_normal (file_pair *pair)
	Compress or decompress using liblzma. More...
static bool	coder_passthru (file_pair *pair)
void	coder_run (const char *filename)
	Compress or decompress the given file. More...
void	coder_free (void)
	Free the memory allocated for the coder and kill the worker threads. More...

Variables
enum operation_mode	opt_mode = MODE_COMPRESS
enum format_type	opt_format = FORMAT_AUTO
bool	opt_auto_adjust = true
bool	opt_single_stream = false
	If true, stop after decoding the first stream. More...
uint64_t	opt_block_size = 0
uint64_t *	opt_block_list = NULL
static lzma_stream	strm = LZMA_STREAM_INIT
	Stream used to communicate with liblzma. More...
static lzma_filter	filters [LZMA_FILTERS_MAX+1]
	Filters needed for all encoding all formats, and also decoding in raw data. More...
static io_buf	in_buf
	Input and output buffers. More...
static io_buf	out_buf
static uint32_t	filters_count = 0
	Number of filters. Zero indicates that we are using a preset. More...
static uint32_t	preset_number = LZMA_PRESET_DEFAULT
	Number of the preset (0-9) More...
static lzma_check	check
	Integrity check type. More...
static bool	check_default = true
	This becomes false if the –check=CHECK option is used. More...

Detailed Description

Compresses or uncompresses a file.

Definition in file coder.c.

Enumeration Type Documentation

coder_init_ret

enum coder_init_ret

Return value type for coder_init().

Enumerator
CODER_INIT_NORMAL
CODER_INIT_PASSTHRU
CODER_INIT_ERROR

Definition at line 17 of file coder.c.

                    {
    CODER_INIT_NORMAL,
    CODER_INIT_PASSTHRU,
    CODER_INIT_ERROR,
};

Function Documentation

coder_add_filter()

void coder_add_filter	(	lzma_vli	id,
		void *	options
	)

Add a filter to the custom filter chain.

Definition at line 107 of file coder.c.

{
    if (filters_count == LZMA_FILTERS_MAX)
        message_fatal(_("Maximum number of filters is four"));
 
    filters[filters_count].id = id;
    filters[filters_count].options = options;
    ++filters_count;
 
    // Setting a custom filter chain makes us forget the preset options.
    // This makes a difference if one specifies e.g. "xz -9 --lzma2 -e"
    // where the custom filter chain resets the preset level back to
    // the default 6, making the example equivalent to "xz -6e".
    preset_number = LZMA_PRESET_DEFAULT;
 
    return;
}

References _, filters, filters_count, id, lzma_filter::id, LZMA_FILTERS_MAX, LZMA_PRESET_DEFAULT, message_fatal(), options, lzma_filter::options, and preset_number.

Referenced by parse_real().

coder_free()

void coder_free

(

void

)

Free the memory allocated for the coder and kill the worker threads.

Definition at line 939 of file coder.c.

{
    lzma_end(&strm);
    return;
}

References strm.

Referenced by main().

coder_init()

static enum coder_init_ret coder_init ( file_pair *  pair )

static

Detect the input file type (for now, this done only when decompressing), and initialize an appropriate coder. Return value indicates if a normal liblzma-based coder was initialized (CODER_INIT_NORMAL), if passthru mode should be used (CODER_INIT_PASSTHRU), or if an error occurred (CODER_INIT_ERROR).

Definition at line 137 of file coder.c.

{
    lzma_ret ret = LZMA_PROG_ERROR;
 
    if (opt_mode == MODE_COMPRESS) {
#ifdef HAVE_ENCODERS
        switch (opt_format) {
        case FORMAT_AUTO:
            // args.c ensures this.
            assert(0);
            break;
 
        case FORMAT_XZ:
#   ifdef MYTHREAD_ENABLED
            if (hardware_threads_get() > 1)
                ret = lzma_stream_encoder_mt(
                        &strm, &mt_options);
            else
#   endif
                ret = lzma_stream_encoder(
                        &strm, filters, check);
            break;
 
        case FORMAT_LZMA:
            ret = lzma_alone_encoder(&strm, filters[0].options);
            break;
 
        case FORMAT_RAW:
            ret = lzma_raw_encoder(&strm, filters);
            break;
        }
#endif
    } else {
#ifdef HAVE_DECODERS
        uint32_t flags = 0;
 
        // It seems silly to warn about unsupported check if the
        // check won't be verified anyway due to --ignore-check.
        if (opt_ignore_check)
            flags |= LZMA_IGNORE_CHECK;
        else
            flags |= LZMA_TELL_UNSUPPORTED_CHECK;
 
        if (!opt_single_stream)
            flags |= LZMA_CONCATENATED;
 
        // We abuse FORMAT_AUTO to indicate unknown file format,
        // for which we may consider passthru mode.
        enum format_type init_format = FORMAT_AUTO;
 
        switch (opt_format) {
        case FORMAT_AUTO:
            if (is_format_xz())
                init_format = FORMAT_XZ;
            else if (is_format_lzma())
                init_format = FORMAT_LZMA;
            break;
 
        case FORMAT_XZ:
            if (is_format_xz())
                init_format = FORMAT_XZ;
            break;
 
        case FORMAT_LZMA:
            if (is_format_lzma())
                init_format = FORMAT_LZMA;
            break;
 
        case FORMAT_RAW:
            init_format = FORMAT_RAW;
            break;
        }
 
        switch (init_format) {
        case FORMAT_AUTO:
            // Unknown file format. If --decompress --stdout
            // --force have been given, then we copy the input
            // as is to stdout. Checking for MODE_DECOMPRESS
            // is needed, because we don't want to do use
            // passthru mode with --test.
            if (opt_mode == MODE_DECOMPRESS
                    && opt_stdout && opt_force)
                return CODER_INIT_PASSTHRU;
 
            ret = LZMA_FORMAT_ERROR;
            break;
 
        case FORMAT_XZ:
            ret = lzma_stream_decoder(&strm,
                    hardware_memlimit_get(
                        MODE_DECOMPRESS), flags);
            break;
 
        case FORMAT_LZMA:
            ret = lzma_alone_decoder(&strm,
                    hardware_memlimit_get(
                        MODE_DECOMPRESS));
            break;
 
        case FORMAT_RAW:
            // Memory usage has already been checked in
            // coder_set_compression_settings().
            ret = lzma_raw_decoder(&strm, filters);
            break;
        }
 
        // Try to decode the headers. This will catch too low
        // memory usage limit in case it happens in the first
        // Block of the first Stream, which is where it very
        // probably will happen if it is going to happen.
        if (ret == LZMA_OK && init_format != FORMAT_RAW) {
            strm.next_out = NULL;
            strm.avail_out = 0;
            ret = lzma_code(&strm, LZMA_RUN);
        }
#endif
    }
 
    if (ret != LZMA_OK) {
        message_error("%s: %s", pair->src_name, message_strm(ret));
        if (ret == LZMA_MEMLIMIT_ERROR)
            message_mem_needed(V_ERROR, lzma_memusage(&strm));
 
        return CODER_INIT_ERROR;
    }
 
    return CODER_INIT_NORMAL;
}

References _, assert(), lzma_mt::block_size, lzma_mt::check, check, check_default, lzma_options_lzma::dict_size, filters, filters_count, FORMAT_LZMA, FORMAT_RAW, FORMAT_XZ, hardware_memlimit_get(), hardware_threads_get(), hardware_threads_set(), i, lzma_filter::id, LZMA_CHECK_CRC32, LZMA_CHECK_CRC64, LZMA_FILTER_DELTA, LZMA_FILTER_LZMA1, LZMA_FILTER_LZMA2, LZMA_VLI_UNKNOWN, message, message_bug(), message_fatal(), message_filters_show(), message_mem_needed(), MODE_COMPRESS, opt_auto_adjust, opt_block_size, opt_flush_timeout, opt_format, opt_lzma, opt_mode, lzma_filter::options, preset_number, PRIu32, round_up_to_mib(), lzma_mt::threads, UINT32_C, UINT64_MAX, uint64_to_str(), V_DEBUG, and V_WARNING.

Referenced by coder_run().

coder_normal()

static bool coder_normal ( file_pair *  pair )

static

Compress or decompress using liblzma.

Definition at line 631 of file coder.c.

{
    // Encoder needs to know when we have given all the input to it.
    // The decoders need to know it too when we are using
    // LZMA_CONCATENATED. We need to check for src_eof here, because
    // the first input chunk has been already read if decompressing,
    // and that may have been the only chunk we will read.
    lzma_action action = pair->src_eof ? LZMA_FINISH : LZMA_RUN;
 
    lzma_ret ret;
 
    // Assume that something goes wrong.
    bool success = false;
 
    // block_remaining indicates how many input bytes to encode before
    // finishing the current .xz Block. The Block size is set with
    // --block-size=SIZE and --block-list. They have an effect only when
    // compressing to the .xz format. If block_remaining == UINT64_MAX,
    // only a single block is created.
    uint64_t block_remaining = UINT64_MAX;
 
    // next_block_remaining for when we are in single-threaded mode and
    // the Block in --block-list is larger than the --block-size=SIZE.
    uint64_t next_block_remaining = 0;
 
    // Position in opt_block_list. Unused if --block-list wasn't used.
    size_t list_pos = 0;
 
    // Handle --block-size for single-threaded mode and the first step
    // of --block-list.
    if (opt_mode == MODE_COMPRESS && opt_format == FORMAT_XZ) {
        // --block-size doesn't do anything here in threaded mode,
        // because the threaded encoder will take care of splitting
        // to fixed-sized Blocks.
        if (hardware_threads_get() == 1 && opt_block_size > 0)
            block_remaining = opt_block_size;
 
        // If --block-list was used, start with the first size.
        //
        // For threaded case, --block-size specifies how big Blocks
        // the encoder needs to be prepared to create at maximum
        // and --block-list will simultaneously cause new Blocks
        // to be started at specified intervals. To keep things
        // logical, the same is done in single-threaded mode. The
        // output is still not identical because in single-threaded
        // mode the size info isn't written into Block Headers.
        if (opt_block_list != NULL) {
            if (block_remaining < opt_block_list[list_pos]) {
                assert(hardware_threads_get() == 1);
                next_block_remaining = opt_block_list[list_pos]
                        - block_remaining;
            } else {
                block_remaining = opt_block_list[list_pos];
            }
        }
    }
 
    strm.next_out = out_buf.u8;
    strm.avail_out = IO_BUFFER_SIZE;
 
    while (!user_abort) {
        // Fill the input buffer if it is empty and we aren't
        // flushing or finishing.
        if (strm.avail_in == 0 && action == LZMA_RUN) {
            strm.next_in = in_buf.u8;
            strm.avail_in = io_read(pair, &in_buf,
                    my_min(block_remaining,
                        IO_BUFFER_SIZE));
 
            if (strm.avail_in == SIZE_MAX)
                break;
 
            if (pair->src_eof) {
                action = LZMA_FINISH;
 
            } else if (block_remaining != UINT64_MAX) {
                // Start a new Block after every
                // opt_block_size bytes of input.
                block_remaining -= strm.avail_in;
                if (block_remaining == 0)
                    action = LZMA_FULL_BARRIER;
            }
 
            if (action == LZMA_RUN && pair->flush_needed)
                action = LZMA_SYNC_FLUSH;
        }
 
        // Let liblzma do the actual work.
        ret = lzma_code(&strm, action);
 
        // Write out if the output buffer became full.
        if (strm.avail_out == 0) {
            if (coder_write_output(pair))
                break;
        }
 
        if (ret == LZMA_STREAM_END && (action == LZMA_SYNC_FLUSH
                || action == LZMA_FULL_BARRIER)) {
            if (action == LZMA_SYNC_FLUSH) {
                // Flushing completed. Write the pending data
                // out immediately so that the reading side
                // can decompress everything compressed so far.
                if (coder_write_output(pair))
                    break;
 
                // Mark that we haven't seen any new input
                // since the previous flush.
                pair->src_has_seen_input = false;
                pair->flush_needed = false;
            } else {
                // Start a new Block after LZMA_FULL_BARRIER.
                if (opt_block_list == NULL) {
                    assert(hardware_threads_get() == 1);
                    assert(opt_block_size > 0);
                    block_remaining = opt_block_size;
                } else {
                    split_block(&block_remaining,
                            &next_block_remaining,
                            &list_pos);
                }
            }
 
            // Start a new Block after LZMA_FULL_FLUSH or continue
            // the same block after LZMA_SYNC_FLUSH.
            action = LZMA_RUN;
 
        } else if (ret != LZMA_OK) {
            // Determine if the return value indicates that we
            // won't continue coding.
            const bool stop = ret != LZMA_NO_CHECK
                    && ret != LZMA_UNSUPPORTED_CHECK;
 
            if (stop) {
                // Write the remaining bytes even if something
                // went wrong, because that way the user gets
                // as much data as possible, which can be good
                // when trying to get at least some useful
                // data out of damaged files.
                if (coder_write_output(pair))
                    break;
            }
 
            if (ret == LZMA_STREAM_END) {
                if (opt_single_stream) {
                    io_fix_src_pos(pair, strm.avail_in);
                    success = true;
                    break;
                }
 
                // Check that there is no trailing garbage.
                // This is needed for LZMA_Alone and raw
                // streams.
                if (strm.avail_in == 0 && !pair->src_eof) {
                    // Try reading one more byte.
                    // Hopefully we don't get any more
                    // input, and thus pair->src_eof
                    // becomes true.
                    strm.avail_in = io_read(
                            pair, &in_buf, 1);
                    if (strm.avail_in == SIZE_MAX)
                        break;
 
                    assert(strm.avail_in == 0
                            || strm.avail_in == 1);
                }
 
                if (strm.avail_in == 0) {
                    assert(pair->src_eof);
                    success = true;
                    break;
                }
 
                // We hadn't reached the end of the file.
                ret = LZMA_DATA_ERROR;
                assert(stop);
            }
 
            // If we get here and stop is true, something went
            // wrong and we print an error. Otherwise it's just
            // a warning and coding can continue.
            if (stop) {
                message_error("%s: %s", pair->src_name,
                        message_strm(ret));
            } else {
                message_warning("%s: %s", pair->src_name,
                        message_strm(ret));
 
                // When compressing, all possible errors set
                // stop to true.
                assert(opt_mode != MODE_COMPRESS);
            }
 
            if (ret == LZMA_MEMLIMIT_ERROR) {
                // Display how much memory it would have
                // actually needed.
                message_mem_needed(V_ERROR,
                        lzma_memusage(&strm));
            }
 
            if (stop)
                break;
        }
 
        // Show progress information under certain conditions.
        message_progress_update();
    }
 
    return success;
}

References test-lz4-speed::action, assert(), lzma_stream::avail_in, lzma_stream::avail_out, coder_write_output(), file_pair::flush_needed, FORMAT_XZ, hardware_threads_get(), in_buf, IO_BUFFER_SIZE, io_fix_src_pos(), io_read(), LZMA_DATA_ERROR, LZMA_FINISH, LZMA_FULL_BARRIER, LZMA_MEMLIMIT_ERROR, LZMA_NO_CHECK, LZMA_OK, LZMA_RUN, LZMA_STREAM_END, LZMA_SYNC_FLUSH, LZMA_UNSUPPORTED_CHECK, message_error(), message_mem_needed(), message_progress_update(), message_strm(), message_warning(), MODE_COMPRESS, my_min, lzma_stream::next_in, lzma_stream::next_out, NULL, opt_block_list, opt_block_size, opt_format, opt_mode, opt_single_stream, out_buf, SIZE_MAX, split_block(), file_pair::src_eof, file_pair::src_has_seen_input, file_pair::src_name, strm, io_buf::u8, UINT64_MAX, user_abort, and V_ERROR.

Referenced by coder_run().

coder_passthru()

static bool coder_passthru ( file_pair *  pair )

static

Copy from input file to output file without processing the data in any way. This is used only when trying to decompress unrecognized files with –decompress –stdout –force, so the output is always stdout.

Definition at line 846 of file coder.c.

{
    while (strm.avail_in != 0) {
        if (user_abort)
            return false;
 
        if (io_write(pair, &in_buf, strm.avail_in))
            return false;
 
        strm.total_in += strm.avail_in;
        strm.total_out = strm.total_in;
        message_progress_update();
 
        strm.avail_in = io_read(pair, &in_buf, IO_BUFFER_SIZE);
        if (strm.avail_in == SIZE_MAX)
            return false;
    }
 
    return true;
}

References lzma_stream::avail_in, in_buf, IO_BUFFER_SIZE, io_read(), io_write(), message_progress_update(), SIZE_MAX, strm, lzma_stream::total_in, lzma_stream::total_out, and user_abort.

Referenced by coder_run().

coder_run()

void coder_run

(

const char *

filename

)

Compress or decompress the given file.

Definition at line 869 of file coder.c.

{
    // Set and possibly print the filename for the progress message.
    message_filename(filename);
 
    // Try to open the input file.
    file_pair *pair = io_open_src(filename);
    if (pair == NULL)
        return;
 
    // Assume that something goes wrong.
    bool success = false;
 
    if (opt_mode == MODE_COMPRESS) {
        strm.next_in = NULL;
        strm.avail_in = 0;
    } else {
        // Read the first chunk of input data. This is needed
        // to detect the input file type.
        strm.next_in = in_buf.u8;
        strm.avail_in = io_read(pair, &in_buf, IO_BUFFER_SIZE);
    }
 
    if (strm.avail_in != SIZE_MAX) {
        // Initialize the coder. This will detect the file format
        // and, in decompression or testing mode, check the memory
        // usage of the first Block too. This way we don't try to
        // open the destination file if we see that coding wouldn't
        // work at all anyway. This also avoids deleting the old
        // "target" file if --force was used.
        const enum coder_init_ret init_ret = coder_init(pair);
 
        if (init_ret != CODER_INIT_ERROR && !user_abort) {
            // Don't open the destination file when --test
            // is used.
            if (opt_mode == MODE_TEST || !io_open_dest(pair)) {
                // Remember the current time. It is needed
                // for progress indicator.
                mytime_set_start_time();
 
                // Initialize the progress indicator.
                const bool is_passthru = init_ret
                        == CODER_INIT_PASSTHRU;
                const uint64_t in_size
                    = pair->src_st.st_size <= 0
                    ? 0 : (uint64_t)(pair->src_st.st_size);
                message_progress_start(&strm,
                        is_passthru, in_size);
 
                // Do the actual coding or passthru.
                if (is_passthru)
                    success = coder_passthru(pair);
                else
                    success = coder_normal(pair);
 
                message_progress_end(success);
            }
        }
    }
 
    // Close the file pair. It needs to know if coding was successful to
    // know if the source or target file should be unlinked.
    io_close(pair, success);
 
    return;
}

References lzma_stream::avail_in, coder_init(), CODER_INIT_ERROR, CODER_INIT_PASSTHRU, coder_normal(), coder_passthru(), in_buf, in_size, IO_BUFFER_SIZE, io_close(), io_open_dest(), io_open_src(), io_read(), message_filename(), message_progress_end(), message_progress_start(), MODE_COMPRESS, MODE_TEST, mytime_set_start_time(), lzma_stream::next_in, NULL, opt_mode, SIZE_MAX, file_pair::src_st, strm, io_buf::u8, and user_abort.

Referenced by main().

coder_set_check()

void coder_set_check

(

lzma_check

new_check

)

Set the integrity check type used when compressing.

Definition at line 64 of file coder.c.

{
    check = new_check;
    check_default = false;
    return;
}

References check, and check_default.

Referenced by parse_real().

coder_set_compression_settings()

void coder_set_compression_settings

(

void

)

Definition at line 137 of file coder.c.

{
    // The default check type is CRC64, but fallback to CRC32
    // if CRC64 isn't supported by the copy of liblzma we are
    // using. CRC32 is always supported.
    if (check_default) {
        check = LZMA_CHECK_CRC64;
        if (!lzma_check_is_supported(check))
            check = LZMA_CHECK_CRC32;
    }
 
    // Options for LZMA1 or LZMA2 in case we are using a preset.
    static lzma_options_lzma opt_lzma;
 
    if (filters_count == 0) {
        // We are using a preset. This is not a good idea in raw mode
        // except when playing around with things. Different versions
        // of this software may use different options in presets, and
        // thus make uncompressing the raw data difficult.
        if (opt_format == FORMAT_RAW) {
            // The message is shown only if warnings are allowed
            // but the exit status isn't changed.
            message(V_WARNING, _("Using a preset in raw mode "
                    "is discouraged."));
            message(V_WARNING, _("The exact options of the "
                    "presets may vary between software "
                    "versions."));
        }
 
        // Get the preset for LZMA1 or LZMA2.
        if (lzma_lzma_preset(&opt_lzma, preset_number))
            message_bug();
 
        // Use LZMA2 except with --format=lzma we use LZMA1.
        filters[0].id = opt_format == FORMAT_LZMA
                ? LZMA_FILTER_LZMA1 : LZMA_FILTER_LZMA2;
        filters[0].options = &opt_lzma;
        filters_count = 1;
    }
 
    // Terminate the filter options array.
    filters[filters_count].id = LZMA_VLI_UNKNOWN;
 
    // If we are using the .lzma format, allow exactly one filter
    // which has to be LZMA1.
    if (opt_format == FORMAT_LZMA && (filters_count != 1
            || filters[0].id != LZMA_FILTER_LZMA1))
        message_fatal(_("The .lzma format supports only "
                "the LZMA1 filter"));
 
    // If we are using the .xz format, make sure that there is no LZMA1
    // filter to prevent LZMA_PROG_ERROR.
    if (opt_format == FORMAT_XZ)
        for (size_t i = 0; i < filters_count; ++i)
            if (filters[i].id == LZMA_FILTER_LZMA1)
                message_fatal(_("LZMA1 cannot be used "
                        "with the .xz format"));
 
    // Print the selected filter chain.
    message_filters_show(V_DEBUG, filters);
 
    // The --flush-timeout option requires LZMA_SYNC_FLUSH support
    // from the filter chain. Currently threaded encoder doesn't support
    // LZMA_SYNC_FLUSH so single-threaded mode must be used.
    if (opt_mode == MODE_COMPRESS && opt_flush_timeout != 0) {
        for (size_t i = 0; i < filters_count; ++i) {
            switch (filters[i].id) {
            case LZMA_FILTER_LZMA2:
            case LZMA_FILTER_DELTA:
                break;
 
            default:
                message_fatal(_("The filter chain is "
                    "incompatible with --flush-timeout"));
            }
        }
 
        if (hardware_threads_get() > 1) {
            message(V_WARNING, _("Switching to single-threaded "
                    "mode due to --flush-timeout"));
            hardware_threads_set(1);
        }
    }
 
    // Get the memory usage. Note that if --format=raw was used,
    // we can be decompressing.
    const uint64_t memory_limit = hardware_memlimit_get(opt_mode);
    uint64_t memory_usage = UINT64_MAX;
    if (opt_mode == MODE_COMPRESS) {
#ifdef HAVE_ENCODERS
#   ifdef MYTHREAD_ENABLED
        if (opt_format == FORMAT_XZ && hardware_threads_get() > 1) {
            mt_options.threads = hardware_threads_get();
            mt_options.block_size = opt_block_size;
            mt_options.check = check;
            memory_usage = lzma_stream_encoder_mt_memusage(
                    &mt_options);
            if (memory_usage != UINT64_MAX)
                message(V_DEBUG, _("Using up to %" PRIu32
                        " threads."),
                        mt_options.threads);
        } else
#   endif
        {
            memory_usage = lzma_raw_encoder_memusage(filters);
        }
#endif
    } else {
#ifdef HAVE_DECODERS
        memory_usage = lzma_raw_decoder_memusage(filters);
#endif
    }
 
    if (memory_usage == UINT64_MAX)
        message_fatal(_("Unsupported filter chain or filter options"));
 
    // Print memory usage info before possible dictionary
    // size auto-adjusting.
    //
    // NOTE: If only encoder support was built, we cannot show the
    // what the decoder memory usage will be.
    message_mem_needed(V_DEBUG, memory_usage);
#ifdef HAVE_DECODERS
    if (opt_mode == MODE_COMPRESS) {
        const uint64_t decmem = lzma_raw_decoder_memusage(filters);
        if (decmem != UINT64_MAX)
            message(V_DEBUG, _("Decompression will need "
                    "%s MiB of memory."), uint64_to_str(
                        round_up_to_mib(decmem), 0));
    }
#endif
 
    if (memory_usage <= memory_limit)
        return;
 
    // If --no-adjust was used or we didn't find LZMA1 or
    // LZMA2 as the last filter, give an error immediately.
    // --format=raw implies --no-adjust.
    if (!opt_auto_adjust || opt_format == FORMAT_RAW)
        memlimit_too_small(memory_usage);
 
    assert(opt_mode == MODE_COMPRESS);
 
#ifdef HAVE_ENCODERS
#   ifdef MYTHREAD_ENABLED
    if (opt_format == FORMAT_XZ && mt_options.threads > 1) {
        // Try to reduce the number of threads before
        // adjusting the compression settings down.
        do {
            // FIXME? The real single-threaded mode has
            // lower memory usage, but it's not comparable
            // because it doesn't write the size info
            // into Block Headers.
            if (--mt_options.threads == 0)
                memlimit_too_small(memory_usage);
 
            memory_usage = lzma_stream_encoder_mt_memusage(
                    &mt_options);
            if (memory_usage == UINT64_MAX)
                message_bug();
 
        } while (memory_usage > memory_limit);
 
        message(V_WARNING, _("Adjusted the number of threads "
            "from %s to %s to not exceed "
            "the memory usage limit of %s MiB"),
            uint64_to_str(hardware_threads_get(), 0),
            uint64_to_str(mt_options.threads, 1),
            uint64_to_str(round_up_to_mib(
                memory_limit), 2));
    }
#   endif
 
    if (memory_usage <= memory_limit)
        return;
 
    // Look for the last filter if it is LZMA2 or LZMA1, so we can make
    // it use less RAM. With other filters we don't know what to do.
    size_t i = 0;
    while (filters[i].id != LZMA_FILTER_LZMA2
            && filters[i].id != LZMA_FILTER_LZMA1) {
        if (filters[i].id == LZMA_VLI_UNKNOWN)
            memlimit_too_small(memory_usage);
 
        ++i;
    }
 
    // Decrease the dictionary size until we meet the memory
    // usage limit. First round down to full mebibytes.
    lzma_options_lzma *opt = filters[i].options;
    const uint32_t orig_dict_size = opt->dict_size;
    opt->dict_size &= ~((UINT32_C(1) << 20) - 1);
    while (true) {
        // If it is below 1 MiB, auto-adjusting failed. We could be
        // more sophisticated and scale it down even more, but let's
        // see if many complain about this version.
        //
        // FIXME: Displays the scaled memory usage instead
        // of the original.
        if (opt->dict_size < (UINT32_C(1) << 20))
            memlimit_too_small(memory_usage);
 
        memory_usage = lzma_raw_encoder_memusage(filters);
        if (memory_usage == UINT64_MAX)
            message_bug();
 
        // Accept it if it is low enough.
        if (memory_usage <= memory_limit)
            break;
 
        // Otherwise 1 MiB down and try again. I hope this
        // isn't too slow method for cases where the original
        // dict_size is very big.
        opt->dict_size -= UINT32_C(1) << 20;
    }
 
    // Tell the user that we decreased the dictionary size.
    message(V_WARNING, _("Adjusted LZMA%c dictionary size "
            "from %s MiB to %s MiB to not exceed "
            "the memory usage limit of %s MiB"),
            filters[i].id == LZMA_FILTER_LZMA2
                ? '2' : '1',
            uint64_to_str(orig_dict_size >> 20, 0),
            uint64_to_str(opt->dict_size >> 20, 1),
            uint64_to_str(round_up_to_mib(memory_limit), 2));
#endif
 
    return;
}

Referenced by args_parse().

coder_set_extreme()

void coder_set_extreme

(

void

)

Enable extreme mode.

Definition at line 98 of file coder.c.

{
    preset_number |= LZMA_PRESET_EXTREME;
    forget_filter_chain();
    return;
}

References forget_filter_chain(), LZMA_PRESET_EXTREME, and preset_number.

Referenced by parse_real().

coder_set_preset()

void coder_set_preset

(

uint32_t

new_preset

)

Set preset number.

Definition at line 88 of file coder.c.

{
    preset_number &= ~LZMA_PRESET_LEVEL_MASK;
    preset_number |= new_preset;
    forget_filter_chain();
    return;
}

References forget_filter_chain(), LZMA_PRESET_LEVEL_MASK, and preset_number.

Referenced by parse_real().

coder_write_output()

static bool coder_write_output ( file_pair *  pair )

static

Definition at line 616 of file coder.c.

{
    if (opt_mode != MODE_TEST) {
        if (io_write(pair, &out_buf, IO_BUFFER_SIZE - strm.avail_out))
            return true;
    }
 
    strm.next_out = out_buf.u8;
    strm.avail_out = IO_BUFFER_SIZE;
    return false;
}

References lzma_stream::avail_out, IO_BUFFER_SIZE, io_write(), MODE_TEST, lzma_stream::next_out, opt_mode, out_buf, strm, and io_buf::u8.

Referenced by coder_normal().

forget_filter_chain()

static void forget_filter_chain ( void  )

static

Definition at line 73 of file coder.c.

{
    // Setting a preset makes us forget a possibly defined custom
    // filter chain.
    while (filters_count > 0) {
        --filters_count;
        free(filters[filters_count].options);
        filters[filters_count].options = NULL;
    }
 
    return;
}

References filters, filters_count, free(), NULL, options, and lzma_filter::options.

Referenced by coder_set_extreme(), and coder_set_preset().

lzma_attribute()

static void lzma_attribute ( (__noreturn__)  )

static

Definition at line 126 of file coder.c.

{
    message(V_ERROR, _("Memory usage limit is too low for the given "
            "filter setup."));
    message_mem_needed(V_ERROR, memory_usage);
    tuklib_exit(E_ERROR, E_ERROR, false);
}

References _, E_ERROR, message, message_mem_needed(), tuklib_exit, and V_ERROR.

split_block()

static void split_block ( uint64_t *  block_remaining, uint64_t *  next_block_remaining, size_t *  list_pos  )

static

Resolve conflicts between opt_block_size and opt_block_list in single threaded mode. We want to default to opt_block_list, except when it is larger than opt_block_size. If this is the case for the current Block at *list_pos, then we break into smaller Blocks. Otherwise advance to the next Block in opt_block_list, and break apart if needed.

Definition at line 571 of file coder.c.

{
    if (*next_block_remaining > 0) {
        // The Block at *list_pos has previously been split up.
        assert(hardware_threads_get() == 1);
        assert(opt_block_size > 0);
        assert(opt_block_list != NULL);
 
        if (*next_block_remaining > opt_block_size) {
            // We have to split the current Block at *list_pos
            // into another opt_block_size length Block.
            *block_remaining = opt_block_size;
        } else {
            // This is the last remaining split Block for the
            // Block at *list_pos.
            *block_remaining = *next_block_remaining;
        }
 
        *next_block_remaining -= *block_remaining;
 
    } else {
        // The Block at *list_pos has been finished. Go to the next
        // entry in the list. If the end of the list has been reached,
        // reuse the size of the last Block.
        if (opt_block_list[*list_pos + 1] != 0)
            ++*list_pos;
 
        *block_remaining = opt_block_list[*list_pos];
 
        // If in single-threaded mode, split up the Block if needed.
        // This is not needed in multi-threaded mode because liblzma
        // will do this due to how threaded encoding works.
        if (hardware_threads_get() == 1 && opt_block_size > 0
                && *block_remaining > opt_block_size) {
            *next_block_remaining
                    = *block_remaining - opt_block_size;
            *block_remaining = opt_block_size;
        }
    }
}

References assert(), hardware_threads_get(), NULL, opt_block_list, and opt_block_size.

Referenced by coder_normal().

Variable Documentation

check

lzma_check check

static

Integrity check type.

Definition at line 49 of file coder.c.

Referenced by coder_init(), and coder_set_check().

check_default

bool check_default = true

static

This becomes false if the –check=CHECK option is used.

Definition at line 52 of file coder.c.

Referenced by coder_init(), and coder_set_check().

filters

lzma_filter filters[LZMA_FILTERS_MAX+1]

static

Filters needed for all encoding all formats, and also decoding in raw data.

Definition at line 36 of file coder.c.

Referenced by coder_add_filter(), coder_init(), and forget_filter_chain().

filters_count

uint32_t filters_count = 0

static

Number of filters. Zero indicates that we are using a preset.

Definition at line 43 of file coder.c.

Referenced by coder_add_filter(), coder_init(), and forget_filter_chain().

in_buf

io_buf in_buf

static

Input and output buffers.

Definition at line 39 of file coder.c.

Referenced by coder_normal(), coder_passthru(), coder_run(), on_rebase_pointer(), rebase_buffer(), rebase_bytes_v1(), rebase_bytes_v2(), rebase_bytes_v3(), uncompress(), xz_compress(), and xz_decompress().

opt_auto_adjust

bool opt_auto_adjust = true

If true, the compression settings are automatically adjusted down if they exceed the memory usage limit.

Definition at line 26 of file coder.c.

Referenced by coder_init(), and parse_real().

opt_block_list

uint64_t* opt_block_list = NULL

This is non-NULL if –block-list was used. This contains the Block sizes as an array that is terminated with 0.

Definition at line 29 of file coder.c.

Referenced by args_free(), coder_normal(), parse_block_list(), and split_block().

opt_block_size

uint64_t opt_block_size = 0

If non-zero, start a new .xz Block after every opt_block_size bytes of input. This has an effect only when compressing to the .xz format.

Definition at line 28 of file coder.c.

Referenced by coder_init(), coder_normal(), parse_real(), and split_block().

opt_format

enum format_type opt_format = FORMAT_AUTO

File format to use when encoding or what format(s) to accept when decoding. This is a global because it's needed also in suffix.c. This is set in args.c.

Definition at line 1 of file coder.c.

Referenced by args_parse(), coder_init(), coder_normal(), compressed_name(), list_file(), parse_real(), and uncompressed_name().

opt_mode

enum operation_mode opt_mode = MODE_COMPRESS

Operation mode of the command line tool. This is set in args.c and read in several files.

Definition at line 1 of file coder.c.

Referenced by args_parse(), coder_init(), coder_normal(), coder_run(), coder_write_output(), io_open_dest_real(), io_open_src_real(), main(), message_filename(), message_mem_needed(), mytime_get_flush_timeout(), parse_real(), print_filename(), progress_pos(), and suffix_get_dest_name().

opt_single_stream

bool opt_single_stream = false

If true, stop after decoding the first stream.

Definition at line 27 of file coder.c.

Referenced by coder_normal(), and parse_real().

out_buf

io_buf out_buf

static

Definition at line 40 of file coder.c.

Referenced by coder_normal(), coder_write_output(), handle_tmp_desc(), rz_core_rtr_gdb_cb(), uncompress(), xz_compress(), and xz_decompress().

preset_number

uint32_t preset_number = LZMA_PRESET_DEFAULT

static

Number of the preset (0-9)

Definition at line 46 of file coder.c.

Referenced by coder_add_filter(), coder_init(), coder_set_extreme(), and coder_set_preset().

strm

lzma_stream strm = LZMA_STREAM_INIT

static

Stream used to communicate with liblzma.

Definition at line 33 of file coder.c.

Referenced by coder_free(), coder_normal(), coder_passthru(), coder_run(), and coder_write_output().