04_compress_easy_mt.c File Reference

Compress in multi-call mode using LZMA2 in multi-threaded mode. More...

#include <stdbool.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <errno.h>
#include <lzma.h>

Go to the source code of this file.

Functions
static bool	init_encoder (lzma_stream *strm)
static bool	compress (lzma_stream *strm, FILE *infile, FILE *outfile)
int	main (void)

Detailed Description

Compress in multi-call mode using LZMA2 in multi-threaded mode.

Usage: ./04_compress_easy_mt < INFILE > OUTFILE

Example: ./04_compress_easy_mt < foo > foo.xz

Definition in file 04_compress_easy_mt.c.

Function Documentation

compress()

static bool compress ( lzma_stream *  strm, FILE *  infile, FILE *  outfile  )

static

Definition at line 118 of file 04_compress_easy_mt.c.

{
    lzma_action action = LZMA_RUN;
 
    uint8_t inbuf[BUFSIZ];
    uint8_t outbuf[BUFSIZ];
 
    strm->next_in = NULL;
    strm->avail_in = 0;
    strm->next_out = outbuf;
    strm->avail_out = sizeof(outbuf);
 
    while (true) {
        if (strm->avail_in == 0 && !feof(infile)) {
            strm->next_in = inbuf;
            strm->avail_in = fread(inbuf, 1, sizeof(inbuf),
                    infile);
 
            if (ferror(infile)) {
                fprintf(stderr, "Read error: %s\n",
                        strerror(errno));
                return false;
            }
 
            if (feof(infile))
                action = LZMA_FINISH;
        }
 
        lzma_ret ret = lzma_code(strm, action);
 
        if (strm->avail_out == 0 || ret == LZMA_STREAM_END) {
            size_t write_size = sizeof(outbuf) - strm->avail_out;
 
            if (fwrite(outbuf, 1, write_size, outfile)
                    != write_size) {
                fprintf(stderr, "Write error: %s\n",
                        strerror(errno));
                return false;
            }
 
            strm->next_out = outbuf;
            strm->avail_out = sizeof(outbuf);
        }
 
        if (ret != LZMA_OK) {
            if (ret == LZMA_STREAM_END)
                return true;
 
            const char *msg;
            switch (ret) {
            case LZMA_MEM_ERROR:
                msg = "Memory allocation failed";
                break;
 
            case LZMA_DATA_ERROR:
                msg = "File size limits exceeded";
                break;
 
            default:
                msg = "Unknown error, possibly a bug";
                break;
            }
 
            fprintf(stderr, "Encoder error: %s (error code %u)\n",
                    msg, ret);
            return false;
        }
    }
}

References test-lz4-speed::action, lzma_stream::avail_in, lzma_stream::avail_out, if(), inbuf, LZMA_DATA_ERROR, LZMA_FINISH, LZMA_MEM_ERROR, LZMA_OK, LZMA_RUN, LZMA_STREAM_END, msg, lzma_stream::next_in, lzma_stream::next_out, NULL, outbuf, outfile, and strm.

Referenced by main().

init_encoder()

static bool init_encoder ( lzma_stream *  strm )

static

Definition at line 26 of file 04_compress_easy_mt.c.

{
    // The threaded encoder takes the options as pointer to
    // a lzma_mt structure.
    lzma_mt mt = {
        // No flags are needed.
        .flags = 0,
 
        // Let liblzma determine a sane block size.
        .block_size = 0,
 
        // Use no timeout for lzma_code() calls by setting timeout
        // to zero. That is, sometimes lzma_code() might block for
        // a long time (from several seconds to even minutes).
        // If this is not OK, for example due to progress indicator
        // needing updates, specify a timeout in milliseconds here.
        // See the documentation of lzma_mt in lzma/container.h for
        // information how to choose a reasonable timeout.
        .timeout = 0,
 
        // Use the default preset (6) for LZMA2.
        // To use a preset, filters must be set to NULL.
        .preset = LZMA_PRESET_DEFAULT,
        .filters = NULL,
 
        // Use CRC64 for integrity checking. See also
        // 01_compress_easy.c about choosing the integrity check.
        .check = LZMA_CHECK_CRC64,
    };
 
    // Detect how many threads the CPU supports.
    mt.threads = lzma_cputhreads();
 
    // If the number of CPU cores/threads cannot be detected,
    // use one thread. Note that this isn't the same as the normal
    // single-threaded mode as this will still split the data into
    // blocks and use more RAM than the normal single-threaded mode.
    // You may want to consider using lzma_easy_encoder() or
    // lzma_stream_encoder() instead of lzma_stream_encoder_mt() if
    // lzma_cputhreads() returns 0 or 1.
    if (mt.threads == 0)
        mt.threads = 1;
 
    // If the number of CPU cores/threads exceeds threads_max,
    // limit the number of threads to keep memory usage lower.
    // The number 8 is arbitrarily chosen and may be too low or
    // high depending on the compression preset and the computer
    // being used.
    //
    // FIXME: A better way could be to check the amount of RAM
    // (or available RAM) and use lzma_stream_encoder_mt_memusage()
    // to determine if the number of threads should be reduced.
    const uint32_t threads_max = 8;
    if (mt.threads > threads_max)
        mt.threads = threads_max;
 
    // Initialize the threaded encoder.
    lzma_ret ret = lzma_stream_encoder_mt(strm, &mt);
 
    if (ret == LZMA_OK)
        return true;
 
    const char *msg;
    switch (ret) {
    case LZMA_MEM_ERROR:
        msg = "Memory allocation failed";
        break;
 
    case LZMA_OPTIONS_ERROR:
        // We are no longer using a plain preset so this error
        // message has been edited accordingly compared to
        // 01_compress_easy.c.
        msg = "Specified filter chain is not supported";
        break;
 
    case LZMA_UNSUPPORTED_CHECK:
        msg = "Specified integrity check is not supported";
        break;
 
    default:
        msg = "Unknown error, possibly a bug";
        break;
    }
 
    fprintf(stderr, "Error initializing the encoder: %s (error code %u)\n",
            msg, ret);
    return false;
}

References lzma_mt::flags, LZMA_CHECK_CRC64, LZMA_MEM_ERROR, LZMA_OK, LZMA_OPTIONS_ERROR, LZMA_PRESET_DEFAULT, LZMA_UNSUPPORTED_CHECK, msg, NULL, strm, lzma_mt::threads, and threads_max.

Referenced by main().

main()

int main

(

void

)

Definition at line 190 of file 04_compress_easy_mt.c.

{
    lzma_stream strm = LZMA_STREAM_INIT;
 
    bool success = init_encoder(&strm);
    if (success)
        success = compress(&strm, stdin, stdout);
 
    lzma_end(&strm);
 
    if (fclose(stdout)) {
        fprintf(stderr, "Write error: %s\n", strerror(errno));
        success = false;
    }
 
    return success ? EXIT_SUCCESS : EXIT_FAILURE;
}

References compress(), init_encoder(), LZMA_STREAM_INIT, and strm.