dictionaryRandomAccess.c File Reference

#include "lz4.h"
#include <stdio.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>

Go to the source code of this file.

Macros
#define	MIN(x, y)   ((x) < (y) ? (x) : (y))

Enumerations
enum	{ BLOCK_BYTES = 1024 , DICTIONARY_BYTES = 1024 , MAX_BLOCKS = 1024 }

Functions
void	write_int (FILE *fp, int i)
void	write_bin (FILE *fp, const void *array, size_t arrayBytes)
void	read_int (FILE *fp, int *i)
size_t	read_bin (FILE *fp, void *array, size_t arrayBytes)
void	seek_bin (FILE *fp, long offset, int origin)
void	test_compress (FILE *outFp, FILE *inpFp, void *dict, int dictSize)
void	test_decompress (FILE *outFp, FILE *inpFp, void *dict, int dictSize, int offset, int length)
int	compare (FILE *fp0, FILE *fp1, int length)
int	main (int argc, char *argv[])

Variables
const char	kTestMagic [] = { 'T', 'E', 'S', 'T' }

Macro Definition Documentation

MIN

#define MIN	(		x,
			y
	)		((x) < (y) ? (x) : (y))

Definition at line 14 of file dictionaryRandomAccess.c.

Enumeration Type Documentation

anonymous enum

anonymous enum

Enumerator
BLOCK_BYTES
DICTIONARY_BYTES
MAX_BLOCKS

Definition at line 16 of file dictionaryRandomAccess.c.

     {
    BLOCK_BYTES = 1024,  /* 1 KiB of uncompressed data in a block */
    DICTIONARY_BYTES = 1024, /* Load a 1 KiB dictionary */
    MAX_BLOCKS = 1024 /* For simplicity of implementation */
};

Function Documentation

compare()

int compare	(	FILE *	fp0,
		FILE *	fp1,
		int	length
	)

Definition at line 173 of file dictionaryRandomAccess.c.

{
    int result = 0;
 
    while(0 == result) {
        char b0[4096];
        char b1[4096];
        const size_t r0 = read_bin(fp0, b0, MIN(length, (int)sizeof(b0)));
        const size_t r1 = read_bin(fp1, b1, MIN(length, (int)sizeof(b1)));
 
        result = (int) r0 - (int) r1;
 
        if(0 == r0 || 0 == r1) {
            break;
        }
        if(0 == result) {
            result = memcmp(b0, b1, r0);
        }
        length -= r0;
    }
 
    return result;
}

References b1, int, length, MIN, r0, r1, and read_bin().

Referenced by main().

main()

int main	(	int	argc,
		char *	argv[]
	)

Definition at line 198 of file dictionaryRandomAccess.c.

{
    char inpFilename[256] = { 0 };
    char lz4Filename[256] = { 0 };
    char decFilename[256] = { 0 };
    char dictFilename[256] = { 0 };
    int offset;
    int length;
    char dict[DICTIONARY_BYTES];
    int dictSize;
 
    if(argc < 5) {
        printf("Usage: %s input dictionary offset length", argv[0]);
        return 0;
    }
 
    snprintf(inpFilename, 256, "%s", argv[1]);
    snprintf(lz4Filename, 256, "%s.lz4s-%d", argv[1], BLOCK_BYTES);
    snprintf(decFilename, 256, "%s.lz4s-%d.dec", argv[1], BLOCK_BYTES);
    snprintf(dictFilename, 256, "%s", argv[2]);
    offset = atoi(argv[3]);
    length = atoi(argv[4]);
 
    printf("inp    = [%s]\n", inpFilename);
    printf("lz4    = [%s]\n", lz4Filename);
    printf("dec    = [%s]\n", decFilename);
    printf("dict   = [%s]\n", dictFilename);
    printf("offset = [%d]\n", offset);
    printf("length = [%d]\n", length);
 
    /* Load dictionary */
    {
        FILE* dictFp = fopen(dictFilename, "rb");
        dictSize = (int)read_bin(dictFp, dict, DICTIONARY_BYTES);
        fclose(dictFp);
    }
 
    /* compress */
    {
        FILE* inpFp = fopen(inpFilename, "rb");
        FILE* outFp = fopen(lz4Filename, "wb");
 
        printf("compress : %s -> %s\n", inpFilename, lz4Filename);
        test_compress(outFp, inpFp, dict, dictSize);
        printf("compress : done\n");
 
        fclose(outFp);
        fclose(inpFp);
    }
 
    /* decompress */
    {
        FILE* inpFp = fopen(lz4Filename, "rb");
        FILE* outFp = fopen(decFilename, "wb");
 
        printf("decompress : %s -> %s\n", lz4Filename, decFilename);
        test_decompress(outFp, inpFp, dict, DICTIONARY_BYTES, offset, length);
        printf("decompress : done\n");
 
        fclose(outFp);
        fclose(inpFp);
    }
 
    /* verify */
    {
        FILE* inpFp = fopen(inpFilename, "rb");
        FILE* decFp = fopen(decFilename, "rb");
        seek_bin(inpFp, offset, SEEK_SET);
 
        printf("verify : %s <-> %s\n", inpFilename, decFilename);
        const int cmp = compare(inpFp, decFp, length);
        if(0 == cmp) {
            printf("verify : OK\n");
        } else {
            printf("verify : NG\n");
        }
 
        fclose(decFp);
        fclose(inpFp);
    }
 
    return 0;
}

References argv, BLOCK_BYTES, cmp(), compare(), DICTIONARY_BYTES, benchmark::FILE, int, length, printf(), read_bin(), seek_bin(), SEEK_SET, snprintf, test_compress(), and test_decompress().

read_bin()

size_t read_bin	(	FILE *	fp,
		void *	array,
		size_t	arrayBytes
	)

Definition at line 45 of file dictionaryRandomAccess.c.

                                                          {
    size_t read = fread(array, 1, arrayBytes, fp);
    if (ferror(fp)) { exit(12); }
    return read;
}

References test-lz4-list::exit, and read().

Referenced by compare(), main(), test_compress(), and test_decompress().

read_int()

void read_int	(	FILE *	fp,
		int *	i
	)

Definition at line 40 of file dictionaryRandomAccess.c.

                                {
    size_t read = fread(i, sizeof(*i), 1, fp);
    if (read != 1) { exit(12); }
}

References test-lz4-list::exit, i, and read().

Referenced by test_decompress().

seek_bin()

void seek_bin	(	FILE *	fp,
		long	offset,
		int	origin
	)

Definition at line 51 of file dictionaryRandomAccess.c.

                                                 {
    if (fseek(fp, offset, origin)) { exit(14); }
}

References test-lz4-list::exit.

Referenced by main(), and test_decompress().

test_compress()

void test_compress	(	FILE *	outFp,
		FILE *	inpFp,
		void *	dict,
		int	dictSize
	)

Definition at line 56 of file dictionaryRandomAccess.c.

{
    LZ4_stream_t lz4Stream_body;
    LZ4_stream_t* lz4Stream = &lz4Stream_body;
 
    char inpBuf[BLOCK_BYTES];
    int offsets[MAX_BLOCKS];
    int *offsetsEnd = offsets;
 
 
    LZ4_initStream(lz4Stream, sizeof(*lz4Stream));
 
    /* Write header magic */
    write_bin(outFp, kTestMagic, sizeof(kTestMagic));
 
    *offsetsEnd++ = sizeof(kTestMagic);
    /* Write compressed data blocks.  Each block contains BLOCK_BYTES of plain
       data except possibly the last. */
    for(;;) {
        const int inpBytes = (int) read_bin(inpFp, inpBuf, BLOCK_BYTES);
        if(0 == inpBytes) {
            break;
        }
 
        /* Forget previously compressed data and load the dictionary */
        LZ4_loadDict(lz4Stream, dict, dictSize);
        {
            char cmpBuf[LZ4_COMPRESSBOUND(BLOCK_BYTES)];
            const int cmpBytes = LZ4_compress_fast_continue(
                lz4Stream, inpBuf, cmpBuf, inpBytes, sizeof(cmpBuf), 1);
            if(cmpBytes <= 0) { exit(1); }
            write_bin(outFp, cmpBuf, (size_t)cmpBytes);
            /* Keep track of the offsets */
            *offsetsEnd = *(offsetsEnd - 1) + cmpBytes;
            ++offsetsEnd;
        }
        if (offsetsEnd - offsets > MAX_BLOCKS) { exit(2); }
    }
    /* Write the tailing jump table */
    {
        int *ptr = offsets;
        while (ptr != offsetsEnd) {
            write_int(outFp, *ptr++);
        }
        write_int(outFp, offsetsEnd - offsets);
    }
}

References BLOCK_BYTES, test-lz4-list::exit, int, kTestMagic, LZ4_compress_fast_continue(), LZ4_COMPRESSBOUND, LZ4_initStream(), LZ4_loadDict(), MAX_BLOCKS, read_bin(), write_bin(), and write_int().

Referenced by main().

test_decompress()

void test_decompress	(	FILE *	outFp,
		FILE *	inpFp,
		void *	dict,
		int	dictSize,
		int	offset,
		int	length
	)

Definition at line 105 of file dictionaryRandomAccess.c.

{
    LZ4_streamDecode_t lz4StreamDecode_body;
    LZ4_streamDecode_t* lz4StreamDecode = &lz4StreamDecode_body;
 
    /* The blocks [currentBlock, endBlock) contain the data we want */
    int currentBlock = offset / BLOCK_BYTES;
    int endBlock = ((offset + length - 1) / BLOCK_BYTES) + 1;
 
    char decBuf[BLOCK_BYTES];
    int offsets[MAX_BLOCKS];
 
    /* Special cases */
    if (length == 0) { return; }
 
    /* Read the magic bytes */
    {
        char magic[sizeof(kTestMagic)];
        size_t read = read_bin(inpFp, magic, sizeof(magic));
        if (read != sizeof(magic)) { exit(1); }
        if (memcmp(kTestMagic, magic, sizeof(magic))) { exit(2); }
    }
 
    /* Read the offsets tail */
    {
        int numOffsets;
        int block;
        int *offsetsPtr = offsets;
        seek_bin(inpFp, -4, SEEK_END);
        read_int(inpFp, &numOffsets);
        if (numOffsets <= endBlock) { exit(3); }
        seek_bin(inpFp, -4 * (numOffsets + 1), SEEK_END);
        for (block = 0; block <= endBlock; ++block) {
            read_int(inpFp, offsetsPtr++);
        }
    }
    /* Seek to the first block to read */
    seek_bin(inpFp, offsets[currentBlock], SEEK_SET);
    offset = offset % BLOCK_BYTES;
 
    /* Start decoding */
    for(; currentBlock < endBlock; ++currentBlock) {
        char cmpBuf[LZ4_COMPRESSBOUND(BLOCK_BYTES)];
        /* The difference in offsets is the size of the block */
        int  cmpBytes = offsets[currentBlock + 1] - offsets[currentBlock];
        {
            const size_t read = read_bin(inpFp, cmpBuf, (size_t)cmpBytes);
            if(read != (size_t)cmpBytes) { exit(4); }
        }
 
        /* Load the dictionary */
        LZ4_setStreamDecode(lz4StreamDecode, dict, dictSize);
        {
            const int decBytes = LZ4_decompress_safe_continue(
                lz4StreamDecode, cmpBuf, decBuf, cmpBytes, BLOCK_BYTES);
            if(decBytes <= 0) { exit(5); }
            {
                /* Write out the part of the data we care about */
                int blockLength = MIN(length, (decBytes - offset));
                write_bin(outFp, decBuf + offset, (size_t)blockLength);
                offset = 0;
                length -= blockLength;
            }
        }
    }
}

References BLOCK_BYTES, test-lz4-list::exit, kTestMagic, length, LZ4_COMPRESSBOUND, LZ4_decompress_safe_continue(), LZ4_setStreamDecode(), MAX_BLOCKS, MIN, read(), read_bin(), read_int(), seek_bin(), SEEK_END, SEEK_SET, and write_bin().

Referenced by main().

write_bin()

void write_bin	(	FILE *	fp,
		const void *	array,
		size_t	arrayBytes
	)

Definition at line 35 of file dictionaryRandomAccess.c.

                                                               {
    size_t written = fwrite(array, 1, arrayBytes, fp);
    if (written != arrayBytes) { exit(11); }
}

References test-lz4-list::exit.

Referenced by test_compress(), and test_decompress().

write_int()

void write_int	(	FILE *	fp,
		int	i
	)

Definition at line 30 of file dictionaryRandomAccess.c.

                                {
    size_t written = fwrite(&i, sizeof(i), 1, fp);
    if (written != 1) { exit(10); }
}

References test-lz4-list::exit, and i.

Referenced by test_compress().

Variable Documentation

kTestMagic

const char kTestMagic[] = { 'T', 'E', 'S', 'T' }

Magic bytes for this test case. This is not a great magic number because it is a common word in ASCII. However, it is important to have some versioning system in your format.

Definition at line 27 of file dictionaryRandomAccess.c.

Referenced by test_compress(), and test_decompress().