#include "platform.h"
#include "util.h"
#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include <time.h>
#include <assert.h>
#include "datagen.h"
#include "xxhash.h"
#include "bench.h"
#include "lz4.h"
#include "lz4hc.h"

Classes
struct	compressionParameters

struct	blockParam_t

Macros
#define	LZ4_STATIC_LINKING_ONLY

#define	LZ4_HC_STATIC_LINKING_ONLY

#define	LZ4_isError(errcode) (errcode==0)

#define	LZ4_GIT_COMMIT_STRING ""

#define	NBSECONDS 3

#define	TIMELOOP_MICROSEC 11000000ULL / 1 second */

#define	TIMELOOP_NANOSEC 11000000000ULL / 1 second */

#define	ACTIVEPERIOD_MICROSEC 701000000ULL / 70 seconds */

#define	COOLPERIOD_SEC 10

#define	DECOMP_MULT 1 /* test decompression DECOMP_MULT times longer than compression */

#define	KB *(1 <<10)

#define	MB *(1 <<20)

#define	GB *(1U<<30)

#define	LZ4_MAX_DICT_SIZE (64 KB)

#define	DISPLAY(...) fprintf(stderr, __VA_ARGS__)

#define	DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); }

#define	DISPLAYUPDATE(l, ...)

#define	DEBUG 0

#define	DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__);

#define	EXM_THROW(error, ...)

#define	MIN(a, b) ((a)<(b) ? (a) : (b))

#define	MAX(a, b) ((a)>(b) ? (a) : (b))

#define	NB_MARKS 4

Functions
static void	LZ4_compressInitNoStream (struct compressionParameters *pThis)

static void	LZ4_compressInitStream (struct compressionParameters *pThis)

static void	LZ4_compressInitStreamHC (struct compressionParameters *pThis)

static void	LZ4_compressResetNoStream (const struct compressionParameters *pThis)

static void	LZ4_compressResetStream (const struct compressionParameters *pThis)

static void	LZ4_compressResetStreamHC (const struct compressionParameters *pThis)

static int	LZ4_compressBlockNoStream (const struct compressionParameters pThis, const char src, char *dst, int srcSize, int dstSize)

static int	LZ4_compressBlockNoStreamHC (const struct compressionParameters pThis, const char src, char *dst, int srcSize, int dstSize)

static int	LZ4_compressBlockStream (const struct compressionParameters pThis, const char src, char *dst, int srcSize, int dstSize)

static int	LZ4_compressBlockStreamHC (const struct compressionParameters pThis, const char src, char *dst, int srcSize, int dstSize)

static void	LZ4_compressCleanupNoStream (const struct compressionParameters *pThis)

static void	LZ4_compressCleanupStream (const struct compressionParameters *pThis)

static void	LZ4_compressCleanupStreamHC (const struct compressionParameters *pThis)

static void	LZ4_buildCompressionParameters (struct compressionParameters pParams, int cLevel, const char dictBuf, int dictSize)

void	BMK_setNotificationLevel (unsigned level)

void	BMK_setAdditionalParam (int additionalParam)

void	BMK_setNbSeconds (unsigned nbSeconds)

void	BMK_setBlockSize (size_t blockSize)

void	BMK_setBenchSeparately (int separate)

static int	BMK_benchMem (const void srcBuffer, size_t srcSize, const char displayName, int cLevel, const size_t fileSizes, U32 nbFiles, const char dictBuf, int dictSize)

static size_t	BMK_findMaxMem (U64 requiredMem)

static void	BMK_benchCLevel (void srcBuffer, size_t benchedSize, const char displayName, int cLevel, int cLevelLast, const size_t fileSizes, unsigned nbFiles, const char dictBuf, int dictSize)

static void	BMK_loadFiles (void buffer, size_t bufferSize, size_t fileSizes, const char **fileNamesTable, unsigned nbFiles)

static void	BMK_benchFileTable (const char *fileNamesTable, unsigned nbFiles, int cLevel, int cLevelLast, const char dictBuf, int dictSize)

static void	BMK_syntheticTest (int cLevel, int cLevelLast, double compressibility, const char *dictBuf, int dictSize)

int	BMK_benchFilesSeparately (const char *fileNamesTable, unsigned nbFiles, int cLevel, int cLevelLast, const char dictBuf, int dictSize)

int	BMK_benchFiles (const char *fileNamesTable, unsigned nbFiles, int cLevel, int cLevelLast, const char dictFileName)

Variables
static const size_t	maxMemory = (sizeof(size_t)==4) ? (2 GB - 64 MB) : (size_t)(1ULL << ((sizeof(size_t)*8)-31))

static U32	g_compressibilityDefault = 50

static U32	g_displayLevel = 2

static const clock_t	refreshRate = CLOCKS_PER_SEC * 15 / 100

static clock_t	g_time = 0

static U32	g_nbSeconds = NBSECONDS

static size_t	g_blockSize = 0

int	g_additionalParam = 0

int	g_benchSeparately = 0

Macro Definition Documentation

◆ ACTIVEPERIOD_MICROSEC

#define ACTIVEPERIOD_MICROSEC 70*1000000ULL /* 70 seconds */

Definition at line 233 of file bench.c.

◆ COOLPERIOD_SEC

#define COOLPERIOD_SEC 10

Definition at line 234 of file bench.c.

◆ DEBUG

#define DEBUG 0

Definition at line 267 of file bench.c.

◆ DEBUGOUTPUT

#define DEBUGOUTPUT ( ... ) if (DEBUG) DISPLAY(__VA_ARGS__);

Definition at line 269 of file bench.c.

◆ DECOMP_MULT

#define DECOMP_MULT 1 /* test decompression DECOMP_MULT times longer than compression */

Definition at line 235 of file bench.c.

◆ DISPLAY

#define DISPLAY ( ... ) fprintf(stderr, __VA_ARGS__)

Definition at line 251 of file bench.c.

◆ DISPLAYLEVEL

#define DISPLAYLEVEL	(	l,
		...
	)	if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); }

Definition at line 252 of file bench.c.

◆ DISPLAYUPDATE

#define DISPLAYUPDATE	(	l,
		...
	)

Value:

            if (g_displayLevel>=l) { \
            if ((clock() - g_time > refreshRate) || (g_displayLevel>=4)) \
            { g_time = clock(); DISPLAY(__VA_ARGS__); \
            if (g_displayLevel>=4) fflush(stdout); } }

Definition at line 255 of file bench.c.

◆ EXM_THROW

#define EXM_THROW	(	error,
		...
	)

Value:

{                                                                         \
    DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \
    DISPLAYLEVEL(1, "Error %i : ", error);                                \
    DISPLAYLEVEL(1, __VA_ARGS__);                                         \
    DISPLAYLEVEL(1, "\n");                                                \
    exit(error);                                                          \
}

Definition at line 270 of file bench.c.

◆ GB

#define GB *(1U<<30)

Definition at line 239 of file bench.c.

◆ KB

#define KB *(1 <<10)

Definition at line 237 of file bench.c.

◆ LZ4_GIT_COMMIT_STRING

#define LZ4_GIT_COMMIT_STRING ""

Definition at line 225 of file bench.c.

◆ LZ4_HC_STATIC_LINKING_ONLY

#define LZ4_HC_STATIC_LINKING_ONLY

Definition at line 52 of file bench.c.

◆ LZ4_isError

#define LZ4_isError ( errcode ) (errcode==0)

Definition at line 218 of file bench.c.

◆ LZ4_MAX_DICT_SIZE

#define LZ4_MAX_DICT_SIZE (64 KB)

Definition at line 241 of file bench.c.

◆ LZ4_STATIC_LINKING_ONLY

#define LZ4_STATIC_LINKING_ONLY

Definition at line 50 of file bench.c.

◆ MAX

#define MAX	(	a,
		b
	)	((a)>(b) ? (a) : (b))

Definition at line 317 of file bench.c.

◆ MB

#define MB *(1 <<20)

Definition at line 238 of file bench.c.

◆ MIN

#define MIN	(	a,
		b
	)	((a)<(b) ? (a) : (b))

Definition at line 316 of file bench.c.

◆ NB_MARKS

#define NB_MARKS 4

◆ NBSECONDS

#define NBSECONDS 3

Definition at line 230 of file bench.c.

◆ TIMELOOP_MICROSEC

#define TIMELOOP_MICROSEC 1*1000000ULL /* 1 second */

Definition at line 231 of file bench.c.

◆ TIMELOOP_NANOSEC

#define TIMELOOP_NANOSEC 1*1000000000ULL /* 1 second */

Definition at line 232 of file bench.c.

Function Documentation

◆ BMK_benchCLevel()

static void BMK_benchCLevel	(	void *	srcBuffer,
		size_t	benchedSize,
		const char *	displayName,
		int	cLevel,
		int	cLevelLast,
		const size_t *	fileSizes,
		unsigned	nbFiles,
		const char *	dictBuf,
		int	dictSize
	)

static

Definition at line 552 of file bench.c.

 {
     int l;
  
     const char* pch = strrchr(displayName, '\\'); /* Windows */
     if (!pch) pch = strrchr(displayName, '/'); /* Linux */
     if (pch) displayName = pch+1;
  
     SET_REALTIME_PRIORITY;
  
     if (g_displayLevel == 1 && !g_additionalParam)
         DISPLAY("bench %s %s: input %u bytes, %u seconds, %u KB blocks\n", LZ4_VERSION_STRING, LZ4_GIT_COMMIT_STRING, (U32)benchedSize, g_nbSeconds, (U32)(g_blockSize>>10));
  
     if (cLevelLast < cLevel) cLevelLast = cLevel;
  
     for (l=cLevel; l <= cLevelLast; l++) {
         BMK_benchMem(srcBuffer, benchedSize,
                      displayName, l,
                      fileSizes, nbFiles,
                      dictBuf, dictSize);
     }
 }

References BMK_benchMem(), compressionParameters::cLevel, compressionParameters::dictBuf, compressionParameters::dictSize, DISPLAY, g_additionalParam, g_blockSize, g_displayLevel, g_nbSeconds, LZ4_GIT_COMMIT_STRING, LZ4_VERSION_STRING, and SET_REALTIME_PRIORITY.

Referenced by BMK_benchFileTable(), and BMK_syntheticTest().

◆ BMK_benchFiles()

int BMK_benchFiles	(	const char **	fileNamesTable,
		unsigned	nbFiles,
		int	cLevel,
		int	cLevelLast,
		const char *	dictFileName
	)

Definition at line 697 of file bench.c.

 {
     double const compressibility = (double)g_compressibilityDefault / 100;
     char* dictBuf = NULL;
     int dictSize = 0;
  
     if (cLevel > LZ4HC_CLEVEL_MAX) cLevel = LZ4HC_CLEVEL_MAX;
     if (cLevelLast > LZ4HC_CLEVEL_MAX) cLevelLast = LZ4HC_CLEVEL_MAX;
     if (cLevelLast < cLevel) cLevelLast = cLevel;
     if (cLevelLast > cLevel) DISPLAYLEVEL(2, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast);
  
     if (dictFileName) {
         FILE* dictFile = NULL;
         U64 dictFileSize = UTIL_getFileSize(dictFileName);
         if (!dictFileSize) EXM_THROW(25, "Dictionary error : could not stat dictionary file");
  
         dictFile = fopen(dictFileName, "rb");
         if (!dictFile) EXM_THROW(25, "Dictionary error : could not open dictionary file");
  
         if (dictFileSize > LZ4_MAX_DICT_SIZE) {
             dictSize = LZ4_MAX_DICT_SIZE;
             if (UTIL_fseek(dictFile, dictFileSize - dictSize, SEEK_SET))
                 EXM_THROW(25, "Dictionary error : could not seek dictionary file");
         } else {
             dictSize = (int)dictFileSize;
         }
  
         dictBuf = (char *)malloc(dictSize);
         if (!dictBuf) EXM_THROW(25, "Allocation error : not enough memory");
  
         if (fread(dictBuf, 1, dictSize, dictFile) != (size_t)dictSize)
             EXM_THROW(25, "Dictionary error : could not read dictionary file");
  
         fclose(dictFile);
     }
  
     if (nbFiles == 0)
         BMK_syntheticTest(cLevel, cLevelLast, compressibility, dictBuf, dictSize);
     else {
         if (g_benchSeparately)
             BMK_benchFilesSeparately(fileNamesTable, nbFiles, cLevel, cLevelLast, dictBuf, dictSize);
         else
             BMK_benchFileTable(fileNamesTable, nbFiles, cLevel, cLevelLast, dictBuf, dictSize);
     }
  
     free(dictBuf);
     return 0;
 }

References BMK_benchFilesSeparately(), BMK_benchFileTable(), BMK_syntheticTest(), compressionParameters::cLevel, compressionParameters::dictBuf, compressionParameters::dictSize, DISPLAYLEVEL, EXM_THROW, benchmark::FILE, free(), g_benchSeparately, g_compressibilityDefault, int, LZ4_MAX_DICT_SIZE, LZ4HC_CLEVEL_MAX, malloc(), NULL, SEEK_SET, UTIL_fseek, and UTIL_getFileSize().

Referenced by main().

◆ BMK_benchFilesSeparately()

int BMK_benchFilesSeparately	(	const char **	fileNamesTable,
		unsigned	nbFiles,
		int	cLevel,
		int	cLevelLast,
		const char *	dictBuf,
		int	dictSize
	)

Definition at line 680 of file bench.c.

 {
     unsigned fileNb;
     if (cLevel > LZ4HC_CLEVEL_MAX) cLevel = LZ4HC_CLEVEL_MAX;
     if (cLevelLast > LZ4HC_CLEVEL_MAX) cLevelLast = LZ4HC_CLEVEL_MAX;
     if (cLevelLast < cLevel) cLevelLast = cLevel;
     if (cLevelLast > cLevel) DISPLAYLEVEL(2, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast);
  
     for (fileNb=0; fileNb<nbFiles; fileNb++)
         BMK_benchFileTable(fileNamesTable+fileNb, 1, cLevel, cLevelLast, dictBuf, dictSize);
  
     return 0;
 }

References BMK_benchFileTable(), compressionParameters::cLevel, compressionParameters::dictBuf, compressionParameters::dictSize, DISPLAYLEVEL, and LZ4HC_CLEVEL_MAX.

Referenced by BMK_benchFiles().

◆ BMK_benchFileTable()

static void BMK_benchFileTable	(	const char **	fileNamesTable,
		unsigned	nbFiles,
		int	cLevel,
		int	cLevelLast,
		const char *	dictBuf,
		int	dictSize
	)

static

Definition at line 614 of file bench.c.

 {
     void* srcBuffer;
     size_t benchedSize;
     size_t* fileSizes = (size_t*)malloc(nbFiles * sizeof(size_t));
     U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, nbFiles);
     char mfName[20] = {0};
  
     if (!fileSizes) EXM_THROW(12, "not enough memory for fileSizes");
  
     /* Memory allocation & restrictions */
     benchedSize = BMK_findMaxMem(totalSizeToLoad * 3) / 3;
     if (benchedSize==0) EXM_THROW(12, "not enough memory");
     if ((U64)benchedSize > totalSizeToLoad) benchedSize = (size_t)totalSizeToLoad;
     if (benchedSize > LZ4_MAX_INPUT_SIZE) {
         benchedSize = LZ4_MAX_INPUT_SIZE;
         DISPLAY("File(s) bigger than LZ4's max input size; testing %u MB only...\n", (U32)(benchedSize >> 20));
     } else {
         if (benchedSize < totalSizeToLoad)
             DISPLAY("Not enough memory; testing %u MB only...\n", (U32)(benchedSize >> 20));
     }
     srcBuffer = malloc(benchedSize + !benchedSize);   /* avoid alloc of zero */
     if (!srcBuffer) EXM_THROW(12, "not enough memory");
  
     /* Load input buffer */
     BMK_loadFiles(srcBuffer, benchedSize, fileSizes, fileNamesTable, nbFiles);
  
     /* Bench */
     snprintf (mfName, sizeof(mfName), " %u files", nbFiles);
     {   const char* displayName = (nbFiles > 1) ? mfName : fileNamesTable[0];
         BMK_benchCLevel(srcBuffer, benchedSize,
                         displayName, cLevel, cLevelLast,
                         fileSizes, nbFiles,
                         dictBuf, dictSize);
     }
  
     /* clean up */
     free(srcBuffer);
     free(fileSizes);
 }

References BMK_benchCLevel(), BMK_findMaxMem(), BMK_loadFiles(), compressionParameters::cLevel, compressionParameters::dictBuf, compressionParameters::dictSize, DISPLAY, EXM_THROW, free(), LZ4_MAX_INPUT_SIZE, malloc(), snprintf, and UTIL_getTotalFileSize().

Referenced by BMK_benchFiles(), and BMK_benchFilesSeparately().

◆ BMK_benchMem()

static int BMK_benchMem	(	const void *	srcBuffer,
		size_t	srcSize,
		const char *	displayName,
		int	cLevel,
		const size_t *	fileSizes,
		U32	nbFiles,
		const char *	dictBuf,
		int	dictSize
	)

static

Definition at line 319 of file bench.c.

 {
     size_t const blockSize = (g_blockSize>=32 ? g_blockSize : srcSize) + (!srcSize) /* avoid div by 0 */ ;
     U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles;
     blockParam_t* const blockTable = (blockParam_t*) malloc(maxNbBlocks * sizeof(blockParam_t));
     size_t const maxCompressedSize = LZ4_compressBound((int)srcSize) + (maxNbBlocks * 1024);   /* add some room for safety */
     void* const compressedBuffer = malloc(maxCompressedSize);
     void* const resultBuffer = malloc(srcSize);
     U32 nbBlocks;
     struct compressionParameters compP;
  
     /* checks */
     if (!compressedBuffer || !resultBuffer || !blockTable)
         EXM_THROW(31, "allocation error : not enough memory");
  
     if (strlen(displayName)>17) displayName += strlen(displayName)-17;   /* can only display 17 characters */
  
     /* init */
     LZ4_buildCompressionParameters(&compP, cLevel, dictBuf, dictSize);
     compP.initFunction(&compP);
  
     /* Init blockTable data */
     {   const char* srcPtr = (const char*)srcBuffer;
         char* cPtr = (char*)compressedBuffer;
         char* resPtr = (char*)resultBuffer;
         U32 fileNb;
         for (nbBlocks=0, fileNb=0; fileNb<nbFiles; fileNb++) {
             size_t remaining = fileSizes[fileNb];
             U32 const nbBlocksforThisFile = (U32)((remaining + (blockSize-1)) / blockSize);
             U32 const blockEnd = nbBlocks + nbBlocksforThisFile;
             for ( ; nbBlocks<blockEnd; nbBlocks++) {
                 size_t const thisBlockSize = MIN(remaining, blockSize);
                 blockTable[nbBlocks].srcPtr = srcPtr;
                 blockTable[nbBlocks].cPtr = cPtr;
                 blockTable[nbBlocks].resPtr = resPtr;
                 blockTable[nbBlocks].srcSize = thisBlockSize;
                 blockTable[nbBlocks].cRoom = (size_t)LZ4_compressBound((int)thisBlockSize);
                 srcPtr += thisBlockSize;
                 cPtr += blockTable[nbBlocks].cRoom;
                 resPtr += thisBlockSize;
                 remaining -= thisBlockSize;
     }   }   }
  
     /* warmimg up memory */
     RDG_genBuffer(compressedBuffer, maxCompressedSize, 0.10, 0.50, 1);
  
     /* Bench */
     {   U64 fastestC = (U64)(-1LL), fastestD = (U64)(-1LL);
         U64 const crcOrig = XXH64(srcBuffer, srcSize, 0);
         UTIL_time_t coolTime;
         U64 const maxTime = (g_nbSeconds * TIMELOOP_NANOSEC) + 100;
         U32 nbCompressionLoops = (U32)((5 MB) / (srcSize+1)) + 1;  /* conservative initial compression speed estimate */
         U32 nbDecodeLoops = (U32)((200 MB) / (srcSize+1)) + 1;  /* conservative initial decode speed estimate */
         U64 totalCTime=0, totalDTime=0;
         U32 cCompleted=0, dCompleted=0;
 #       define NB_MARKS 4
         const char* const marks[NB_MARKS] = { " |", " /", " =",  "\\" };
         U32 markNb = 0;
         size_t cSize = 0;
         double ratio = 0.;
  
         coolTime = UTIL_getTime();
         DISPLAYLEVEL(2, "\r%79s\r", "");
         while (!cCompleted || !dCompleted) {
             /* overheat protection */
             if (UTIL_clockSpanMicro(coolTime) > ACTIVEPERIOD_MICROSEC) {
                 DISPLAYLEVEL(2, "\rcooling down ...    \r");
                 UTIL_sleep(COOLPERIOD_SEC);
                 coolTime = UTIL_getTime();
             }
  
             /* Compression */
             DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->\r", marks[markNb], displayName, (U32)srcSize);
             if (!cCompleted) memset(compressedBuffer, 0xE5, maxCompressedSize);  /* warm up and erase result buffer */
  
             UTIL_sleepMilli(1);  /* give processor time to other processes */
             UTIL_waitForNextTick();
  
             if (!cCompleted) {   /* still some time to do compression tests */
                 UTIL_time_t const clockStart = UTIL_getTime();
                 U32 nbLoops;
                 for (nbLoops=0; nbLoops < nbCompressionLoops; nbLoops++) {
                     U32 blockNb;
                     compP.resetFunction(&compP);
                     for (blockNb=0; blockNb<nbBlocks; blockNb++) {
                         size_t const rSize = (size_t)compP.blockFunction(
                             &compP,
                             blockTable[blockNb].srcPtr, blockTable[blockNb].cPtr,
                             (int)blockTable[blockNb].srcSize, (int)blockTable[blockNb].cRoom);
                         if (LZ4_isError(rSize)) EXM_THROW(1, "LZ4 compression failed");
                         blockTable[blockNb].cSize = rSize;
                 }   }
                 {   U64 const clockSpan = UTIL_clockSpanNano(clockStart);
                     if (clockSpan > 0) {
                         if (clockSpan < fastestC * nbCompressionLoops)
                             fastestC = clockSpan / nbCompressionLoops;
                         assert(fastestC > 0);
                         nbCompressionLoops = (U32)(TIMELOOP_NANOSEC / fastestC) + 1;  /* aim for ~1sec */
                     } else {
                         assert(nbCompressionLoops < 40000000);   /* avoid overflow */
                         nbCompressionLoops *= 100;
                     }
                     totalCTime += clockSpan;
                     cCompleted = totalCTime>maxTime;
             }   }
  
             cSize = 0;
             { U32 blockNb; for (blockNb=0; blockNb<nbBlocks; blockNb++) cSize += blockTable[blockNb].cSize; }
             cSize += !cSize;  /* avoid div by 0 */
             ratio = (double)srcSize / (double)cSize;
             markNb = (markNb+1) % NB_MARKS;
             DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s\r",
                     marks[markNb], displayName, (U32)srcSize, (U32)cSize, ratio,
                     ((double)srcSize / fastestC) * 1000 );
  
             (void)fastestD; (void)crcOrig;   /*  unused when decompression disabled */
 #if 1
             /* Decompression */
             if (!dCompleted) memset(resultBuffer, 0xD6, srcSize);  /* warm result buffer */
  
             UTIL_sleepMilli(5); /* give processor time to other processes */
             UTIL_waitForNextTick();
  
             if (!dCompleted) {
                 UTIL_time_t const clockStart = UTIL_getTime();
                 U32 nbLoops;
                 for (nbLoops=0; nbLoops < nbDecodeLoops; nbLoops++) {
                     U32 blockNb;
                     for (blockNb=0; blockNb<nbBlocks; blockNb++) {
                         int const regenSize = LZ4_decompress_safe_usingDict(
                             blockTable[blockNb].cPtr, blockTable[blockNb].resPtr,
                             (int)blockTable[blockNb].cSize, (int)blockTable[blockNb].srcSize,
                             dictBuf, dictSize);
                         if (regenSize < 0) {
                             DISPLAY("LZ4_decompress_safe_usingDict() failed on block %u \n", blockNb);
                             break;
                         }
                         blockTable[blockNb].resSize = (size_t)regenSize;
                 }   }
                 {   U64 const clockSpan = UTIL_clockSpanNano(clockStart);
                     if (clockSpan > 0) {
                         if (clockSpan < fastestD * nbDecodeLoops)
                             fastestD = clockSpan / nbDecodeLoops;
                         assert(fastestD > 0);
                         nbDecodeLoops = (U32)(TIMELOOP_NANOSEC / fastestD) + 1;  /* aim for ~1sec */
                     } else {
                         assert(nbDecodeLoops < 40000000);   /* avoid overflow */
                         nbDecodeLoops *= 100;
                     }
                     totalDTime += clockSpan;
                     dCompleted = totalDTime > (DECOMP_MULT*maxTime);
             }   }
  
             markNb = (markNb+1) % NB_MARKS;
             DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s ,%6.1f MB/s\r",
                     marks[markNb], displayName, (U32)srcSize, (U32)cSize, ratio,
                     ((double)srcSize / fastestC) * 1000,
                     ((double)srcSize / fastestD) * 1000);
  
             /* CRC Checking */
             {   U64 const crcCheck = XXH64(resultBuffer, srcSize, 0);
                 if (crcOrig!=crcCheck) {
                     size_t u;
                     DISPLAY("\n!!! WARNING !!! %17s : Invalid Checksum : %x != %x   \n", displayName, (unsigned)crcOrig, (unsigned)crcCheck);
                     for (u=0; u<srcSize; u++) {
                         if (((const BYTE*)srcBuffer)[u] != ((const BYTE*)resultBuffer)[u]) {
                             U32 segNb, bNb, pos;
                             size_t bacc = 0;
                             DISPLAY("Decoding error at pos %u ", (U32)u);
                             for (segNb = 0; segNb < nbBlocks; segNb++) {
                                 if (bacc + blockTable[segNb].srcSize > u) break;
                                 bacc += blockTable[segNb].srcSize;
                             }
                             pos = (U32)(u - bacc);
                             bNb = pos / (128 KB);
                             DISPLAY("(block %u, sub %u, pos %u) \n", segNb, bNb, pos);
                             break;
                         }
                         if (u==srcSize-1) {  /* should never happen */
                             DISPLAY("no difference detected\n");
                     }   }
                     break;
             }   }   /* CRC Checking */
 #endif
         }   /* for (testNb = 1; testNb <= (g_nbSeconds + !g_nbSeconds); testNb++) */
  
         if (g_displayLevel == 1) {
             double const cSpeed = ((double)srcSize / fastestC) * 1000;
             double const dSpeed = ((double)srcSize / fastestD) * 1000;
             if (g_additionalParam)
                 DISPLAY("-%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s  %s (param=%d)\n", cLevel, (int)cSize, ratio, cSpeed, dSpeed, displayName, g_additionalParam);
             else
                 DISPLAY("-%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s  %s\n", cLevel, (int)cSize, ratio, cSpeed, dSpeed, displayName);
         }
         DISPLAYLEVEL(2, "%2i#\n", cLevel);
     }   /* Bench */
  
     /* clean up */
     compP.cleanupFunction(&compP);
     free(blockTable);
     free(compressedBuffer);
     free(resultBuffer);
     return 0;
 }

References ACTIVEPERIOD_MICROSEC, assert(), compressionParameters::blockFunction, compressionParameters::cleanupFunction, compressionParameters::cLevel, COOLPERIOD_SEC, blockParam_t::cPtr, blockParam_t::cRoom, blockParam_t::cSize, DECOMP_MULT, compressionParameters::dictBuf, compressionParameters::dictSize, DISPLAY, DISPLAYLEVEL, EXM_THROW, free(), g_additionalParam, g_blockSize, g_displayLevel, g_nbSeconds, compressionParameters::initFunction, int, KB, LZ4_buildCompressionParameters(), LZ4_compressBound(), LZ4_decompress_safe_usingDict(), LZ4_isError, malloc(), MB, memset(), MIN, NB_MARKS, pos, RDG_genBuffer(), compressionParameters::resetFunction, blockParam_t::resPtr, blockParam_t::resSize, blockParam_t::srcPtr, srcSize, blockParam_t::srcSize, TIMELOOP_NANOSEC, U32, U64, UTIL_clockSpanMicro(), UTIL_clockSpanNano(), UTIL_getTime(), UTIL_sleep, UTIL_sleepMilli, UTIL_waitForNextTick(), and XXH64().

Referenced by BMK_benchCLevel().

◆ BMK_findMaxMem()

static size_t BMK_findMaxMem ( U64 requiredMem )

static

Definition at line 528 of file bench.c.

 {
     size_t step = 64 MB;
     BYTE* testmem=NULL;
  
     requiredMem = (((requiredMem >> 26) + 1) << 26);
     requiredMem += 2*step;
     if (requiredMem > maxMemory) requiredMem = maxMemory;
  
     while (!testmem) {
         if (requiredMem > step) requiredMem -= step;
         else requiredMem >>= 1;
         testmem = (BYTE*) malloc ((size_t)requiredMem);
     }
     free (testmem);
  
     /* keep some space available */
     if (requiredMem > step) requiredMem -= step;
     else requiredMem >>= 1;
  
     return (size_t)requiredMem;
 }

References free(), malloc(), maxMemory, MB, NULL, and step().

Referenced by BMK_benchFileTable().

◆ BMK_loadFiles()

static void BMK_loadFiles	(	void *	buffer,
		size_t	bufferSize,
		size_t *	fileSizes,
		const char **	fileNamesTable,
		unsigned	nbFiles
	)

static

BMK_loadFiles() : Loads buffer with content of files listed within fileNamesTable. At most, fills buffer entirely

Definition at line 582 of file bench.c.

 {
     size_t pos = 0, totalSize = 0;
     unsigned n;
     for (n=0; n<nbFiles; n++) {
         FILE* f;
         U64 fileSize = UTIL_getFileSize(fileNamesTable[n]);
         if (UTIL_isDirectory(fileNamesTable[n])) {
             DISPLAYLEVEL(2, "Ignoring %s directory...       \n", fileNamesTable[n]);
             fileSizes[n] = 0;
             continue;
         }
         f = fopen(fileNamesTable[n], "rb");
         if (f==NULL) EXM_THROW(10, "impossible to open file %s", fileNamesTable[n]);
         DISPLAYUPDATE(2, "Loading %s...       \r", fileNamesTable[n]);
         if (fileSize > bufferSize-pos) { /* buffer too small - stop after this file */
             fileSize = bufferSize-pos;
             nbFiles=n;
         }
         { size_t const readSize = fread(((char*)buffer)+pos, 1, (size_t)fileSize, f);
           if (readSize != (size_t)fileSize) EXM_THROW(11, "could not read %s", fileNamesTable[n]);
           pos += readSize; }
         fileSizes[n] = (size_t)fileSize;
         totalSize += (size_t)fileSize;
         fclose(f);
     }
  
     if (totalSize == 0) EXM_THROW(12, "no data to bench");
 }

References DISPLAYLEVEL, DISPLAYUPDATE, EXM_THROW, f, benchmark::FILE, n, NULL, pos, UTIL_getFileSize(), and UTIL_isDirectory().

Referenced by BMK_benchFileTable().

◆ BMK_setAdditionalParam()

void BMK_setAdditionalParam ( int additionalParam )

Definition at line 290 of file bench.c.

290 { g_additionalParam=additionalParam; }

References g_additionalParam.

◆ BMK_setBenchSeparately()

void BMK_setBenchSeparately ( int separate )

Definition at line 300 of file bench.c.

300 { g_benchSeparately = (separate!=0); }

References g_benchSeparately.

Referenced by main().

◆ BMK_setBlockSize()

void BMK_setBlockSize ( size_t blockSize )

Definition at line 298 of file bench.c.

298 { g_blockSize = blockSize; }

References g_blockSize.

Referenced by main().

◆ BMK_setNbSeconds()

void BMK_setNbSeconds ( unsigned nbSeconds )

Definition at line 292 of file bench.c.

 {
     g_nbSeconds = nbSeconds;
     DISPLAYLEVEL(3, "- test >= %u seconds per compression / decompression -\n", g_nbSeconds);
 }

References DISPLAYLEVEL, and g_nbSeconds.

Referenced by main().

◆ BMK_setNotificationLevel()

void BMK_setNotificationLevel ( unsigned level )

Definition at line 288 of file bench.c.

288 { g_displayLevel=level; }

level

static int level

Definition: vmenus.c:2424

References g_displayLevel, and level.

Referenced by main().

◆ BMK_syntheticTest()

static void BMK_syntheticTest	(	int	cLevel,
		int	cLevelLast,
		double	compressibility,
		const char *	dictBuf,
		int	dictSize
	)

static

Definition at line 658 of file bench.c.

 {
     char name[20] = {0};
     size_t benchedSize = 10000000;
     void* const srcBuffer = malloc(benchedSize);
  
     /* Memory allocation */
     if (!srcBuffer) EXM_THROW(21, "not enough memory");
  
     /* Fill input buffer */
     RDG_genBuffer(srcBuffer, benchedSize, compressibility, 0.0, 0);
  
     /* Bench */
     snprintf (name, sizeof(name), "Synthetic %2u%%", (unsigned)(compressibility*100));
     BMK_benchCLevel(srcBuffer, benchedSize, name, cLevel, cLevelLast, &benchedSize, 1, dictBuf, dictSize);
  
     /* clean up */
     free(srcBuffer);
 }

References BMK_benchCLevel(), compressionParameters::cLevel, compressionParameters::dictBuf, compressionParameters::dictSize, EXM_THROW, free(), malloc(), RDG_genBuffer(), and snprintf.

Referenced by BMK_benchFiles().

◆ LZ4_buildCompressionParameters()

static void LZ4_buildCompressionParameters	(	struct compressionParameters *	pParams,
		int	cLevel,
		const char *	dictBuf,
		int	dictSize
	)

static

Definition at line 185 of file bench.c.

 {
     pParams->cLevel = cLevel;
     pParams->dictBuf = dictBuf;
     pParams->dictSize = dictSize;
  
     if (dictSize) {
         if (cLevel < LZ4HC_CLEVEL_MIN) {
             pParams->initFunction = LZ4_compressInitStream;
             pParams->resetFunction = LZ4_compressResetStream;
             pParams->blockFunction = LZ4_compressBlockStream;
             pParams->cleanupFunction = LZ4_compressCleanupStream;
         } else {
             pParams->initFunction = LZ4_compressInitStreamHC;
             pParams->resetFunction = LZ4_compressResetStreamHC;
             pParams->blockFunction = LZ4_compressBlockStreamHC;
             pParams->cleanupFunction = LZ4_compressCleanupStreamHC;
         }
     } else {
         pParams->initFunction = LZ4_compressInitNoStream;
         pParams->resetFunction = LZ4_compressResetNoStream;
         pParams->cleanupFunction = LZ4_compressCleanupNoStream;
  
         if (cLevel < LZ4HC_CLEVEL_MIN) {
             pParams->blockFunction = LZ4_compressBlockNoStream;
         } else {
             pParams->blockFunction = LZ4_compressBlockNoStreamHC;
         }
     }
 }

References compressionParameters::blockFunction, compressionParameters::cleanupFunction, compressionParameters::cLevel, compressionParameters::dictBuf, compressionParameters::dictSize, compressionParameters::initFunction, LZ4_compressBlockNoStream(), LZ4_compressBlockNoStreamHC(), LZ4_compressBlockStream(), LZ4_compressBlockStreamHC(), LZ4_compressCleanupNoStream(), LZ4_compressCleanupStream(), LZ4_compressCleanupStreamHC(), LZ4_compressInitNoStream(), LZ4_compressInitStream(), LZ4_compressInitStreamHC(), LZ4_compressResetNoStream(), LZ4_compressResetStream(), LZ4_compressResetStreamHC(), LZ4HC_CLEVEL_MIN, and compressionParameters::resetFunction.

Referenced by BMK_benchMem().

◆ LZ4_compressBlockNoStream()

static int LZ4_compressBlockNoStream	(	const struct compressionParameters *	pThis,
		const char *	src,
		char *	dst,
		int	srcSize,
		int	dstSize
	)

static

Definition at line 131 of file bench.c.

 {
     int const acceleration = (pThis->cLevel < 0) ? -pThis->cLevel + 1 : 1;
     return LZ4_compress_fast(src, dst, srcSize, dstSize, acceleration);
 }

References compressionParameters::cLevel, dst, LZ4_compress_fast(), src, and srcSize.

Referenced by LZ4_buildCompressionParameters().

◆ LZ4_compressBlockNoStreamHC()

static int LZ4_compressBlockNoStreamHC	(	const struct compressionParameters *	pThis,
		const char *	src,
		char *	dst,
		int	srcSize,
		int	dstSize
	)

static

Definition at line 140 of file bench.c.

 {
     return LZ4_compress_HC(src, dst, srcSize, dstSize, pThis->cLevel);
 }

References compressionParameters::cLevel, dst, LZ4_compress_HC(), src, and srcSize.

Referenced by LZ4_buildCompressionParameters().

◆ LZ4_compressBlockStream()

static int LZ4_compressBlockStream	(	const struct compressionParameters *	pThis,
		const char *	src,
		char *	dst,
		int	srcSize,
		int	dstSize
	)

static

Definition at line 148 of file bench.c.

 {
     int const acceleration = (pThis->cLevel < 0) ? -pThis->cLevel + 1 : 1;
     return LZ4_compress_fast_continue(pThis->LZ4_stream, src, dst, srcSize, dstSize, acceleration);
 }

References compressionParameters::cLevel, dst, LZ4_compress_fast_continue(), compressionParameters::LZ4_stream, src, and srcSize.

Referenced by LZ4_buildCompressionParameters().

◆ LZ4_compressBlockStreamHC()

static int LZ4_compressBlockStreamHC	(	const struct compressionParameters *	pThis,
		const char *	src,
		char *	dst,
		int	srcSize,
		int	dstSize
	)

static

Definition at line 157 of file bench.c.

 {
     return LZ4_compress_HC_continue(pThis->LZ4_streamHC, src, dst, srcSize, dstSize);
 }

References dst, LZ4_compress_HC_continue(), compressionParameters::LZ4_streamHC, src, and srcSize.

Referenced by LZ4_buildCompressionParameters().

◆ LZ4_compressCleanupNoStream()

static void LZ4_compressCleanupNoStream ( const struct compressionParameters * pThis )

static

Definition at line 165 of file bench.c.

 {
     (void)pThis;
 }

Referenced by LZ4_buildCompressionParameters().

◆ LZ4_compressCleanupStream()

static void LZ4_compressCleanupStream ( const struct compressionParameters * pThis )

static

Definition at line 171 of file bench.c.

 {
     LZ4_freeStream(pThis->LZ4_stream);
     LZ4_freeStream(pThis->LZ4_dictStream);
 }

References compressionParameters::LZ4_dictStream, LZ4_freeStream(), and compressionParameters::LZ4_stream.

Referenced by LZ4_buildCompressionParameters().

◆ LZ4_compressCleanupStreamHC()

static void LZ4_compressCleanupStreamHC ( const struct compressionParameters * pThis )

static

Definition at line 178 of file bench.c.

 {
     LZ4_freeStreamHC(pThis->LZ4_streamHC);
     LZ4_freeStreamHC(pThis->LZ4_dictStreamHC);
 }

References compressionParameters::LZ4_dictStreamHC, LZ4_freeStreamHC(), and compressionParameters::LZ4_streamHC.

Referenced by LZ4_buildCompressionParameters().

◆ LZ4_compressInitNoStream()

static void LZ4_compressInitNoStream ( struct compressionParameters * pThis )

static

Definition at line 82 of file bench.c.

 {
     pThis->LZ4_stream = NULL;
     pThis->LZ4_dictStream = NULL;
     pThis->LZ4_streamHC = NULL;
     pThis->LZ4_dictStreamHC = NULL;
 }

References compressionParameters::LZ4_dictStream, compressionParameters::LZ4_dictStreamHC, compressionParameters::LZ4_stream, compressionParameters::LZ4_streamHC, and NULL.

Referenced by LZ4_buildCompressionParameters().

◆ LZ4_compressInitStream()

static void LZ4_compressInitStream ( struct compressionParameters * pThis )

static

Definition at line 91 of file bench.c.

 {
     pThis->LZ4_stream = LZ4_createStream();
     pThis->LZ4_dictStream = LZ4_createStream();
     pThis->LZ4_streamHC = NULL;
     pThis->LZ4_dictStreamHC = NULL;
     LZ4_loadDict(pThis->LZ4_dictStream, pThis->dictBuf, pThis->dictSize);
 }

References compressionParameters::dictBuf, compressionParameters::dictSize, LZ4_createStream(), compressionParameters::LZ4_dictStream, compressionParameters::LZ4_dictStreamHC, LZ4_loadDict(), compressionParameters::LZ4_stream, compressionParameters::LZ4_streamHC, and NULL.

Referenced by LZ4_buildCompressionParameters().

◆ LZ4_compressInitStreamHC()

static void LZ4_compressInitStreamHC ( struct compressionParameters * pThis )

static

Definition at line 101 of file bench.c.

 {
     pThis->LZ4_stream = NULL;
     pThis->LZ4_dictStream = NULL;
     pThis->LZ4_streamHC = LZ4_createStreamHC();
     pThis->LZ4_dictStreamHC = LZ4_createStreamHC();
     LZ4_loadDictHC(pThis->LZ4_dictStreamHC, pThis->dictBuf, pThis->dictSize);
 }

References compressionParameters::dictBuf, compressionParameters::dictSize, LZ4_createStreamHC(), compressionParameters::LZ4_dictStream, compressionParameters::LZ4_dictStreamHC, LZ4_loadDictHC(), compressionParameters::LZ4_stream, compressionParameters::LZ4_streamHC, and NULL.

Referenced by LZ4_buildCompressionParameters().

◆ LZ4_compressResetNoStream()

static void LZ4_compressResetNoStream ( const struct compressionParameters * pThis )

static

Definition at line 111 of file bench.c.

 {
     (void)pThis;
 }

Referenced by LZ4_buildCompressionParameters().

◆ LZ4_compressResetStream()

static void LZ4_compressResetStream ( const struct compressionParameters * pThis )

static

Definition at line 117 of file bench.c.

 {
     LZ4_resetStream_fast(pThis->LZ4_stream);
     LZ4_attach_dictionary(pThis->LZ4_stream, pThis->LZ4_dictStream);
 }

References LZ4_attach_dictionary(), compressionParameters::LZ4_dictStream, LZ4_resetStream_fast(), and compressionParameters::LZ4_stream.

Referenced by LZ4_buildCompressionParameters().

◆ LZ4_compressResetStreamHC()

static void LZ4_compressResetStreamHC ( const struct compressionParameters * pThis )

static

Definition at line 124 of file bench.c.

 {
     LZ4_resetStreamHC_fast(pThis->LZ4_streamHC, pThis->cLevel);
     LZ4_attach_HC_dictionary(pThis->LZ4_streamHC, pThis->LZ4_dictStreamHC);
 }

References compressionParameters::cLevel, LZ4_attach_HC_dictionary(), compressionParameters::LZ4_dictStreamHC, LZ4_resetStreamHC_fast(), and compressionParameters::LZ4_streamHC.

Referenced by LZ4_buildCompressionParameters().

Variable Documentation

◆ g_additionalParam

int g_additionalParam = 0

Definition at line 285 of file bench.c.

Referenced by BMK_benchCLevel(), BMK_benchMem(), and BMK_setAdditionalParam().

◆ g_benchSeparately

int g_benchSeparately = 0

Definition at line 286 of file bench.c.

Referenced by BMK_benchFiles(), and BMK_setBenchSeparately().

◆ g_blockSize

size_t g_blockSize = 0

static

Definition at line 284 of file bench.c.

Referenced by BMK_benchCLevel(), BMK_benchMem(), and BMK_setBlockSize().

◆ g_compressibilityDefault

U32 g_compressibilityDefault = 50

static

Definition at line 245 of file bench.c.

Referenced by BMK_benchFiles().

◆ g_displayLevel

U32 g_displayLevel = 2

static

Definition at line 253 of file bench.c.

Referenced by BMK_benchCLevel(), BMK_benchMem(), and BMK_setNotificationLevel().

◆ g_nbSeconds

U32 g_nbSeconds = NBSECONDS

static

Definition at line 283 of file bench.c.

Referenced by BMK_benchCLevel(), BMK_benchMem(), and BMK_setNbSeconds().

◆ g_time

clock_t g_time = 0

static

Definition at line 260 of file bench.c.

Referenced by FUZ_displayUpdate().

◆ maxMemory

const size_t maxMemory = (sizeof(size_t)==4) ? (2 GB - 64 MB) : (size_t)(1ULL << ((sizeof(size_t)*8)-31))

static

Definition at line 243 of file bench.c.

Referenced by BMK_findMaxMem().

◆ refreshRate

const clock_t refreshRate = CLOCKS_PER_SEC * 15 / 100

static

Definition at line 259 of file bench.c.

Classes

Macros

Functions

Variables

Macro Definition Documentation

◆ ACTIVEPERIOD_MICROSEC

◆ COOLPERIOD_SEC

◆ DEBUG

◆ DEBUGOUTPUT

◆ DECOMP_MULT

◆ DISPLAY

◆ DISPLAYLEVEL

◆ DISPLAYUPDATE

◆ EXM_THROW

◆ GB

◆ KB

◆ LZ4_GIT_COMMIT_STRING

◆ LZ4_HC_STATIC_LINKING_ONLY

◆ LZ4_isError

◆ LZ4_MAX_DICT_SIZE

◆ LZ4_STATIC_LINKING_ONLY

◆ MAX

◆ MB

◆ MIN

◆ NB_MARKS

◆ NBSECONDS

◆ TIMELOOP_MICROSEC

◆ TIMELOOP_NANOSEC

Function Documentation

◆ BMK_benchCLevel()

◆ BMK_benchFiles()

◆ BMK_benchFilesSeparately()

◆ BMK_benchFileTable()

◆ BMK_benchMem()

◆ BMK_findMaxMem()

◆ BMK_loadFiles()

◆ BMK_setAdditionalParam()

◆ BMK_setBenchSeparately()

◆ BMK_setBlockSize()

◆ BMK_setNbSeconds()

◆ BMK_setNotificationLevel()

◆ BMK_syntheticTest()

◆ LZ4_buildCompressionParameters()

◆ LZ4_compressBlockNoStream()

◆ LZ4_compressBlockNoStreamHC()

◆ LZ4_compressBlockStream()

◆ LZ4_compressBlockStreamHC()

◆ LZ4_compressCleanupNoStream()

◆ LZ4_compressCleanupStream()

◆ LZ4_compressCleanupStreamHC()

◆ LZ4_compressInitNoStream()

◆ LZ4_compressInitStream()

◆ LZ4_compressInitStreamHC()

◆ LZ4_compressResetNoStream()

◆ LZ4_compressResetStream()

◆ LZ4_compressResetStreamHC()

Variable Documentation

◆ g_additionalParam

◆ g_benchSeparately

◆ g_blockSize

◆ g_compressibilityDefault

◆ g_displayLevel

◆ g_nbSeconds

◆ g_time

◆ maxMemory

◆ refreshRate