utf8.c File Reference

#include <rz_types.h>
#include <rz_util.h>
#include <rz_windows.h>

Go to the source code of this file.

Macros
#define	UTF_LAST_BLOCK   (281)
#define	UTF_BLOCKS_COUNT   RZ_ARRAY_SIZE(utf_blocks)
#define	UTF_NONPRINTABLE_RANGES_COUNT   RZ_ARRAY_SIZE(nonprintable_ranges)

Functions
RZ_API const char *	rz_utf_block_name (int idx)
RZ_API int	rz_utf8_decode (const ut8 *ptr, int ptrlen, RzRune *ch)
RZ_API int	rz_mutf8_decode (const ut8 *ptr, int ptrlen, RzRune *ch)
RZ_API int	rz_utf8_encode (ut8 *ptr, const RzRune ch)
RZ_API int	rz_utf8_encode_str (const RzRune *str, ut8 *dst, const int dst_length)
RZ_API int	rz_utf8_size (const ut8 *ptr)
RZ_API int	rz_utf8_strlen (const ut8 *str)
RZ_API bool	rz_rune_is_printable (const RzRune c)
	Returns true when the RzRune is a printable symbol. More...
RZ_API int	rz_utf_block_idx (RzRune ch)
RZ_API int *	rz_utf_block_list (const ut8 *str, int len, int **freq_list)
RZ_API RzStrEnc	rz_utf_bom_encoding (const ut8 *ptr, int ptrlen)

Variables
struct {
ut32   from
ut32   to
}	nonprintable_ranges []
const RUtfBlock	utf_blocks []

Macro Definition Documentation

UTF_BLOCKS_COUNT

#define UTF_BLOCKS_COUNT   RZ_ARRAY_SIZE(utf_blocks)

Definition at line 11 of file utf8.c.

UTF_LAST_BLOCK

#define UTF_LAST_BLOCK   (281)

Definition at line 10 of file utf8.c.

UTF_NONPRINTABLE_RANGES_COUNT

#define UTF_NONPRINTABLE_RANGES_COUNT   RZ_ARRAY_SIZE(nonprintable_ranges)

Definition at line 12 of file utf8.c.

Function Documentation

rz_mutf8_decode()

RZ_API int rz_mutf8_decode	(	const ut8 *	ptr,
		int	ptrlen,
		RzRune *	ch
	)

Definition at line 524 of file utf8.c.

                                                                   {
    if (ptrlen > 1 && ptr[0] == 0xc0 && ptr[1] == 0x80) {
        if (ch) {
            *ch = 0;
        }
        return 2;
    }
    return rz_utf8_decode(ptr, ptrlen, ch);
}

References rz_utf8_decode().

Referenced by escape_utf8_for_json().

rz_rune_is_printable()

RZ_API bool rz_rune_is_printable

(

const RzRune

c

)

Returns true when the RzRune is a printable symbol.

Parameters

c	RzRune value to test

Returns

true if the rune is printable, otherwise false

Definition at line 606 of file utf8.c.

                                                 {
    // RzRunes are most commonly single byte... We can early out with this common case.
    if (c < 0x34F) {
        /*
        manually copied from top, please update if this ever changes
        { 0x0000, 0x001F }, { 0x007F, 0x009F }, { 0x034F, 0x034F },
        could do a linear search, but that's a lot slower than a few compare
        */
        return !(c <= 0x1F || (c >= 0x7F && c <= 0x9F));
    }
 
    const int last = UTF_NONPRINTABLE_RANGES_COUNT;
 
    int low = 0;
    int hi = last - 1;
 
    do {
        int mid = (low + hi) >> 1;
        if (c >= nonprintable_ranges[mid].from && c <= nonprintable_ranges[mid].to) {
            return false;
        }
        if (mid < last && c > nonprintable_ranges[mid].to) {
            low = mid + 1;
        }
        if (mid < last && c < nonprintable_ranges[mid].from) {
            hi = mid - 1;
        }
    } while (low <= hi);
 
    return true;
}

References c, from, hi(), nonprintable_ranges, to, and UTF_NONPRINTABLE_RANGES_COUNT.

Referenced by escape_utf8_for_json(), process_one_string(), rz_scan_strings_raw(), and rz_str_stringify_raw_buffer().

rz_utf8_decode()

RZ_API int rz_utf8_decode	(	const ut8 *	ptr,
		int	ptrlen,
		RzRune *	ch
	)

Definition at line 492 of file utf8.c.

                                                                  {
    if (ptrlen < 1) {
        return 0;
    }
    if (ptr[0] < 0x80) {
        if (ch) {
            *ch = (ut32)ptr[0];
        }
        return 1;
    } else if (ptrlen > 1 && (ptr[0] & 0xe0) == 0xc0 && (ptr[1] & 0xc0) == 0x80) {
        RzRune rune = (ptr[0] & 0x1f) << 6 | (ptr[1] & 0x3f);
        if (ch) {
            *ch = rune;
        }
        return rune < 0x80 ? 0 : 2;
    } else if (ptrlen > 2 && (ptr[0] & 0xf0) == 0xe0 && (ptr[1] & 0xc0) == 0x80 && (ptr[2] & 0xc0) == 0x80) {
        RzRune rune = (ptr[0] & 0xf) << 12 | (ptr[1] & 0x3f) << 6 | (ptr[2] & 0x3f);
        if (ch) {
            *ch = rune;
        }
        return rune < 0x800 ? 0 : 3;
    } else if (ptrlen > 3 && (ptr[0] & 0xf8) == 0xf0 && (ptr[1] & 0xc0) == 0x80 && (ptr[2] & 0xc0) == 0x80 && (ptr[3] & 0xc0) == 0x80) {
        RzRune rune = (ptr[0] & 7) << 18 | (ptr[1] & 0x3f) << 12 | (ptr[2] & 0x3f) << 6 | (ptr[3] & 0x3f);
        if (ch) {
            *ch = rune;
        }
        return rune < 0x10000 ? 0 : 4;
    }
    return 0;
}

Referenced by calculate_utf8_string_info(), can_be_utf16_le(), can_be_utf32_le(), ds_esc_str(), escape_utf8_for_json(), process_one_string(), rz_mutf8_decode(), rz_scan_strings_raw(), rz_search_keyword_new_wide(), rz_str_escape_utf(), rz_str_is_printable(), rz_str_is_printable_incl_newlines(), rz_str_is_printable_limited(), rz_str_is_utf8(), rz_str_stringify_raw_buffer(), rz_utf_block_list(), and sanitize_cab_filename().

rz_utf8_encode()

RZ_API int rz_utf8_encode	(	ut8 *	ptr,
		const RzRune	ch
	)

Definition at line 535 of file utf8.c.

                                                     {
    if (ch < 0x80) {
        ptr[0] = (ut8)ch;
        return 1;
    } else if (ch < 0x800) {
        ptr[0] = 0xc0 | (ch >> 6);
        ptr[1] = 0x80 | (ch & 0x3f);
        return 2;
    } else if (ch < 0x10000) {
        ptr[0] = 0xe0 | (ch >> 12);
        ptr[1] = 0x80 | ((ch >> 6) & 0x3f);
        ptr[2] = 0x80 | (ch & 0x3f);
        return 3;
    } else if (ch < 0x200000) {
        ptr[0] = 0xf0 | (ch >> 18);
        ptr[1] = 0x80 | ((ch >> 12) & 0x3f);
        ptr[2] = 0x80 | ((ch >> 6) & 0x3f);
        ptr[3] = 0x80 | (ch & 0x3f);
        return 4;
    }
    return 0;
}

References ut8.

Referenced by process_one_string(), rz_str_escape_utf(), rz_str_stringify_raw_buffer(), rz_utf8_encode_str(), sanitize_cab_filename(), and unescape_string().

rz_utf8_encode_str()

RZ_API int rz_utf8_encode_str	(	const RzRune *	str,
		ut8 *	dst,
		const int	dst_length
	)

Definition at line 559 of file utf8.c.

                                                                                 {
    if (!str || !dst) {
        return -1;
    }
 
    int pos = 0;
    for (size_t i = 0; i < sizeof(str) - 1 && str[i] && pos < dst_length - 1; i++) {
        pos += rz_utf8_encode(&dst[pos], str[i]);
    }
 
    dst[pos++] = '\0';
    return pos;
}

References dst, i, pos, rz_utf8_encode(), and cmd_descs_generate::str.

Referenced by rz_core_analysis_hasrefs_to_depth().

rz_utf8_size()

RZ_API int rz_utf8_size

(

const ut8 *

ptr

)

Definition at line 574 of file utf8.c.

                                        {
    const int utf8_size[] = {
        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
        2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, // 0xC0-0xCF
        2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, // 0xD0-0xDF
        3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, // 0xE0-0xEF
        4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, // 0xF0-0xFF
    };
    return (ptr[0] & 0x80) ? utf8_size[ptr[0] ^ 0x80] : 1;
}

rz_utf8_strlen()

RZ_API int rz_utf8_strlen

(

const ut8 *

str

)

Definition at line 588 of file utf8.c.

                                          {
    int len = 0;
 
    for (int i = 0; str[i]; i++) {
        if ((str[i] & 0xc0) != 0x80) {
            len++;
        }
    }
 
    return len;
}

References i, len, and cmd_descs_generate::str.

Referenced by ds_print_relocs().

rz_utf_block_idx()

RZ_API int rz_utf_block_idx

(

RzRune

ch

)

Definition at line 733 of file utf8.c.

                                       {
    const int last = UTF_BLOCKS_COUNT;
    int low = 0, hi = last - 1, mid = 0;
 
    do {
        mid = (low + hi) >> 1;
        if (ch >= utf_blocks[mid].from && ch <= utf_blocks[mid].to) {
            return mid;
        }
        if (mid < last && ch > utf_blocks[mid].to) {
            low = mid + 1;
        }
        if (mid < last && ch < utf_blocks[mid].from) {
            hi = mid - 1;
        }
    } while (low <= hi);
 
    return UTF_BLOCKS_COUNT - 1; /* index for "No_Block" */
}

References from, hi(), to, utf_blocks, and UTF_BLOCKS_COUNT.

Referenced by rz_utf_block_list().

rz_utf_block_list()

RZ_API int* rz_utf_block_list	(	const ut8 *	str,
		int	len,
		int **	freq_list
	)

Definition at line 754 of file utf8.c.

                                                                        {
    if (!str) {
        return NULL;
    }
    if (len < 0) {
        len = strlen((const char *)str);
    }
    int block_freq[UTF_BLOCKS_COUNT] = { 0 };
    int *list = RZ_NEWS0(int, len + 1);
    if (!list) {
        return NULL;
    }
    int *freq_list_ptr = NULL;
    if (freq_list) {
        *freq_list = RZ_NEWS0(int, len + 1);
        if (!*freq_list) {
            free(list);
            return NULL;
        }
        freq_list_ptr = *freq_list;
    }
    int *list_ptr = list;
    const ut8 *str_ptr = str;
    const ut8 *str_end = str + len;
    RzRune ch = 0;
    while (str_ptr < str_end) {
        int block_idx;
        int ch_bytes = rz_utf8_decode(str_ptr, str_end - str_ptr, &ch);
        if (!ch_bytes) {
            block_idx = UTF_BLOCKS_COUNT - 1;
            ch_bytes = 1;
        } else {
            block_idx = rz_utf_block_idx(ch);
        }
        if (!block_freq[block_idx]) {
            *list_ptr = block_idx;
            list_ptr++;
        }
        block_freq[block_idx]++;
        str_ptr += ch_bytes;
    }
    *list_ptr = -1;
    if (freq_list_ptr) {
        for (list_ptr = list; *list_ptr != -1; list_ptr++) {
            *freq_list_ptr = block_freq[*list_ptr];
            freq_list_ptr++;
        }
        *freq_list_ptr = -1;
    }
    for (list_ptr = list; *list_ptr != -1; list_ptr++) {
        block_freq[*list_ptr] = 0;
    }
    return list;
}

References free(), len, list(), NULL, RZ_NEWS0, rz_utf8_decode(), rz_utf_block_idx(), cmd_descs_generate::str, and UTF_BLOCKS_COUNT.

Referenced by reduce_false_positives(), and strings_print().

rz_utf_block_name()

RZ_API const char* rz_utf_block_name

(

int

idx

)

Definition at line 484 of file utf8.c.

                                              {
    if (idx < 0 || idx >= UTF_LAST_BLOCK) {
        return NULL;
    }
    return utf_blocks[idx].name;
}

References setup::idx, RUtfBlock::name, NULL, utf_blocks, and UTF_LAST_BLOCK.

Referenced by strings_print().

rz_utf_bom_encoding()

RZ_API RzStrEnc rz_utf_bom_encoding	(	const ut8 *	ptr,
		int	ptrlen
	)

Definition at line 809 of file utf8.c.

                                                                {
    if (ptrlen > 3) {
        if (ptr[0] == 0xff && ptr[1] == 0xfe && !ptr[2] && !ptr[3]) {
            return RZ_STRING_ENC_UTF32LE;
        }
        if (!ptr[0] && !ptr[1] && ptr[2] == 0xfe && ptr[3] == 0xff) {
            return RZ_STRING_ENC_UTF32BE;
        }
    }
    if (ptrlen > 2) {
        if (ptr[0] == 0xef && ptr[1] == 0xbb && ptr[2] == 0xbf) {
            return RZ_STRING_ENC_UTF8;
        }
    }
    if (ptrlen > 1) {
        if (ptr[0] == 0xff && ptr[1] == 0xfe) {
            return RZ_STRING_ENC_UTF16LE;
        }
        if (ptr[0] == 0xfe && ptr[1] == 0xff) {
            return RZ_STRING_ENC_UTF16BE;
        }
    }
    return RZ_STRING_ENC_GUESS;
}

References RZ_STRING_ENC_GUESS, RZ_STRING_ENC_UTF16BE, RZ_STRING_ENC_UTF16LE, RZ_STRING_ENC_UTF32BE, RZ_STRING_ENC_UTF32LE, and RZ_STRING_ENC_UTF8.

Referenced by ds_esc_str(), ds_print_ptr(), and rz_str_guess_encoding_from_buffer().

Variable Documentation

from

ut32 from

Definition at line 14 of file utf8.c.

Referenced by rz_rune_is_printable(), and rz_utf_block_idx().

const { ... } nonprintable_ranges[]

Referenced by rz_rune_is_printable().

to

ut32 to

Definition at line 14 of file utf8.c.

Referenced by rz_rune_is_printable(), and rz_utf_block_idx().

utf_blocks

const RUtfBlock utf_blocks[]

Definition at line 200 of file utf8.c.

Referenced by rz_utf_block_idx(), and rz_utf_block_name().