|
|
|
#include <stdio.h>
|
|
|
|
#include <stdlib.h>
|
|
|
|
#include <string.h>
|
|
|
|
#include <inttypes.h>
|
|
|
|
#include <sys/types.h>
|
|
|
|
#include <sys/stat.h>
|
|
|
|
#include <sys/time.h>
|
|
|
|
#include <fcntl.h>
|
|
|
|
#include <unistd.h>
|
|
|
|
|
|
|
|
int utf8_to16_iconv(const unsigned char *buf8, size_t len8,
|
|
|
|
unsigned short *buf16, size_t *len16);
|
|
|
|
int utf8_to16_naive(const unsigned char *buf8, size_t len8,
|
|
|
|
unsigned short *buf16, size_t *len16);
|
|
|
|
|
|
|
|
static struct ftab {
|
|
|
|
const char *name;
|
|
|
|
int (*func)(const unsigned char *buf8, size_t len8,
|
|
|
|
unsigned short *buf16, size_t *len16);
|
|
|
|
} ftab[] = {
|
|
|
|
{
|
|
|
|
.name = "iconv",
|
|
|
|
.func = utf8_to16_iconv,
|
|
|
|
}, {
|
|
|
|
.name = "naive",
|
|
|
|
.func = utf8_to16_naive,
|
|
|
|
},
|
|
|
|
};
|
|
|
|
|
|
|
|
static unsigned char *load_test_buf(int len)
|
|
|
|
{
|
|
|
|
const char utf8[] = "\xF0\x90\xBF\x80";
|
|
|
|
const int utf8_len = sizeof(utf8)/sizeof(utf8[0]) - 1;
|
|
|
|
|
|
|
|
unsigned char *data = malloc(len);
|
|
|
|
unsigned char *p = data;
|
|
|
|
|
|
|
|
while (len >= utf8_len) {
|
|
|
|
memcpy(p, utf8, utf8_len);
|
|
|
|
p += utf8_len;
|
|
|
|
len -= utf8_len;
|
|
|
|
}
|
|
|
|
|
|
|
|
while (len--)
|
|
|
|
*p++ = 0x7F;
|
|
|
|
|
|
|
|
return data;
|
|
|
|
}
|
|
|
|
|
|
|
|
static unsigned char *load_test_file(int *len)
|
|
|
|
{
|
|
|
|
unsigned char *data;
|
|
|
|
int fd;
|
|
|
|
struct stat stat;
|
|
|
|
|
|
|
|
fd = open("../UTF-8-demo.txt", O_RDONLY);
|
|
|
|
if (fd == -1) {
|
|
|
|
printf("Failed to open ../UTF-8-demo.txt!\n");
|
|
|
|
exit(1);
|
|
|
|
}
|
|
|
|
if (fstat(fd, &stat) == -1) {
|
|
|
|
printf("Failed to get file size!\n");
|
|
|
|
exit(1);
|
|
|
|
}
|
|
|
|
|
|
|
|
*len = stat.st_size;
|
|
|
|
data = malloc(*len);
|
|
|
|
if (read(fd, data, *len) != *len) {
|
|
|
|
printf("Failed to read file!\n");
|
|
|
|
exit(1);
|
|
|
|
}
|
|
|
|
|
|
|
|
close(fd);
|
|
|
|
|
|
|
|
return data;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void print_test(const unsigned char *data, int len)
|
|
|
|
{
|
|
|
|
printf(" [len=%d] \"", len);
|
|
|
|
while (len--)
|
|
|
|
printf("\\x%02X", *data++);
|
|
|
|
|
|
|
|
printf("\"\n");
|
|
|
|
}
|
|
|
|
|
|
|
|
struct test {
|
|
|
|
const unsigned char *data;
|
|
|
|
int len;
|
|
|
|
};
|
|
|
|
|
|
|
|
static void prepare_test_buf(unsigned char *buf, const struct test *pos,
|
|
|
|
int pos_len, int pos_idx)
|
|
|
|
{
|
|
|
|
/* Round concatenate correct tokens to 1024 bytes */
|
|
|
|
int buf_idx = 0;
|
|
|
|
while (buf_idx < 1024) {
|
|
|
|
int buf_len = 1024 - buf_idx;
|
|
|
|
|
|
|
|
if (buf_len >= pos[pos_idx].len) {
|
|
|
|
memcpy(buf+buf_idx, pos[pos_idx].data, pos[pos_idx].len);
|
|
|
|
buf_idx += pos[pos_idx].len;
|
|
|
|
} else {
|
|
|
|
memset(buf+buf_idx, 0, buf_len);
|
|
|
|
buf_idx += buf_len;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (++pos_idx == pos_len)
|
|
|
|
pos_idx = 0;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Return 0 on success, -1 on error */
|
|
|
|
static int test_manual(const struct ftab *ftab, unsigned short *buf16,
|
|
|
|
unsigned short *_buf16)
|
|
|
|
{
|
|
|
|
#define LEN16 4096
|
|
|
|
|
|
|
|
#pragma GCC diagnostic push
|
|
|
|
#pragma GCC diagnostic ignored "-Wpointer-sign"
|
|
|
|
/* positive tests */
|
|
|
|
static const struct test pos[] = {
|
|
|
|
{"", 0},
|
|
|
|
{"\x00", 1},
|
|
|
|
{"\x66", 1},
|
|
|
|
{"\x7F", 1},
|
|
|
|
{"\x00\x7F", 2},
|
|
|
|
{"\x7F\x00", 2},
|
|
|
|
{"\xC2\x80", 2},
|
|
|
|
{"\xDF\xBF", 2},
|
|
|
|
{"\xE0\xA0\x80", 3},
|
|
|
|
{"\xE0\xA0\xBF", 3},
|
|
|
|
{"\xED\x9F\x80", 3},
|
|
|
|
{"\xEF\x80\xBF", 3},
|
|
|
|
{"\xF0\x90\xBF\x80", 4},
|
|
|
|
{"\xF2\x81\xBE\x99", 4},
|
|
|
|
{"\xF4\x8F\x88\xAA", 4},
|
|
|
|
};
|
|
|
|
|
|
|
|
/* negative tests */
|
|
|
|
static const struct test neg[] = {
|
|
|
|
{"\x80", 1},
|
|
|
|
{"\xBF", 1},
|
|
|
|
{"\xC0\x80", 2},
|
|
|
|
{"\xC1\x00", 2},
|
|
|
|
{"\xC2\x7F", 2},
|
|
|
|
{"\xDF\xC0", 2},
|
|
|
|
{"\xE0\x9F\x80", 3},
|
|
|
|
{"\xE0\xC2\x80", 3},
|
|
|
|
{"\xED\xA0\x80", 3},
|
|
|
|
{"\xED\x7F\x80", 3},
|
|
|
|
{"\xEF\x80\x00", 3},
|
|
|
|
{"\xF0\x8F\x80\x80", 4},
|
|
|
|
{"\xF0\xEE\x80\x80", 4},
|
|
|
|
{"\xF2\x90\x91\x7F", 4},
|
|
|
|
{"\xF4\x90\x88\xAA", 4},
|
|
|
|
{"\xF4\x00\xBF\xBF", 4},
|
|
|
|
{"\x00\x00\x00\x00\x00\xC2\x80\x00\x00\x00\xE1\x80\x80\x00\x00\xC2" \
|
|
|
|
"\xC2\x80\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00",
|
|
|
|
32},
|
|
|
|
{"\x00\x00\x00\x00\x00\xC2\xC2\x80\x00\x00\xE1\x80\x80\x00\x00\x00",
|
|
|
|
16},
|
|
|
|
{"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" \
|
|
|
|
"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xF1\x80",
|
|
|
|
32},
|
|
|
|
{"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" \
|
|
|
|
"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xF1",
|
|
|
|
32},
|
|
|
|
{"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" \
|
|
|
|
"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xF1\x80" \
|
|
|
|
"\x80", 33},
|
|
|
|
{"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" \
|
|
|
|
"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xF1\x80" \
|
|
|
|
"\xC2\x80", 34},
|
|
|
|
{"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" \
|
|
|
|
"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xF0" \
|
|
|
|
"\x80\x80\x80", 35},
|
|
|
|
};
|
|
|
|
#pragma GCC diagnostic push
|
|
|
|
|
|
|
|
size_t len16 = LEN16, _len16 = LEN16;
|
|
|
|
int ret, _ret;
|
|
|
|
|
|
|
|
/* Test single token */
|
|
|
|
for (int i = 0; i < sizeof(pos)/sizeof(pos[0]); ++i) {
|
|
|
|
ret = ftab->func(pos[i].data, pos[i].len, buf16, &len16);
|
|
|
|
_ret = utf8_to16_iconv(pos[i].data, pos[i].len, _buf16, &_len16);
|
|
|
|
if (ret != _ret || len16 != _len16 || memcmp(buf16, _buf16, len16)) {
|
|
|
|
printf("FAILED positive test(%d:%d, %lu:%lu): ",
|
|
|
|
ret, _ret, len16, _len16);
|
|
|
|
print_test(pos[i].data, pos[i].len);
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
len16 = _len16 = LEN16;
|
|
|
|
}
|
|
|
|
for (int i = 0; i < sizeof(neg)/sizeof(neg[0]); ++i) {
|
|
|
|
ret = ftab->func(neg[i].data, neg[i].len, buf16, &len16);
|
|
|
|
_ret = utf8_to16_iconv(neg[i].data, neg[i].len, _buf16, &_len16);
|
|
|
|
if (ret != _ret || len16 != _len16 || memcmp(buf16, _buf16, len16)) {
|
|
|
|
printf("FAILED negative test(%d:%d, %lu:%lu): ",
|
|
|
|
ret, _ret, len16, _len16);
|
|
|
|
print_test(neg[i].data, neg[i].len);
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
len16 = _len16 = LEN16;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Test shifted buffer to cover 1k length */
|
|
|
|
/* buffer size must be greater than 1024 + 16 + max(test string length) */
|
|
|
|
const int max_size = 1024*2;
|
|
|
|
uint64_t buf64[max_size/8 + 2];
|
|
|
|
/* Offset 8 bytes by 1 byte */
|
|
|
|
unsigned char *buf = ((unsigned char *)buf64) + 1;
|
|
|
|
int buf_len;
|
|
|
|
|
|
|
|
for (int i = 0; i < sizeof(pos)/sizeof(pos[0]); ++i) {
|
|
|
|
/* Positive test: shift 16 bytes, validate each shift */
|
|
|
|
prepare_test_buf(buf, pos, sizeof(pos)/sizeof(pos[0]), i);
|
|
|
|
buf_len = 1024;
|
|
|
|
for (int j = 0; j < 16; ++j) {
|
|
|
|
ret = ftab->func(buf, buf_len, buf16, &len16);
|
|
|
|
_ret = utf8_to16_iconv(buf, buf_len, _buf16, &_len16);
|
|
|
|
if (ret != _ret || len16 != _len16 || \
|
|
|
|
memcmp(buf16, _buf16, len16)) {
|
|
|
|
printf("FAILED positive test(%d:%d, %lu:%lu): ",
|
|
|
|
ret, _ret, len16, _len16);
|
|
|
|
print_test(buf, buf_len);
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
len16 = _len16 = LEN16;
|
|
|
|
for (int k = buf_len; k >= 1; --k)
|
|
|
|
buf[k] = buf[k-1];
|
|
|
|
buf[0] = '\x55';
|
|
|
|
++buf_len;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Negative test: trunk last non ascii */
|
|
|
|
while (buf_len >= 1 && buf[buf_len-1] <= 0x7F)
|
|
|
|
--buf_len;
|
|
|
|
if (buf_len) {
|
|
|
|
ret = ftab->func(buf, buf_len-1, buf16, &len16);
|
|
|
|
_ret = utf8_to16_iconv(buf, buf_len-1, _buf16, &_len16);
|
|
|
|
if (ret != _ret || len16 != _len16 || \
|
|
|
|
memcmp(buf16, _buf16, len16)) {
|
|
|
|
printf("FAILED negative test(%d:%d, %lu:%lu): ",
|
|
|
|
ret, _ret, len16, _len16);
|
|
|
|
print_test(buf, buf_len-1);
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
len16 = _len16 = LEN16;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Negative test */
|
|
|
|
for (int i = 0; i < sizeof(neg)/sizeof(neg[0]); ++i) {
|
|
|
|
/* Append one error token, shift 16 bytes, validate each shift */
|
|
|
|
int pos_idx = i % (sizeof(pos)/sizeof(pos[0]));
|
|
|
|
prepare_test_buf(buf, pos, sizeof(pos)/sizeof(pos[0]), pos_idx);
|
|
|
|
memcpy(buf+1024, neg[i].data, neg[i].len);
|
|
|
|
buf_len = 1024 + neg[i].len;
|
|
|
|
for (int j = 0; j < 16; ++j) {
|
|
|
|
ret = ftab->func(buf, buf_len, buf16, &len16);
|
|
|
|
_ret = utf8_to16_iconv(buf, buf_len, _buf16, &_len16);
|
|
|
|
if (ret != _ret || len16 != _len16 || \
|
|
|
|
memcmp(buf16, _buf16, len16)) {
|
|
|
|
printf("FAILED negative test(%d:%d, %lu:%lu): ",
|
|
|
|
ret, _ret, len16, _len16);
|
|
|
|
print_test(buf, buf_len);
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
len16 = _len16 = LEN16;
|
|
|
|
for (int k = buf_len; k >= 1; --k)
|
|
|
|
buf[k] = buf[k-1];
|
|
|
|
buf[0] = '\x66';
|
|
|
|
++buf_len;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void test(const unsigned char *buf8, size_t len8,
|
|
|
|
unsigned short *buf16, size_t len16, const struct ftab *ftab)
|
|
|
|
{
|
|
|
|
/* Use iconv as the reference answer */
|
|
|
|
if (strcmp(ftab->name, "iconv") == 0)
|
|
|
|
return;
|
|
|
|
|
|
|
|
printf("%s\n", ftab->name);
|
|
|
|
|
|
|
|
/* Test file or buffer */
|
|
|
|
size_t _len16 = len16;
|
|
|
|
unsigned short *_buf16 = (unsigned short *)malloc(_len16);
|
|
|
|
if (utf8_to16_iconv(buf8, len8, _buf16, &_len16)) {
|
|
|
|
printf("Invalid test file or buffer!\n");
|
|
|
|
exit(1);
|
|
|
|
}
|
|
|
|
printf("standard test: ");
|
|
|
|
if (ftab->func(buf8, len8, buf16, &len16) || len16 != _len16 || \
|
|
|
|
memcmp(buf16, _buf16, len16) != 0)
|
|
|
|
printf("FAIL\n");
|
|
|
|
else
|
|
|
|
printf("pass\n");
|
|
|
|
free(_buf16);
|
|
|
|
|
|
|
|
/* Manual cases */
|
|
|
|
unsigned short *mbuf8 = (unsigned short *)malloc(LEN16);
|
|
|
|
unsigned short *mbuf16 = (unsigned short *)malloc(LEN16);
|
|
|
|
printf("manual test: %s\n",
|
|
|
|
test_manual(ftab, mbuf8, mbuf16) ? "FAIL" : "pass");
|
|
|
|
free(mbuf8);
|
|
|
|
free(mbuf16);
|
|
|
|
printf("\n");
|
|
|
|
}
|
|
|
|
|
|
|
|
static void bench(const unsigned char *buf8, size_t len8,
|
|
|
|
unsigned short *buf16, size_t len16, const struct ftab *ftab)
|
|
|
|
{
|
|
|
|
const int loops = 1024*1024*1024/len8;
|
|
|
|
int ret = 0;
|
|
|
|
double time, size;
|
|
|
|
struct timeval tv1, tv2;
|
|
|
|
|
|
|
|
fprintf(stderr, "bench %s... ", ftab->name);
|
|
|
|
gettimeofday(&tv1, 0);
|
|
|
|
for (int i = 0; i < loops; ++i)
|
|
|
|
ret |= ftab->func(buf8, len8, buf16, &len16);
|
|
|
|
gettimeofday(&tv2, 0);
|
|
|
|
printf("%s\n", ret?"FAIL":"pass");
|
|
|
|
|
|
|
|
time = tv2.tv_usec - tv1.tv_usec;
|
|
|
|
time = time / 1000000 + tv2.tv_sec - tv1.tv_sec;
|
|
|
|
size = ((double)len8 * loops) / (1024*1024);
|
|
|
|
printf("time: %.4f s\n", time);
|
|
|
|
printf("data: %.0f MB\n", size);
|
|
|
|
printf("BW: %.2f MB/s\n", size / time);
|
|
|
|
printf("\n");
|
|
|
|
}
|
|
|
|
|
|
|
|
static void usage(const char *bin)
|
|
|
|
{
|
|
|
|
printf("Usage:\n");
|
|
|
|
printf("%s test [alg] ==> test all or one algorithm\n", bin);
|
|
|
|
printf("%s bench [alg] ==> benchmark all or one algorithm\n", bin);
|
|
|
|
printf("%s bench size NUM ==> benchmark with specific buffer size\n", bin);
|
|
|
|
printf("alg = ");
|
|
|
|
for (int i = 0; i < sizeof(ftab)/sizeof(ftab[0]); ++i)
|
|
|
|
printf("%s ", ftab[i].name);
|
|
|
|
printf("\nNUM = buffer size in bytes, 1 ~ 67108864(64M)\n");
|
|
|
|
}
|
|
|
|
|
|
|
|
int main(int argc, char *argv[])
|
|
|
|
{
|
|
|
|
int len8 = 0, len16;
|
|
|
|
unsigned char *buf8;
|
|
|
|
unsigned short *buf16;
|
|
|
|
const char *alg = NULL;
|
|
|
|
void (*tb)(const unsigned char *buf8, size_t len8,
|
|
|
|
unsigned short *buf16, size_t len16, const struct ftab *ftab);
|
|
|
|
|
|
|
|
tb = NULL;
|
|
|
|
if (argc >= 2) {
|
|
|
|
if (strcmp(argv[1], "test") == 0)
|
|
|
|
tb = test;
|
|
|
|
else if (strcmp(argv[1], "bench") == 0)
|
|
|
|
tb = bench;
|
|
|
|
if (argc >= 3) {
|
|
|
|
alg = argv[2];
|
|
|
|
if (strcmp(alg, "size") == 0) {
|
|
|
|
if (argc < 4) {
|
|
|
|
tb = NULL;
|
|
|
|
} else {
|
|
|
|
alg = NULL;
|
|
|
|
len8 = atoi(argv[3]);
|
|
|
|
if (len8 <= 0 || len8 > 67108864) {
|
|
|
|
printf("Buffer size error!\n\n");
|
|
|
|
tb = NULL;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (tb == NULL) {
|
|
|
|
usage(argv[0]);
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Load UTF8 test buffer */
|
|
|
|
if (len8)
|
|
|
|
buf8 = load_test_buf(len8);
|
|
|
|
else
|
|
|
|
buf8 = load_test_file(&len8);
|
|
|
|
|
|
|
|
/* Prepare UTF16 buffer large enough */
|
|
|
|
len16 = len8 * 2;
|
|
|
|
buf16 = (unsigned short *)malloc(len16);
|
|
|
|
|
|
|
|
if (tb == bench)
|
|
|
|
printf("============== Bench UTF8 (%d bytes) ==============\n", len8);
|
|
|
|
for (int i = 0; i < sizeof(ftab)/sizeof(ftab[0]); ++i) {
|
|
|
|
if (alg && strcmp(alg, ftab[i].name) != 0)
|
|
|
|
continue;
|
|
|
|
tb((const unsigned char *)buf8, len8, buf16, len16, &ftab[i]);
|
|
|
|
}
|
|
|
|
|
|
|
|
#if 0
|
|
|
|
if (tb == bench) {
|
|
|
|
printf("==================== Bench ASCII ====================\n");
|
|
|
|
/* Change test buffer to ascii */
|
|
|
|
for (int i = 0; i < len; i++)
|
|
|
|
data[i] &= 0x7F;
|
|
|
|
|
|
|
|
for (int i = 0; i < sizeof(ftab)/sizeof(ftab[0]); ++i) {
|
|
|
|
if (alg && strcmp(alg, ftab[i].name) != 0)
|
|
|
|
continue;
|
|
|
|
tb((const unsigned char *)data, len, &ftab[i]);
|
|
|
|
printf("\n");
|
|
|
|
}
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|