/**************************************************** * PCRE maintainers' helper program: UTF-8 converter * ****************************************************/ /* This is a test program for converting character code points to UTF-8 and vice versa. Note that this program conforms to the original definition of UTF-8, which allows codepoints up to 7fffffff. The more recent definition limits the validity of Unicode UTF-8 codepoints to a maximum of 10ffffff, and forbids the "surrogate" code points. This program now gives warnings for these invalid code points. The arguments are either single code point values written as U+hh.. or 0xhh.. for conversion to UTF-8, or sequences of hex values, written without 0x and optionally including spaces (but such arguments must be quoted), for conversion from UTF-8 to codepoints. For example: ./utf8 0x1234 U+00001234 => e1 88 b4 ./utf8 "e1 88 b4" U+00001234 <= e1 88 b4 In the second case, a number of UTF-8 characters can be present in one argument. In other words, each such argument is interpreted (after ignoring spaces) as a string of UTF-8 bytes representing a string of characters: ./utf8 "65 e188b4 77" 0x00000065 <= 65 0x00001234 <= e1 88 b4 0x00000077 <= 77 If the option -s is given, the sequence of UTF-bytes is written out between angle brackets at the end of the line. On a UTF-8 terminal, this will show the appropriate graphic for the code point. Errors provoke error messages, but the program carries on with the next argument. The return code is always zero. Philip Hazel Original creation data: unknown Code extended and tidied to avoid compiler warnings: 26 March 2020 */ #include #include #include #include /* The valid ranges for UTF-8 characters are: 0000 0000 to 0000 007f 1 byte (ascii) 0000 0080 to 0000 07ff 2 bytes 0000 0800 to 0000 ffff 3 bytes 0001 0000 to 001f ffff 4 bytes 0020 0000 to 03ff ffff 5 bytes 0400 0000 to 7fff ffff 6 bytes */ static const unsigned int utf8_table1[] = { 0x0000007f, 0x000007ff, 0x0000ffff, 0x001fffff, 0x03ffffff, 0x7fffffff}; static const int utf8_table2[] = { 0, 0xc0, 0xe0, 0xf0, 0xf8, 0xfc}; static const int utf8_table3[] = { 0xff, 0x1f, 0x0f, 0x07, 0x03, 0x01}; /************************************************* * Convert character value to UTF-8 * *************************************************/ /* This function takes an unsigned long integer value in the range 0 - 0x7fffffff and encodes it as a UTF-8 character in 1 to 6 bytes. Arguments: cvalue the character value buffer pointer to buffer for result - at least 6 bytes long Returns: number of bytes placed in the buffer 0 if input code point is too big */ static size_t ord2utf8(unsigned long int cvalue, unsigned char *buffer) { size_t i, j; for (i = 0; i < sizeof(utf8_table1)/sizeof(int); i++) if (cvalue <= utf8_table1[i]) break; if (i >= sizeof(utf8_table1)/sizeof(int)) return 0; buffer += i; for (j = i; j > 0; j--) { *buffer-- = 0x80 | (cvalue & 0x3f); cvalue >>= 6; } *buffer = utf8_table2[i] | cvalue; return i + 1; } /************************************************* * Convert UTF-8 string to value * *************************************************/ /* This function takes one or more bytes that represent a UTF-8 character from the start of a string of bytes. It returns the value of the character, or the offset of a malformation. For an overlong encoding that works but is not the correct (shortest) one, the error offset is just after the last byte. Argument: buffer a pointer to the byte vector buffend a pointer to the end of the buffer vptr a pointer to a variable to receive the value lenptr a pointer to a variable to receive the offset when error detected Returns: > 0 => the number of bytes consumed 0 => invalid UTF-8: first byte missing 0x40 bit -1 => invalid UTF-8: first byte has too many high-order 1-bits -2 => incomplete sequence at end of string -3 => incomplete sequence within string -4 => overlong code sequence */ static int utf82ord(unsigned char *buffer, unsigned char *buffend, long unsigned int *vptr, int *lenptr) { unsigned int c = *buffer++; unsigned int d = c; int i, j, s; /* Check for an ASCII character, or find the number of additional bytes in a multibyte character. */ for (i = -1; i < 6; i++) { if ((d & 0x80) == 0) break; d <<= 1; } switch (i) { case -1: /* ASCII character; first byte does not have 0x80 bit */ *vptr = c; return 1; case 0: /* First byte has 0x80 but is missing 0x40 bit */ *lenptr = 0; return 0; case 6: *lenptr = 0; /* Too many high bits */ return -1; default: break; } /* i now has a value in the range 1-5 */ s = 6*i; d = (c & utf8_table3[i]) << s; for (j = 0; j < i; j++) { if (buffer >= buffend) { *lenptr = j + 1; return -2; } c = *buffer++; if ((c & 0xc0) != 0x80) { *lenptr = j + 1; return -3; } s -= 6; d |= (c & 0x3f) << s; } /* Valid UTF-8 syntax */ *vptr = d; /* Check that encoding was the correct one, not overlong */ for (j = 0; j < (int)(sizeof(utf8_table1)/sizeof(int)); j++) if (d <= utf8_table1[j]) break; if (j != i) { *lenptr = i + 1; return -4; } /* Valid value */ return i + 1; } /************************************************* * Main Program * *************************************************/ int main(int argc, char **argv) { int i = 1; int show = 0; unsigned char buffer[64]; if (argc > 1 && strcmp(argv[1], "-s") == 0) { show = 1; i = 2; } for (; i < argc; i++) { char *x = argv[i]; char *endptr; if (strncmp(x, "0x", 2) == 0 || strncmp(x, "U+", 2) == 0) { size_t rc, j; unsigned long int d = strtoul(x+2, &endptr, 16); if (*endptr != 0) { printf("** Invalid hex number %s\n", x); continue; /* With next argument */ } rc = ord2utf8(d, buffer); printf("U+%08lx => ", d); if (rc == 0) printf("** Code point greater than 0x7fffffff cannot be encoded"); else { for (j = 0; j < rc; j++) printf("%02x ", buffer[j]); if (show) { printf(">"); for (j = 0; j < rc; j++) printf("%c", buffer[j]); printf("< "); } if (d >= 0xd800 && d <= 0xdfff) printf("** Invalid Unicode (surrogate)"); else if (d > 0x10ffff) printf("** Invalid Unicode (greater than U+10ffff)"); } printf("\n"); } else { unsigned char *bptr; unsigned char *buffend; int len = 0; int y = 0; int z = 0; for (;;) { while (*x == ' ') x++; if (*x == 0 && !z) break; if (!isxdigit(*x)) { printf("** Malformed hex string: %s\n", argv[i]); len = -1; break; } y = y * 16 + tolower(*x) - ((isdigit(*x))? '0' : 'W'); x++; if (z) { buffer[len++] = y; y = 0; } z ^= 1; } if (len < 0) continue; /* With next argument after malformation */ bptr = buffer; buffend = buffer + len; while (bptr < buffend) { unsigned long int d; int j; int offset; int rc = utf82ord(bptr, buffend, &d, &offset); if (rc > 0) { printf("U+%08lx <= ", d); for (j = 0; j < rc; j++) printf("%02x ", bptr[j]); if (show) { printf(">"); for (j = 0; j < rc; j++) printf("%c", bptr[j]); printf("<"); } printf("\n"); bptr += rc; } else if (rc == -4) { printf("U+%08lx <= ", d); for (j = 0; j < offset; j++) printf("%02x ", bptr[j]); printf("** Overlong UTF-8 sequence\n"); bptr += offset; } else { switch (rc) { case 0: printf("** First byte missing 0x40 bit"); break; case -1: printf("** First byte has too many high-order bits"); break; case -2: printf("** Incomplete UTF-8 sequence at end of string"); break; case -3: printf("** Incomplete UTF-8 sequence"); break; default: printf("** Unexpected return %d from utf82ord()", rc); break; } printf(" at offset %d in string ", offset); while (bptr < buffend) printf("%02x ", *bptr++); printf("\n"); break; } } } } return 0; } /* End */