mirror of https://github.com/leahneukirchen/mblaze
revamp utf-8 handling code
parent
8b50a794b0
commit
73641367db
@ -1,76 +1,57 @@
|
|||||||
#include <stdint.h>
|
#include <stdint.h>
|
||||||
#include <stdio.h>
|
#include <stdio.h>
|
||||||
|
|
||||||
|
#include "u8decode.h"
|
||||||
|
|
||||||
void
|
void
|
||||||
safe_u8putstr(char *s0, size_t l, FILE *stream)
|
safe_u8putstr(char *s0, size_t l, FILE *stream)
|
||||||
{
|
{
|
||||||
// tty-safe output of s, with relaxed utf-8 semantics:
|
// tty-safe output of s, with relaxed utf-8 semantics:
|
||||||
// - C0 and C1 are displayed as escape sequences
|
// - C0 and C1 are displayed as escape sequences
|
||||||
// - valid utf8 is printed as is
|
// - valid utf-8 is printed as is
|
||||||
// - rest is assumed to be latin1, and translated into utf8
|
// - rest is assumed to be latin-1, and translated into utf-8
|
||||||
// - translate CRLF to CR
|
// - translate CRLF to CR
|
||||||
|
|
||||||
unsigned char *s = (unsigned char *)s0;
|
unsigned char *s = (unsigned char *)s0;
|
||||||
unsigned char *e = s + l;
|
unsigned char *e = s + l;
|
||||||
|
uint32_t c;
|
||||||
|
|
||||||
while (s < e) {
|
while (s < e) {
|
||||||
if ((*s & 0x80) == 0) {
|
int l = u8decode((char *)s, &c);
|
||||||
if (*s < 32 &&
|
if (l == -1) {
|
||||||
*s != ' ' && *s != '\t' && *s != '\n' && *s != '\r') {
|
l = 1;
|
||||||
// C0
|
if (*s <= 0x9fu) {
|
||||||
|
// C1
|
||||||
fputc(0xe2, stream);
|
fputc(0xe2, stream);
|
||||||
fputc(0x90, stream);
|
fputc(0x90, stream);
|
||||||
fputc(0x80+*s, stream);
|
fputc(0x80+0x1b, stream);
|
||||||
} else if (*s == 127) {
|
|
||||||
// DEL
|
|
||||||
fputc(0xe2, stream);
|
fputc(0xe2, stream);
|
||||||
fputc(0x90, stream);
|
fputc(0x90, stream);
|
||||||
fputc(0xa1, stream);
|
|
||||||
} else if (*s == '\r') {
|
|
||||||
if (e - s > 1 && s[1] == '\n')
|
|
||||||
s++;
|
|
||||||
fputc(*s, stream);
|
fputc(*s, stream);
|
||||||
} else {
|
} else {
|
||||||
// safe ASCII
|
/* invalid utf-8, assume it was latin-1 */
|
||||||
fputc(*s, stream);
|
fputc(0xc0 | (*s >> 6), stream);
|
||||||
|
fputc(0x80 | (*s & 0x3f), stream);
|
||||||
}
|
}
|
||||||
} else if ((*s & 0xc0) == 0x80) {
|
} else if (c < 32 &&
|
||||||
if (*s >= 0xa0)
|
*s != ' ' && *s != '\t' && *s != '\n' && *s != '\r') {
|
||||||
goto latin1;
|
// C0
|
||||||
|
|
||||||
// C1
|
|
||||||
fputc(0xe2, stream);
|
fputc(0xe2, stream);
|
||||||
fputc(0x90, stream);
|
fputc(0x90, stream);
|
||||||
fputc(0x80+0x1b, stream);
|
fputc(0x80+*s, stream);
|
||||||
|
} else if (c == 127) {
|
||||||
|
// DEL
|
||||||
fputc(0xe2, stream);
|
fputc(0xe2, stream);
|
||||||
fputc(0x90, stream);
|
fputc(0x90, stream);
|
||||||
|
fputc(0xa1, stream);
|
||||||
|
} else if (c == '\r') {
|
||||||
|
if (e - s > 1 && s[1] == '\n')
|
||||||
|
s++;
|
||||||
fputc(*s, stream);
|
fputc(*s, stream);
|
||||||
} else {
|
} else {
|
||||||
uint32_t f = 0;
|
fwrite(s, 1, l, stream);
|
||||||
if (e - s >= 4)
|
|
||||||
f = (s[0]<<24) | (s[1]<<16) | (s[2]<<8) | s[3];
|
|
||||||
else if (e - s == 3)
|
|
||||||
f = (s[0]<<24) | (s[1]<<16) | (s[2]<<8);
|
|
||||||
else if (e - s == 2)
|
|
||||||
f = (s[0]<<24) | (s[1]<<16);
|
|
||||||
else if (e - s == 1)
|
|
||||||
f = (s[0]<<24);
|
|
||||||
|
|
||||||
if ((f & 0xe0c00000) == 0xc0800000) goto u2;
|
|
||||||
else if ((f & 0xf0c0c000) == 0xe0808000) goto u3;
|
|
||||||
else if ((f & 0xf8c0c0c0) == 0xf0808080) {
|
|
||||||
fputc(*s++, stream);
|
|
||||||
u3: fputc(*s++, stream);
|
|
||||||
u2: fputc(*s++, stream);
|
|
||||||
fputc(*s, stream);
|
|
||||||
} else {
|
|
||||||
latin1:
|
|
||||||
/* invalid utf8, assume it was latin1 */
|
|
||||||
fputc(0xc0 | (*s >> 6), stream);
|
|
||||||
fputc(0x80 | (*s & 0x3f), stream);
|
|
||||||
}
|
}
|
||||||
}
|
s += l;
|
||||||
s++;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -0,0 +1,25 @@
|
|||||||
|
#include <stdint.h>
|
||||||
|
|
||||||
|
// Decode one UTF-8 codepoint into cp, return number of bytes to next one.
|
||||||
|
// On invalid UTF-8, return -1, and do not change cp.
|
||||||
|
// Overlong sequences, surrogates and invalid codepoints are not checked.
|
||||||
|
//
|
||||||
|
// This code is meant to be inlined, if cp is unused it can be optimized away.
|
||||||
|
static int
|
||||||
|
u8decode(const char *cs, uint32_t *cp)
|
||||||
|
{
|
||||||
|
const uint8_t *s = (uint8_t *)cs;
|
||||||
|
|
||||||
|
if (*s == 0) { *cp = 0; return 0; }
|
||||||
|
if (*s < 0x80) { *cp = *s; return 1; }
|
||||||
|
if (*s < 0xc0) { return -1; }
|
||||||
|
if (*s < 0xe0) { *cp = *s & 0x1f; goto u2; }
|
||||||
|
if (*s < 0xf0) { *cp = *s & 0x0f; goto u3; }
|
||||||
|
if (*s < 0xf8) { *cp = *s & 0x07; goto u4; }
|
||||||
|
return -1;
|
||||||
|
|
||||||
|
u4: if ((*++s & 0xc0) != 0x80) return -1; *cp = (*cp << 6) | (*s & 0x3f);
|
||||||
|
u3: if ((*++s & 0xc0) != 0x80) return -1; *cp = (*cp << 6) | (*s & 0x3f);
|
||||||
|
u2: if ((*++s & 0xc0) != 0x80) return -1; *cp = (*cp << 6) | (*s & 0x3f);
|
||||||
|
return s - (uint8_t *)cs + 1;
|
||||||
|
}
|
Loading…
Reference in New Issue