1 1ac119fb 2024-01-23 op /* Copyright (c) 2008-2009 Bjoern Hoehrmann <bjoern@hoehrmann.de>
3 1ac119fb 2024-01-23 op * Permission is hereby granted, free of charge, to any person
4 1ac119fb 2024-01-23 op * obtaining a copy of this software and associated documentation
5 1ac119fb 2024-01-23 op * files (the "Software"), to deal in the Software without
6 1ac119fb 2024-01-23 op * restriction, including without limitation the rights to use, copy,
7 1ac119fb 2024-01-23 op * modify, merge, publish, distribute, sublicense, and/or sell copies
8 1ac119fb 2024-01-23 op * of the Software, and to permit persons to whom the Software is
9 1ac119fb 2024-01-23 op * furnished to do so, subject to the following conditions:
11 1ac119fb 2024-01-23 op * The above copyright notice and this permission notice shall be
12 1ac119fb 2024-01-23 op * included in all copies or substantial portions of the Software.
14 1ac119fb 2024-01-23 op * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
15 1ac119fb 2024-01-23 op * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
16 1ac119fb 2024-01-23 op * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
17 1ac119fb 2024-01-23 op * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
18 1ac119fb 2024-01-23 op * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
19 1ac119fb 2024-01-23 op * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
20 1ac119fb 2024-01-23 op * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
24 1ac119fb 2024-01-23 op #include "compat.h"
26 1ac119fb 2024-01-23 op #include <assert.h>
27 1ac119fb 2024-01-23 op #include <stddef.h>
28 1ac119fb 2024-01-23 op #include <stdint.h>
29 1ac119fb 2024-01-23 op #include <wchar.h>
31 1ac119fb 2024-01-23 op #include "utf8.h"
33 1ac119fb 2024-01-23 op #define UTF8_ACCEPT 0
34 1ac119fb 2024-01-23 op #define UTF8_REJECT 1
36 1ac119fb 2024-01-23 op static const uint8_t utf8d[] = {
37 1ac119fb 2024-01-23 op 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 00..1f
38 1ac119fb 2024-01-23 op 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 20..3f
39 1ac119fb 2024-01-23 op 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 40..5f
40 1ac119fb 2024-01-23 op 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 60..7f
41 1ac119fb 2024-01-23 op 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9, // 80..9f
42 1ac119fb 2024-01-23 op 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7, // a0..bf
43 1ac119fb 2024-01-23 op 8,8,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, // c0..df
44 1ac119fb 2024-01-23 op 0xa,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x4,0x3,0x3, // e0..ef
45 1ac119fb 2024-01-23 op 0xb,0x6,0x6,0x6,0x5,0x8,0x8,0x8,0x8,0x8,0x8,0x8,0x8,0x8,0x8,0x8, // f0..ff
46 1ac119fb 2024-01-23 op 0x0,0x1,0x2,0x3,0x5,0x8,0x7,0x1,0x1,0x1,0x4,0x6,0x1,0x1,0x1,0x1, // s0..s0
47 1ac119fb 2024-01-23 op 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,0,1,0,1,1,1,1,1,1, // s1..s2
48 1ac119fb 2024-01-23 op 1,2,1,1,1,1,1,2,1,2,1,1,1,1,1,1,1,1,1,1,1,1,1,2,1,1,1,1,1,1,1,1, // s3..s4
49 1ac119fb 2024-01-23 op 1,2,1,1,1,1,1,1,1,2,1,1,1,1,1,1,1,1,1,1,1,1,1,3,1,3,1,1,1,1,1,1, // s5..s6
50 1ac119fb 2024-01-23 op 1,3,1,1,1,1,1,3,1,3,1,1,1,1,1,1,1,3,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // s7..s8
53 1ac119fb 2024-01-23 op static inline uint32_t
54 1ac119fb 2024-01-23 op decode(uint32_t* restrict state, uint32_t* restrict codep, uint8_t byte)
56 1ac119fb 2024-01-23 op uint32_t type = utf8d[byte];
58 1ac119fb 2024-01-23 op *codep = (*state != UTF8_ACCEPT) ?
59 1ac119fb 2024-01-23 op (byte & 0x3fu) | (*codep << 6) :
60 1ac119fb 2024-01-23 op (0xff >> type) & (byte);
62 1ac119fb 2024-01-23 op *state = utf8d[256 + *state*16 + type];
63 1ac119fb 2024-01-23 op return *state;
67 1ac119fb 2024-01-23 op /* end of the converter, utility functions ahead */
69 1ac119fb 2024-01-23 op #define ZERO_WIDTH_SPACE 0x200B
71 1ac119fb 2024-01-23 op /* public version of decode */
73 1ac119fb 2024-01-23 op utf8_decode(uint32_t* restrict state, uint32_t* restrict codep, uint8_t byte)
75 1ac119fb 2024-01-23 op return decode(state, codep, byte);
78 1ac119fb 2024-01-23 op /* encode cp in s. s must be at least 4 bytes wide */
80 1ac119fb 2024-01-23 op utf8_encode(uint32_t cp, char *s)
82 1ac119fb 2024-01-23 op if (cp <= 0x7F) {
83 1ac119fb 2024-01-23 op *s = (uint8_t)cp;
85 1ac119fb 2024-01-23 op } else if (cp <= 0x7FF) {
86 1ac119fb 2024-01-23 op s[1] = (uint8_t)(( cp & 0x3F ) + 0x80);
87 1ac119fb 2024-01-23 op s[0] = (uint8_t)(((cp >> 6) & 0x1F) + 0xC0);
89 1ac119fb 2024-01-23 op } else if (cp <= 0xFFFF) {
90 1ac119fb 2024-01-23 op s[2] = (uint8_t)(( cp & 0x3F) + 0x80);
91 1ac119fb 2024-01-23 op s[1] = (uint8_t)(((cp >> 6) & 0x3F) + 0x80);
92 1ac119fb 2024-01-23 op s[0] = (uint8_t)(((cp >> 12) & 0x0F) + 0xE0);
94 1ac119fb 2024-01-23 op } else if (cp <= 0x10FFFF) {
95 1ac119fb 2024-01-23 op s[3] = (uint8_t)(( cp & 0x3F) + 0x80);
96 1ac119fb 2024-01-23 op s[2] = (uint8_t)(((cp >> 6) & 0x3F) + 0x80);
97 1ac119fb 2024-01-23 op s[1] = (uint8_t)(((cp >> 12) & 0x3F) + 0x80);
98 1ac119fb 2024-01-23 op s[0] = (uint8_t)(((cp >> 18) & 0x07) + 0xF0);
107 1ac119fb 2024-01-23 op utf8_nth(char *s, size_t n)
110 1ac119fb 2024-01-23 op uint32_t cp = 0, state = 0;
112 1ac119fb 2024-01-23 op for (i = 0; *s && i < n; ++s)
113 1ac119fb 2024-01-23 op if (!decode(&state, &cp, *s))
116 1ac119fb 2024-01-23 op if (state != UTF8_ACCEPT)
124 1ac119fb 2024-01-23 op utf8_cplen(char *s)
126 1ac119fb 2024-01-23 op uint32_t cp = 0, state = 0;
130 1ac119fb 2024-01-23 op for (; *s; ++s)
131 1ac119fb 2024-01-23 op if (!decode(&state, &cp, *s))
137 1ac119fb 2024-01-23 op utf8_ncplen(const char *s, size_t slen)
139 1ac119fb 2024-01-23 op uint32_t cp = 0, state = 0;
140 1ac119fb 2024-01-23 op size_t len = 0;
142 1ac119fb 2024-01-23 op for (; slen > 0 && *s; ++s, --slen)
143 1ac119fb 2024-01-23 op if (!decode(&state, &cp, *s))
148 1ac119fb 2024-01-23 op /* returns only 0, 1, 2 or 8. assumes sizeof(wchar_t) is 4 */
150 1ac119fb 2024-01-23 op utf8_chwidth(uint32_t cp)
152 1ac119fb 2024-01-23 op /* XXX: if we're running on a platform where sizeof(wchar_t)
153 1ac119fb 2024-01-23 op * == 2 what to do? The manpage for wcwidth and wcs isn't
154 1ac119fb 2024-01-23 op * clear about the encoding, but if it's 16 bit wide I assume
155 1ac119fb 2024-01-23 op * it must use UTF-16... right? */
156 1ac119fb 2024-01-23 op assert(sizeof(wchar_t) == 4);
159 1ac119fb 2024-01-23 op * quick and dirty fix for the tabs. In the future we may
160 1ac119fb 2024-01-23 op * want to expand tabs into N spaces, but for the time being
161 1ac119fb 2024-01-23 op * this seems to be good enough (tm).
163 1ac119fb 2024-01-23 op if (cp == '\t')
166 1ac119fb 2024-01-23 op return wcwidth((wchar_t)cp);
169 1ac119fb 2024-01-23 op /* NOTE: n is the number of codepoints, NOT the byte length. In
170 1ac119fb 2024-01-23 op * other words, s MUST be NUL-terminated. */
172 1ac119fb 2024-01-23 op utf8_snwidth(const char *s, size_t n)
174 1ac119fb 2024-01-23 op size_t i, tot;
175 1ac119fb 2024-01-23 op uint32_t cp = 0, state = 0;
178 1ac119fb 2024-01-23 op for (i = 0; *s && i < n; ++s)
179 1ac119fb 2024-01-23 op if (!decode(&state, &cp, *s)) {
181 1ac119fb 2024-01-23 op tot += utf8_chwidth(cp);
188 1ac119fb 2024-01-23 op utf8_swidth(const char *s)
191 1ac119fb 2024-01-23 op uint32_t cp = 0, state = 0;
194 1ac119fb 2024-01-23 op for (; *s; ++s)
195 1ac119fb 2024-01-23 op if (!decode(&state, &cp, *s))
196 1ac119fb 2024-01-23 op tot += utf8_chwidth(cp);
202 1ac119fb 2024-01-23 op utf8_swidth_between(const char *str, const char *end)
205 1ac119fb 2024-01-23 op uint32_t cp = 0, state = 0;
208 1ac119fb 2024-01-23 op for (; *str && str < end; ++str)
209 1ac119fb 2024-01-23 op if (!decode(&state, &cp, *str))
210 1ac119fb 2024-01-23 op tot += utf8_chwidth(cp);
215 1ac119fb 2024-01-23 op utf8_next_cp(const char *s)
217 1ac119fb 2024-01-23 op uint32_t cp = 0, state = 0;
219 1ac119fb 2024-01-23 op for (; *s; ++s)
220 1ac119fb 2024-01-23 op if (!decode(&state, &cp, *s))
222 1ac119fb 2024-01-23 op return (char*)s+1;
226 1ac119fb 2024-01-23 op utf8_prev_cp(const char *start, const char *base)
230 1ac119fb 2024-01-23 op for (; start > base; start--) {
232 1ac119fb 2024-01-23 op if ((c & 0xC0) != 0x80)
233 1ac119fb 2024-01-23 op return (char*)start;
236 1ac119fb 2024-01-23 op return (char*)base;
240 1ac119fb 2024-01-23 op * XXX: This is not correct. There are codepoints classified as
241 1ac119fb 2024-01-23 op * "emoji", but these can be joined toghether to form more complex
242 1ac119fb 2024-01-23 op * emoji. There is an official list of what these valid combinations
243 1ac119fb 2024-01-23 op * are, but it would require a costly lookup (a trie can be used to
244 1ac119fb 2024-01-23 op * reduce the times, but...). The following approach is conceptually
245 1ac119fb 2024-01-23 op * simpler: if there is a sequence of "emoji codepoints" (or ZWS) and
246 1ac119fb 2024-01-23 op * then a space, consider everything before the space a single emoji.
247 1ac119fb 2024-01-23 op * It needs a special check for numbers (yes, 0..9 and # are
248 1ac119fb 2024-01-23 op * technically speaking emojis) but otherwise seems to work well in
252 1ac119fb 2024-01-23 op emojied_line(const char *s, const char **space_ret)
254 1ac119fb 2024-01-23 op uint32_t cp = 0, state = 0;
255 1ac119fb 2024-01-23 op int only_numbers = 1;
257 1ac119fb 2024-01-23 op for (; *s; ++s) {
258 1ac119fb 2024-01-23 op if (!decode(&state, &cp, *s)) {
259 1ac119fb 2024-01-23 op if (cp == ZERO_WIDTH_SPACE)
261 1ac119fb 2024-01-23 op if (cp == ' ') {
262 1ac119fb 2024-01-23 op *space_ret = s;
263 1ac119fb 2024-01-23 op return !only_numbers;
265 1ac119fb 2024-01-23 op if (!is_emoji(cp))
267 1ac119fb 2024-01-23 op if (cp < '0' || cp > '9')
268 1ac119fb 2024-01-23 op only_numbers = 0;