Blob


1 /* Copyright (c) 2008-2009 Bjoern Hoehrmann <bjoern@hoehrmann.de>
2 *
3 * Permission is hereby granted, free of charge, to any person
4 * obtaining a copy of this software and associated documentation
5 * files (the "Software"), to deal in the Software without
6 * restriction, including without limitation the rights to use, copy,
7 * modify, merge, publish, distribute, sublicense, and/or sell copies
8 * of the Software, and to permit persons to whom the Software is
9 * furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be
12 * included in all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
15 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
16 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
17 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
18 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
19 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
20 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 * SOFTWARE.
22 */
24 #include "compat.h"
26 #include <assert.h>
27 #include <stddef.h>
28 #include <stdint.h>
29 #include <wchar.h>
31 #include "telescope.h"
32 #include "utf8.h"
34 #define UTF8_ACCEPT 0
35 #define UTF8_REJECT 1
37 static const uint8_t utf8d[] = {
38 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 00..1f
39 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 20..3f
40 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 40..5f
41 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 60..7f
42 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9, // 80..9f
43 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7, // a0..bf
44 8,8,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, // c0..df
45 0xa,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x4,0x3,0x3, // e0..ef
46 0xb,0x6,0x6,0x6,0x5,0x8,0x8,0x8,0x8,0x8,0x8,0x8,0x8,0x8,0x8,0x8, // f0..ff
47 0x0,0x1,0x2,0x3,0x5,0x8,0x7,0x1,0x1,0x1,0x4,0x6,0x1,0x1,0x1,0x1, // s0..s0
48 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,0,1,0,1,1,1,1,1,1, // s1..s2
49 1,2,1,1,1,1,1,2,1,2,1,1,1,1,1,1,1,1,1,1,1,1,1,2,1,1,1,1,1,1,1,1, // s3..s4
50 1,2,1,1,1,1,1,1,1,2,1,1,1,1,1,1,1,1,1,1,1,1,1,3,1,3,1,1,1,1,1,1, // s5..s6
51 1,3,1,1,1,1,1,3,1,3,1,1,1,1,1,1,1,3,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // s7..s8
52 };
54 static inline uint32_t
55 decode(uint32_t* restrict state, uint32_t* restrict codep, uint8_t byte)
56 {
57 uint32_t type = utf8d[byte];
59 *codep = (*state != UTF8_ACCEPT) ?
60 (byte & 0x3fu) | (*codep << 6) :
61 (0xff >> type) & (byte);
63 *state = utf8d[256 + *state*16 + type];
64 return *state;
65 }
68 /* end of the converter, utility functions ahead */
70 #define ZERO_WIDTH_SPACE 0x200B
72 /* public version of decode */
73 uint32_t
74 utf8_decode(uint32_t* restrict state, uint32_t* restrict codep, uint8_t byte)
75 {
76 return decode(state, codep, byte);
77 }
79 /* encode cp in s. s must be at least 4 bytes wide */
80 size_t
81 utf8_encode(uint32_t cp, char *s)
82 {
83 if (cp <= 0x7F) {
84 *s = (uint8_t)cp;
85 return 1;
86 } else if (cp <= 0x7FF) {
87 s[1] = (uint8_t)(( cp & 0x3F ) + 0x80);
88 s[0] = (uint8_t)(((cp >> 6) & 0x1F) + 0xC0);
89 return 2;
90 } else if (cp <= 0xFFFF) {
91 s[2] = (uint8_t)(( cp & 0x3F) + 0x80);
92 s[1] = (uint8_t)(((cp >> 6) & 0x3F) + 0x80);
93 s[0] = (uint8_t)(((cp >> 12) & 0x0F) + 0xE0);
94 return 3;
95 } else if (cp <= 0x10FFFF) {
96 s[3] = (uint8_t)(( cp & 0x3F) + 0x80);
97 s[2] = (uint8_t)(((cp >> 6) & 0x3F) + 0x80);
98 s[1] = (uint8_t)(((cp >> 12) & 0x3F) + 0x80);
99 s[0] = (uint8_t)(((cp >> 18) & 0x07) + 0xF0);
100 return 4;
101 } else {
102 s[0] = '\0';
103 return 0;
107 char *
108 utf8_nth(char *s, size_t n)
110 size_t i;
111 uint32_t cp = 0, state = 0;
113 for (i = 0; *s && i < n; ++s)
114 if (!decode(&state, &cp, *s))
115 ++i;
117 if (state != UTF8_ACCEPT)
118 return NULL;
119 if (i == n)
120 return s;
121 return NULL;
124 size_t
125 utf8_cplen(char *s)
127 uint32_t cp = 0, state = 0;
128 size_t len;
130 len = 0;
131 for (; *s; ++s)
132 if (!decode(&state, &cp, *s))
133 len++;
134 return len;
137 /* returns only 0, 1, 2 or 8. assumes sizeof(wchar_t) is 4 */
138 size_t
139 utf8_chwidth(uint32_t cp)
141 /* XXX: if we're running on a platform where sizeof(wchar_t)
142 * == 2 what to do? The manpage for wcwidth and wcs isn't
143 * clear about the encoding, but if it's 16 bit wide I assume
144 * it must use UTF-16... right? */
145 assert(sizeof(wchar_t) == 4);
147 /*
148 * quick and dirty fix for the tabs. In the future we may
149 * want to expand tabs into N spaces, but for the time being
150 * this seems to be good enough (tm).
151 */
152 if (cp == '\t')
153 return 8;
155 return wcwidth((wchar_t)cp);
158 /* NOTE: n is the number of codepoints, NOT the byte length. In
159 * other words, s MUST be NUL-terminated. */
160 size_t
161 utf8_snwidth(const char *s, size_t n)
163 size_t i, tot;
164 uint32_t cp = 0, state = 0;
166 tot = 0;
167 for (i = 0; *s && i < n; ++s)
168 if (!decode(&state, &cp, *s)) {
169 i++;
170 tot += utf8_chwidth(cp);
173 return tot;
176 size_t
177 utf8_swidth(const char *s)
179 size_t tot;
180 uint32_t cp = 0, state = 0;
182 tot = 0;
183 for (; *s; ++s)
184 if (!decode(&state, &cp, *s))
185 tot += utf8_chwidth(cp);
187 return tot;
190 size_t
191 utf8_swidth_between(const char *str, const char *end)
193 size_t tot;
194 uint32_t cp = 0, state = 0;
196 tot = 0;
197 for (; *str && str < end; ++str)
198 if (!decode(&state, &cp, *str))
199 tot += utf8_chwidth(cp);
200 return tot;
203 char *
204 utf8_next_cp(const char *s)
206 uint32_t cp = 0, state = 0;
208 for (; *s; ++s)
209 if (!decode(&state, &cp, *s))
210 break;
211 return (char*)s+1;
214 char *
215 utf8_prev_cp(const char *start, const char *base)
217 uint8_t c;
219 for (; start > base; start--) {
220 c = *start;
221 if ((c & 0xC0) != 0x80)
222 return (char*)start;
225 return (char*)base;
228 /*
229 * XXX: This is not correct. There are codepoints classified as
230 * "emoji", but these can be joined toghether to form more complex
231 * emoji. There is an official list of what these valid combinations
232 * are, but it would require a costly lookup (a trie can be used to
233 * reduce the times, but...). The following approach is conceptually
234 * simpler: if there is a sequence of "emoji codepoints" (or ZWS) and
235 * then a space, consider everything before the space a single emoji.
236 * It needs a special check for numbers (yes, 0..9 and # are
237 * technically speaking emojis) but otherwise seems to work well in
238 * practice.
239 */
240 int
241 emojied_line(const char *s, const char **space_ret)
243 uint32_t cp = 0, state = 0;
244 int only_numbers = 1;
246 for (; *s; ++s) {
247 if (!decode(&state, &cp, *s)) {
248 if (cp == ZERO_WIDTH_SPACE)
249 continue;
250 if (cp == ' ') {
251 *space_ret = s;
252 return !only_numbers;
254 if (!is_emoji(cp))
255 return 0;
256 if (cp < '0' || cp > '9')
257 only_numbers = 0;
261 return 0;