Blame


1 5e11c00c 2021-03-02 op /* Copyright (c) 2008-2009 Bjoern Hoehrmann <bjoern@hoehrmann.de>
2 5e11c00c 2021-03-02 op *
3 5e11c00c 2021-03-02 op * Permission is hereby granted, free of charge, to any person
4 5e11c00c 2021-03-02 op * obtaining a copy of this software and associated documentation
5 5e11c00c 2021-03-02 op * files (the "Software"), to deal in the Software without
6 5e11c00c 2021-03-02 op * restriction, including without limitation the rights to use, copy,
7 5e11c00c 2021-03-02 op * modify, merge, publish, distribute, sublicense, and/or sell copies
8 5e11c00c 2021-03-02 op * of the Software, and to permit persons to whom the Software is
9 5e11c00c 2021-03-02 op * furnished to do so, subject to the following conditions:
10 5e11c00c 2021-03-02 op *
11 5e11c00c 2021-03-02 op * The above copyright notice and this permission notice shall be
12 5e11c00c 2021-03-02 op * included in all copies or substantial portions of the Software.
13 5e11c00c 2021-03-02 op *
14 5e11c00c 2021-03-02 op * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
15 5e11c00c 2021-03-02 op * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
16 5e11c00c 2021-03-02 op * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
17 5e11c00c 2021-03-02 op * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
18 5e11c00c 2021-03-02 op * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
19 5e11c00c 2021-03-02 op * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
20 5e11c00c 2021-03-02 op * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 5e11c00c 2021-03-02 op * SOFTWARE.
22 5e11c00c 2021-03-02 op */
23 786e6deb 2021-07-21 op
24 786e6deb 2021-07-21 op #include "compat.h"
25 5e11c00c 2021-03-02 op
26 174b3cdf 2021-03-21 op #include <assert.h>
27 5e11c00c 2021-03-02 op #include <stddef.h>
28 5e11c00c 2021-03-02 op #include <stdint.h>
29 174b3cdf 2021-03-21 op #include <wchar.h>
30 5e11c00c 2021-03-02 op
31 5caf7d67 2021-07-12 op #include "telescope.h"
32 5caf7d67 2021-07-12 op #include "utf8.h"
33 5caf7d67 2021-07-12 op
34 5e11c00c 2021-03-02 op #define UTF8_ACCEPT 0
35 5e11c00c 2021-03-02 op #define UTF8_REJECT 1
36 5e11c00c 2021-03-02 op
37 5e11c00c 2021-03-02 op static const uint8_t utf8d[] = {
38 5e11c00c 2021-03-02 op 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 00..1f
39 5e11c00c 2021-03-02 op 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 20..3f
40 5e11c00c 2021-03-02 op 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 40..5f
41 5e11c00c 2021-03-02 op 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 60..7f
42 5e11c00c 2021-03-02 op 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9, // 80..9f
43 5e11c00c 2021-03-02 op 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7, // a0..bf
44 5e11c00c 2021-03-02 op 8,8,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, // c0..df
45 5e11c00c 2021-03-02 op 0xa,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x4,0x3,0x3, // e0..ef
46 5e11c00c 2021-03-02 op 0xb,0x6,0x6,0x6,0x5,0x8,0x8,0x8,0x8,0x8,0x8,0x8,0x8,0x8,0x8,0x8, // f0..ff
47 5e11c00c 2021-03-02 op 0x0,0x1,0x2,0x3,0x5,0x8,0x7,0x1,0x1,0x1,0x4,0x6,0x1,0x1,0x1,0x1, // s0..s0
48 5e11c00c 2021-03-02 op 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,0,1,0,1,1,1,1,1,1, // s1..s2
49 5e11c00c 2021-03-02 op 1,2,1,1,1,1,1,2,1,2,1,1,1,1,1,1,1,1,1,1,1,1,1,2,1,1,1,1,1,1,1,1, // s3..s4
50 5e11c00c 2021-03-02 op 1,2,1,1,1,1,1,1,1,2,1,1,1,1,1,1,1,1,1,1,1,1,1,3,1,3,1,1,1,1,1,1, // s5..s6
51 5e11c00c 2021-03-02 op 1,3,1,1,1,1,1,3,1,3,1,1,1,1,1,1,1,3,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // s7..s8
52 5e11c00c 2021-03-02 op };
53 5e11c00c 2021-03-02 op
54 a1e5a814 2021-07-15 op static inline uint32_t
55 a1e5a814 2021-07-15 op decode(uint32_t* restrict state, uint32_t* restrict codep, uint8_t byte)
56 a1e5a814 2021-07-15 op {
57 5e11c00c 2021-03-02 op uint32_t type = utf8d[byte];
58 5e11c00c 2021-03-02 op
59 5e11c00c 2021-03-02 op *codep = (*state != UTF8_ACCEPT) ?
60 5e11c00c 2021-03-02 op (byte & 0x3fu) | (*codep << 6) :
61 5e11c00c 2021-03-02 op (0xff >> type) & (byte);
62 5e11c00c 2021-03-02 op
63 5e11c00c 2021-03-02 op *state = utf8d[256 + *state*16 + type];
64 5e11c00c 2021-03-02 op return *state;
65 5e11c00c 2021-03-02 op }
66 5e11c00c 2021-03-02 op
67 174b3cdf 2021-03-21 op
68 174b3cdf 2021-03-21 op /* end of the converter, utility functions ahead */
69 5e11c00c 2021-03-02 op
70 ab2d4a19 2021-07-16 op #define ZERO_WIDTH_SPACE 0x200B
71 ab2d4a19 2021-07-16 op
72 a1e5a814 2021-07-15 op /* public version of decode */
73 a1e5a814 2021-07-15 op uint32_t
74 a1e5a814 2021-07-15 op utf8_decode(uint32_t* restrict state, uint32_t* restrict codep, uint8_t byte)
75 a1e5a814 2021-07-15 op {
76 a1e5a814 2021-07-15 op return decode(state, codep, byte);
77 a1e5a814 2021-07-15 op }
78 a1e5a814 2021-07-15 op
79 8947c1f2 2021-03-21 op /* encode cp in s. s must be at least 4 bytes wide */
80 17e5106b 2021-03-21 op size_t
81 8947c1f2 2021-03-21 op utf8_encode(uint32_t cp, char *s)
82 8947c1f2 2021-03-21 op {
83 8947c1f2 2021-03-21 op if (cp <= 0x7F) {
84 95a8c791 2021-08-26 op *s = (uint8_t)cp;
85 17e5106b 2021-03-21 op return 1;
86 8947c1f2 2021-03-21 op } else if (cp <= 0x7FF) {
87 95a8c791 2021-08-26 op s[1] = (uint8_t)(( cp & 0x3F ) + 0x80);
88 8947c1f2 2021-03-21 op s[0] = (uint8_t)(((cp >> 6) & 0x1F) + 0xC0);
89 17e5106b 2021-03-21 op return 2;
90 8947c1f2 2021-03-21 op } else if (cp <= 0xFFFF) {
91 95a8c791 2021-08-26 op s[2] = (uint8_t)(( cp & 0x3F) + 0x80);
92 8947c1f2 2021-03-21 op s[1] = (uint8_t)(((cp >> 6) & 0x3F) + 0x80);
93 8947c1f2 2021-03-21 op s[0] = (uint8_t)(((cp >> 12) & 0x0F) + 0xE0);
94 17e5106b 2021-03-21 op return 3;
95 8947c1f2 2021-03-21 op } else if (cp <= 0x10FFFF) {
96 95a8c791 2021-08-26 op s[3] = (uint8_t)(( cp & 0x3F) + 0x80);
97 8947c1f2 2021-03-21 op s[2] = (uint8_t)(((cp >> 6) & 0x3F) + 0x80);
98 8947c1f2 2021-03-21 op s[1] = (uint8_t)(((cp >> 12) & 0x3F) + 0x80);
99 8947c1f2 2021-03-21 op s[0] = (uint8_t)(((cp >> 18) & 0x07) + 0xF0);
100 17e5106b 2021-03-21 op return 4;
101 17e5106b 2021-03-21 op } else {
102 17e5106b 2021-03-21 op s[0] = '\0';
103 17e5106b 2021-03-21 op return 0;
104 17e5106b 2021-03-21 op }
105 8947c1f2 2021-03-21 op }
106 8947c1f2 2021-03-21 op
107 5e11c00c 2021-03-02 op char *
108 5e11c00c 2021-03-02 op utf8_nth(char *s, size_t n)
109 5e11c00c 2021-03-02 op {
110 5e11c00c 2021-03-02 op size_t i;
111 5e11c00c 2021-03-02 op uint32_t cp = 0, state = 0;
112 5e11c00c 2021-03-02 op
113 5e11c00c 2021-03-02 op for (i = 0; *s && i < n; ++s)
114 a1e5a814 2021-07-15 op if (!decode(&state, &cp, *s))
115 5e11c00c 2021-03-02 op ++i;
116 5e11c00c 2021-03-02 op
117 5e11c00c 2021-03-02 op if (state != UTF8_ACCEPT)
118 5e11c00c 2021-03-02 op return NULL;
119 5e11c00c 2021-03-02 op if (i == n)
120 5e11c00c 2021-03-02 op return s;
121 5e11c00c 2021-03-02 op return NULL;
122 5e11c00c 2021-03-02 op }
123 174b3cdf 2021-03-21 op
124 174b3cdf 2021-03-21 op size_t
125 174b3cdf 2021-03-21 op utf8_cplen(char *s)
126 174b3cdf 2021-03-21 op {
127 174b3cdf 2021-03-21 op uint32_t cp = 0, state = 0;
128 174b3cdf 2021-03-21 op size_t len;
129 174b3cdf 2021-03-21 op
130 174b3cdf 2021-03-21 op len = 0;
131 174b3cdf 2021-03-21 op for (; *s; ++s)
132 a1e5a814 2021-07-15 op if (!decode(&state, &cp, *s))
133 174b3cdf 2021-03-21 op len++;
134 174b3cdf 2021-03-21 op return len;
135 174b3cdf 2021-03-21 op }
136 174b3cdf 2021-03-21 op
137 421bca4a 2021-06-12 op /* returns only 0, 1, 2 or 8. assumes sizeof(wchar_t) is 4 */
138 174b3cdf 2021-03-21 op size_t
139 174b3cdf 2021-03-21 op utf8_chwidth(uint32_t cp)
140 174b3cdf 2021-03-21 op {
141 174b3cdf 2021-03-21 op /* XXX: if we're running on a platform where sizeof(wchar_t)
142 174b3cdf 2021-03-21 op * == 2 what to do? The manpage for wcwidth and wcs isn't
143 174b3cdf 2021-03-21 op * clear about the encoding, but if it's 16 bit wide I assume
144 174b3cdf 2021-03-21 op * it must use UTF-16... right? */
145 174b3cdf 2021-03-21 op assert(sizeof(wchar_t) == 4);
146 174b3cdf 2021-03-21 op
147 421bca4a 2021-06-12 op /*
148 421bca4a 2021-06-12 op * quick and dirty fix for the tabs. In the future we may
149 421bca4a 2021-06-12 op * want to expand tabs into N spaces, but for the time being
150 421bca4a 2021-06-12 op * this seems to be good enough (tm).
151 421bca4a 2021-06-12 op */
152 421bca4a 2021-06-12 op if (cp == '\t')
153 421bca4a 2021-06-12 op return 8;
154 421bca4a 2021-06-12 op
155 174b3cdf 2021-03-21 op return wcwidth((wchar_t)cp);
156 174b3cdf 2021-03-21 op }
157 174b3cdf 2021-03-21 op
158 174b3cdf 2021-03-21 op /* NOTE: n is the number of codepoints, NOT the byte length. In
159 174b3cdf 2021-03-21 op * other words, s MUST be NUL-terminated. */
160 174b3cdf 2021-03-21 op size_t
161 174b3cdf 2021-03-21 op utf8_snwidth(const char *s, size_t n)
162 174b3cdf 2021-03-21 op {
163 174b3cdf 2021-03-21 op size_t i, tot;
164 174b3cdf 2021-03-21 op uint32_t cp = 0, state = 0;
165 174b3cdf 2021-03-21 op
166 174b3cdf 2021-03-21 op tot = 0;
167 174b3cdf 2021-03-21 op for (i = 0; *s && i < n; ++s)
168 a1e5a814 2021-07-15 op if (!decode(&state, &cp, *s)) {
169 174b3cdf 2021-03-21 op i++;
170 174b3cdf 2021-03-21 op tot += utf8_chwidth(cp);
171 174b3cdf 2021-03-21 op }
172 174b3cdf 2021-03-21 op
173 174b3cdf 2021-03-21 op return tot;
174 174b3cdf 2021-03-21 op }
175 174b3cdf 2021-03-21 op
176 174b3cdf 2021-03-21 op size_t
177 174b3cdf 2021-03-21 op utf8_swidth(const char *s)
178 174b3cdf 2021-03-21 op {
179 174b3cdf 2021-03-21 op size_t tot;
180 174b3cdf 2021-03-21 op uint32_t cp = 0, state = 0;
181 174b3cdf 2021-03-21 op
182 174b3cdf 2021-03-21 op tot = 0;
183 174b3cdf 2021-03-21 op for (; *s; ++s)
184 a1e5a814 2021-07-15 op if (!decode(&state, &cp, *s))
185 174b3cdf 2021-03-21 op tot += utf8_chwidth(cp);
186 174b3cdf 2021-03-21 op
187 174b3cdf 2021-03-21 op return tot;
188 174b3cdf 2021-03-21 op }
189 17e5106b 2021-03-21 op
190 17e5106b 2021-03-21 op size_t
191 17e5106b 2021-03-21 op utf8_swidth_between(const char *str, const char *end)
192 17e5106b 2021-03-21 op {
193 17e5106b 2021-03-21 op size_t tot;
194 17e5106b 2021-03-21 op uint32_t cp = 0, state = 0;
195 17e5106b 2021-03-21 op
196 17e5106b 2021-03-21 op tot = 0;
197 17e5106b 2021-03-21 op for (; *str && str < end; ++str)
198 a1e5a814 2021-07-15 op if (!decode(&state, &cp, *str))
199 17e5106b 2021-03-21 op tot += utf8_chwidth(cp);
200 17e5106b 2021-03-21 op return tot;
201 17e5106b 2021-03-21 op }
202 17e5106b 2021-03-21 op
203 17e5106b 2021-03-21 op char *
204 2ba66cea 2021-03-22 op utf8_next_cp(const char *s)
205 17e5106b 2021-03-21 op {
206 17e5106b 2021-03-21 op uint32_t cp = 0, state = 0;
207 17e5106b 2021-03-21 op
208 17e5106b 2021-03-21 op for (; *s; ++s)
209 a1e5a814 2021-07-15 op if (!decode(&state, &cp, *s))
210 17e5106b 2021-03-21 op break;
211 2ba66cea 2021-03-22 op return (char*)s+1;
212 17e5106b 2021-03-21 op }
213 17e5106b 2021-03-21 op
214 17e5106b 2021-03-21 op char *
215 2ba66cea 2021-03-22 op utf8_prev_cp(const char *start, const char *base)
216 17e5106b 2021-03-21 op {
217 17e5106b 2021-03-21 op uint8_t c;
218 17e5106b 2021-03-21 op
219 17e5106b 2021-03-21 op for (; start > base; start--) {
220 17e5106b 2021-03-21 op c = *start;
221 95a8c791 2021-08-26 op if ((c & 0xC0) != 0x80)
222 2ba66cea 2021-03-22 op return (char*)start;
223 17e5106b 2021-03-21 op }
224 17e5106b 2021-03-21 op
225 2ba66cea 2021-03-22 op return (char*)base;
226 17e5106b 2021-03-21 op }
227 ab2d4a19 2021-07-16 op
228 4c015bee 2021-07-17 op /*
229 4c015bee 2021-07-17 op * XXX: This is not correct. There are codepoints classified as
230 4c015bee 2021-07-17 op * "emoji", but these can be joined toghether to form more complex
231 4c015bee 2021-07-17 op * emoji. There is an ufficial list of what these valid combinations
232 4c015bee 2021-07-17 op * are, but it would require a costly lookup (a trie can be used to
233 4c015bee 2021-07-17 op * reduce the times, but...). The following approach is conceptually
234 4c015bee 2021-07-17 op * simpler: if there is a sequence of "emoji codepoints" (or ZWS) and
235 4c015bee 2021-07-17 op * then a space, consider everything before the space a single emoji.
236 4c015bee 2021-07-17 op * It needs a special check for numbers (yes, 0..9 and # are
237 4c015bee 2021-07-17 op * technically speaking emojis) but otherwise seems to work well in
238 4c015bee 2021-07-17 op * practice.
239 4c015bee 2021-07-17 op */
240 ab2d4a19 2021-07-16 op int
241 ab2d4a19 2021-07-16 op emojied_line(const char *s, const char **space_ret)
242 ab2d4a19 2021-07-16 op {
243 ab2d4a19 2021-07-16 op uint32_t cp = 0, state = 0;
244 1c6e356a 2021-07-17 op int only_numbers = 1;
245 ab2d4a19 2021-07-16 op
246 ab2d4a19 2021-07-16 op for (; *s; ++s) {
247 ab2d4a19 2021-07-16 op if (!decode(&state, &cp, *s)) {
248 ab2d4a19 2021-07-16 op if (cp == ZERO_WIDTH_SPACE)
249 ab2d4a19 2021-07-16 op continue;
250 ab2d4a19 2021-07-16 op if (cp == ' ') {
251 ab2d4a19 2021-07-16 op *space_ret = s;
252 1c6e356a 2021-07-17 op return !only_numbers;
253 ab2d4a19 2021-07-16 op }
254 ab2d4a19 2021-07-16 op if (!is_emoji(cp))
255 ab2d4a19 2021-07-16 op return 0;
256 1c6e356a 2021-07-17 op if (cp < '0' || cp > '9')
257 1c6e356a 2021-07-17 op only_numbers = 0;
258 ab2d4a19 2021-07-16 op }
259 ab2d4a19 2021-07-16 op }
260 ab2d4a19 2021-07-16 op
261 ab2d4a19 2021-07-16 op return 0;
262 ab2d4a19 2021-07-16 op }