1 3448adb0 2022-11-02 op /* See LICENSE file for copyright and license details. */
2 3448adb0 2022-11-02 op #include <limits.h>
3 3448adb0 2022-11-02 op #include <stdbool.h>
4 3448adb0 2022-11-02 op #include <stddef.h>
5 3448adb0 2022-11-02 op #include <stdint.h>
7 3448adb0 2022-11-02 op #include "../gen/types.h"
8 3448adb0 2022-11-02 op #include "../grapheme.h"
9 3448adb0 2022-11-02 op #include "util.h"
12 3448adb0 2022-11-02 op herodotus_reader_init(HERODOTUS_READER *r, enum herodotus_type type,
13 3448adb0 2022-11-02 op const void *src, size_t srclen)
17 3448adb0 2022-11-02 op r->type = type;
19 3448adb0 2022-11-02 op r->srclen = srclen;
21 3448adb0 2022-11-02 op r->terminated_by_null = false;
23 3448adb0 2022-11-02 op for (i = 0; i < LEN(r->soft_limit); i++) {
24 3448adb0 2022-11-02 op r->soft_limit[i] = SIZE_MAX;
29 3448adb0 2022-11-02 op herodotus_reader_copy(const HERODOTUS_READER *src, HERODOTUS_READER *dest)
34 3448adb0 2022-11-02 op * we copy such that we have a "fresh" start and build on the
35 3448adb0 2022-11-02 op * fact that src->soft_limit[i] for any i and src->srclen are
36 3448adb0 2022-11-02 op * always larger or equal to src->off
38 3448adb0 2022-11-02 op dest->type = src->type;
39 3448adb0 2022-11-02 op if (src->type == HERODOTUS_TYPE_CODEPOINT) {
40 3448adb0 2022-11-02 op dest->src = (src->src == NULL) ? NULL :
41 3448adb0 2022-11-02 op ((const uint_least32_t *)(src->src)) + src->off;
42 3448adb0 2022-11-02 op } else { /* src->type == HERODOTUS_TYPE_UTF8 */
43 3448adb0 2022-11-02 op dest->src = (src->src == NULL) ? NULL :
44 3448adb0 2022-11-02 op ((const char *)(src->src)) + src->off;
46 3448adb0 2022-11-02 op if (src->srclen == SIZE_MAX) {
47 3448adb0 2022-11-02 op dest->srclen = SIZE_MAX;
49 3448adb0 2022-11-02 op dest->srclen = (src->off < src->srclen) ? src->srclen - src->off : 0;
51 3448adb0 2022-11-02 op dest->off = 0;
52 3448adb0 2022-11-02 op dest->terminated_by_null = src->terminated_by_null;
54 3448adb0 2022-11-02 op for (i = 0; i < LEN(src->soft_limit); i++) {
55 3448adb0 2022-11-02 op if (src->soft_limit[i] == SIZE_MAX) {
56 3448adb0 2022-11-02 op dest->soft_limit[i] = SIZE_MAX;
59 3448adb0 2022-11-02 op * if we have a degenerate case where the offset is
60 3448adb0 2022-11-02 op * higher than the soft-limit, we simply clamp the
61 3448adb0 2022-11-02 op * soft-limit to zero given we can't decide here
62 3448adb0 2022-11-02 op * to release the limit and, instead, we just
63 3448adb0 2022-11-02 op * prevent any more reads
65 3448adb0 2022-11-02 op dest->soft_limit[i] = (src->off < src->soft_limit[i]) ?
66 3448adb0 2022-11-02 op src->soft_limit[i] - src->off : 0;
72 3448adb0 2022-11-02 op herodotus_reader_push_advance_limit(HERODOTUS_READER *r, size_t count)
76 3448adb0 2022-11-02 op for (i = LEN(r->soft_limit) - 1; i >= 1; i--) {
77 3448adb0 2022-11-02 op r->soft_limit[i] = r->soft_limit[i - 1];
79 3448adb0 2022-11-02 op r->soft_limit[0] = r->off + count;
83 3448adb0 2022-11-02 op herodotus_reader_pop_limit(HERODOTUS_READER *r)
87 3448adb0 2022-11-02 op for (i = 0; i < LEN(r->soft_limit) - 1; i++) {
88 3448adb0 2022-11-02 op r->soft_limit[i] = r->soft_limit[i + 1];
90 3448adb0 2022-11-02 op r->soft_limit[LEN(r->soft_limit) - 1] = SIZE_MAX;
94 3448adb0 2022-11-02 op herodotus_reader_next_word_break(const HERODOTUS_READER *r)
96 3448adb0 2022-11-02 op if (r->type == HERODOTUS_TYPE_CODEPOINT) {
97 3448adb0 2022-11-02 op return grapheme_next_word_break(
98 3448adb0 2022-11-02 op (const uint_least32_t *)(r->src) + r->off,
99 3448adb0 2022-11-02 op MIN(r->srclen, r->soft_limit[0]) - r->off);
100 3448adb0 2022-11-02 op } else { /* r->type == HERODOTUS_TYPE_UTF8 */
101 3448adb0 2022-11-02 op return grapheme_next_word_break_utf8(
102 3448adb0 2022-11-02 op (const char *)(r->src) + r->off,
103 3448adb0 2022-11-02 op MIN(r->srclen, r->soft_limit[0]) - r->off);
108 3448adb0 2022-11-02 op herodotus_reader_next_codepoint_break(const HERODOTUS_READER *r)
110 3448adb0 2022-11-02 op if (r->type == HERODOTUS_TYPE_CODEPOINT) {
111 3448adb0 2022-11-02 op return (r->off < MIN(r->srclen, r->soft_limit[0])) ? 1 : 0;
112 3448adb0 2022-11-02 op } else { /* r->type == HERODOTUS_TYPE_UTF8 */
113 3448adb0 2022-11-02 op return grapheme_decode_utf8(
114 3448adb0 2022-11-02 op (const char *)(r->src) + r->off,
115 3448adb0 2022-11-02 op MIN(r->srclen, r->soft_limit[0]) - r->off, NULL);
120 3448adb0 2022-11-02 op herodotus_reader_number_read(const HERODOTUS_READER *r)
122 3448adb0 2022-11-02 op return r->off;
125 3448adb0 2022-11-02 op enum herodotus_status
126 3448adb0 2022-11-02 op herodotus_read_codepoint(HERODOTUS_READER *r, bool advance, uint_least32_t *cp)
130 3448adb0 2022-11-02 op if (r->terminated_by_null || r->off >= r->srclen || r->src == NULL) {
131 3448adb0 2022-11-02 op *cp = GRAPHEME_INVALID_CODEPOINT;
132 3448adb0 2022-11-02 op return HERODOTUS_STATUS_END_OF_BUFFER;
135 3448adb0 2022-11-02 op if (r->off >= r->soft_limit[0]) {
136 3448adb0 2022-11-02 op *cp = GRAPHEME_INVALID_CODEPOINT;
137 3448adb0 2022-11-02 op return HERODOTUS_STATUS_SOFT_LIMIT_REACHED;
140 3448adb0 2022-11-02 op if (r->type == HERODOTUS_TYPE_CODEPOINT) {
141 3448adb0 2022-11-02 op *cp = ((const uint_least32_t *)(r->src))[r->off];
143 3448adb0 2022-11-02 op } else { /* r->type == HERODOTUS_TYPE_UTF8 */
144 3448adb0 2022-11-02 op ret = grapheme_decode_utf8((const char *)r->src + r->off,
145 3448adb0 2022-11-02 op MIN(r->srclen, r->soft_limit[0]) -
149 3448adb0 2022-11-02 op if (unlikely(r->srclen == SIZE_MAX && *cp == 0)) {
151 3448adb0 2022-11-02 op * We encountered a null-codepoint. Don't increment
152 3448adb0 2022-11-02 op * offset and return as if the buffer had ended here all
155 3448adb0 2022-11-02 op r->terminated_by_null = true;
156 3448adb0 2022-11-02 op return HERODOTUS_STATUS_END_OF_BUFFER;
159 3448adb0 2022-11-02 op if (r->off + ret > MIN(r->srclen, r->soft_limit[0])) {
161 3448adb0 2022-11-02 op * we want more than we have; instead of returning
162 3448adb0 2022-11-02 op * garbage we terminate here.
164 3448adb0 2022-11-02 op return HERODOTUS_STATUS_END_OF_BUFFER;
168 3448adb0 2022-11-02 op * Increase offset which we now know won't surpass the limits,
169 3448adb0 2022-11-02 op * unless we got told otherwise
171 3448adb0 2022-11-02 op if (advance) {
172 3448adb0 2022-11-02 op r->off += ret;
175 3448adb0 2022-11-02 op return HERODOTUS_STATUS_SUCCESS;
179 3448adb0 2022-11-02 op herodotus_writer_init(HERODOTUS_WRITER *w, enum herodotus_type type,
180 3448adb0 2022-11-02 op void *dest, size_t destlen)
182 3448adb0 2022-11-02 op w->type = type;
183 3448adb0 2022-11-02 op w->dest = dest;
184 3448adb0 2022-11-02 op w->destlen = destlen;
186 3448adb0 2022-11-02 op w->first_unwritable_offset = SIZE_MAX;
190 3448adb0 2022-11-02 op herodotus_writer_nul_terminate(HERODOTUS_WRITER *w)
192 3448adb0 2022-11-02 op if (w->dest == NULL) {
196 3448adb0 2022-11-02 op if (w->off < w->destlen) {
197 3448adb0 2022-11-02 op /* We still have space in the buffer. Simply use it */
198 3448adb0 2022-11-02 op if (w->type == HERODOTUS_TYPE_CODEPOINT) {
199 3448adb0 2022-11-02 op ((uint_least32_t *)(w->dest))[w->off] = 0;
200 3448adb0 2022-11-02 op } else { /* w->type == HERODOTUS_TYPE_UTF8 */
201 3448adb0 2022-11-02 op ((char *)(w->dest))[w->off] = '\0';
203 3448adb0 2022-11-02 op } else if (w->first_unwritable_offset < w->destlen) {
205 3448adb0 2022-11-02 op * There is no more space in the buffer. However,
206 3448adb0 2022-11-02 op * we have noted down the first offset we couldn't
207 3448adb0 2022-11-02 op * use to write into the buffer and it's smaller than
208 3448adb0 2022-11-02 op * destlen. Thus we bailed writing into the
209 3448adb0 2022-11-02 op * destination when a multibyte-codepoint couldn't be
210 3448adb0 2022-11-02 op * written. So the last "real" byte might be at
211 3448adb0 2022-11-02 op * destlen-4, destlen-3, destlen-2 or destlen-1
212 3448adb0 2022-11-02 op * (the last case meaning truncation).
214 3448adb0 2022-11-02 op if (w->type == HERODOTUS_TYPE_CODEPOINT) {
215 3448adb0 2022-11-02 op ((uint_least32_t *)(w->dest))
216 3448adb0 2022-11-02 op [w->first_unwritable_offset] = 0;
217 3448adb0 2022-11-02 op } else { /* w->type == HERODOTUS_TYPE_UTF8 */
218 3448adb0 2022-11-02 op ((char *)(w->dest))[w->first_unwritable_offset] = '\0';
220 3448adb0 2022-11-02 op } else if (w->destlen > 0) {
222 3448adb0 2022-11-02 op * In this case, there is no more space in the buffer and
223 3448adb0 2022-11-02 op * the last unwritable offset is larger than
224 3448adb0 2022-11-02 op * or equal to the destination buffer length. This means
225 3448adb0 2022-11-02 op * that we are forced to simply write into the last
228 3448adb0 2022-11-02 op if (w->type == HERODOTUS_TYPE_CODEPOINT) {
229 3448adb0 2022-11-02 op ((uint_least32_t *)(w->dest))
230 3448adb0 2022-11-02 op [w->destlen - 1] = 0;
231 3448adb0 2022-11-02 op } else { /* w->type == HERODOTUS_TYPE_UTF8 */
232 3448adb0 2022-11-02 op ((char *)(w->dest))[w->destlen - 1] = '\0';
236 3448adb0 2022-11-02 op /* w->off is not incremented in any case */
240 3448adb0 2022-11-02 op herodotus_writer_number_written(const HERODOTUS_WRITER *w)
242 3448adb0 2022-11-02 op return w->off;
246 3448adb0 2022-11-02 op herodotus_write_codepoint(HERODOTUS_WRITER *w, uint_least32_t cp)
251 3448adb0 2022-11-02 op * This function will always faithfully say how many codepoints
252 3448adb0 2022-11-02 op * were written, even if the buffer ends. This is used to enable
253 3448adb0 2022-11-02 op * truncation detection.
255 3448adb0 2022-11-02 op if (w->type == HERODOTUS_TYPE_CODEPOINT) {
256 3448adb0 2022-11-02 op if (w->dest != NULL && w->off < w->destlen) {
257 3448adb0 2022-11-02 op ((uint_least32_t *)(w->dest))[w->off] = cp;
261 3448adb0 2022-11-02 op } else { /* w->type == HERODOTUS_TYPE_UTF8 */
263 3448adb0 2022-11-02 op * First determine how many bytes we need to encode the
266 3448adb0 2022-11-02 op ret = grapheme_encode_utf8(cp, NULL, 0);
268 3448adb0 2022-11-02 op if (w->dest != NULL && w->off + ret < w->destlen) {
269 3448adb0 2022-11-02 op /* we still have enough room in the buffer */
270 3448adb0 2022-11-02 op grapheme_encode_utf8(cp, (char *)(w->dest) +
271 3448adb0 2022-11-02 op w->off, w->destlen - w->off);
272 3448adb0 2022-11-02 op } else if (w->first_unwritable_offset == SIZE_MAX) {
274 3448adb0 2022-11-02 op * the first unwritable offset has not been
275 3448adb0 2022-11-02 op * noted down, so this is the first time we can't
276 3448adb0 2022-11-02 op * write (completely) to an offset
278 3448adb0 2022-11-02 op w->first_unwritable_offset = w->off;
281 3448adb0 2022-11-02 op w->off += ret;
286 3448adb0 2022-11-02 op proper_init(const HERODOTUS_READER *r, void *state, uint_least8_t no_prop,
287 3448adb0 2022-11-02 op uint_least8_t (*get_break_prop)(uint_least32_t),
288 3448adb0 2022-11-02 op bool (*is_skippable_prop)(uint_least8_t),
289 3448adb0 2022-11-02 op void (*skip_shift_callback)(uint_least8_t, void *),
290 3448adb0 2022-11-02 op struct proper *p)
292 3448adb0 2022-11-02 op uint_least8_t prop;
293 3448adb0 2022-11-02 op uint_least32_t cp;
296 3448adb0 2022-11-02 op /* set internal variables */
297 3448adb0 2022-11-02 op p->state = state;
298 3448adb0 2022-11-02 op p->no_prop = no_prop;
299 3448adb0 2022-11-02 op p->get_break_prop = get_break_prop;
300 3448adb0 2022-11-02 op p->is_skippable_prop = is_skippable_prop;
301 3448adb0 2022-11-02 op p->skip_shift_callback = skip_shift_callback;
304 3448adb0 2022-11-02 op * Initialize mid-reader, which is basically just there
305 3448adb0 2022-11-02 op * to reflect the current position of the viewing-line
307 3448adb0 2022-11-02 op herodotus_reader_copy(r, &(p->mid_reader));
310 3448adb0 2022-11-02 op * In the initialization, we simply (try to) fill in next_prop.
311 3448adb0 2022-11-02 op * If we cannot read in more (due to the buffer ending), we
312 3448adb0 2022-11-02 op * fill in the prop as invalid
316 3448adb0 2022-11-02 op * initialize the previous properties to have no property
317 3448adb0 2022-11-02 op * (given we are at the start of the buffer)
319 3448adb0 2022-11-02 op p->raw.prev_prop[1] = p->raw.prev_prop[0] = p->no_prop;
320 3448adb0 2022-11-02 op p->skip.prev_prop[1] = p->skip.prev_prop[0] = p->no_prop;
323 3448adb0 2022-11-02 op * initialize the next properties
326 3448adb0 2022-11-02 op /* initialize the raw reader */
327 3448adb0 2022-11-02 op herodotus_reader_copy(r, &(p->raw_reader));
329 3448adb0 2022-11-02 op /* fill in the two next raw properties (after no-initialization) */
330 3448adb0 2022-11-02 op p->raw.next_prop[0] = p->raw.next_prop[1] = p->no_prop;
331 3448adb0 2022-11-02 op for (i = 0; i < 2 && herodotus_read_codepoint(&(p->raw_reader), true, &cp) ==
332 3448adb0 2022-11-02 op HERODOTUS_STATUS_SUCCESS; ) {
333 3448adb0 2022-11-02 op p->raw.next_prop[i++] = p->get_break_prop(cp);
336 3448adb0 2022-11-02 op /* initialize the skip reader */
337 3448adb0 2022-11-02 op herodotus_reader_copy(r, &(p->skip_reader));
339 3448adb0 2022-11-02 op /* fill in the two next skip properties (after no-initialization) */
340 3448adb0 2022-11-02 op p->skip.next_prop[0] = p->skip.next_prop[1] = p->no_prop;
341 3448adb0 2022-11-02 op for (i = 0; i < 2 && herodotus_read_codepoint(&(p->skip_reader), true, &cp) ==
342 3448adb0 2022-11-02 op HERODOTUS_STATUS_SUCCESS; ) {
343 3448adb0 2022-11-02 op prop = p->get_break_prop(cp);
344 3448adb0 2022-11-02 op if (!p->is_skippable_prop(prop)) {
345 3448adb0 2022-11-02 op p->skip.next_prop[i++] = prop;
351 3448adb0 2022-11-02 op proper_advance(struct proper *p)
353 3448adb0 2022-11-02 op uint_least8_t prop;
354 3448adb0 2022-11-02 op uint_least32_t cp;
356 3448adb0 2022-11-02 op /* read in next "raw" property */
357 3448adb0 2022-11-02 op if (herodotus_read_codepoint(&(p->raw_reader), true, &cp) ==
358 3448adb0 2022-11-02 op HERODOTUS_STATUS_SUCCESS) {
359 3448adb0 2022-11-02 op prop = p->get_break_prop(cp);
361 3448adb0 2022-11-02 op prop = p->no_prop;
365 3448adb0 2022-11-02 op * do a shift-in, unless we find that the property that is to
366 3448adb0 2022-11-02 op * be moved past the "raw-viewing-line" (this property is stored
367 3448adb0 2022-11-02 op * in p->raw.next_prop[0]) is a no_prop, indicating that
368 3448adb0 2022-11-02 op * we are at the end of the buffer.
370 3448adb0 2022-11-02 op if (p->raw.next_prop[0] == p->no_prop) {
374 3448adb0 2022-11-02 op /* shift in the properties */
375 3448adb0 2022-11-02 op p->raw.prev_prop[1] = p->raw.prev_prop[0];
376 3448adb0 2022-11-02 op p->raw.prev_prop[0] = p->raw.next_prop[0];
377 3448adb0 2022-11-02 op p->raw.next_prop[0] = p->raw.next_prop[1];
378 3448adb0 2022-11-02 op p->raw.next_prop[1] = prop;
380 3448adb0 2022-11-02 op /* advance the middle reader viewing-line */
381 3448adb0 2022-11-02 op (void)herodotus_read_codepoint(&(p->mid_reader), true, &cp);
383 3448adb0 2022-11-02 op /* check skippability-property */
384 3448adb0 2022-11-02 op if (!p->is_skippable_prop(p->raw.prev_prop[0])) {
386 3448adb0 2022-11-02 op * the property that has moved past the "raw-viewing-line"
387 3448adb0 2022-11-02 op * (this property is now (after the raw-shift) stored in
388 3448adb0 2022-11-02 op * p->raw.prev_prop[0] and guaranteed not to be a no-prop,
389 3448adb0 2022-11-02 op * guaranteeing that we won't shift a no-prop past the
390 3448adb0 2022-11-02 op * "viewing-line" in the skip-properties) is not a skippable
391 3448adb0 2022-11-02 op * property, thus we need to shift the skip property as well.
393 3448adb0 2022-11-02 op p->skip.prev_prop[1] = p->skip.prev_prop[0];
394 3448adb0 2022-11-02 op p->skip.prev_prop[0] = p->skip.next_prop[0];
395 3448adb0 2022-11-02 op p->skip.next_prop[0] = p->skip.next_prop[1];
398 3448adb0 2022-11-02 op * call the skip-shift-callback on the property that
399 3448adb0 2022-11-02 op * passed the skip-viewing-line (this property is now
400 3448adb0 2022-11-02 op * stored in p->skip.prev_prop[0]).
402 3448adb0 2022-11-02 op p->skip_shift_callback(p->skip.prev_prop[0], p->state);
404 3448adb0 2022-11-02 op /* determine the next shift property */
405 3448adb0 2022-11-02 op p->skip.next_prop[1] = p->no_prop;
406 3448adb0 2022-11-02 op while (herodotus_read_codepoint(&(p->skip_reader), true, &cp) ==
407 3448adb0 2022-11-02 op HERODOTUS_STATUS_SUCCESS) {
408 3448adb0 2022-11-02 op prop = p->get_break_prop(cp);
409 3448adb0 2022-11-02 op if (!p->is_skippable_prop(prop)) {
410 3448adb0 2022-11-02 op p->skip.next_prop[1] = prop;
