commit 16b32c386a9b4419fb4c16544dd03e9d57ca5a93
from: Omar Polo <op@omarpolo.com>
date: Mon Apr 11 18:55:09 2022 UTC

tokenize: always return a valid NULL-terminated array

if we couldn't extract even a single word from the string, make sure
to return something that's not NULL otherwise the callers don't know
whether we succeeded or not.

commit - 5e180e92d05844c704c9dfe3596aed85099ea647
commit + 16b32c386a9b4419fb4c16544dd03e9d57ca5a93
blob - d07eb30cb1b222a478848edaccfeefadee59a8d9
blob + fa8612c8d282e6cb2dd540f9863a76ceb13fe730
--- lib/tokenize.c
+++ lib/tokenize.c
@@ -61,6 +61,8 @@ tokenize(const char *s)
 	}
 
 	free(dup);
+	if (tok == NULL)
+		return calloc(1, sizeof(char *));
 	return tok;
 
 err:
blob - a2fa1f2751eaa0c92f8c65cfe384e8ed9f4f018d
blob + b991e4533263af4315924f508b2b6ece37fdc621
--- mkftsidx/wiki.c
+++ mkftsidx/wiki.c
@@ -143,11 +143,11 @@ el_end(void *data, const char *element)
 	if (r == -1)
 		err(1, "asprintf");
 
-	if ((toks = tokenize(doc)) != NULL) {
-		if (!dictionary_add_words(d->dict, toks, d->len-1))
-			err(1, "dictionary_add_words");
-		freetoks(toks);
-	}
+	if ((toks = tokenize(doc)) == NULL)
+		err(1, "tokenize");
+	if (!dictionary_add_words(d->dict, toks, d->len-1))
+		err(1, "dictionary_add_words");
+	freetoks(toks);
 	free(doc);
 
 	free(d->title);