1 # when raw index has a lot of entries like
2 # 1578324 problematico, a, ci, che
3 # apply this algorithm:
4 # treat things after comma as suffixes
6 # if single letter, replace last letter
7 # else search backwards for beginning of suffix
8 # and if it leads to an old suffix of approximately
9 # the same length, put replace that suffix
10 # This will still leave some commas to fix by hand
11 # Usage: awk -F' ' -f comfix.awk rawindex > newrawindex
15 if(i == 0 || length($2) == 0)
18 n = split($2, a, /,[ ]*/)
20 printf "%s\t%s\n", $1, w
21 for(i = 2; i <= n; i++) {
23 m = matchsuflen(w, suf)
25 nw = substr(w, 1, length(w)-m) suf
26 printf "%s\t%s\n", $1, nw
28 printf "%s\t%s\n", $1, w ", " suf
36 function matchsuflen(w, suf, wlen,suflen,c,pat,k,d)
44 for (k = 1; k <= wlen ; k++)
45 if(substr(w, wlen-k+1, 1) == c)