1 08708877 2003-11-25 devnull # when raw index has a lot of entries like
2 08708877 2003-11-25 devnull # 1578324 problematico, a, ci, che
3 08708877 2003-11-25 devnull # apply this algorithm:
4 08708877 2003-11-25 devnull # treat things after comma as suffixes
5 08708877 2003-11-25 devnull # for each suffix:
6 08708877 2003-11-25 devnull # if single letter, replace last letter
7 08708877 2003-11-25 devnull # else search backwards for beginning of suffix
8 08708877 2003-11-25 devnull # and if it leads to an old suffix of approximately
9 08708877 2003-11-25 devnull # the same length, put replace that suffix
10 08708877 2003-11-25 devnull # This will still leave some commas to fix by hand
11 08708877 2003-11-25 devnull # Usage: awk -F' ' -f comfix.awk rawindex > newrawindex
13 08708877 2003-11-25 devnull NF == 2 {
14 08708877 2003-11-25 devnull i = index($2, ",")
15 08708877 2003-11-25 devnull if(i == 0 || length($2) == 0)
18 08708877 2003-11-25 devnull n = split($2, a, /,[ ]*/)
20 08708877 2003-11-25 devnull printf "%s\t%s\n", $1, w
21 08708877 2003-11-25 devnull for(i = 2; i <= n; i++) {
22 08708877 2003-11-25 devnull suf = a[i]
23 08708877 2003-11-25 devnull m = matchsuflen(w, suf)
25 08708877 2003-11-25 devnull nw = substr(w, 1, length(w)-m) suf
26 08708877 2003-11-25 devnull printf "%s\t%s\n", $1, nw
28 08708877 2003-11-25 devnull printf "%s\t%s\n", $1, w ", " suf
32 08708877 2003-11-25 devnull NF != 2 {
36 08708877 2003-11-25 devnull function matchsuflen(w, suf, wlen,suflen,c,pat,k,d)
38 08708877 2003-11-25 devnull wlen = length(w)
39 08708877 2003-11-25 devnull suflen = length(suf)
40 08708877 2003-11-25 devnull if(suflen == 1)
43 08708877 2003-11-25 devnull c = substr(suf, 1, 1)
44 08708877 2003-11-25 devnull for (k = 1; k <= wlen ; k++)
45 08708877 2003-11-25 devnull if(substr(w, wlen-k+1, 1) == c)
47 08708877 2003-11-25 devnull if(k > wlen)
49 08708877 2003-11-25 devnull d = k-suflen
50 08708877 2003-11-25 devnull if(d < 0)
52 08708877 2003-11-25 devnull if(d > 3)