1 # when raw index has a lot of entries like
2 # 1578324	problematico, a, ci, che
3 # apply this algorithm:
4 #  treat things after comma as suffixes
5 #  for each suffix:
6 #      if single letter, replace last letter
7 #      else search backwards for beginning of suffix
8 #      and if it leads to an old suffix of approximately
9 #      the same length, put replace that suffix
10 # This will still leave some commas to fix by hand
11 # Usage: awk -F'	' -f comfix.awk rawindex > newrawindex
12 
13 NF == 2	{
14 		i = index($2, ",")
15 		if(i == 0 || length($2) == 0)
16 			print $0
17 		else {
18 			n = split($2, a, /,[ ]*/)
19 			w = a[1]
20 			printf "%s\t%s\n", $1, w
21 			for(i = 2; i <= n; i++) {
22 				suf = a[i]
23 				m = matchsuflen(w, suf)
24 				if(m) {
25 					nw = substr(w, 1, length(w)-m) suf
26 					printf "%s\t%s\n", $1, nw
27 				} else
28 					printf "%s\t%s\n", $1, w ", " suf
29 			}
30 		}
31 	}
32 NF != 2 {
33 	print $0
34 	}
35 
36 function matchsuflen(w, suf,		wlen,suflen,c,pat,k,d)
37 {
38 	wlen = length(w)
39 	suflen = length(suf)
40 	if(suflen == 1)
41 		return 1
42 	else {
43 		c = substr(suf, 1, 1)
44 		for (k = 1; k <= wlen ; k++)
45 			if(substr(w, wlen-k+1, 1) == c)
46 				break
47 		if(k > wlen)
48 			return 0
49 		d = k-suflen
50 		if(d < 0)
51 			d = -d
52 		if(d > 3)
53 			return 0
54 		return k
55 	}
56 }