1 a0d146ed 2005-07-12 devnull #include "stdinc.h"
2 a0d146ed 2005-07-12 devnull #include "dat.h"
3 a0d146ed 2005-07-12 devnull #include "fns.h"
8 a0d146ed 2005-07-12 devnull usage(void)
10 a0d146ed 2005-07-12 devnull fprint(2, "usage: fmtbloom [-s size] [-n nblocks | -N nhash] file\n");
11 a0d146ed 2005-07-12 devnull threadexitsall(0);
15 a0d146ed 2005-07-12 devnull threadmain(int argc, char *argv[])
17 a0d146ed 2005-07-12 devnull Part *part;
18 a0d146ed 2005-07-12 devnull char *file;
19 a0d146ed 2005-07-12 devnull vlong bits, size, size2;
20 a0d146ed 2005-07-12 devnull int nhash;
21 a0d146ed 2005-07-12 devnull vlong nblocks;
23 a0d146ed 2005-07-12 devnull ventifmtinstall();
24 a0d146ed 2005-07-12 devnull statsinit();
26 a0d146ed 2005-07-12 devnull size = 0;
27 3d46902d 2005-10-29 devnull nhash = 0;
28 3d46902d 2005-10-29 devnull nblocks = 0;
29 a0d146ed 2005-07-12 devnull ARGBEGIN{
30 a0d146ed 2005-07-12 devnull case 'n':
31 a0d146ed 2005-07-12 devnull if(nhash || nblocks)
33 a0d146ed 2005-07-12 devnull nblocks = unittoull(EARGF(usage()));
35 a0d146ed 2005-07-12 devnull case 'N':
36 a0d146ed 2005-07-12 devnull if(nhash || nblocks)
38 a0d146ed 2005-07-12 devnull nhash = unittoull(EARGF(usage()));
39 a0d146ed 2005-07-12 devnull if(nhash > BloomMaxHash){
40 a0d146ed 2005-07-12 devnull fprint(2, "maximum possible is -N %d", BloomMaxHash);
44 a0d146ed 2005-07-12 devnull case 's':
45 a0d146ed 2005-07-12 devnull size = unittoull(ARGF());
46 a0d146ed 2005-07-12 devnull if(size == ~0)
54 a0d146ed 2005-07-12 devnull if(argc != 1)
57 a0d146ed 2005-07-12 devnull file = argv[0];
59 a0d146ed 2005-07-12 devnull part = initpart(file, ORDWR|ODIRECT);
60 a0d146ed 2005-07-12 devnull if(part == nil)
61 a0d146ed 2005-07-12 devnull sysfatal("can't open partition %s: %r", file);
63 a0d146ed 2005-07-12 devnull if(size == 0)
64 a0d146ed 2005-07-12 devnull size = part->size;
66 a0d146ed 2005-07-12 devnull if(size < 1024*1024)
67 a0d146ed 2005-07-12 devnull sysfatal("bloom filter too small");
69 a0d146ed 2005-07-12 devnull if(size > MaxBloomSize){
70 a0d146ed 2005-07-12 devnull fprint(2, "warning: not using entire %,lld bytes; using only %,lld bytes\n",
71 3d46902d 2005-10-29 devnull size, (vlong)MaxBloomSize);
72 a0d146ed 2005-07-12 devnull size = MaxBloomSize;
74 a0d146ed 2005-07-12 devnull if(size&(size-1)){
75 a0d146ed 2005-07-12 devnull for(size2=1; size2<size; size2*=2)
77 a0d146ed 2005-07-12 devnull size = size2/2;
78 a0d146ed 2005-07-12 devnull fprint(2, "warning: size not a power of 2; only using %lldMB\n", size/1024/1024);
81 a0d146ed 2005-07-12 devnull if(nblocks){
83 a0d146ed 2005-07-12 devnull * no use for more than 32 bits per block
84 a0d146ed 2005-07-12 devnull * shoot for less than 64 bits per block
86 a0d146ed 2005-07-12 devnull size2 = size;
87 a0d146ed 2005-07-12 devnull while(size2*8 >= nblocks*64)
88 a0d146ed 2005-07-12 devnull size2 >>= 1;
89 a0d146ed 2005-07-12 devnull if(size2 != size){
90 a0d146ed 2005-07-12 devnull size = size2;
91 a0d146ed 2005-07-12 devnull fprint(2, "warning: using only %lldMB - not enough blocks to warrant more\n",
92 a0d146ed 2005-07-12 devnull size/1024/1024);
96 a0d146ed 2005-07-12 devnull * optimal is to use ln 2 times as many hash functions as we have bits per blocks.
98 a0d146ed 2005-07-12 devnull bits = (8*size)/nblocks;
99 a0d146ed 2005-07-12 devnull nhash = bits*7/10;
100 a0d146ed 2005-07-12 devnull if(nhash > BloomMaxHash)
101 a0d146ed 2005-07-12 devnull nhash = BloomMaxHash;
103 a0d146ed 2005-07-12 devnull if(!nhash)
104 a0d146ed 2005-07-12 devnull nhash = BloomMaxHash;
105 a0d146ed 2005-07-12 devnull if(bloominit(&b, size, nil) < 0)
106 a0d146ed 2005-07-12 devnull sysfatal("bloominit: %r");
107 a0d146ed 2005-07-12 devnull b.nhash = nhash;
108 a0d146ed 2005-07-12 devnull bits = nhash*10/7;
109 a0d146ed 2005-07-12 devnull nblocks = (8*size)/bits;
110 a0d146ed 2005-07-12 devnull fprint(2, "fmtbloom: using %lldMB, %d hashes/score, best up to %,lld blocks\n", size, nhash, nblocks);
111 a0d146ed 2005-07-12 devnull b.data = vtmallocz(size);
112 a0d146ed 2005-07-12 devnull b.part = part;
113 a0d146ed 2005-07-12 devnull if(writebloom(&b) < 0)
114 a0d146ed 2005-07-12 devnull sysfatal("writing %s: %r", file);
115 a0d146ed 2005-07-12 devnull threadexitsall(0);