Use bloom filter in rocksdb. Set counters for posting lists cache hit and miss.

posting/lists.go

@@ -55,6 +55,7 @@ var lcache map[uint64]*entry
 var pstore *store.Store
 var clog *commit.Logger
 var ch chan uint64
+var lc *lcounters
 
 func Init(posting *store.Store, log *commit.Logger) {
 	lmutex.Lock()
@@ -64,6 +65,8 @@ func Init(posting *store.Store, log *commit.Logger) {
 	pstore = posting
 	clog = log
 	ch = make(chan uint64, 10000)
+	lc = new(lcounters)
+	go lc.periodicLog()
 }
 
 func get(k uint64) *List {
@@ -75,12 +78,29 @@ func get(k uint64) *List {
 	return nil
 }
 
+type lcounters struct {
+	hit     uint64
+	miss    uint64
+	misshit uint64
+}
+
+func (lc *lcounters) periodicLog() {
+	for _ = range time.Tick(10 * time.Second) {
+		glog.WithFields(logrus.Fields{
+			"hit":     atomic.LoadUint64(&lc.hit),
+			"miss":    atomic.LoadUint64(&lc.miss),
+			"misshit": atomic.LoadUint64(&lc.misshit),
+		}).Info("Lists counters")
+	}
+}
+
 func Get(key []byte) *List {
 	// Acquire read lock and check if list is available.
 	lmutex.RLock()
 	uid := farm.Fingerprint64(key)
 	if e, ok := lcache[uid]; ok {
 		lmutex.RUnlock()
+		atomic.AddUint64(&lc.hit, 1)
 		return e.l
 	}
 	lmutex.RUnlock()
@@ -90,9 +110,11 @@ func Get(key []byte) *List {
 	defer lmutex.Unlock()
 	// Check again after acquiring write lock.
 	if e, ok := lcache[uid]; ok {
+		atomic.AddUint64(&lc.misshit, 1)
 		return e.l
 	}
 
+	atomic.AddUint64(&lc.miss, 1)
 	e := new(entry)
 	e.l = new(List)
 	e.l.init(key, pstore, clog)

store/rocksdb/db.go

@@ -76,6 +76,7 @@ func Open(dbname string, o *Options) (*DB, error) {
 	ldbname := C.CString(dbname)
 	defer C.free(unsafe.Pointer(ldbname))
 
+	C.rocksdb_options_set_block_based_table_factory(o.Opt, o.Bopt)
 	rocksdb := C.rocksdb_open(o.Opt, ldbname, &errStr)
 	if errStr != nil {
 		gs := C.GoString(errStr)

store/rocksdb/options.go

@@ -22,7 +22,8 @@ const (
 // To prevent memory leaks, Close must be called on an Options when the
 // program no longer needs it.
 type Options struct {
-	Opt *C.rocksdb_options_t
+	Opt  *C.rocksdb_options_t
+	Bopt *C.rocksdb_block_based_table_options_t
 }
 
 // ReadOptions represent all of the available options when reading from a
@@ -45,8 +46,10 @@ type WriteOptions struct {
 
 // NewOptions allocates a new Options object.
 func NewOptions() *Options {
-	opt := C.rocksdb_options_create()
-	return &Options{opt}
+	o := new(Options)
+	o.Opt = C.rocksdb_options_create()
+	o.Bopt = C.rocksdb_block_based_options_create()
+	return o
 }
 
 // NewReadOptions allocates a new ReadOptions object.
@@ -64,6 +67,7 @@ func NewWriteOptions() *WriteOptions {
 // Close deallocates the Options, freeing its underlying C struct.
 func (o *Options) Close() {
 	C.rocksdb_options_destroy(o.Opt)
+	C.rocksdb_block_based_options_destroy(o.Bopt)
 }
 
 // SetComparator sets the comparator to be used for all read and write
@@ -170,15 +174,13 @@ func (o *Options) SetCreateIfMissing(b bool) {
 
 // SetFilterPolicy causes Open to create a new database that will uses filter
 // created from the filter policy passed in.
-/*
 func (o *Options) SetFilterPolicy(fp *FilterPolicy) {
 	var policy *C.rocksdb_filterpolicy_t
 	if fp != nil {
 		policy = fp.Policy
 	}
-	C.rocksdb_options_set_filter_policy(o.Opt, policy)
+	C.rocksdb_block_based_options_set_filter_policy(o.Bopt, policy)
 }
-*/
 
 // Close deallocates the ReadOptions, freeing its underlying C struct.
 func (ro *ReadOptions) Close() {

store/store.go

@@ -35,6 +35,8 @@ type Store struct {
 func (s *Store) Init(filepath string) {
 	s.opt = rocksdb.NewOptions()
 	s.opt.SetCreateIfMissing(true)
+	fp := rocksdb.NewBloomFilter(16)
+	s.opt.SetFilterPolicy(fp)
 
 	s.ropt = rocksdb.NewReadOptions()
 	s.wopt = rocksdb.NewWriteOptions()