From 192be4c262c27cd727ae658107c94231abca9689 Mon Sep 17 00:00:00 2001
From: Manish R Jain <manishrjain@gmail.com>
Date: Tue, 17 Nov 2015 19:52:20 +1100
Subject: [PATCH] Use bloom filter in rocksdb. Set counters for posting lists
cache hit and miss.
---
posting/lists.go | 22 ++++++++++++++++++++++
store/rocksdb/db.go | 1 +
store/rocksdb/options.go | 14 ++++++++------
store/store.go | 2 ++
4 files changed, 33 insertions(+), 6 deletions(-)
diff --git a/posting/lists.go b/posting/lists.go
index 0ceb28c6..be5c27e0 100644
--- a/posting/lists.go
+++ b/posting/lists.go
@@ -55,6 +55,7 @@ var lcache map[uint64]*entry
var pstore *store.Store
var clog *commit.Logger
var ch chan uint64
+var lc *lcounters
func Init(posting *store.Store, log *commit.Logger) {
lmutex.Lock()
@@ -64,6 +65,8 @@ func Init(posting *store.Store, log *commit.Logger) {
pstore = posting
clog = log
ch = make(chan uint64, 10000)
+ lc = new(lcounters)
+ go lc.periodicLog()
}
func get(k uint64) *List {
@@ -75,12 +78,29 @@ func get(k uint64) *List {
return nil
}
+type lcounters struct {
+ hit uint64
+ miss uint64
+ misshit uint64
+}
+
+func (lc *lcounters) periodicLog() {
+ for _ = range time.Tick(10 * time.Second) {
+ glog.WithFields(logrus.Fields{
+ "hit": atomic.LoadUint64(&lc.hit),
+ "miss": atomic.LoadUint64(&lc.miss),
+ "misshit": atomic.LoadUint64(&lc.misshit),
+ }).Info("Lists counters")
+ }
+}
+
func Get(key []byte) *List {
// Acquire read lock and check if list is available.
lmutex.RLock()
uid := farm.Fingerprint64(key)
if e, ok := lcache[uid]; ok {
lmutex.RUnlock()
+ atomic.AddUint64(&lc.hit, 1)
return e.l
}
lmutex.RUnlock()
@@ -90,9 +110,11 @@ func Get(key []byte) *List {
defer lmutex.Unlock()
// Check again after acquiring write lock.
if e, ok := lcache[uid]; ok {
+ atomic.AddUint64(&lc.misshit, 1)
return e.l
}
+ atomic.AddUint64(&lc.miss, 1)
e := new(entry)
e.l = new(List)
e.l.init(key, pstore, clog)
diff --git a/store/rocksdb/db.go b/store/rocksdb/db.go
index 5b6a4f5b..5ee0775a 100644
--- a/store/rocksdb/db.go
+++ b/store/rocksdb/db.go
@@ -76,6 +76,7 @@ func Open(dbname string, o *Options) (*DB, error) {
ldbname := C.CString(dbname)
defer C.free(unsafe.Pointer(ldbname))
+ C.rocksdb_options_set_block_based_table_factory(o.Opt, o.Bopt)
rocksdb := C.rocksdb_open(o.Opt, ldbname, &errStr)
if errStr != nil {
gs := C.GoString(errStr)
diff --git a/store/rocksdb/options.go b/store/rocksdb/options.go
index a5b07fc4..e31df4cf 100644
--- a/store/rocksdb/options.go
+++ b/store/rocksdb/options.go
@@ -22,7 +22,8 @@ const (
// To prevent memory leaks, Close must be called on an Options when the
// program no longer needs it.
type Options struct {
- Opt *C.rocksdb_options_t
+ Opt *C.rocksdb_options_t
+ Bopt *C.rocksdb_block_based_table_options_t
}
// ReadOptions represent all of the available options when reading from a
@@ -45,8 +46,10 @@ type WriteOptions struct {
// NewOptions allocates a new Options object.
func NewOptions() *Options {
- opt := C.rocksdb_options_create()
- return &Options{opt}
+ o := new(Options)
+ o.Opt = C.rocksdb_options_create()
+ o.Bopt = C.rocksdb_block_based_options_create()
+ return o
}
// NewReadOptions allocates a new ReadOptions object.
@@ -64,6 +67,7 @@ func NewWriteOptions() *WriteOptions {
// Close deallocates the Options, freeing its underlying C struct.
func (o *Options) Close() {
C.rocksdb_options_destroy(o.Opt)
+ C.rocksdb_block_based_options_destroy(o.Bopt)
}
// SetComparator sets the comparator to be used for all read and write
@@ -170,15 +174,13 @@ func (o *Options) SetCreateIfMissing(b bool) {
// SetFilterPolicy causes Open to create a new database that will uses filter
// created from the filter policy passed in.
-/*
func (o *Options) SetFilterPolicy(fp *FilterPolicy) {
var policy *C.rocksdb_filterpolicy_t
if fp != nil {
policy = fp.Policy
}
- C.rocksdb_options_set_filter_policy(o.Opt, policy)
+ C.rocksdb_block_based_options_set_filter_policy(o.Bopt, policy)
}
-*/
// Close deallocates the ReadOptions, freeing its underlying C struct.
func (ro *ReadOptions) Close() {
diff --git a/store/store.go b/store/store.go
index 4656e3ec..01f2785f 100644
--- a/store/store.go
+++ b/store/store.go
@@ -35,6 +35,8 @@ type Store struct {
func (s *Store) Init(filepath string) {
s.opt = rocksdb.NewOptions()
s.opt.SetCreateIfMissing(true)
+ fp := rocksdb.NewBloomFilter(16)
+ s.opt.SetFilterPolicy(fp)
s.ropt = rocksdb.NewReadOptions()
s.wopt = rocksdb.NewWriteOptions()
--
GitLab