Skip to content
Snippets Groups Projects
Commit 192be4c2 authored by Manish R Jain's avatar Manish R Jain
Browse files

Use bloom filter in rocksdb. Set counters for posting lists cache hit and miss.

parent 6f460e6d
No related branches found
No related tags found
No related merge requests found
......@@ -55,6 +55,7 @@ var lcache map[uint64]*entry
var pstore *store.Store
var clog *commit.Logger
var ch chan uint64
var lc *lcounters
func Init(posting *store.Store, log *commit.Logger) {
lmutex.Lock()
......@@ -64,6 +65,8 @@ func Init(posting *store.Store, log *commit.Logger) {
pstore = posting
clog = log
ch = make(chan uint64, 10000)
lc = new(lcounters)
go lc.periodicLog()
}
func get(k uint64) *List {
......@@ -75,12 +78,29 @@ func get(k uint64) *List {
return nil
}
type lcounters struct {
hit uint64
miss uint64
misshit uint64
}
func (lc *lcounters) periodicLog() {
for _ = range time.Tick(10 * time.Second) {
glog.WithFields(logrus.Fields{
"hit": atomic.LoadUint64(&lc.hit),
"miss": atomic.LoadUint64(&lc.miss),
"misshit": atomic.LoadUint64(&lc.misshit),
}).Info("Lists counters")
}
}
func Get(key []byte) *List {
// Acquire read lock and check if list is available.
lmutex.RLock()
uid := farm.Fingerprint64(key)
if e, ok := lcache[uid]; ok {
lmutex.RUnlock()
atomic.AddUint64(&lc.hit, 1)
return e.l
}
lmutex.RUnlock()
......@@ -90,9 +110,11 @@ func Get(key []byte) *List {
defer lmutex.Unlock()
// Check again after acquiring write lock.
if e, ok := lcache[uid]; ok {
atomic.AddUint64(&lc.misshit, 1)
return e.l
}
atomic.AddUint64(&lc.miss, 1)
e := new(entry)
e.l = new(List)
e.l.init(key, pstore, clog)
......
......@@ -76,6 +76,7 @@ func Open(dbname string, o *Options) (*DB, error) {
ldbname := C.CString(dbname)
defer C.free(unsafe.Pointer(ldbname))
C.rocksdb_options_set_block_based_table_factory(o.Opt, o.Bopt)
rocksdb := C.rocksdb_open(o.Opt, ldbname, &errStr)
if errStr != nil {
gs := C.GoString(errStr)
......
......@@ -22,7 +22,8 @@ const (
// To prevent memory leaks, Close must be called on an Options when the
// program no longer needs it.
type Options struct {
Opt *C.rocksdb_options_t
Opt *C.rocksdb_options_t
Bopt *C.rocksdb_block_based_table_options_t
}
// ReadOptions represent all of the available options when reading from a
......@@ -45,8 +46,10 @@ type WriteOptions struct {
// NewOptions allocates a new Options object.
func NewOptions() *Options {
opt := C.rocksdb_options_create()
return &Options{opt}
o := new(Options)
o.Opt = C.rocksdb_options_create()
o.Bopt = C.rocksdb_block_based_options_create()
return o
}
// NewReadOptions allocates a new ReadOptions object.
......@@ -64,6 +67,7 @@ func NewWriteOptions() *WriteOptions {
// Close deallocates the Options, freeing its underlying C struct.
func (o *Options) Close() {
C.rocksdb_options_destroy(o.Opt)
C.rocksdb_block_based_options_destroy(o.Bopt)
}
// SetComparator sets the comparator to be used for all read and write
......@@ -170,15 +174,13 @@ func (o *Options) SetCreateIfMissing(b bool) {
// SetFilterPolicy causes Open to create a new database that will uses filter
// created from the filter policy passed in.
/*
func (o *Options) SetFilterPolicy(fp *FilterPolicy) {
var policy *C.rocksdb_filterpolicy_t
if fp != nil {
policy = fp.Policy
}
C.rocksdb_options_set_filter_policy(o.Opt, policy)
C.rocksdb_block_based_options_set_filter_policy(o.Bopt, policy)
}
*/
// Close deallocates the ReadOptions, freeing its underlying C struct.
func (ro *ReadOptions) Close() {
......
......@@ -35,6 +35,8 @@ type Store struct {
func (s *Store) Init(filepath string) {
s.opt = rocksdb.NewOptions()
s.opt.SetCreateIfMissing(true)
fp := rocksdb.NewBloomFilter(16)
s.opt.SetFilterPolicy(fp)
s.ropt = rocksdb.NewReadOptions()
s.wopt = rocksdb.NewWriteOptions()
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment