diff --git a/tools/merge/main.go b/tools/merge/main.go index 8312b2f18a72d292ffa5b8f086853d794bede3ce..6a5a6e9b04a554e0c69f2bd0c95ee7b9eb07b462 100644 --- a/tools/merge/main.go +++ b/tools/merge/main.go @@ -21,6 +21,7 @@ import ( "bytes" "container/heap" "flag" + "fmt" "io/ioutil" "math" "path" @@ -96,7 +97,8 @@ func mergeFolders(mergePath, destPath string) { opt.SetCreateIfMissing(true) ropt = rocksdb.NewReadOptions() wopt = rocksdb.NewWriteOptions() - wopt.SetSync(true) + wopt.SetSync(false) + wb := rocksdb.NewWriteBatch() pq = make(PriorityQueue, 0) heap.Init(&pq) @@ -134,6 +136,7 @@ func mergeFolders(mergePath, destPath string) { } var lastKey, lastValue []byte + count := 0 for pq.Len() > 0 { top := heap.Pop(&pq).(*Item) @@ -143,7 +146,12 @@ func mergeFolders(mergePath, destPath string) { Fatal("different value for same key") } } - db.Put(wopt, top.key, top.value) + wb.Put(top.key, top.value) + count++ + if count%1000 == 0 { + db.Write(wopt, wb) + wb.Clear() + } lastKey = top.key lastValue = top.value @@ -158,6 +166,11 @@ func mergeFolders(mergePath, destPath string) { } heap.Push(&pq, item) } + + db.Write(wopt, wb) + wb.Close() + + fmt.Println("Count : ", count) } func main() {