From 8eac2743860f97282789e320ecc5008c3dacc43f Mon Sep 17 00:00:00 2001
From: Ashwin <ashwin2007ray@gmail.com>
Date: Tue, 8 Mar 2016 18:06:15 +1100
Subject: [PATCH] batch write while merging
---
tools/merge/main.go | 17 +++++++++++++++--
1 file changed, 15 insertions(+), 2 deletions(-)
diff --git a/tools/merge/main.go b/tools/merge/main.go
index 8312b2f1..6a5a6e9b 100644
--- a/tools/merge/main.go
+++ b/tools/merge/main.go
@@ -21,6 +21,7 @@ import (
"bytes"
"container/heap"
"flag"
+ "fmt"
"io/ioutil"
"math"
"path"
@@ -96,7 +97,8 @@ func mergeFolders(mergePath, destPath string) {
opt.SetCreateIfMissing(true)
ropt = rocksdb.NewReadOptions()
wopt = rocksdb.NewWriteOptions()
- wopt.SetSync(true)
+ wopt.SetSync(false)
+ wb := rocksdb.NewWriteBatch()
pq = make(PriorityQueue, 0)
heap.Init(&pq)
@@ -134,6 +136,7 @@ func mergeFolders(mergePath, destPath string) {
}
var lastKey, lastValue []byte
+ count := 0
for pq.Len() > 0 {
top := heap.Pop(&pq).(*Item)
@@ -143,7 +146,12 @@ func mergeFolders(mergePath, destPath string) {
Fatal("different value for same key")
}
}
- db.Put(wopt, top.key, top.value)
+ wb.Put(top.key, top.value)
+ count++
+ if count%1000 == 0 {
+ db.Write(wopt, wb)
+ wb.Clear()
+ }
lastKey = top.key
lastValue = top.value
@@ -158,6 +166,11 @@ func mergeFolders(mergePath, destPath string) {
}
heap.Push(&pq, item)
}
+
+ db.Write(wopt, wb)
+ wb.Close()
+
+ fmt.Println("Count : ", count)
}
func main() {
--
GitLab