diff --git a/tools/merge/main b/tools/merge/main new file mode 100755 index 0000000000000000000000000000000000000000..c8e43b052bd40b98394aeba94d8f2ad85c0f308b Binary files /dev/null and b/tools/merge/main differ diff --git a/tools/merge/rocksmerge.go b/tools/merge/main.go similarity index 89% rename from tools/merge/rocksmerge.go rename to tools/merge/main.go index 6000ede8a3af569dab0769deaa8dc18652d3c3b4..92c0439d5bca7a8c07de4b2b144305dc038983a1 100644 --- a/tools/merge/rocksmerge.go +++ b/tools/merge/main.go @@ -78,7 +78,7 @@ func equalValue(a, b interface{}) bool { aUid := x.Uid() bUid := y.Uid() - return aUid == bUid || + return (x.ValueBytes() == nil && y.ValueBytes() == nil && (aUid == bUid)) || ((x.Uid() == math.MaxUint64 && y.Uid() == math.MaxUint64) && bytes.Compare(x.ValueBytes(), y.ValueBytes()) == 0) } @@ -98,12 +98,8 @@ func MergeFolders(mergePath, destPath string) { wopt = rocksdb.NewWriteOptions() wopt.SetSync(true) - count := 0 - for range dirList { - count++ - } - - pq = make(PriorityQueue, count) + pq = make(PriorityQueue, 0) + heap.Init(&pq) var storeIters []*rocksdb.Iterator for i, dir := range dirList { mPath := path.Join(mergePath, dir.Name()) @@ -120,33 +116,24 @@ func MergeFolders(mergePath, destPath string) { glog.WithField("path", mPath).Info("Store empty") continue } - pq[i] = &Item{ + item := &Item{ key: it.Key(), value: it.Value(), storeIdx: i, } + heap.Push(&pq, item) storeIters = append(storeIters, it) } - heap.Init(&pq) - mergeUsingHeap(destPath, storeIters) -} - -func mergeUsingHeap(destPath string, storeIters []*rocksdb.Iterator) { - var opt *rocksdb.Options - var wopt *rocksdb.WriteOptions - opt = rocksdb.NewOptions() - opt.SetCreateIfMissing(true) - wopt = rocksdb.NewWriteOptions() - wopt.SetSync(true) var db *rocksdb.DB - db, err := rocksdb.Open(destPath, opt) + db, err = rocksdb.Open(destPath, opt) defer db.Close() if err != nil { glog.WithField("filepath", destPath). Fatal("While opening store") } + heap.Init(&pq) var lastKey, lastValue []byte for pq.Len() > 0 { top := heap.Pop(&pq).(*Item) diff --git a/tools/merge/rocksmerge_test.go b/tools/merge/main_test.go similarity index 94% rename from tools/merge/rocksmerge_test.go rename to tools/merge/main_test.go index 611c234d3cc469b13e5445e1da77bd542bd3ec31..6de6150333480b905a3d7a442dd480c84f987368 100644 --- a/tools/merge/rocksmerge_test.go +++ b/tools/merge/main_test.go @@ -11,12 +11,11 @@ import ( "github.com/dgraph-io/dgraph/posting" "github.com/dgraph-io/dgraph/store" "github.com/dgraph-io/dgraph/store/rocksdb" - "github.com/dgraph-io/dgraph/tools/merge" "github.com/dgraph-io/dgraph/uid" "github.com/dgryski/go-farm" ) -func TestQuery(t *testing.T) { +func TestMergeFolders(t *testing.T) { logrus.SetLevel(logrus.DebugLevel) rootDir, err := ioutil.TempDir("", "storetest_") if err != nil { @@ -82,7 +81,7 @@ func TestQuery(t *testing.T) { ps1.Close() ps2.Close() - main.MergeFolders(rootDir, destDir) + MergeFolders(rootDir, destDir) var opt *rocksdb.Options var ropt *rocksdb.ReadOptions