From 7205275cc2f66a974febf78a812f9a1ef30243db Mon Sep 17 00:00:00 2001 From: Manish R Jain <manish@dgraph.io> Date: Fri, 18 May 2018 09:19:26 -0700 Subject: [PATCH] Fix the build (#2400) * Fix the UID bug. * If we find a deleted version, stop iteration. * Rename dummyPostingList to emptyPostingList, which is what it is. * Instead of adding a dummy key, remove a dummy key, so dummy predicate doesn't show up in our states. * Use require for testing, instead of assert. * Be able to pick up Facets as well, via json.Number. --- contrib/integration/testtxn/main_test.go | 2 +- edgraph/nquads_from_json.go | 48 ++++++++++++++++-------- posting/lists.go | 18 +++++---- posting/mvcc.go | 4 +- 4 files changed, 46 insertions(+), 26 deletions(-) diff --git a/contrib/integration/testtxn/main_test.go b/contrib/integration/testtxn/main_test.go index dbc4e624..190d2b36 100644 --- a/contrib/integration/testtxn/main_test.go +++ b/contrib/integration/testtxn/main_test.go @@ -467,7 +467,7 @@ func TestIgnoreIndexConflict(t *testing.T) { log.Fatalf("Error while running query: %v\n", err) } expectedResp := []byte(fmt.Sprintf(`{"me":[{"uid":"%s"},{"uid":"%s"}]}`, uid1, uid2)) - x.AssertTrue(bytes.Equal(resp.Json, expectedResp)) + require.Equal(t, expectedResp, resp.Json) } func TestReadIndexKeySameTxn(t *testing.T) { diff --git a/edgraph/nquads_from_json.go b/edgraph/nquads_from_json.go index ab18e4d5..a6bb551f 100644 --- a/edgraph/nquads_from_json.go +++ b/edgraph/nquads_from_json.go @@ -23,8 +23,6 @@ import ( "github.com/twpayne/go-geom/encoding/geojson" ) -// TODO(pawan) - Refactor code here to make it simpler. - func parseFacets(m map[string]interface{}, prefix string) ([]*api.Facet, error) { // This happens at root. if prefix == "" { @@ -55,10 +53,23 @@ func parseFacets(m map[string]interface{}, prefix string) ([]*api.Facet, error) f.ValType = api.Facet_STRING fv = v } - case float64: - // Could be int too, but we just store it as float. - fv = v - f.ValType = api.Facet_FLOAT + case json.Number: + valn := facetVal.(json.Number) + if strings.Index(valn.String(), ".") >= 0 { + if vf, err := valn.Float64(); err != nil { + return nil, err + } else { + fv = vf + f.ValType = api.Facet_FLOAT + } + } else { + if vi, err := valn.Int64(); err != nil { + return nil, err + } else { + fv = vi + f.ValType = api.Facet_INT + } + } case bool: fv = v f.ValType = api.Facet_BOOL @@ -176,12 +187,21 @@ func mapToNquads(m map[string]interface{}, idx *int, op int, parentPred string) // Check field in map. if uidVal, ok := m["uid"]; ok { var uid uint64 - if id, ok := uidVal.(float64); ok { - uid = uint64(id) - // We need to check for length of id as empty string would give an error while - // calling ParseUint. We should assign a new uid if len == 0. - } else if id, ok := uidVal.(string); ok && len(id) > 0 { - if ok := strings.HasPrefix(id, "_:"); ok { + + switch uidVal.(type) { + case json.Number: + uidn := uidVal.(json.Number) + ui, err := uidn.Int64() + if err != nil { + return mr, err + } + uid = uint64(ui) + + case string: + id := uidVal.(string) + if len(id) == 0 { + uid = 0 + } else if ok := strings.HasPrefix(id, "_:"); ok { mr.uid = id } else if u, err := strconv.ParseUint(id, 0, 64); err != nil { return mr, err @@ -189,11 +209,9 @@ func mapToNquads(m map[string]interface{}, idx *int, op int, parentPred string) uid = u } } - if uid > 0 { mr.uid = fmt.Sprintf("%d", uid) } - } if len(mr.uid) == 0 { @@ -250,7 +268,7 @@ func mapToNquads(m map[string]interface{}, idx *int, op int, parentPred string) } switch v.(type) { - case string, float64, bool, json.Number: + case string, json.Number, bool: if err := handleBasicType(pred, v, op, &nq); err != nil { return mr, err } diff --git a/posting/lists.go b/posting/lists.go index d33c3d2d..7717d58d 100644 --- a/posting/lists.go +++ b/posting/lists.go @@ -31,7 +31,7 @@ import ( ) var ( - dummyPostingList []byte // Used for indexing. + emptyPostingList []byte // Used for indexing. elog trace.EventLog ) @@ -58,7 +58,7 @@ func init() { Checksum: h.Sum(nil), } var err error - dummyPostingList, err = pl.Marshal() + emptyPostingList, err = pl.Marshal() x.Check(err) }) elog = trace.NewEventLog("Memory", "") @@ -300,12 +300,14 @@ func CommitLists(commit func(key []byte) bool) { close(workChan) wg.Wait() - // Consider using sync in syncIfDirty instead of async. - // Hacky solution for now, ensures that everything is flushed to disk before we return. + // The following hack ensures that all the asynchrously run commits above would have been done + // before this completes. Badger now actually gets rid of keys, which are deleted. So, we can + // use the Delete function. txn := pstore.NewTransactionAt(1, true) defer txn.Discard() - // Code is written with assumption that nothing is deleted in dgraph, so don't - // use delete - txn.SetWithMeta(x.DataKey("dummy", 1), nil, BitEmptyPosting) - txn.CommitAt(1, nil) + x.Check(txn.Delete(x.DataKey("dummy", 1))) + // Nothing is being read, so there can't be an ErrConflict. This should go to disk. + if err := txn.CommitAt(1, nil); err != nil { + x.Printf("Commit unexpectedly failed with error: %v", err) + } } diff --git a/posting/mvcc.go b/posting/mvcc.go index 0f79f884..4cd0bf9e 100644 --- a/posting/mvcc.go +++ b/posting/mvcc.go @@ -394,8 +394,8 @@ func ReadPostingList(key []byte, it *badger.Iterator) (*List, error) { for it.Valid() { item := it.Item() if item.IsDeletedOrExpired() { - it.Next() - continue + // Don't consider any more versions. + break } if !bytes.Equal(item.Key(), l.key) { break -- GitLab