diff --git a/lex/lexer.go b/lex/lexer.go index 00cebc2da671f844aa86896a1b007ab50f356b95..c05fb98e3e943894505ece51fb76859037cd9aca 100644 --- a/lex/lexer.go +++ b/lex/lexer.go @@ -18,18 +18,12 @@ package lex import ( "fmt" - "sync" "unicode/utf8" "github.com/dgraph-io/dgraph/x" ) var glog = x.Log("lexer") -var LexerPool = sync.Pool{ - New: func() interface{} { - return &Lexer{} - }, -} const EOF = -1 @@ -72,12 +66,9 @@ type Lexer struct { Mode int // mode based on information so far. } -func NewLexer(input string) *Lexer { - l := LexerPool.Get().(*Lexer) - *l = Lexer{} +func (l *Lexer) Init(input string) { l.Input = input l.Items = make(chan item, 5) - return l } func (l *Lexer) Errorf(format string, diff --git a/rdf/README.txt b/rdf/README.txt index deff864d5e7fc72589fe641633e2953bba78cb61..13ec847c20968e38e3b70a57f1750e6e386b6ab2 100644 --- a/rdf/README.txt +++ b/rdf/README.txt @@ -51,3 +51,26 @@ Showing top 10 nodes out of 62 (cum >= 18180150) 25895676 4.62% 68.91% 25895676 4.62% github.com/zond/gotomic.(*element).search 18546971 3.31% 72.22% 72863016 13.00% github.com/dgraph-io/dgraph/loader.(*state).parseStream 18090764 3.23% 75.45% 18180150 3.24% github.com/dgraph-io/dgraph/loader.(*state).readLines + +After a few more discussions, I realized that lexer didn't need to be allocated on the heap. +So, I switched it to be allocated on stack. These are the results. + +$ go tool pprof uidassigner heap.prof +Entering interactive mode (type "help" for commands) +(pprof) top10 +1308.70MB of 1696.59MB total (77.14%) +Dropped 73 nodes (cum <= 8.48MB) +Showing top 10 nodes out of 52 (cum >= 161.50MB) + flat flat% sum% cum cum% + 304.56MB 17.95% 17.95% 304.56MB 17.95% github.com/dgraph-io/dgraph/posting.NewList + 209.55MB 12.35% 30.30% 209.55MB 12.35% github.com/Sirupsen/logrus.(*Entry).WithFields + 207.55MB 12.23% 42.54% 417.10MB 24.58% github.com/Sirupsen/logrus.(*Entry).WithField + 108MB 6.37% 48.90% 108MB 6.37% github.com/dgraph-io/dgraph/uid.(*lockManager).newOrExisting + 88MB 5.19% 54.09% 88MB 5.19% github.com/zond/gotomic.newMockEntry + 85.51MB 5.04% 59.13% 85.51MB 5.04% github.com/google/flatbuffers/go.(*Builder).growByteBuffer + 78.01MB 4.60% 63.73% 78.01MB 4.60% github.com/dgraph-io/dgraph/posting.Key + 78.01MB 4.60% 68.32% 78.51MB 4.63% github.com/dgraph-io/dgraph/uid.stringKey + 76MB 4.48% 72.80% 76MB 4.48% github.com/zond/gotomic.newRealEntryWithHashCode + 73.50MB 4.33% 77.14% 161.50MB 9.52% github.com/zond/gotomic.(*Hash).getBucketByIndex + +Now, rdf.Parse is no longer shows up in memory profiler. Win! diff --git a/rdf/parse.go b/rdf/parse.go index 2dfaf002c8e76f35bd8bf493d615033ab1f0b1a8..732e851bfb8857e937a9ea226e768a0648756374 100644 --- a/rdf/parse.go +++ b/rdf/parse.go @@ -120,8 +120,8 @@ func stripBracketsIfPresent(val string) string { } func Parse(line string) (rnq NQuad, rerr error) { - l := lex.NewLexer(line) - defer lex.LexerPool.Put(l) + l := &lex.Lexer{} + l.Init(line) go run(l) var oval string