Skip to content
Snippets Groups Projects
Commit fe21a84d authored by MichaelJCompton's avatar MichaelJCompton
Browse files

remove example from clients, point to godoc and longer examples, add longer examples to v0.8

parent aafb29ff
Branches release/v0.8.0
No related tags found
No related merge requests found
Expand all Hide whitespace changes
Inline Side-by-side
Showing
with 1755 additions and 292 deletions
client/client.go
+ 1
− 1
View file @ fe21a84d
......@@ -455,7 +455,7 @@ func (e *Edge) SetValueBytes(val []byte) error {
return nil
}
// AddFacet adds the key, value pair as facets on Edge e. No checking is done.
// AddFacet adds the key, value pair as facets on Edge e. No checking is done.
func (e *Edge) AddFacet(key, val string) {
e.nq.Facets = append(e.nq.Facets, &protos.Facet{
Key: key,
......
wiki/content/clients/index.md
+ 11
− 291
View file @ fe21a84d
......@@ -5,9 +5,10 @@ title = "Clients"
## Implementation
Clients communicate with the server using [Protocol Buffers](https://developers.google.com/protocol-buffers) over [gRPC](http://www.grpc.io/). This requires [defining services](http://www.grpc.io/docs/#defining-a-service) and data types in a ''proto'' file and then generating client and server side code using the [protoc compiler](https://github.com/google/protobuf).
All clients can communicate with the server via the HTTP endpoint (set with option `--port` when starting Dgraph). Queries and mutations can be submitted and JSON is returned.
Go clients can use the clients package and communicate with the server over [gRPC](http://www.grpc.io/). Internally this uses [Protocol Buffers](https://developers.google.com/protocol-buffers) and the proto file used by Dgraph is located at [graphresponse.proto](https://github.com/dgraph-io/dgraph/blob/master/protos/graphresponse.proto).
The proto file used by Dgraph is located at [graphresponse.proto](https://github.com/dgraph-io/dgraph/blob/master/protos/graphp/graphresponse.proto).
## Languages
......@@ -15,306 +16,25 @@ The proto file used by Dgraph is located at [graphresponse.proto](https://github
[![GoDoc](https://godoc.org/github.com/dgraph-io/dgraph/client?status.svg)](https://godoc.org/github.com/dgraph-io/dgraph/client)
After you have the followed [Get started]({{< relref "get-started/index.md">}}) and got the server running on `127.0.0.1:8080` and (for gRPC) `127.0.0.1:9080`, you can use the Go client to run queries and mutations as shown in the example below.
{{% notice "note" %}}The example below would store values with the correct types only if the
correct [schema type]({{< relref "query-language/index.md#schema" >}}) is specified in the mutation, otherwise everything would be converted to schema type and stored. Schema is derived based on first mutation received by the server. If first mutation is of default type then everything would be stored as default type.{{% /notice %}}
The go client communicates with the server on the grpc port (set with option `--grpc_port` when starting Dgraph).
#### Installation
To get the Go client, you can run
Go get the client:
```
go get -u -v github.com/dgraph-io/dgraph/client github.com/dgraph-io/dgraph/protos
go get -u -v github.com/dgraph-io/dgraph/client
```
#### Example
The working example below shows the common operations that one would perform on a graph.
#### Examples
```
package main
import (
"context"
"flag"
"fmt"
"time"
"github.com/dgraph-io/dgraph/client"
"github.com/dgraph-io/dgraph/x"
"google.golang.org/grpc"
)
var (
dgraph = flag.String("d", "127.0.0.1:9080", "Dgraph server address")
)
type nameFacets struct {
Since time.Time `dgraph:"since"`
Alias string `dgraph:"alias"`
}
The client [GoDoc](https://godoc.org/github.com/dgraph-io/dgraph/client) has specifications of all functions and examples.
type friendFacets struct {
Close bool `dgraph:"close"`
}
Larger examples can be found [here](https://github.com/dgraph-io/dgraph/tree/master/wiki/resources/examples/goclient). And [this](https://open.dgraph.io/post/client0.8.0) blog post explores the examples further.
type Person struct {
Name string `dgraph:"name"`
NameFacets nameFacets `dgraph:"name@facets"`
Birthday time.Time `dgraph:"birthday"`
Location []byte `dgraph:"loc"`
Salary float64 `dgraph:"salary"`
Age int `dgraph:"age"`
Married bool `dgraph:"married"`
ByteDob []byte `dgraph:"bytedob"`
Friends []Person `dgraph:"friend"`
FriendFacets []friendFacets `dgraph:"@facets"`
}
The app [dgraphloader](https://github.com/dgraph-io/dgraph/tree/master/cmd/dgraphloader) uses the client interface to batch concurrent mutations.
type Res struct {
Root Person `dgraph:"me"`
}
func main() {
conn, err := grpc.Dial(*dgraph, grpc.WithInsecure())
x.Checkf(err, "While trying to dial gRPC")
defer conn.Close()
dgraphClient := client.NewDgraphClient([]*grpc.ClientConn{conn}, client.DefaultOptions)
req := client.Req{}
person1, err := dgraphClient.NodeBlank("person1")
x.Check(err)
// Creating a person node, and adding a name attribute to it.
e := person1.Edge("name")
err = e.SetValueString("Steven Spielberg")
x.Check(err)
e.AddFacet("since", "2006-01-02T15:04:05")
e.AddFacet("alias", `"Steve"`)
req.Set(e)
e = person1.Edge("birthday")
err = e.SetValueDatetime(time.Date(1991, 2, 1, 0, 0, 0, 0, time.UTC))
x.Check(err)
req.Set(e)
e = person1.Edge("loc")
err = e.SetValueGeoJson(`{"type":"Point","coordinates":[-122.2207184,37.72129059]}`)
x.Check(err)
req.Set(e)
e = person1.Edge("salary")
err = e.SetValueFloat(13333.6161)
x.Check(err)
req.Set(e)
e = person1.Edge("age")
err = e.SetValueInt(25)
x.Check(err)
req.Set(e)
e = person1.Edge("married")
err = e.SetValueBool(true)
x.Check(err)
req.Set(e)
e = person1.Edge("bytedob")
err = e.SetValueBytes([]byte("01-02-1991"))
x.Check(err)
req.Set(e)
person2, err := dgraphClient.NodeBlank("person2")
x.Check(err)
e = person2.Edge("name")
err = e.SetValueString("William Jones")
x.Check(err)
req.Set(e)
e = person1.Edge("friend")
err = e.ConnectTo(person2)
x.Check(err)
e.AddFacet("close", "true")
req.Set(e)
resp, err := dgraphClient.Run(context.Background(), &req)
x.Check(err)
req = client.Req{}
req.SetQuery(fmt.Sprintf(`{
me(func: uid(%v)) {
_uid_
name @facets
now
birthday
loc
salary
age
married
bytedob
friend @facets {
_uid_
name
}
}
}`, person1))
resp, err = dgraphClient.Run(context.Background(), &req)
x.Check(err)
var r Res
err = client.Unmarshal(resp.N, &r)
fmt.Printf("Steven: %+v\n\n", r.Root)
fmt.Printf("William: %+v\n", r.Root.Friends[0])
req = client.Req{}
// Here is an example of deleting an edge
e = person1.Edge("friend")
err = e.ConnectTo(person2)
x.Check(err)
req.Delete(e)
resp, err = dgraphClient.Run(context.Background(), &req)
x.Check(err)
err = dgraphClient.Close()
x.Check(err)
}
```
{{% notice "note" %}}Type for the facets are automatically interpreted from the value. If you want it to
be interpreted as string, it has to be a raw string literal with `""` as shown above for `alias` facet.{{% /notice %}}
An appropriate schema for the above example would be
```
curl localhost:8080/query -XPOST -d $'
mutation {
schema {
now: dateTime .
birthday: date .
age: int .
salary: float .
name: string .
loc: geo .
married: bool .
}
}' | python -m json.tool | less
```
Apart from the above syntax, you could perform all the queries and mutations listed on [Query Language]({{< relref "query-language/index.md" >}}). For example you could do this.
```
req := client.Req{}
req.SetQuery(`
mutation {
set {
<class1> <student> _:x .
<class1> <name> "awesome class" .
_:x <name> "alice" .
_:x <planet> "Mars" .
_:x <friend> _:y .
_:y <name> "bob" .
}
}
{
class(id:class1) {
name
student {
name
planet
friend {
name
}
}
}
}`)
resp, err := c.Run(context.Background(), req.Request())
if err != nil {
log.Fatalf("Error in getting response from server, %s", err)
}
fmt.Printf("%+v\n", resp)
```
Here is an example of how you could use the Dgraph client to do batch mutations concurrently.
This is what the **dgraphloader** uses internally.
```
package main
import (
"bufio"
"bytes"
"compress/gzip"
"io"
"log"
"os"
"google.golang.org/grpc"
"github.com/dgraph-io/dgraph/client"
"github.com/dgraph-io/dgraph/rdf"
"github.com/dgraph-io/dgraph/x"
)
func ExampleBatchMutation() {
conn, err := grpc.Dial("127.0.0.1:9080", grpc.WithInsecure())
x.Checkf(err, "While trying to dial gRPC")
defer conn.Close()
// Start a new batch with batch size 1000 and 100 concurrent requests.
bmOpts := client.BatchMutationOptions{
Size: 1000,
Pending: 100,
PrintCounters: false,
}
dgraphClient := client.NewDgraphClient(conn, bmOpts)
// Process your file, convert data to a protos.NQuad and add it to the batch.
// For each edge, do a BatchSet (this would typically be done in a loop
// after processing the data into edges). Here we show example of reading a
// file with RDF data, converting it to edges and adding it to the batch.
f, err := os.Open("goldendata.rdf.gz")
x.Check(err)
defer f.Close()
gr, err := gzip.NewReader(f)
x.Check(err)
var buf bytes.Buffer
bufReader := bufio.NewReader(gr)
var line int
for {
err = x.ReadLine(bufReader, &buf)
if err != nil {
break
}
line++
nq, err := rdf.Parse(buf.String())
if err == rdf.ErrEmpty { // special case: comment/empty line
buf.Reset()
continue
} else if err != nil {
log.Fatalf("Error while parsing RDF: %v, on line:%v %v", err, line, buf.String())
}
buf.Reset()
nq.Subject = Node(nq.Subject, dgraphClient)
if len(nq.ObjectId) > 0 {
nq.ObjectId = Node(nq.ObjectId, dgraphClient)
}
if err = dgraphClient.BatchSet(client.NewEdge(nq)); err != nil {
log.Fatal("While adding mutation to batch: ", err)
}
}
if err != io.EOF {
x.Checkf(err, "Error while reading file")
}
// Wait for all requests to complete. This is very important, else some
// data might not be sent to server.
dgraphClient.BatchFlush()
err = dgraphClient.Close()
x.Check(err)
}
```
{{% notice "note" %}}As with mutations through a mutation block, [schema type]({{< relref "query-language/index.md#schema" >}}) needs to be set for the edges, or schema is derived based on first mutation received by the server. {{% /notice %}}
### Python
{{% notice "incomplete" %}}A lot of development has gone into the Go client and the Python client is not up to date with it. We are looking for help from contributors to bring it up to date.{{% /notice %}}
......
wiki/resources/examples/goclient/README.md 0 → 100644
+ 9
− 0
View file @ fe21a84d
# go client examples
Examples of using the go client with Dgraph. Built for Dgraph version 0.8.
* crawlerRDF queries Dgraph, unmarshals results into custom structs and writes out an RDF file that can be loaded with dgraphloader.
* crawlermutation multithreaded crawler that shows how to use client with go routines. Concurrent crawlers query a source Dgraph, unmarshal into custom structs and write to a target Dgraph. Shows how to issue mutations using requests in client.
* movielensbatch reads from CSV and uses the client batch interface to write to Dgraph.
Check out the [blog post](https://open.dgraph.io/post/client0.8.0) about these examples.
\ No newline at end of file
wiki/resources/examples/goclient/crawlerRDF/.gitignore 0 → 100644
+ 2
− 0
View file @ fe21a84d
/crawler
/crawl.rdf.gz
wiki/resources/examples/goclient/crawlerRDF/crawler.go 0 → 100644
+ 682
− 0
View file @ fe21a84d
This diff is collapsed.
wiki/resources/examples/goclient/crawlermutations/.gitignore 0 → 100644
+ 2
− 0
View file @ fe21a84d
/crawler
wiki/resources/examples/goclient/crawlermutations/crawler.go 0 → 100644
+ 726
− 0
View file @ fe21a84d
This diff is collapsed.
wiki/resources/examples/goclient/movielensbatch/.gitignore 0 → 100644
+ 2
− 0
View file @ fe21a84d
/mlbatch
/movielensbatch
wiki/resources/examples/goclient/movielensbatch/mlbatch.go 0 → 100644
+ 320
− 0
View file @ fe21a84d
/*
* Copyright 2016 Dgraph Labs, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/*
*** Check out the blog post about this code : https://open.dgraph.io/post/client0.8.0 ***
Go routines concurrently read from text input files and batch writes using the go client.
Another version of this program outputed gzipped RDF
https://github.com/dgraph-io/benchmarks/tree/master/movielens/conv100k
It's output was used in blog posts about building recommendation systems in Dgraph
https://open.dgraph.io/post/recommendation/
https://open.dgraph.io/post/recommendation2/
To run the program:
- build with
go build mlbatch.go
- start a dgraph instance, e.g for default options
dgraph
- run in a directory with the movielens data unpacked
./mlbatch
Use --help on dgraph or mlbatch to see the options - dgraph's --grpc_port must match mlbatch's --d
and the unpacked movieLens data must be in dir ./ml-100k or --genre, etc given as options.
You can add a schema with
mutation {
schema {
name: string @index(term, exact) .
rated: uid @reverse @count .
genre: uid @reverse .
}
}
and run queries like
{
q(func: eq(name, "Toy Story (1995)")) {
name
genre {
name
}
count(~rated)
}
}
or see the queries at
https://open.dgraph.io/post/recommendation/ and
https://open.dgraph.io/post/recommendation2/
(though your UIDs will be different)
*** Check out the blog post about this code : https://open.dgraph.io/post/client0.8.0 ***
*/
package main
import (
"bufio"
"flag"
"io"
"io/ioutil"
"log"
"os"
"strconv"
"strings"
"sync"
"google.golang.org/grpc"
"github.com/dgraph-io/dgraph/client"
"github.com/dgraph-io/dgraph/x"
)
var (
dgraph = flag.String("d", "127.0.0.1:9080", "Dgraph gRPC server address")
concurrent = flag.Int("c", 10, "Number of concurrent requests to make to Dgraph")
numRdf = flag.Int("m", 100, "Number of RDF N-Quads to send as part of a mutation.")
genre = flag.String("genre", "ml-100k/u.genre", "")
data = flag.String("rating", "ml-100k/u.data", "")
users = flag.String("user", "ml-100k/u.user", "")
movie = flag.String("movie", "ml-100k/u.item", "")
)
func main() {
log.SetFlags(log.LstdFlags | log.Lshortfile)
flag.Parse()
if !flag.Parsed() {
log.Fatal("Unable to parse flags")
}
conn, err := grpc.Dial(*dgraph, grpc.WithInsecure())
x.Check(err)
defer conn.Close()
// The Dgraph client needs a directory in which to write its blank node maps.
// We'll use blank nodes below to record users and movies, when we need them again
// we can look them up in the client's map and get the right node without having
// to record or issue a query.
//
// The blank node map records blank-node-name -> Node, so everytime in the load we refer
// to a node we can grab it from the map. These names aren't presisted in Dgraph (if we
// wanted that we'd use an XID).
clientDir, err := ioutil.TempDir("", "client_")
x.Check(err)
defer os.RemoveAll(clientDir)
// The client builds Pending concurrent batchs each of size Size and submits to the server
// as each batch fills.
bmOpts := client.BatchMutationOptions{
Size: *numRdf,
Pending: *concurrent,
PrintCounters: true,
}
dgraphClient := client.NewDgraphClient([]*grpc.ClientConn{conn}, bmOpts, clientDir)
defer dgraphClient.Close()
// We aren't writing any schema here. You can load the schema at
// https://github.com/dgraph-io/benchmarks/blob/master/movielens/movielens.schema
// using draphloader (--s option with no data file), or by running a mutation
// mutation { schema { <content of schema file> } }
// dgraphloader function processSchemaFile() also shows an example of how to load
// a schema file using the client.
gf, err := os.Open(*genre)
x.Check(err)
defer gf.Close()
uf, err := os.Open(*users)
x.Check(err)
defer gf.Close()
df, err := os.Open(*data)
x.Check(err)
defer df.Close()
mf, err := os.Open(*movie)
x.Check(err)
defer mf.Close()
var wg sync.WaitGroup
wg.Add(4)
// The blank node names are used accross 4 go routines. It doesn't mater which go routine
// uses the node name first - a "user<userID>" node may be first used in the go routine reading
// users, or the one setting ratings - the Dgraph client ensures that all the routines are
// talking about the same nodes. So no matter what the creation order, or the order the client
// submits the batches in the background, the graph is connected properly.
go func() {
defer wg.Done()
// Each line in genre file looks like
// genre-name|genreID
// We'll use a blank node named "genre<genreID>" to identify each genre node
br := bufio.NewReader(gf)
log.Println("Reading genre file")
for {
line, err := br.ReadString('\n')
if err != nil && err == io.EOF {
break
}
line = strings.Trim(line, "\n")
csv := strings.Split(line, "|")
if len(csv) != 2 {
continue
}
n, err := dgraphClient.NodeBlank("genre" + csv[1])
x.Check(err)
e := n.Edge("name")
x.Check(e.SetValueString(csv[0]))
x.Check(dgraphClient.BatchSet(e))
}
}()
go func() {
defer wg.Done()
// Each line in the user file looks like
// userID|age|genre|occupation|ZIPcode
// We'll use a blank node named "user<userID>" to identify each user node
br := bufio.NewReader(uf)
log.Println("Reading user file")
for {
line, err := br.ReadString('\n')
if err != nil && err == io.EOF {
break
}
line = strings.Trim(line, "\n")
csv := strings.Split(line, "|")
if len(csv) != 5 {
continue
}
n, err := dgraphClient.NodeBlank("user" + csv[0])
x.Check(err)
e := n.Edge("age")
age, err := strconv.ParseInt(csv[1], 10, 32)
x.Check(err)
x.Check(e.SetValueInt(age))
x.Check(dgraphClient.BatchSet(e))
e = n.Edge("gender")
x.Check(e.SetValueString(csv[2]))
x.Check(dgraphClient.BatchSet(e))
e = n.Edge("occupation")
x.Check(e.SetValueString(csv[3]))
x.Check(dgraphClient.BatchSet(e))
e = n.Edge("zipcode")
x.Check(e.SetValueString(csv[4]))
x.Check(dgraphClient.BatchSet(e))
}
}()
go func() {
defer wg.Done()
// The lines of the movie file look like
// movieID|movie-name|date||imdb-address|genre0?|genre1?|...|genre18?
// We'll use "movie<movieID>" as the blank node name
br := bufio.NewReader(mf)
log.Println("Reading movies file")
for {
line, err := br.ReadString('\n')
if err != nil && err == io.EOF {
break
}
line = strings.Trim(line, "\n")
csv := strings.Split(line, "|")
if len(csv) != 24 {
continue
}
n, err := dgraphClient.NodeBlank("movie" + csv[0])
x.Check(err)
e := n.Edge("name")
x.Check(e.SetValueString(csv[1]))
x.Check(dgraphClient.BatchSet(e))
// 1 means the movie has the corresponding genre
for i := 5; i < 24; i++ {
if csv[i] == "0" {
continue
}
g, err := dgraphClient.NodeBlank("genre" + strconv.Itoa(i-5))
x.Check(err)
e = n.ConnectTo("genre", g)
x.Check(dgraphClient.BatchSet(e))
}
}
}()
go func() {
defer wg.Done()
// Each line in the rating file looks like
// userID movieID rating timestamp
br := bufio.NewReader(df)
log.Println("Reading rating file")
for {
line, err := br.ReadString('\n')
if err != nil && err == io.EOF {
break
}
line = strings.Trim(line, "\n")
csv := strings.Split(line, "\t")
if len(csv) != 4 {
continue
}
u, err := dgraphClient.NodeBlank("user" + csv[0])
x.Check(err)
m, err := dgraphClient.NodeBlank("movie" + csv[1])
x.Check(err)
e := u.ConnectTo("rated", m)
e.AddFacet("rating", csv[2])
x.Check(dgraphClient.BatchSet(e))
}
}()
wg.Wait()
// This must be called at the end of a batch to ensure all batched mutations are sent to the store.
dgraphClient.BatchFlush()
log.Println("Finised.")
}
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment