HTTP endpoint to export policy results

cmd/infohub/main.go

@@ -4,16 +4,20 @@ import (
 	"context"
 	"errors"
 	"log"
+	"net"
 	"net/http"
 	"time"
 
 	"github.com/kelseyhightower/envconfig"
+	"go.mongodb.org/mongo-driver/mongo"
+	"go.mongodb.org/mongo-driver/mongo/options"
 	"go.uber.org/zap"
 	"go.uber.org/zap/zapcore"
 	goahttp "goa.design/goa/v3/http"
 	goa "goa.design/goa/v3/pkg"
 	"golang.org/x/sync/errgroup"
 
+	"code.vereign.com/gaiax/tsa/golib/cache"
 	"code.vereign.com/gaiax/tsa/golib/graceful"
 	goahealth "code.vereign.com/gaiax/tsa/infohub/gen/health"
 	goahealthsrv "code.vereign.com/gaiax/tsa/infohub/gen/http/health/server"
@@ -21,10 +25,14 @@ import (
 	goaopenapisrv "code.vereign.com/gaiax/tsa/infohub/gen/http/openapi/server"
 	goainfohub "code.vereign.com/gaiax/tsa/infohub/gen/infohub"
 	"code.vereign.com/gaiax/tsa/infohub/gen/openapi"
+	"code.vereign.com/gaiax/tsa/infohub/internal/clients/policy"
+	"code.vereign.com/gaiax/tsa/infohub/internal/clients/vault"
 	"code.vereign.com/gaiax/tsa/infohub/internal/config"
+	"code.vereign.com/gaiax/tsa/infohub/internal/credential"
 	"code.vereign.com/gaiax/tsa/infohub/internal/service"
 	"code.vereign.com/gaiax/tsa/infohub/internal/service/health"
 	"code.vereign.com/gaiax/tsa/infohub/internal/service/infohub"
+	"code.vereign.com/gaiax/tsa/infohub/internal/storage"
 )
 
 var Version = "0.0.0+development"
@@ -45,13 +53,46 @@ func main() {
 
 	logger.Info("infohub service started", zap.String("version", Version), zap.String("goa", goa.Version()))
 
+	// connect to mongo db
+	db, err := mongo.Connect(
+		context.Background(),
+		options.Client().ApplyURI(cfg.Mongo.Addr).SetAuth(options.Credential{
+			Username: cfg.Mongo.User,
+			Password: cfg.Mongo.Pass,
+		}),
+	)
+	if err != nil {
+		logger.Fatal("error connecting to mongodb", zap.Error(err))
+	}
+	defer db.Disconnect(context.Background()) //nolint:errcheck
+
+	// create storage
+	storage, err := storage.New(db, cfg.Mongo.DB, cfg.Mongo.Collection, logger)
+	if err != nil {
+		logger.Fatal("error connecting to database", zap.Error(err))
+	}
+
+	vault, err := vault.New(cfg.Vault.Addr, cfg.Vault.Token, cfg.Vault.Keyname)
+	if err != nil {
+		logger.Fatal("error creating vault client", zap.Error(err))
+	}
+
+	httpClient := httpClient()
+	credentials := credential.NewIssuer(cfg.Credential.IssuerName, cfg.Credential.Keyname, vault, httpClient)
+
+	// create policy client
+	policy := policy.New(cfg.Policy.Addr, httpClient)
+
+	// create cache client
+	cache := cache.New(cfg.Cache.Addr)
+
 	// create services
 	var (
 		infohubSvc goainfohub.Service
 		healthSvc  goahealth.Service
 	)
 	{
-		infohubSvc = infohub.New(logger)
+		infohubSvc = infohub.New(storage, policy, cache, credentials, logger)
 		healthSvc = health.New()
 	}
 
@@ -145,18 +186,18 @@ func errFormatter(e error) goahttp.Statuser {
 	return service.NewErrorResponse(e)
 }
 
-//func httpClient() *http.Client {
-//	return &http.Client{
-//		Transport: &http.Transport{
-//			Proxy: http.ProxyFromEnvironment,
-//			DialContext: (&net.Dialer{
-//				Timeout: 30 * time.Second,
-//			}).DialContext,
-//			MaxIdleConns:        100,
-//			MaxIdleConnsPerHost: 100,
-//			TLSHandshakeTimeout: 10 * time.Second,
-//			IdleConnTimeout:     60 * time.Second,
-//		},
-//		Timeout: 30 * time.Second,
-//	}
-//}
+func httpClient() *http.Client {
+	return &http.Client{
+		Transport: &http.Transport{
+			Proxy: http.ProxyFromEnvironment,
+			DialContext: (&net.Dialer{
+				Timeout: 30 * time.Second,
+			}).DialContext,
+			MaxIdleConns:        100,
+			MaxIdleConnsPerHost: 100,
+			TLSHandshakeTimeout: 10 * time.Second,
+			IdleConnTimeout:     60 * time.Second,
+		},
+		Timeout: 30 * time.Second,
+	}
+}

deployment/ci/Dockerfile

+FROM golang:1.17.10-alpine3.15 as builder
+
+ENV GOPRIVATE=code.vereign.com
+
+RUN apk add git
+
+WORKDIR /go/src/code.vereign.com/gaiax/tsa/infohub
+
+ADD . .
+
+RUN CGO_ENABLED=0 GOOS=linux go build -ldflags="-X main.Version=$(git describe --tags --always)" -mod=vendor -o /tmp/infohub ./cmd/infohub/...
+
+FROM alpine:3.15 as runner
+
+COPY --from=builder /tmp/infohub /opt/infohub
+
+WORKDIR /opt
+
+CMD ["./infohub"]

deployment/compose/Dockerfile

+FROM golang:1.17.10
+
+ENV GO111MODULE=on
+ENV GOPRIVATE=code.vereign.com
+
+RUN go install github.com/canthefason/go-watcher/cmd/watcher@v0.2.4
+
+ADD . /go/src/code.vereign.com/gaiax/tsa/infohub
+
+WORKDIR /go/src/code.vereign.com/gaiax/tsa/infohub
+
+RUN go install -mod=vendor ./cmd/infohub/...
+
+EXPOSE 8080
+
+ENTRYPOINT ["sh", "-c", "/go/bin/watcher -run code.vereign.com/gaiax/tsa/infohub/cmd/infohub -watch code.vereign.com/gaiax/tsa/infohub"]

internal/clients/policy/client.go

+package policy
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+
+	"code.vereign.com/gaiax/tsa/golib/errors"
+)
+
+const headerEvaluationID = "x-evaluation-id"
+
+type Client struct {
+	addr       string
+	httpClient *http.Client
+}
+
+func New(addr string, httpClient *http.Client) *Client {
+	return &Client{
+		addr:       addr,
+		httpClient: httpClient,
+	}
+}
+
+// Evaluate calls the policy service to execute the given policy.
+// The policy is expected as a string path uniquely identifying the
+// policy that has to be evaluated. For example, with policy = `gaiax/didResolve/1.0`,
+// the client will do HTTP request to http://policyhost/policy/gaiax/didResolve/1.0/evaluation.
+func (c *Client) Evaluate(ctx context.Context, policy string, data interface{}, evaluationID string) ([]byte, error) {
+	uri := c.addr + "/policy/" + policy + "/evaluation"
+	policyURL, err := url.ParseRequestURI(uri)
+	if err != nil {
+		return nil, errors.New(errors.BadRequest, "invalid policy evaluation URL", err)
+	}
+
+	jsonData, err := json.Marshal(data)
+	if err != nil {
+		return nil, err
+	}
+
+	req, err := http.NewRequest(http.MethodPost, policyURL.String(), bytes.NewReader(jsonData))
+	if err != nil {
+		return nil, err
+	}
+
+	if evaluationID != "" {
+		req.Header.Set(headerEvaluationID, evaluationID)
+	}
+
+	resp, err := c.httpClient.Do(req.WithContext(ctx))
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close() // nolint:errcheck
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("unexpected response on policy evaluation: %s", resp.Status)
+	}
+
+	return io.ReadAll(resp.Body)
+}

internal/clients/vault/client.go

+package vault
+
+import (
+	"context"
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"net/http"
+
+	vaultpkg "github.com/hashicorp/vault/api"
+)
+
+type Client struct {
+	keyname string
+	client  *vaultpkg.Client
+}
+
+func New(addr string, token string, keyname string) (*Client, error) {
+	cfg := vaultpkg.DefaultConfig()
+	cfg.Address = addr
+	client, err := vaultpkg.NewClient(cfg)
+	if err != nil {
+		return nil, err
+	}
+
+	client.SetToken(token)
+
+	return &Client{
+		client:  client,
+		keyname: keyname,
+	}, nil
+}
+
+func (c *Client) Sign(data []byte) ([]byte, error) {
+	body := map[string]interface{}{
+		"input": base64.StdEncoding.EncodeToString(data),
+	}
+
+	req := c.client.NewRequest(http.MethodPost, "/v1/transit/sign/"+c.keyname)
+	if err := req.SetJSONBody(body); err != nil {
+		return nil, err
+	}
+
+	// nolint:staticcheck
+	res, err := c.client.RawRequestWithContext(context.Background(), req)
+	if err != nil {
+		return nil, err
+	}
+	defer res.Body.Close()
+
+	if res.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("unexpected response from vault: %s", res.Status)
+	}
+
+	// expected response from the sign operation
+	var response struct {
+		Data struct {
+			Signature string `json:"signature"`
+		} `json:"data"`
+	}
+	if err := json.NewDecoder(res.Body).Decode(&response); err != nil {
+		return nil, err
+	}
+
+	if len(response.Data.Signature) == 0 {
+		return nil, fmt.Errorf("unexpected response: no signature")
+	}
+
+	return []byte(response.Data.Signature), nil
+}

internal/config/config.go

@@ -3,7 +3,13 @@ package config
 import "time"
 
 type Config struct {
-	HTTP     httpConfig
+	HTTP       httpConfig
+	Mongo      mongoConfig
+	Policy     policyConfig
+	Cache      cacheConfig
+	Vault      vaultConfig
+	Credential credentialConfig
+
 	LogLevel string `envconfig:"LOG_LEVEL" default:"INFO"`
 }
 
@@ -14,3 +20,30 @@ type httpConfig struct {
 	ReadTimeout  time.Duration `envconfig:"HTTP_READ_TIMEOUT" default:"10s"`
 	WriteTimeout time.Duration `envconfig:"HTTP_WRITE_TIMEOUT" default:"10s"`
 }
+
+type mongoConfig struct {
+	Addr       string `envconfig:"MONGO_ADDR" required:"true"`
+	User       string `envconfig:"MONGO_USER" required:"true"`
+	Pass       string `envconfig:"MONGO_PASS" required:"true"`
+	DB         string `envconfig:"MONGO_DBNAME" default:"infohub"`
+	Collection string `envconfig:"MONGO_COLLECTION" default:"exports"`
+}
+
+type vaultConfig struct {
+	Addr    string `envconfig:"VAULT_ADDR" required:"true"`
+	Token   string `envconfig:"VAULT_TOKEN" required:"true"`
+	Keyname string `envconfig:"VAULT_KEYNAME" required:"true"`
+}
+
+type credentialConfig struct {
+	IssuerName string `envconfig:"CRED_ISSUER_NAME" required:"true"`
+	Keyname    string `envconfig:"CRED_KEYNAME" required:"true"`
+}
+
+type policyConfig struct {
+	Addr string `envconfig:"POLICY_ADDR" required:"true"`
+}
+
+type cacheConfig struct {
+	Addr string `envconfig:"CACHE_ADDR" required:"true"`
+}

internal/credential/issuer.go

+package credential
+
+import (
+	"net/http"
+	"time"
+
+	"github.com/hyperledger/aries-framework-go/pkg/doc/signature/jsonld"
+	"github.com/hyperledger/aries-framework-go/pkg/doc/signature/suite"
+	"github.com/hyperledger/aries-framework-go/pkg/doc/signature/suite/ed25519signature2018"
+	"github.com/hyperledger/aries-framework-go/pkg/doc/util"
+	"github.com/hyperledger/aries-framework-go/pkg/doc/verifiable"
+	"github.com/piprate/json-gold/ld"
+)
+
+type Signer interface {
+	Sign(data []byte) ([]byte, error)
+}
+
+type Issuer struct {
+	issuerName string
+	signer     Signer
+	keyname    string
+
+	// proofContext is used to generate linked data proof
+	proofContext *verifiable.LinkedDataProofContext
+	docLoader    *ld.CachingDocumentLoader
+}
+
+func NewIssuer(issuerName string, keyname string, signer Signer, httpClient *http.Client) *Issuer {
+	sigSuite := ed25519signature2018.New(
+		suite.WithSigner(signer),
+		suite.WithVerifier(ed25519signature2018.NewPublicKeyVerifier()))
+
+	proofContext := &verifiable.LinkedDataProofContext{
+		Suite:                   sigSuite,
+		SignatureType:           ed25519signature2018.SignatureType,
+		SignatureRepresentation: verifiable.SignatureProofValue,
+		VerificationMethod:      keyname,
+	}
+
+	loader := ld.NewDefaultDocumentLoader(httpClient)
+
+	return &Issuer{
+		issuerName:   issuerName,
+		signer:       signer,
+		keyname:      keyname,
+		docLoader:    ld.NewCachingDocumentLoader(loader),
+		proofContext: proofContext,
+	}
+}
+
+func (i *Issuer) NewCredential(contexts []string, subjectID string, subject map[string]interface{}, proof bool) (*verifiable.Credential, error) {
+	jsonldContexts := []string{"https://www.w3.org/2018/credentials/v1"}
+	jsonldContexts = append(jsonldContexts, contexts...)
+
+	vc := &verifiable.Credential{
+		Context: jsonldContexts,
+		Types:   []string{verifiable.VCType},
+		Issuer:  verifiable.Issuer{ID: i.issuerName},
+		Issued:  &util.TimeWrapper{Time: time.Now()},
+		Subject: verifiable.Subject{
+			ID:           subjectID,
+			CustomFields: subject,
+		},
+	}
+
+	if proof {
+		if err := vc.AddLinkedDataProof(i.proofContext, jsonld.WithDocumentLoader(i.docLoader)); err != nil {
+			return nil, err
+		}
+	}
+
+	return vc, nil
+}
+
+func (i *Issuer) NewPresentation(contexts []string, vc ...*verifiable.Credential) (*verifiable.Presentation, error) {
+	jsonldContexts := []string{"https://www.w3.org/2018/credentials/v1"}
+	jsonldContexts = append(jsonldContexts, contexts...)
+
+	vp, err := verifiable.NewPresentation(verifiable.WithCredentials(vc...))
+	if err != nil {
+		return nil, err
+	}
+	vp.Context = jsonldContexts
+	vp.ID = i.issuerName
+	vp.Type = []string{verifiable.VPType}
+
+	if err := vp.AddLinkedDataProof(i.proofContext, jsonld.WithDocumentLoader(i.docLoader)); err != nil {
+		return nil, err
+	}
+
+	return vp, nil
+}

internal/service/infohub/service.go

@@ -2,21 +2,119 @@ package infohub
 
 import (
 	"context"
-	"fmt"
+	"encoding/json"
 
+	"github.com/hyperledger/aries-framework-go/pkg/doc/verifiable"
 	"go.uber.org/zap"
 
+	"code.vereign.com/gaiax/tsa/golib/errors"
 	"code.vereign.com/gaiax/tsa/infohub/gen/infohub"
+	"code.vereign.com/gaiax/tsa/infohub/internal/storage"
 )
 
+var exportAccepted = map[string]interface{}{"result": "accepted"}
+
+type Storage interface {
+	ExportConfiguration(ctx context.Context, exportName string) (*storage.ExportConfiguration, error)
+}
+
+type Policy interface {
+	Evaluate(ctx context.Context, policy string, data interface{}, evaluationID string) ([]byte, error)
+}
+
+type Cache interface {
+	Get(ctx context.Context, key, namespace, scope string) ([]byte, error)
+}
+
+type Credentials interface {
+	NewCredential(contexts []string, subjectID string, subject map[string]interface{}, proof bool) (*verifiable.Credential, error)
+	NewPresentation(contexts []string, credentials ...*verifiable.Credential) (*verifiable.Presentation, error)
+}
+
 type Service struct {
-	logger *zap.Logger
+	storage     Storage
+	policy      Policy
+	cache       Cache
+	credentials Credentials
+	logger      *zap.Logger
 }
 
-func New(logger *zap.Logger) *Service {
-	return &Service{logger: logger}
+func New(storage Storage, policy Policy, cache Cache, cred Credentials, logger *zap.Logger) *Service {
+	return &Service{
+		storage:     storage,
+		policy:      policy,
+		cache:       cache,
+		credentials: cred,
+		logger:      logger,
+	}
 }
 
 func (s *Service) Export(ctx context.Context, req *infohub.ExportRequest) (interface{}, error) {
-	return nil, fmt.Errorf("not implemented")
+	logger := s.logger.With(zap.String("exportName", req.ExportName))
+	exportCfg, err := s.storage.ExportConfiguration(ctx, req.ExportName)
+	if err != nil {
+		logger.Error("error getting export configuration", zap.Error(err))
+		return nil, err
+	}
+
+	// get the results of all policies configured in the export
+	results := make(map[string][]byte)
+	for policy := range exportCfg.Policies {
+		res, err := s.cache.Get(ctx, exportCacheKey(req.ExportName, policy), "", "")
+		if err != nil {
+			if errors.Is(errors.NotFound, err) {
+				if err := s.triggerExport(ctx, exportCfg); err != nil {
+					logger.Error("error performing export", zap.Error(err))
+					return nil, err
+				}
+				return exportAccepted, nil
+			}
+			logger.Error("failed to get policy result from cache", zap.Error(err))
+			return nil, err
+		}
+		results[policy] = res
+	}
+
+	// create separate verifiable credential for each policy result
+	var creds []*verifiable.Credential
+	for policy, result := range results {
+		var res map[string]interface{}
+		if err := json.Unmarshal(result, &res); err != nil {
+			logger.Error("error decoding policy result as json", zap.Error(err))
+			return nil, errors.New("error creating export", err)
+		}
+
+		// credentials do not include proof, because the final VP will include a proof for all
+		cred, err := s.credentials.NewCredential(exportCfg.Contexts, policy, res, false)
+		if err != nil {
+			logger.Error("failed to create verifiable credential", zap.Error(err))
+			return nil, errors.New("error creating export", err)
+		}
+		creds = append(creds, cred)
+	}
+
+	// bundle all credentials in a verifiable presentation with proof
+	vp, err := s.credentials.NewPresentation(exportCfg.Contexts, creds...)
+	if err != nil {
+		logger.Error("failed to create verifiable presentation", zap.Error(err))
+		return nil, errors.New("error creating export", err)
+	}
+
+	return vp, nil
+}
+
+func (s *Service) triggerExport(ctx context.Context, exportCfg *storage.ExportConfiguration) error {
+	s.logger.Info("export triggered", zap.String("exportName", exportCfg.ExportName))
+	for policy, input := range exportCfg.Policies {
+		cacheKey := exportCacheKey(exportCfg.ExportName, policy)
+		_, err := s.policy.Evaluate(ctx, policy, input, cacheKey)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func exportCacheKey(exportName string, policyName string) string {
+	return exportName + ":" + policyName
 }

internal/storage/storage.go

+package storage
+
+import (
+	"context"
+	"strings"
+
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/mongo"
+	"go.uber.org/zap"
+
+	"code.vereign.com/gaiax/tsa/golib/errors"
+)
+
+type ExportConfiguration struct {
+	ExportName string
+	Contexts   []string
+	Policies   map[string]interface{}
+}
+
+type Storage struct {
+	exportConfig *mongo.Collection
+	logger       *zap.Logger
+}
+
+func New(db *mongo.Client, dbname, collection string, logger *zap.Logger) (*Storage, error) {
+	if err := db.Ping(context.Background(), nil); err != nil {
+		return nil, err
+	}
+	return &Storage{
+		exportConfig: db.Database(dbname).Collection(collection),
+		logger:       logger,
+	}, nil
+}
+
+func (s *Storage) ExportConfiguration(ctx context.Context, exportName string) (*ExportConfiguration, error) {
+	result := s.exportConfig.FindOne(ctx, bson.M{
+		"exportName": exportName,
+	})
+
+	if result.Err() != nil {
+		if strings.Contains(result.Err().Error(), "no documents in result") {
+			return nil, errors.New(errors.NotFound, "export configuration not found")
+		}
+		return nil, result.Err()
+	}
+
+	var expcfg ExportConfiguration
+	if err := result.Decode(&expcfg); err != nil {
+		return nil, err
+	}
+
+	return &expcfg, nil
+}