Compare revisions

Becker, Christoph · Lyuben Penkovski · Lyuben Penkovski · Lyuben Penkovski · Kalin Daskalov · Kalin Daskalov
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
 variables:
  HELPERS_FILE: docker-build.yml
-  HELM_HELPERS_FILE: helm.yml
  APP_HELM_NAME: task
  DOCKER_FILE: deployment/ci/Dockerfile

@@ -8,15 +7,13 @@ stages:
  - compile
  - test
  - build
-  - helm
+  - manifest
  - deploy

 include:
  - project: '${HELPERS_PATH}'
    file: '${HELPERS_FILE}'
  - template: 'Workflows/Branch-Pipelines.gitlab-ci.yml'
-  - project: "$HELM_HELPERS_PATH}"
-    file: "${HELM_HELPERS_FILE}"

 lint:
  image: golangci/golangci-lint:v1.50.1
@@ -59,8 +56,14 @@ amd64:
  tags:
    - amd64-docker

-helm-lint:
-  extends: .helm-lint
-  stage: helm
-  tags:
-    - amd64-docker
+manifest:
+  extends: .manifest-amd64
+  stage: manifest
+
+cloud:
+  extends: .manifest-cloud
+  stage: manifest
+
+release:
+  extends: .manifest-release
+  stage: manifest
--- a/.gitlab-ci.yml.old
+++ b/.gitlab-ci.yml.old
-stages:
-  - test
-
-before_script:
-  - ln -s /builds /go/src/gitlab.com
-  - cd /go/src/gitlab.com/${CI_PROJECT_PATH}
-
-lint:
-  image: golangci/golangci-lint:v1.44.2
-  stage: test
-  tags:
-    - amd64-docker
-  script:
-    - golangci-lint --version
-    - golangci-lint run
-
-unit tests:
-  image: golang:1.17.8
-  stage: test
-  tags:
-    - amd64-docker
-  script:
-    - go version
-    - go test -race ./... -coverprofile=coverage.out
-    - go tool cover -func=coverage.out
--- a/GDPR.md
+++ b/GDPR.md
+# GDPR Compliance Document
+The objective of this document is to detail, the data being stored and proccessed by the Trust Service API.
+
+# Issuing Verifiable Credential 
+
+## What information is stored
+### Source User Information (Private)
+The Open Id connect claims MAY contain all sorts of personal data (like email, name, age and others), typically received from an external source.
+
+### Technical User Information (Public)
+
+- Schema information (public)
+- Credential/credential definition ids and states
+- DID of issuer
+- DID of holder
+- Created/updated dates
+- Offered credential attributes and attachments
+
+## How is the information stored and used
+### Source User Information
+Source User Information is encrypted using the Private Key of the organizational deployment, thereby creating the Verifiable Credential. This Verifiable Credential is shared with the legitimate recipient. Subsequently Source User Information(including the Verifiable Credential), is permanently erased from organizational deployment. 
+
+### Technical User Information (Public)
+Technical User Information is used to send the Verifiable credential to legitimate recipient. After successful issuance of the Verifiable Credential, per default Technical User information is permenetly erased from organizational deployment.
+
+
+## Who can access the information
+The Source User Information and Technical User Information both are accessible only by the system administrators of the organizational deployment.
+
+## How long will the information stay 
+### Source User Information
+The Source User Information is wiped out once the Verifiable Credential is issued.
+
+### Technical User Information (Public)
+The Technical User Information is wiped out per default after Vereifiable Credential is isssued or optionally stored according to retention periods (not defined yet).
+
+# Receiving Verifiable Credential 
+
+## What information is stored
+### Source User Information (Private)
+The Open Id connect claims MAY contain all sorts of personal data (like email, name, age and others), typically received from an external source.
+
+### Technical User Information (Public)
+
+- Schema information (public)
+- Credential/credential definition ids and states
+- DID of issuer
+- DID of holder
+- Created/updated dates
+- Offered credential attributes and attachments
+
+## How is the information stored and used
+### Source User Information
+Source User Information is decrypted. Per default received Verifiable Credential is not stored permanently. In case this is changed within a specific organizational deployment, an amendment of this GDPR Compliance Document will be necessary. This is the due to the fact that these details depend on the specific use cases and intentions.
+
+
+### Technical User Information (Public)
+Technical User Information is used to received the Verifiable credential from legitimate sender. After successful acceptance of the Verifiable Credential, per default Technical User information is permanently erased from the organizational deployment.
+
+## Who can access the information
+The Source User Information and Technical User Information both are accessible only by the system administrators of the organizational deployment.
+
+## How long will the information stay 
+### Source User Information
+The Source User Information is wiped out per default once the Verifiable Credential is received.
+
+### Technical User Information (Public)
+The Technical User Information is wiped out per default after Vereifiable Credential is received or stored according to retention periods (not defined yet).
--- a/README.md
+++ b/README.md
@@ -2,6 +2,7 @@
 [![coverage report](https://gitlab.eclipse.org/eclipse/xfsc/tsa/task/badges/main/coverage.svg)](https://gitlab.eclipse.org/eclipse/xfsc/tsa/task/-/commits/main)

 # Task Service
+This project has been migrated to Eclipse Foundation, and it can be found under https://gitlab.eclipse.org/eclipse/xfsc/

 The task service provides an HTTP interface for executing asynchronous (HTTP) tasks and task lists.

@@ -17,18 +18,21 @@ environment, the Swagger URL is available at http://localhost:8082/swagger-ui/
 ```mermaid  
 flowchart LR  
 	A([client]) -- HTTP --> B[Task API] 
-	subgraph task 
-		B --> C[(tasks DB)] 
-		C --> D[Executor] 
+	subgraph task
+        B --- C[(Storage)]
+        C --- E[Executor]
+        B --> D[(Queue)]
+        D --> E[Executor]
 	end 
-	D --> E[Policy]
-	D --> F[Cache]
+	E --> F[Policy]
+	E --> G[Cache]
 ```

-Tasks are created by clients making HTTP requests. The newly created tasks are
-stored in a persistent database which is used like Queue. An executor component
-is retrieving tasks from the Queue for execution. Clients receive an 
-immediate response with the `taskID` for the created task and can later query
+Tasks are created by clients making HTTP requests.
+In order to create a Task, a TaskTemplate must be present in the Storage. The newly created tasks are
+added to a Queue. Current implementation of the [Queue interface](internal/service/queue.go)
+is a persistent database. An executor component is retrieving tasks from the Queue for execution.
+Clients receive an immediate response with the `taskID` for the created task and can later query
 the state of task and retrieve its result either by directly querying the Cache
 service, or by querying the task HTTP interface for task results.

@@ -36,10 +40,17 @@ service, or by querying the task HTTP interface for task results.
 * [Tasks](docs/task.md)
 * [Task lists](docs/task-list.md)
 * [Queue](docs/queue.md)
+* [Storage](docs/storage.md)


+### Cache events

+Task service is able to subscribe for events produced by the Cache service
+and create a Task for every received event. Current implementation uses
+[NATS](https://nats.io/) for messaging system.

+##### More information
+* [Cache Event Task](docs/cache-event-task.md)

 ### Tests and Linters

@@ -65,6 +76,11 @@ go mod vendor

 [Dependencies](go.mod)

+## GDPR
+<hr/>
+
+[GDPR](GDPR.md)
+
 ## License
 <hr/>


--- a/cmd/task/main.go
+++ b/cmd/task/main.go
@@ -32,6 +32,7 @@ import (
 	goatask "gitlab.eclipse.org/eclipse/xfsc/tsa/task/gen/task"
 	goatasklist "gitlab.eclipse.org/eclipse/xfsc/tsa/task/gen/task_list"
 	"gitlab.eclipse.org/eclipse/xfsc/tsa/task/internal/clients/cache"
+	"gitlab.eclipse.org/eclipse/xfsc/tsa/task/internal/clients/event"
 	"gitlab.eclipse.org/eclipse/xfsc/tsa/task/internal/clients/policy"
 	"gitlab.eclipse.org/eclipse/xfsc/tsa/task/internal/config"
 	"gitlab.eclipse.org/eclipse/xfsc/tsa/task/internal/executor"
@@ -63,8 +64,9 @@ func main() {
 	db, err := mongo.Connect(
 		context.Background(),
 		options.Client().ApplyURI(cfg.Mongo.Addr).SetAuth(options.Credential{
-			Username: cfg.Mongo.User,
-			Password: cfg.Mongo.Pass,
+			Username:      cfg.Mongo.User,
+			Password:      cfg.Mongo.Pass,
+			AuthMechanism: cfg.Mongo.AuthMechanism,
 		}),
 	)
 	if err != nil {
@@ -91,6 +93,17 @@ func main() {
 	// create cache client
 	cache := cache.New(cfg.Cache.Addr, cache.WithHTTPClient(oauthClient))

+	var events *event.Client
+	if cfg.Nats.Addr != "" {
+		events, err = event.New(storage, storage, cfg.Nats.Addr, cfg.Nats.Subject)
+		if err != nil {
+			logger.Fatal("failed to create events client", zap.Error(err))
+		}
+		defer events.Close(context.Background()) //nolint:errcheck
+	} else {
+		logger.Info("task service is not able to subscribe for cache events")
+	}
+
 	// create task executor
 	executor := executor.New(
 		storage,
@@ -124,7 +137,7 @@ func main() {
 	{
 		taskSvc = task.New(storage, storage, cache, logger)
 		taskListSvc = tasklist.New(storage, storage, cache, logger)
-		healthSvc = health.New()
+		healthSvc = health.New(Version)
 	}

 	// create endpoints
@@ -213,6 +226,11 @@ func main() {
 	g.Go(func() error {
 		return listExecutor.Start(ctx)
 	})
+	if events != nil {
+		g.Go(func() error {
+			return events.Start(ctx)
+		})
+	}
 	if err := g.Wait(); err != nil {
 		logger.Error("run group stopped", zap.Error(err))
 	}

--- a/deployment/ci/Dockerfile
+++ b/deployment/ci/Dockerfile
-FROM golang:1.21.0-alpine3.17 as builder
+FROM golang:1.21.5-alpine3.17 as builder

 RUN apk add git

 WORKDIR /go/src/gitlab.eclipse.org/eclipse/xfsc/tsa/task

+ARG APP_REPO_TAG
+
 ADD . .

-RUN CGO_ENABLED=0 GOOS=linux go build -ldflags="-X main.Version=$(git describe --tags --always)" -mod=vendor -o /tmp/task ./cmd/task/...
+RUN CGO_ENABLED=0 GOOS=linux go build -ldflags="-X main.Version=$APP_REPO_TAG" -mod=vendor -o /tmp/task ./cmd/task/...

 FROM alpine:3.17 as runner


--- a/deployment/compose/Dockerfile
+++ b/deployment/compose/Dockerfile
-FROM golang:1.21.0
+FROM golang:1.21.5

 RUN go install github.com/ysmood/kit/cmd/guard@v0.25.11


--- a/deployment/helm/Chart.yaml
+++ b/deployment/helm/Chart.yaml
@@ -3,4 +3,3 @@ appVersion: v1.0.1-rc
 description: task deployment
 name: task
 version: 1.0.1
-icon: "https://www.vereign.com/wp-content/themes/vereign2020/images/vereign-logo.svg"
--- a/deployment/helm/README.md
+++ b/deployment/helm/README.md
-# task
-
-![Version: 1.0.1](https://img.shields.io/badge/Version-1.0.1-informational?style=flat-square) ![AppVersion: v1.0.1-rc](https://img.shields.io/badge/AppVersion-v1.0.1--rc-informational?style=flat-square)
-
-task deployment
-
-## Values
-
-| Key | Type | Default | Description |
-|-----|------|---------|-------------|
-| addresses.cache | string | `"http://cache:8080"` |  |
-| addresses.policy | string | `"http://policy:8080"` |  |
-| autoscaling.enabled | bool | `false` | Enable autoscaling |
-| autoscaling.maxReplicas | int | `3` | Maximum replicas |
-| autoscaling.minReplicas | int | `1` | Minimum replicas |
-| autoscaling.targetCPUUtilizationPercentage | int | `70` | CPU target for autoscaling trigger |
-| autoscaling.targetMemoryUtilizationPercentage | int | `70` | Memory target for autoscaling trigger |
-| image.name | string | `"gaiax/task"` | Image name |
-| image.pullPolicy | string | `"IfNotPresent"` | Image pull policy |
-| image.pullSecrets | string | `"deployment-key-light"` | Image pull secret when internal image is used |
-| image.repository | string | `"eu.gcr.io/vrgn-infra-prj"` |  |
-| image.sha | string | `""` | Image sha, usually generated by the CI Uses image.tag if empty |
-| image.tag | string | `""` | Image tag Uses .Chart.AppVersion if empty |
-| ingress.annotations."cert-manager.io/cluster-issuer" | string | `"letsencrypt-production-http"` |  |
-| ingress.annotations."kubernetes.io/ingress.class" | string | `"nginx"` |  |
-| ingress.annotations."kubernetes.io/ingress.global-static-ip-name" | string | `"dev-light-public"` |  |
-| ingress.annotations."nginx.ingress.kubernetes.io/rewrite-target" | string | `"/$2"` |  |
-| ingress.enabled | bool | `true` |  |
-| ingress.frontendDomain | string | `"gaiax.vereign.com"` |  |
-| ingress.frontendTlsSecretName | string | `"cert-manager-tls"` |  |
-| ingress.tlsEnabled | bool | `true` |  |
-| log.encoding | string | `"json"` |  |
-| log.level | string | `"debug"` |  |
-| metrics.enabled | bool | `true` | Enable prometheus metrics |
-| metrics.port | int | `2112` | Port for prometheus metrics |
-| mongo.addr | string | `"mongodb://mongodb-mongodb-replicaset.infra:27017/task?replicaSet=rs0&authSource=admin"` |  |
-| mongo.dbname | string | `"task"` |  |
-| mongo.pass | string | `""` |  |
-| mongo.user | string | `""` |  |
-| name | string | `"task"` | Application name |
-| nameOverride | string | `""` | Ovverwrites application name |
-| podAnnotations | object | `{}` |  |
-| replicaCount | int | `1` | Default number of instances to start  |
-| resources.limits.cpu | string | `"150m"` |  |
-| resources.limits.memory | string | `"128Mi"` |  |
-| resources.requests.cpu | string | `"25m"` |  |
-| resources.requests.memory | string | `"64Mi"` |  |
-| security.runAsGid | int | `0` | Group used by the apps |
-| security.runAsNonRoot | bool | `false` | by default, apps run as non-root |
-| security.runAsUid | int | `0` | User used by the apps |
-| service.port | int | `8080` |  |
-| task.http.host | string | `""` |  |
-| task.http.port | int | `8080` |  |
-| task.http.timeout.idle | string | `"120s"` |  |
-| task.http.timeout.read | string | `"10s"` |  |
-| task.http.timeout.write | string | `"10s"` |  |
-
----------------------------------------------
-Autogenerated from chart metadata using [helm-docs v1.10.0](https://github.com/norwoodj/helm-docs/releases/v1.10.0)
--- a/deployment/helm/ci/argocd.yaml
+++ b/deployment/helm/ci/argocd.yaml
+# ArgoCD Application Definition
+source:
+  repoURL: "{{ .RepoURL }}"
+  path: "{{ .Path }}"
+  targetRevision: "{{ .TargetRevision }}"
+
+applications:
+  - name: apps-of-apps
+    namespace: argocd
+    enabled: true
+    k8sAPI: https://kubernetes.default.svc
+    project: my-project
+    sourcePath: "."
+    sourceRef: "main"
+    chartName: "my-name"
+    ignoreDifferences:
+      - group: admissionregistration.k8s.io
+        kind: ValidatingWebhookConfiguration
+        name: cert-manager-webhook
+        jsonPointers:
+          - /webhooks/0/namespaceSelector/matchExpressions/2
+    helmValues:
+      - my_chart/values.yaml # Include values from the Helm chart
+
+  - name: example-app
+    enabled: true
+    sourcePath: "."
+    helmValues:
+      - my_chart/values.yaml # Include values from the Helm chart
+      - my_chart/example-values.yaml # Include additional example-specific values
--- a/deployment/helm/templates/argocd/argo-application.yaml
+++ b/deployment/helm/templates/argocd/argo-application.yaml
--- a/deployment/helm/templates/argocd/argo-project.yaml
+++ b/deployment/helm/templates/argocd/argo-project.yaml
--- a/deployment/helm/templates/deployment.yaml
+++ b/deployment/helm/templates/deployment.yaml
@@ -50,18 +50,35 @@ spec:
            value: {{ .Values.task.http.timeout.read | quote }}
          - name: HTTP_WRITE_TIMEOUT
            value: {{ .Values.task.http.timeout.write | quote }}
+          {{- if .Values.mongo.addr }}
          - name: MONGO_ADDR
            value: {{ .Values.mongo.addr | quote }}
+          {{- end }}
+          {{- if .Values.mongo.user }}
          - name: MONGO_USER
            value: {{ .Values.mongo.user | quote }}
+          {{- end }}
+          {{- if .Values.mongo.pass }}
          - name: MONGO_PASS
            value: {{ .Values.mongo.pass | quote }}
+          {{- end }}
+          {{- if .Values.mongo.dbname }}
          - name: MONGO_DB
            value: {{ .Values.mongo.dbname | quote }}
+          {{- end }}
          - name: CACHE_ADDR
            value: {{ .Values.addresses.cache | quote }}
          - name: POLICY_ADDR
            value: {{ .Values.addresses.policy | quote }}
+          {{- if .Values.secretEnv }}
+          {{- range $key, $value := .Values.secretEnv }}
+          - name: "{{ $key }}"
+            valueFrom:
+              secretKeyRef:
+                name: "{{ $value.name }}"
+                key: "{{ $value.key }}"
+          {{- end }}
+          {{- end }}
          {{- if .Values.extraVars }}
          {{- toYaml .Values.extraVars | indent 10 }}
          {{- end }}

--- a/deployment/helm/templates/ingress.yaml
+++ b/deployment/helm/templates/ingress.yaml
@@ -19,7 +19,7 @@ spec:
    - host: {{ .Values.ingress.frontendDomain }}
      http:
        paths:
-          - path: /{{ .Release.Namespace }}/{{ template "app.name" . }}(/|$)(.*)
+          - path: /{{ template "app.name" . }}(/|$)(.*)
            pathType: Prefix
            backend:
              service:

--- a/deployment/helm/templates/istio/autorization-rules.yaml
+++ b/deployment/helm/templates/istio/autorization-rules.yaml
+# templates/istio/authorization-rules.yaml
+apiVersion: security.istio.io/v1beta1
+kind: AuthorizationPolicy
+metadata:
+  name: { { include "my_chart.fullname" . } }
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: { { include "my_chart.name" . } }
+      app.kubernetes.io/instance: { { .Release.Name } }
+  action: ALLOW
+  rules:
+    - from:
+        - source:
+            principals: ["cluster.local/ns/default/sa/default"]
+      to:
+        - operation:
+            methods: ["GET"]
--- a/deployment/helm/templates/istio/gateway.yaml
+++ b/deployment/helm/templates/istio/gateway.yaml
+# templates/istio/gateway.yaml
+apiVersion: networking.istio.io/v1alpha3
+kind: Gateway
+metadata:
+  name: my-chart-gateway
+spec:
+  selector:
+    istio: ingressgateway
+  servers:
+    - port:
+        number: 80
+        name: http
+        protocol: HTTP
+      hosts:
+        - "*"
--- a/deployment/helm/templates/istio/virtual-service.yaml
+++ b/deployment/helm/templates/istio/virtual-service.yaml
+# templates/istio/virtual-service.yaml
+apiVersion: networking.istio.io/v1alpha3
+kind: VirtualService
+metadata:
+  name: { { include "my_chart.fullname" . } }
+spec:
+  hosts:
+    - "*"
+  gateways:
+    - my-chart-gateway
+  http:
+    - route:
+        - destination:
+            host: { { include "my_chart.fullname" . } }
+            port:
+              number: 80
--- a/deployment/helm/values.yaml
+++ b/deployment/helm/values.yaml
@@ -8,7 +8,7 @@ nameOverride: ""
 image:
  repository: eu.gcr.io/vrgn-infra-prj
  # -- Image name
-  name: gaiax/task
+  name: tsa/task
  # -- Image tag
  # Uses .Chart.AppVersion if empty
  tag: ""
@@ -96,7 +96,7 @@ task:
      write: 10s

 mongo:
-  addr: "mongodb://mongodb-mongodb-replicaset.infra:27017/task?replicaSet=rs0&authSource=admin"
+  addr: "mongodb://mongodb-0.mongodb:27017/task?replicaSet=rs0&authSource=admin"
  user: ""
  pass: ""
  dbname: task
@@ -110,7 +110,6 @@ ingress:
  annotations:
    kubernetes.io/ingress.class: nginx
    nginx.ingress.kubernetes.io/rewrite-target: /$2
-  tlsEnabled: true
-  frontendDomain: gaiax.vereign.com
+  frontendDomain: tsa.xfsc.dev
  frontendTlsSecretName: cert-manager-tls

--- a/design/design.go
+++ b/design/design.go
@@ -97,7 +97,7 @@ var _ = Service("health", func() {

 	Method("Liveness", func() {
 		Payload(Empty)
-		Result(Empty)
+		Result(HealthResponse)
 		HTTP(func() {
 			GET("/liveness")
 			Response(StatusOK)
@@ -106,7 +106,7 @@ var _ = Service("health", func() {

 	Method("Readiness", func() {
 		Payload(Empty)
-		Result(Empty)
+		Result(HealthResponse)
 		HTTP(func() {
 			GET("/readiness")
 			Response(StatusOK)

--- a/design/types.go
+++ b/design/types.go
@@ -68,3 +68,10 @@ var TaskStatus = Type("TaskStatus", func() {
 		Example("done")
 	})
 })
+
+var HealthResponse = Type("HealthResponse", func() {
+	Field(1, "service", String, "Service name.")
+	Field(2, "status", String, "Status message.")
+	Field(3, "version", String, "Service runtime version.")
+	Required("service", "status", "version")
+})
No results found