Merge branch 'localbuild' into '16-refactor-deprecated-functions'

Localbuild

See merge request !53

.gitlab-ci.yml

@@ -32,7 +32,7 @@ ppc64le:
   tags:
     - ppc64le
   script:
-    - docker build --build-arg CI_COMMIT_SHA=$CI_COMMIT_SHA --build-arg CI_COMMIT_REF_NAME=$CI_COMMIT_REF_NAME --build-arg CI_PROJECT_NAME=$CI_PROJECT_NAME --build-arg CI_JOB_TOKEN=$CI_JOB_TOKEN   --pull -t $IMAGE_ppc64le -f Dockerfile .
+    - docker build --build-arg CI_COMMIT_SHA=$CI_COMMIT_SHA --build-arg CI_COMMIT_REF_NAME=$CI_COMMIT_REF_NAME --build-arg CI_PROJECT_NAME=$CI_PROJECT_NAME --build-arg GITLAB_LOGIN=gitlab-ci-token --build-arg GITLAB_PASSWORD=$CI_JOB_TOKEN   --pull -t $IMAGE_ppc64le -f Dockerfile .
     - docker push $IMAGE_ppc64le
 
 amd64:
@@ -41,7 +41,7 @@ amd64:
   tags:
     - amd64
   script:
-    - docker build --build-arg CI_COMMIT_SHA=$CI_COMMIT_SHA --build-arg CI_COMMIT_REF_NAME=$CI_COMMIT_REF_NAME --build-arg CI_PROJECT_NAME=$CI_PROJECT_NAME --build-arg CI_JOB_TOKEN=$CI_JOB_TOKEN --pull -t $IMAGE_amd64 -f Dockerfile .
+    - docker build --build-arg CI_COMMIT_SHA=$CI_COMMIT_SHA --build-arg CI_COMMIT_REF_NAME=$CI_COMMIT_REF_NAME --build-arg CI_PROJECT_NAME=$CI_PROJECT_NAME --build-arg GITLAB_LOGIN=gitlab-ci-token --build-arg GITLAB_PASSWORD=$CI_JOB_TOKEN --pull -t $IMAGE_amd64 -f Dockerfile .
     - docker push $IMAGE_amd64
 
 manifest:
@@ -63,8 +63,9 @@ deploy:
     - master
   script:
     - apk add curl
-    - for ref in $DEPLOYENVS; do curl -X POST -F token=$TRIGGER_TOKEN -F ref=$ref -F variables[branch]=$CI_COMMIT_REF_NAME $TRIGGER_CODE_BUILD; done
-    
+    - for ref in $DEPLOYENVS; do curl -X POST -F token=$TRIGGER_TOKEN     -F ref=$ref -F variables[branch]=$CI_COMMIT_REF_NAME $TRIGGER_CODE_BUILD; done
+    -                            curl -X POST -F token=$TRIGGER_TOKEN_VCL -F ref=$CI_COMMIT_REF_NAME                           $TRIGGER_CODE_VCL
+
 duplicate:
   stage: tech_registry
   tags:

Dockerfile

-FROM registry.vereign.com/docker/go-with-dep:master
+FROM registry.vereign.com/docker/go-with-dep:with-codecgen
 
-#RUN echo deb http://deb.debian.org/debian buster main > /etc/apt/sources.list.d/buster.list \
-#    && apt-get update \
-#    && apt-get install -y --no-install-recommends npm zlib1g-dev 
+ARG GITLAB_LOGIN
+ARG GITLAB_PASSWORD
 
 ENV GOPATH /srv/
-ENV SERVER_OUT "bin/server"
-ENV SERVER_PKG_BUILD ${PKG}
 
 ENV PATH "$PATH:/srv/bin/"
 
-ARG CI_JOB_TOKEN
 ARG CI_PROJECT_NAME
 ARG CI_COMMIT_REF_NAME
 ARG CI_COMMIT_SHA
 
 ENV CI_PROJECT_NAME ${CI_PROJECT_NAME} 
 
-RUN echo project is $CI_PROJECT_NAME && 	mkdir -p /srv/src/code.vereign.com/code/$CI_PROJECT_NAME
+RUN mkdir -p /srv/src/code.vereign.com/code/$CI_PROJECT_NAME
 
 COPY . /srv/src/code.vereign.com/code/$CI_PROJECT_NAME/
 
 RUN cd /srv/src/code.vereign.com/code/$CI_PROJECT_NAME && \
-    git config --global credential.https://code.vereign.com.username gitlab-ci-token && \
-    git config --global credential.helper "store --file /tmp/store" && \
-    echo https://gitlab-ci-token:$CI_JOB_TOKEN@code.vereign.com > /tmp/store && cat /tmp/store && \
-	export PKG=code.vereign.com/code/$CI_PROJECT_NAME && \
-    go get -u github.com/ugorji/go/codec/codecgen && \
-	echo PATH $PATH && \
-	make && rm /tmp/store 
-
-#FROM debian:stretch 
-#RUN mkdir -p /srv/src/code.vereign.com/code/ 
-#COPY --from=0 /srv/src/code.vereign.com/code/data-storage-agent /srv/src/code.vereign.com/code/
-#RUN find /srv/src/code.vereign.com/code/
-	
-ENTRYPOINT /srv/src/code.vereign.com/code/$CI_PROJECT_NAME/bin/server
+    git config --global url."https://$GITLAB_LOGIN:$GITLAB_PASSWORD@code.vereign.com".insteadOf "https://code.vereign.com" && \
+    dep ensure && \
+    go generate vendor/github.com/coreos/etcd/client/keys.go && \
+    go build -ldflags="-X code.vereign.com/code/${CI_PROJECT_NAME}/handler.version=${CI_COMMIT_REF_NAME}.${CI_COMMIT_SHA}" -i -v -o /srv/${CI_PROJECT_NAME} code.vereign.com/code/${CI_PROJECT_NAME}
+
+FROM buildpack-deps:buster-curl
+
+ARG CI_PROJECT_NAME
+ENV CI_PROJECT_NAME ${CI_PROJECT_NAME} 
+
+COPY --from=0 /srv/${CI_PROJECT_NAME} /srv/${CI_PROJECT_NAME}
+
+WORKDIR /srv
+
+RUN mkdir -p /srv/src/code.vereign.com/code/key-storage-agent/bin/ && ln /srv/key-storage-agent /srv/src/code.vereign.com/code/key-storage-agent/bin/server && ln -s /srv/ /go
+
+ENTRYPOINT ["/srv/key-storage-agent"]
 

Dockerfile-x86

-FROM registry.vereign.com/docker/go-with-dep:master-x86
-
-#RUN echo deb http://deb.debian.org/debian buster main > /etc/apt/sources.list.d/buster.list \
-#    && apt-get update \
-#    && apt-get install -y --no-install-recommends npm zlib1g-dev 
-
-ENV GOPATH /srv/
-ENV SERVER_OUT "bin/server"
-ENV SERVER_PKG_BUILD ${PKG}
-
-ARG CI_JOB_TOKEN
-ARG CI_PROJECT_NAME
-ARG CI_COMMIT_REF_NAME
-ARG CI_COMMIT_SHA
-
-ENV CI_PROJECT_NAME ${CI_PROJECT_NAME} 
-
-RUN echo project is $CI_PROJECT_NAME && 	mkdir -p /srv/src/code.vereign.com/code/$CI_PROJECT_NAME
-
-COPY . /srv/src/code.vereign.com/code/$CI_PROJECT_NAME/
-
-RUN cd /srv/src/code.vereign.com/code/$CI_PROJECT_NAME && \
-    git config --global credential.https://code.vereign.com.username gitlab-ci-token && \
-    git config --global credential.helper "store --file /tmp/store" && \
-    echo https://gitlab-ci-token:$CI_JOB_TOKEN@code.vereign.com > /tmp/store && cat /tmp/store && \
-	export PKG=code.vereign.com/code/$CI_PROJECT_NAME && \
-	make && rm /tmp/store 
-
-#FROM debian:stretch 
-#RUN mkdir -p /srv/src/code.vereign.com/code/ 
-#COPY --from=0 /srv/src/code.vereign.com/code/data-storage-agent /srv/src/code.vereign.com/code/
-#RUN find /srv/src/code.vereign.com/code/
-	
-ENTRYPOINT /srv/src/code.vereign.com/code/$CI_PROJECT_NAME/bin/server
-

config.yaml.sample

@@ -6,6 +6,8 @@
 dataStorageUrl: localhost:7777
 grpcListenAddress: localhost:7877
 restListenAddress: localhost:7878
+# This one using by rest-gateway. Must contain the same domain name with issued certificate
+grpcURL: key-storage-agent:7877
 
 # VIAM Variables
 viamUUID: viam-system

config/configs.go

@@ -29,6 +29,7 @@ var MaxMessageSize int
 
 var GrpcListenAddress string
 var RestListenAddress string
+var GrpcURL string
 var DataStorageUrl string
 var EntitiesManagerUrl string
 var CertDir string
@@ -37,10 +38,11 @@ func SetConfigValues(configFile, etcdURL string) {
 	// Set Default Values For Config Variables
 
 	// Vereign API Related
-	viper.SetDefault("grpcListenAddress", "localhost:7877")
-	viper.SetDefault("restListenAddress", "localhost:7878")
-	viper.SetDefault("dataStorageUrl", "localhost:7777")
-	viper.SetDefault("entitiesManagerUrl", "localhost:7779")
+	viper.SetDefault("grpcListenAddress", ":7877")
+	viper.SetDefault("restListenAddress", ":7878")
+	viper.SetDefault("dataStorageUrl", "data-storage-agent:7777")
+	viper.SetDefault("entitiesManagerUrl", "entities-management-agent:7779")
+	viper.SetDefault("grpcURL", "key-storage-agent:7877")
 
 	viper.SetDefault("viamUUID", "viam-system")
 	viper.SetDefault("viamSession", "viam-session")
@@ -122,6 +124,7 @@ func SetConfigValues(configFile, etcdURL string) {
 	}
 
 	GrpcListenAddress = viper.GetString("grpcListenAddress")
+	GrpcURL = viper.GetString("grpcURL")
 	RestListenAddress = viper.GetString("restListenAddress")
 
 	DataStorageUrl = viper.GetString("dataStorageUrl")

main.go

@@ -42,6 +42,7 @@ func main() {
 
 	grpcAddress := config.GrpcListenAddress
 	restAddress := config.RestListenAddress
+	grpcURL := config.GrpcURL
 	dataStorageAddress := config.DataStorageUrl
 	certPem := config.CertificatePEM
 	keyPem := config.PrivateKeyPEM
@@ -62,7 +63,7 @@ func main() {
 
 	// fire the REST server in a goroutine
 	go func() {
-		err := server.StartRESTServer(restAddress, grpcAddress, certPem)
+		err := server.StartRESTServer(restAddress, grpcURL, certPem, keyPem)
 		if err != nil {
 			log.Fatalf("failed to start gRPC server: %s", err)
 		}

run.sh

@@ -2,7 +2,7 @@
 [ -d cert ] || mkdir cert
 
 openssl genrsa -out cert/server.key 2048
-openssl req -new -key cert/server.key -out cert/server.csr -subj "/CN=localhost"
+openssl req -new -key cert/server.key -out cert/server.csr -subj "/CN=key-storage-agent"
 openssl x509 -req -days 365 -in cert/server.csr -signkey cert/server.key -out cert/server.crt
 
 

server/server.go

@@ -54,6 +54,9 @@ func credMatcher(headerName string) (mdName string, ok bool) {
 	if headerName == "Session" {
 		return headerName, true
 	}
+	if headerName == "Uuid" {
+		return headerName, true
+	}
 	return "", false
 }
 
@@ -177,7 +180,8 @@ func StartGRPCServer(address string, certPEM, privateKeyPEM, caCertPEM, vereignC
 	return nil
 }
 
-func StartRESTServer(address, grpcAddress string, certPEM []byte) error {
+func StartRESTServer(address, grpcURL string, certPEM, keyPEM []byte) error {
+	log.Println("grpcAddress: ", grpcURL)
 	ctx := context.Background()
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
@@ -200,14 +204,31 @@ func StartRESTServer(address, grpcAddress string, certPEM []byte) error {
 	opts := []grpc.DialOption{grpc.WithTransportCredentials(creds)}
 
 	// Register RedisStorageServer
-	err = api.RegisterKeyStorageHandlerFromEndpoint(ctx, mux, grpcAddress, opts)
+	err = api.RegisterKeyStorageHandlerFromEndpoint(ctx, mux, grpcURL, opts)
 	if err != nil {
 		log.Printf("Error: %v", err)
 		return fmt.Errorf("could not register service RedisStorageServer: %s", err)
 	}
 
-	log.Printf("starting HTTP/1.1 REST server on %s", address)
-	http.ListenAndServe(address, mux)
+	// server certificate
+	certificate, err := tls.X509KeyPair(certPEM, keyPEM)
+	if err != nil {
+		log.Printf("Error: %v", err)
+		return fmt.Errorf("could not load server key pair: %s", err)
+	}
+
+	serverTLSConfig := &tls.Config{
+		Certificates: []tls.Certificate{certificate},
+	}
+
+	tlsServer := &http.Server{
+		Addr: address,
+		Handler: mux,
+		TLSConfig: serverTLSConfig,
+	}
+
+	log.Printf("starting HTTP/1.1 REST server on %s", address)	
+	tlsServer.ListenAndServeTLS("","")
 
 	return nil
 }