Refactor building and packing pipeline

.drone.yml

-workspace:
-  base: /go/src
-  path: github.com/cortezaproject/corteza-server
-
 kind: pipeline
-name: docker image build
+name: Build
+type: docker
 
 steps:
-  - name: unit-tests
+  - name: "Quality Control"
     image: cortezaproject/corteza-server-builder:latest
     pull: always
-    environment:
-      CGO_ENABLED: 0
-      GOOS: linux
-      GOARCH: amd64
-      CI: circleci
     commands:
       - make test.unit
 
-  - name: corteza-server
-    image: plugins/docker
-    depends_on: [ "unit-tests" ]
-    settings:
-      repo: cortezaproject/corteza-server
-      auto_tag: true # generate tag names automatically based on git branch and git tag
-      dockerfile: Dockerfile.corteza-server
-      username: { from_secret: docker_hub_username }
-      password: { from_secret: docker_hub_password }
-
-  - name: corteza-server-system
-    image: plugins/docker
-    depends_on: [ "corteza-server" ]
-    settings:
-      repo: cortezaproject/corteza-server-system
-      auto_tag: true # generate tag names automatically based on git branch and git tag
-      dockerfile: Dockerfile.corteza-server-system
-      username: { from_secret: docker_hub_username }
-      password: { from_secret: docker_hub_password }
-    when:
-      ref: { include: [ "refs/tags/20??.*.*", "refs/heads/master" ] }
-      event: [ "push", "tag" ]
-
-
-  - name: corteza-server-compose
-    image: plugins/docker
-    depends_on: [ "corteza-server" ]
-    settings:
-      repo: cortezaproject/corteza-server-compose
-      auto_tag: true # generate tag names automatically based on git branch and git tag
-      dockerfile: Dockerfile.corteza-server-compose
-      username: { from_secret: docker_hub_username }
-      password: { from_secret: docker_hub_password }
+  - name: "Build release (tag)"
+    image: cortezaproject/corteza-server-builder:latest
+    environment:
+      RELEASE_SFTP_KEY: { from_secret: RELEASE_SFTP_KEY }
+      RELEASE_SFTP_URI: { from_secret: RELEASE_SFTP_URI }
+    commands:
+    - make release-clean release -j4 BUILD_OS=linux BUILDARCH=amd64 BUILD_VERSION=${DRONE_TAG}
+#    - make release-clean release -j4 BUILD_OS=darwin BUILDARCH=amd64 BUILD_VERSION=${DRONE_TAG}
+#    - make release-clean release -j4 BUILD_OS=windows BUILDARCH=amd64 BUILD_VERSION=${DRONE_TAG}
+    - make upload
     when:
-      ref: { include: [ "refs/tags/20??.*.*", "refs/heads/master" ] }
-      event: [ "push", "tag" ]
-
-
-  - name: corteza-server-messaging
-    depends_on: [ "corteza-server" ]
-    image: plugins/docker
-    settings:
-      repo: cortezaproject/corteza-server-messaging
-      auto_tag: true # generate tag names automatically based on git branch and git tag
-      dockerfile: Dockerfile.corteza-server-messaging
-      username: { from_secret: docker_hub_username }
-      password: { from_secret: docker_hub_password }
+      event: [ tag ]
+      ref:
+      - refs/tags/20??.3.*
+      - refs/tags/20??.6.*
+      - refs/tags/20??.9.*
+      - refs/tags/20??.12.*
+
+  - name: "Build unstable release (develop)"
+    image: cortezaproject/corteza-server-builder:latest
+    environment:
+      RELEASE_SFTP_KEY: { from_secret: RELEASE_SFTP_KEY }
+      RELEASE_SFTP_URI: { from_secret: RELEASE_SFTP_URI }
+    commands:
+    - make release-clean release -j4 BUILD_OS=linux BUILDARCH=amd64 BUILD_VERSION=unstable
+    - make upload
     when:
-      ref: { include: [ "refs/tags/20??.*.*", "refs/heads/master" ] }
-      event: [ "push", "tag" ]
-
-trigger:
-  event:
-    # Pipeline will be executed from branch event,
-    # this ensures we do not run duplicates.
-    exclude: [ "pull_request" ]
-
-
+      event: [ push ]
+      branch: [ develop ]
 
 ---
-workspace:
-  base: /go/src
-  path: github.com/cortezaproject/corteza-server
 
 kind: pipeline
-name: integration
+name: Integration
+type: docker
 
 steps:
   - name: test
@@ -125,7 +86,4 @@ services:
 
 trigger:
   event:
-    # Pipeline will be executed from branch event,
-    # this ensures we do not run duplicates.
-    exclude: [ "pull_request" ]
-
+    exclude: [ pull_request ]

.project

-cortezaproject/corteza

Dockerfile.corteza-server

-## builder image
-FROM cortezaproject/corteza-server-builder:latest AS builder
-
-WORKDIR /cortrza/src
-
-COPY . .
-
-RUN scripts/builder-make-bin.sh monolith /tmp/corteza-server
-
-## == target image ==
-
-FROM alpine:3.7
-
-RUN apk add --no-cache ca-certificates
-
-COPY --from=builder /tmp/corteza-server /bin
-
-ENV STORAGE_PATH   /data
-
-ENV CORREDOR_ADDR corredor:80
-
-VOLUME /data
-
-EXPOSE 80
-ENTRYPOINT ["/bin/corteza-server"]
-CMD ["serve-api"]

Dockerfile.corteza-server-compose

-## builder image
-FROM cortezaproject/corteza-server-builder:latest AS builder
-
-WORKDIR /corteza/src
-
-COPY . .
-
-RUN scripts/builder-make-bin.sh compose /tmp/corteza-server-compose
-
-## == target image ==
-
-FROM alpine:3.7
-
-RUN apk add --no-cache ca-certificates
-
-COPY --from=builder /tmp/corteza-server-compose /bin
-
-ENV STORAGE_PATH /data
-ENV CORREDOR_ADDR corredor:80
-
-VOLUME /data
-
-EXPOSE 80
-ENTRYPOINT ["/bin/corteza-server-compose"]
-
-CMD ["serve-api"]
-

Dockerfile.corteza-server-messaging

-## builder image
-FROM cortezaproject/corteza-server-builder:latest AS builder
-
-WORKDIR /corteza/src
-
-COPY . .
-
-RUN scripts/builder-make-bin.sh messaging /tmp/corteza-server-messaging
-
-## == target image ==
-
-FROM alpine:3.7
-
-RUN apk add --no-cache ca-certificates
-
-COPY --from=builder /tmp/corteza-server-messaging /bin
-
-ENV STORAGE_PATH /data
-ENV CORREDOR_ADDR corredor:80
-
-VOLUME /data
-
-EXPOSE 80
-ENTRYPOINT ["/bin/corteza-server-messaging"]
-
-CMD ["serve-api"]
-

Dockerfile.corteza-server-system

-## builder image
-FROM cortezaproject/corteza-server-builder:latest AS builder
-
-WORKDIR /corteza/src
-
-COPY . .
-
-RUN scripts/builder-make-bin.sh system /tmp/corteza-server-system
-
-## == target image ==
-
-FROM alpine:3.7
-
-RUN apk add --no-cache ca-certificates
-
-COPY --from=builder /tmp/corteza-server-system /bin
-
-ENV STORAGE_PATH /data
-ENV CORREDOR_ADDR corredor:80
-
-VOLUME /data
-
-
-EXPOSE 80
-ENTRYPOINT ["/bin/corteza-server-system"]
-
-CMD ["serve-api"]
-

Makefile

-.PHONY: help docker docker-push realize qa critic vet codegen integration
+.PHONY: pack build help realize qa critic vet codegen integration
 
-GO        = go
-GOGET     = $(GO) get -u
+GO         = go
+GOGET      = $(GO) get -u
 GOTEST    ?= go test
+GOFLAGS   ?= -mod=vendor
+
+export GOFLAGS
+
+BUILD_FLAVOUR         ?= corteza
+BUILD_APPS            ?= system compose messaging monolith
+BUILD_TIME            ?= $(shell date +%FT%T%z)
+BUILD_VERSION         ?= $(shell git describe --tags --abbrev=0)
+BUILD_ARCH            ?= $(shell go env GOARCH)
+BUILD_OS              ?= $(shell go env GOOS)
+BUILD_OS_is_windows    = $(filter windows,$(BUILD_OS))
+BUILD_DEST_DIR        ?= build
+BUILD_NAME             = $(BUILD_FLAVOUR)-server-$*-$(BUILD_VERSION)-$(BUILD_OS)-$(BUILD_ARCH)
+BUILD_BIN_NAME         = $(BUILD_NAME)$(if $(BUILD_OS_is_windows),.exe,)
+
+RELEASE_BASEDIR        = $(BUILD_DEST_DIR)/pkg/$(BUILD_FLAVOUR)-server-$*
+RELEASE_NAME           = $(BUILD_NAME).tar.gz
+RELEASE_EXTRA_FILES   ?= README.md LICENSE CONTRIBUTING.md DCO .env.example
+RELEASE_PKEY          ?= .upload-rsa
+
+
+LDFLAGS_BUILD_TIME     = -X github.com/cortezaproject/corteza-server/pkg/version.BuildTime=$(BUILD_TIME)
+LDFLAGS_VERSION        = -X github.com/cortezaproject/corteza-server/pkg/version.Version=$(BUILD_VERSION)
+LDFLAGS_EXTRA         ?=
+LDFLAGS                = -ldflags "$(LDFLAGS_BUILD_TIME) $(LDFLAGS_GIT_TAG) $(LDFLAGS_EXTRA)"
 
-BASEPKGS = system compose messaging
-IMAGES   = corteza-server-system corteza-server-compose corteza-server-messaging corteza-server
-TESTABLE = messaging system compose pkg internal
 
 # Run watcher with a different event-trigger delay, eg:
 # $> WATCH_DELAY=5s make watch.test.integration
@@ -35,8 +57,12 @@ TEST_SUITE_unit        = $(TEST_SUITE_pkg) $(TEST_SUITE_services)
 TEST_SUITE_integration = ./tests/...
 TEST_SUITE_all         = $(TEST_SUITE_unit) $(TEST_SUITE_integration)
 
-DEV_MINIO_PORT ?= 9000
+# Dev Support apps settings
+DEV_MINIO_PORT        ?= 9000
+DEV_MAILHOG_SMTP_ADDR ?= 1025
+DEV_MAILHOG_HTTP_ADDR ?= 8025
 
+DOCKER                ?= docker
 
 ########################################################################################################################
 # Tool bins
@@ -55,29 +81,43 @@ help:
 	@echo
 	@echo Usage: make [target]
 	@echo
-	@echo - docker-images: builds docker images locally
-	@echo - docker-push:   push built images
+	@echo - build             build all apps
+	@echo - build.<app>       build a specific app
+	@echo - vet               run go vet on all code
+	@echo - critic            run go critic on all code
+	@echo - test.all          run all tests
+	@echo - test.unit         run all unit tests
+	@echo - test.integration  run all integration tests
 	@echo
-	@echo - vet - run go vet on all code
-	@echo - critic - run go critic on all code
-	@echo - test.all - run all tests
-	@echo - test.unit - run all unit tests
-	@echo - test.integration - run all integration tests
-	@echo
-	@echo See tests/README.md for more info
+	@echo See tests/README.md for more info on running tests
 	@echo
 
+########################################################################################################################
+# Building & packing
+
+build: $(addprefix build., $(BUILD_APPS))
+
+build.%: cmd/%
+	GOOS=$(BUILD_OS) GOARCH=$(BUILD_ARCH) go build $(LDFLAGS) -o $(BUILD_DEST_DIR)/$(BUILD_BIN_NAME) cmd/$*/main.go
 
-docker-images: $(IMAGES:%=docker-image.%)
+release.%: $(addprefix build., %)
+	@ mkdir -p $(RELEASE_BASEDIR) $(RELEASE_BASEDIR)/bin
+	@ cp $(RELEASE_EXTRA_FILES) $(RELEASE_BASEDIR)/
+	@ cp $(BUILD_DEST_DIR)/$(BUILD_BIN_NAME) $(RELEASE_BASEDIR)/bin/$(BUILD_FLAVOUR)-server-$*
+	@ tar -C $(dir $(RELEASE_BASEDIR)) -czf $(BUILD_DEST_DIR)/$(RELEASE_NAME) $(notdir $(RELEASE_BASEDIR))
 
-docker-image.%: Dockerfile.%
-	@ docker build --no-cache --rm -f Dockerfile.$* -t cortezaproject/$*:latest .
+release: $(addprefix release.,$(BUILD_APPS))
 
-docker-push: $(IMAGES:%=docker-push.%)
+release-clean:
+	@ rm -rf $(RELEASE_BASEDIR)
 
-docker-push.%: Dockerfile.%
-	@ docker push cortezaproject/$*:latest
+upload: $(RELEASE_PKEY)
+	@ echo "put $(BUILD_DEST_DIR)/*.tar.gz" | sftp -q -i $(RELEASE_PKEY) $(RELEASE_SFTP_URI)
+	@ rm -f $(RELEASE_PKEY)
 
+$(RELEASE_PKEY):
+	@ echo $(RELEASE_SFTP_KEY) | base64 -d > $(RELEASE_PKEY)
+	@ chmod 0400 $@
 
 ########################################################################################################################
 # Development
@@ -89,12 +129,12 @@ codegen: $(PROTOGEN)
 	./codegen.sh
 
 mailhog.up:
-	docker run --rm --publish 8025:8025 --publish 1025:1025 mailhog/mailhog
+	$(DOCKER) run --rm --publish $(DEV_MAILHOG_SMTP_ADDR):8025 --publish $(DEV_MAILHOG_HTTP_ADDR):1025 mailhog/mailhog
 
 minio.up:
 	# Runs temp minio server
-	# No volume because we do not want the data to persist
-	docker run --rm --publish 9000:$(DEV_MINIO_PORT) --env-file .env minio/minio server /data
+	# No volume mounts because we do not want the data to persist
+	$(DOCKER) run --rm --publish $(DEV_MINIO_PORT):9000 --env-file .env minio/minio server /data
 
 watch.test.%: $(NODEMON)
 	# Development helper - watches for file

build.sh

-#!/bin/bash
-set -e
-PROJECT=$(<.project)
-NAMES=$(find cmd/* -type d | xargs -n1 basename)
-if [ ! -z "$1" ]; then
-	NAMES="$1"
-fi
-for NAME in $NAMES; do
-	OSES=${OSS:-"linux"}
-	ARCHS=${ARCHS:-"amd64"}
-	for ARCH in $ARCHS; do
-		for OS in $OSES; do
-			echo $OS $ARCH $NAME
-			GOOS=${OS} GOARCH=${ARCH} CGO_ENABLED=0 GOARM=7 go build -o build/${NAME}-${OS}-${ARCH} cmd/${NAME}/*.go
-			if [ $? -eq 0 ]; then
-				echo OK
-			fi
-			if [ "$OS" == "windows" ]; then
-				mv build/${NAME}-${OS}-${ARCH} build/${NAME}-${OS}-${ARCH}.exe
-			fi
-		done
-	done
-done

scripts/builder-make-bin.sh

-#!/usr/bin/env sh
-
-# This file is intended to be used from Dockerfile,
-# presumably from cortezaproject/corteza-server-builder.
-
-set -eu
-
-BUILD_TIME=${BUILD_TIME:-$(date +%FT%T%z)}
-GIT_TAG=${GIT_TAG:-$(git describe --always --tags)}
-APP=${1}
-DST=${2:-"/bin/corteza-server-${APP}"}
-
-LDFLAGS=""
-LDFLAGS="${LDFLAGS} -X github.com/cortezaproject/corteza-server/pkg/version.BuildTime=${BUILD_TIME}"
-LDFLAGS="${LDFLAGS} -X github.com/cortezaproject/corteza-server/pkg/version.Version=${GIT_TAG}"
-
-go build -ldflags "${LDFLAGS}" -o $DST ./cmd/$APP/*.go