Update Build and Release workflows (#1349)

* Fix K8S-Proxy image build and tag

* Update build script to remove unused PUSH and TAG features

* Update workflows

- Remove build.sh from root of repo as it's no longer used
- Update workflows to use matrix feature for parallelization

* Add Docker image build step and other projects

* Remove csproj filenames from steps

* Fix typo

* Fix directory for k8s-proxy image

* Move Docker stub and uploads out of matrix

* Change to block-style sequence for service_name matrix

* Add missing quote character

* Fix working directories of Dotnet steps

* Update step name

* Add checkout repo step to Upload job

* Add 'dotnet tool restore' step

* Fix Swagger directory

* Fix Swagger build step

* Fix Swagger build step (again)

.github/workflows/build.yml

@@ -10,12 +10,13 @@ on:
 
 jobs:
   cloc:
+    name: CLOC
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repo
         uses: actions/checkout@5a4ac9002d0be2fb38bd78e4b4dbde5606d7042f
 
-      - name: Set up cloc
+      - name: Install cloc
         run: |
           sudo apt-get update
           sudo apt-get -y install cloc
@@ -24,12 +25,13 @@ jobs:
         run: cloc --include-lang C#,SQL,Razor,"Bourne Shell",PowerShell,HTML,CSS,Sass,JavaScript,TypeScript --vcs git
 
   testing:
+    name: Testing
     runs-on: windows-latest
     steps:
       - name: Set up NuGet
         uses: nuget/setup-nuget@04b0c2b8d1b97922f67eca497d7cf0bf17b8ffe1
         with:
-          nuget-version: 'latest'
+          nuget-version: '5'
 
       - name: Set up MSBuild
         uses: microsoft/setup-msbuild@c26a08ba26249b81327e26f6ef381897b6a8754d
@@ -48,9 +50,6 @@ jobs:
           npm --version
           echo "GitHub ref: $GITHUB_REF"
           echo "GitHub event: $GITHUB_EVENT"
-        env:
-          GITHUB_REF: ${{ github.ref }}
-          GITHUB_EVENT: ${{ github.event_name }}
 
       - name: Checkout repo
         uses: actions/checkout@5a4ac9002d0be2fb38bd78e4b4dbde5606d7042f
@@ -68,17 +67,66 @@ jobs:
         shell: pwsh
 
   build:
+    name: Build
     runs-on: ubuntu-latest
     needs: testing
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - service_name: Admin
+            base_path: ./src
+            dotnet: true
+            gulp: true
+          - service_name: Api
+            base_path: ./src
+            dotnet: true
+          - service_name: Attachments
+            base_path: ./util
+          - service_name: Billing
+            base_path: ./src
+            dotnet: true
+          - service_name: Events
+            base_path: ./src
+            dotnet: true
+          - service_name: Icons
+            base_path: ./src
+            dotnet: true
+          - service_name: Identity
+            base_path: ./src
+            dotnet: true
+          - service_name: K8S-Proxy
+            base_path: ./util
+          - service_name: MsSql
+            base_path: ./util
+          - service_name: Nginx
+            base_path: ./util
+          - service_name: Notifications
+            base_path: ./src
+            dotnet: true
+          - service_name: Portal
+            base_path: ./bitwarden_license/src
+            dotnet: true
+            gulp: true
+          - service_name: Server
+            base_path: ./util
+            dotnet: true
+          - service_name: Setup
+            base_path: ./util
+            dotnet: true
+          - service_name: Sso
+            base_path: ./bitwarden_license/src
+            dotnet: true
+            gulp: true
     steps:
+      - name: Checkout repo
+        uses: actions/checkout@5a4ac9002d0be2fb38bd78e4b4dbde5606d7042f
+
       - name: Set up Node
         uses: actions/setup-node@46071b5c7a2e0c34e49c3cb8a0e792e86e18d5ea
         with:
           node-version: '14'
 
-      - name: Set up gulp
-        run: npm i -g gulp
-
       - name: Print environment
         run: |
           whoami
@@ -89,9 +137,6 @@ jobs:
           docker --version
           echo "GitHub ref: $GITHUB_REF"
           echo "GitHub event: $GITHUB_EVENT"
-        env:
-          GITHUB_REF: ${{ github.ref }}
-          GITHUB_EVENT: ${{ github.event_name }}
 
       - name: Login to Azure
         uses: Azure/login@77f1b2e3fb80c0e8645114159d17008b8a2e475a
@@ -108,7 +153,7 @@ jobs:
                     dct-delegate-2-repo-passphrase,
                     dct-delegate-2-key"
 
-      - name: Log into docker
+      - name: Log into Docker
         if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/rc'
         run: echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
         env:
@@ -125,44 +170,99 @@ jobs:
           DCT_DELEGATION_KEY_ID: "c9bde8ec820701516491e5e03d3a6354e7bd66d05fa3df2b0062f68b116dc59c"
           DCT_DELEGATE_KEY: ${{ steps.retrieve-secrets.outputs.dct-delegate-2-key }}
 
-      - name: Checkout repo
+      - name: Set up Gulp
-        uses: actions/checkout@5a4ac9002d0be2fb38bd78e4b4dbde5606d7042f
+        if: ${{ matrix.gulp }}
-
+        working-directory: ${{ matrix.base_path }}/${{ matrix.service_name }}
-      - name: Restore
-        run: dotnet tool restore
-
-      - name: Build
         run: |
-          chmod +x ./build.sh
+          npm install -g gulp
-          ./build.sh
 
-      - name: Tag rc branch
+      - name: Restore/Clean service
+        if: ${{ matrix.dotnet }}
+        working-directory: ${{ matrix.base_path }}/${{ matrix.service_name }}
+        run: |
+          echo "Restore"
+          dotnet restore
+          echo "Clean"
+          dotnet clean -c "Release" -o obj/Azure/publish
+
+      - name: Execute Gulp
+        if: ${{ matrix.gulp }}
+        working-directory: ${{ matrix.base_path }}/${{ matrix.service_name }}
+        run: |
+          npm install
+          gulp --gulpfile gulpfile.js build
+
+      - name: Publish service
+        if: ${{ matrix.dotnet }}
+        working-directory: ${{ matrix.base_path }}/${{ matrix.service_name }}
+        run: |
+          echo "Publish"
+          dotnet publish -c "Release" -o obj/Azure/publish
+
+          cd obj/Azure/publish
+          zip -r ${{ matrix.service_name }}.zip .
+          mv ${{ matrix.service_name }}.zip ../../../
+
+      - name: Build Docker images
+        if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/rc'
+        run: |
+          SERVICE_NAME=$(echo "${{ matrix.service_name }}" | tr '[:upper:]' '[:lower:]')
+          if [ "$SERVICE_NAME" = "k8s-proxy" ]; then
+            docker build -f ${{ matrix.base_path }}/Nginx/Dockerfile-k8s -t bitwarden/$SERVICE_NAME ${{ matrix.base_path }}/Nginx
+          else
+            docker build -t bitwarden/$SERVICE_NAME ${{ matrix.base_path }}/$SERVICE_NAME
+          fi
+
+      - name: Tag rc
         if: github.ref == 'refs/heads/rc'
-        run: ./build.sh tag rc
+        run: |
+          SERVICE_NAME=$(echo "${{ matrix.service_name }}" | tr '[:upper:]' '[:lower:]')
+          docker tag bitwarden/$SERVICE_NAME bitwarden/$SERVICE_NAME:rc
 
       - name: Tag dev
         if: github.ref == 'refs/heads/master'
-        run: ./build.sh tag dev
+        run: |
+          SERVICE_NAME=$(echo "${{ matrix.service_name }}" | tr '[:upper:]' '[:lower:]')
+          docker tag bitwarden/$SERVICE_NAME bitwarden/$SERVICE_NAME:dev
 
-      - name: List docker images
+      - name: List Docker images
         if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/rc'
         run: docker images
 
       - name: Push rc images
         if: github.ref == 'refs/heads/rc'
-        run: ./build.sh push rc 
+        run: |
+          SERVICE_NAME=$(echo "${{ matrix.service_name }}" | tr '[:upper:]' '[:lower:]')
+          docker push bitwarden/$SERVICE_NAME:rc
         env:
           DOCKER_CONTENT_TRUST: 1
           DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE: ${{ steps.retrieve-secrets.outputs.dct-delegate-2-repo-passphrase }}
 
       - name: Push dev images
         if: github.ref == 'refs/heads/master'
-        run: ./build.sh push dev
+        run: |
+          SERVICE_NAME=$(echo "${{ matrix.service_name }}" | tr '[:upper:]' '[:lower:]')
+          docker push bitwarden/$SERVICE_NAME:dev
         env:
           DOCKER_CONTENT_TRUST: 1
           DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE: ${{ steps.retrieve-secrets.outputs.dct-delegate-2-repo-passphrase }}
 
-      - name: Make docker stub
+      - name: Log out of Docker
+        if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/rc'
+        run: docker logout
+
+  upload:
+    name: Upload
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@5a4ac9002d0be2fb38bd78e4b4dbde5606d7042f
+
+      - name: Restore
+        run: dotnet tool restore
+
+      - name: Make Docker stub
         run: |
           STUB_OUTPUT=$(pwd)/docker-stub
           docker run -i --rm --name setup -v $STUB_OUTPUT:/bitwarden bitwarden/setup:dev \
@@ -173,29 +273,31 @@ jobs:
           touch $STUB_OUTPUT/env/uid.env
           cd docker-stub; zip -r ../docker-stub.zip *; cd ..
 
-      - name: Upload docker stub artifact
+      - name: Upload Docker stub artifact
         uses: actions/upload-artifact@ee69f02b3dfdecd58bb31b4d133da38ba6fe3700
         with:
           name: docker-stub.zip
           path: ./docker-stub.zip
 
-      - name: Build swagger
+      - name: Build Swagger
         run: |
           cd ./src/Api
-          dotnet swagger tofile --output ../../swagger.json --host https://api.bitwarden.com `
+          echo "Restore"
-            ./obj/Docker/publish/Api/Api.dll public
+          dotnet restore
+          echo "Clean"
+          dotnet clean -c "Release" -o obj/Azure/publish
+          echo "Publish"
+          dotnet publish -c "Release" -o obj/Azure/publish
+
+          dotnet swagger tofile --output ../../swagger.json --host https://api.bitwarden.com \
+            ./obj/Azure/publish/Api.dll public
           cd ../..
-        shell: pwsh
         env:
           ASPNETCORE_ENVIRONMENT: Production
           swaggerGen: 'True'
 
-      - name: Upload swagger artifact
+      - name: Upload Swagger artifact
         uses: actions/upload-artifact@ee69f02b3dfdecd58bb31b4d133da38ba6fe3700
         with:
           name: swagger.json
           path: ./swagger.json
-
-      - name: Log out of docker
-        if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/rc'
-        run: docker logout

.github/workflows/release.yml

@@ -9,6 +9,7 @@ on:
 
 jobs:
   setup:
+    name: Setup
     runs-on: ubuntu-latest
     outputs:
       release_upload_url: ${{ steps.create_release.outputs.upload_url }}
@@ -53,8 +54,27 @@ jobs:
           prerelease: false
 
   release:
+    name: Release
     runs-on: ubuntu-latest
     needs: setup
+    strategy:
+      fail-fast: false
+      matrix:
+        service_name:
+          - admin
+          - api
+          - attachments
+          - events
+          - icons
+          - identity
+          - k8s-proxy
+          - mssql
+          - nginx
+          - notifications
+          - portal
+          - server
+          - setup
+          - sso
     env:
       RELEASE_VERSION: ${{ needs.setup.outputs.release_version }}
       TAG_VERSION: ${{ needs.setup.outputs.tag_version }}
@@ -65,9 +85,6 @@ jobs:
           docker --version
           echo "GitHub ref: $GITHUB_REF"
           echo "GitHub event: $GITHUB_EVENT"
-        env:
-          GITHUB_REF: ${{ github.ref }}
-          GITHUB_EVENT: ${{ github.event_name }}
 
       - name: Login to Azure
         uses: Azure/login@77f1b2e3fb80c0e8645114159d17008b8a2e475a
@@ -84,7 +101,7 @@ jobs:
                     dct-delegate-2-repo-passphrase,
                     dct-delegate-2-key"
 
-      - name: Log into docker
+      - name: Log into Docker
         if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/rc'
         run: echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
         env:
@@ -92,6 +109,7 @@ jobs:
           DOCKER_PASSWORD: ${{ steps.retrieve-secrets.outputs.docker-password }}
 
       - name: Setup Docker Trust
+        if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/rc'
         run: |
           mkdir -p ~/.docker/trust/private
 
@@ -103,73 +121,46 @@ jobs:
       - name: Checkout repo
         uses: actions/checkout@5a4ac9002d0be2fb38bd78e4b4dbde5606d7042f
 
+      - name: Pull Docker images
+        run: |
+          docker pull bitwarden/${{ matrix.service_name }}:rc
+        env:
+          DOCKER_CONTENT_TRUST: 1
+
+      - name: Re-tag Docker images
+        run: |
+          docker tag bitwarden/${{ matrix.service_name }}:rc bitwarden/${{ matrix.service_name }}:latest
+          docker tag bitwarden/${{ matrix.service_name }}:rc bitwarden/${{ matrix.service_name }}:$RELEASE_VERSION
+
+      - name: List Docker images
+        run: docker images
+
+      - name: Push Docker images
+        if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/rc'
+        run: |
+          docker push bitwarden/${{ matrix.service_name }}:latest
+          docker push bitwarden/${{ matrix.service_name }}:$RELEASE_VERSION
+        env:
+          DOCKER_CONTENT_TRUST: 1
+          DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE: ${{ steps.retrieve-secrets.outputs.dct-delegate-2-repo-passphrase }}
+
+      - name: Log out of Docker
+        if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/rc'
+        run: docker logout
+
+
+  upload:
+    name: Upload
+    runs-on: ubuntu-latest
+    needs: release
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@5a4ac9002d0be2fb38bd78e4b4dbde5606d7042f
+
       - name: Restore
         run: dotnet tool restore
 
-      - name: pull docker images
+      - name: Make Docker stub
-        run: |
-          docker pull bitwarden/api:rc
-          docker pull bitwarden/identity:rc
-          docker pull bitwarden/server:rc
-          docker pull bitwarden/attachments:rc
-          docker pull bitwarden/icons:rc
-          docker pull bitwarden/notifications:rc
-          docker pull bitwarden/events:rc
-          docker pull bitwarden/admin:rc
-          docker pull bitwarden/nginx:rc
-          docker pull bitwarden/nginx:rc
-          docker pull bitwarden/sso:rc
-          docker pull bitwarden/portal:rc
-          docker pull bitwarden/mssql:rc
-          docker pull bitwarden/setup:rc
-        env:
-          DOCKER_CONTENT_TRUST: 1
-
-
-      - name: re-tag docker images
-        run: |
-          tags=( latest beta ${RELEASE_VERSION} )
-          for TAG in "${tags[@]}"
-          do
-            docker tag bitwarden/api:rc bitwarden/api:$TAG
-            docker tag bitwarden/identity:rc bitwarden/identity:$TAG
-            docker tag bitwarden/server:rc bitwarden/server:$TAG
-            docker tag bitwarden/attachments:rc bitwarden/attachments:$TAG
-            docker tag bitwarden/icons:rc bitwarden/icons:$TAG
-            docker tag bitwarden/notifications:rc bitwarden/notifications:$TAG
-            docker tag bitwarden/events:rc bitwarden/events:$TAG
-            docker tag bitwarden/admin:rc bitwarden/admin:$TAG
-            docker tag bitwarden/nginx:rc bitwarden/nginx:$TAG
-            docker tag bitwarden/nginx:rc bitwarden/k8s-proxy:$TAG
-            docker tag bitwarden/sso:rc bitwarden/sso:$TAG
-            docker tag bitwarden/portal:rc bitwarden/portal:$TAG
-            docker tag bitwarden/mssql:rc bitwarden/mssql:$TAG
-            docker tag bitwarden/setup:rc bitwarden/setup:$TAG
-          done 
-
-      - name: List docker images
-        run: docker images
-
-      - name: Push beta images
-        run: ./build.sh push beta
-        env:
-          DOCKER_CONTENT_TRUST: 1
-          DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE: ${{ steps.retrieve-secrets.outputs.dct-delegate-2-repo-passphrase }}
-
-      - name: Push latest images
-        run: ./build.sh push latest
-        env:
-          DOCKER_CONTENT_TRUST: 1
-          DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE: ${{ steps.retrieve-secrets.outputs.dct-delegate-2-repo-passphrase }}
-
-      - name: Push version images
-        run: ./build.sh push $($env:$RELEASE_VERSION)
-        shell: pwsh
-        env:
-          DOCKER_CONTENT_TRUST: 1
-          DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE: ${{ steps.retrieve-secrets.outputs.dct-delegate-2-repo-passphrase }}
-
-      - name: Make docker stub
         run: |
           STUB_OUTPUT=$(pwd)/docker-stub
           docker run -i --rm --name setup -v $STUB_OUTPUT:/bitwarden bitwarden/setup:dev \
@@ -180,39 +171,35 @@ jobs:
           touch $STUB_OUTPUT/env/uid.env
           cd docker-stub; zip -r ../docker-stub.zip *; cd ..
 
-      - name: Upload docker stub artifact
+      - name: Upload Docker stub artifact
         uses: actions/upload-artifact@ee69f02b3dfdecd58bb31b4d133da38ba6fe3700
         with:
           name: docker-stub.zip
           path: ./docker-stub.zip
 
-      - name: Build swagger
+      - name: Build Swagger
         run: |
           cd ./src/Api
           echo "Restore"
-          dotnet restore "Api.csproj"
+          dotnet restore
           echo "Clean"
-          dotnet clean "Api.csproj" -c "Release" -o "obj/Docker/publish/Api"
+          dotnet clean -c "Release" -o obj/Azure/publish
           echo "Publish"
-          dotnet publish "Api.csproj" -c "Release" -o "obj/Docker/publish/Api"
+          dotnet publish -c "Release" -o obj/Azure/publish
 
-          dotnet swagger tofile --output ../../swagger.json --host https://api.bitwarden.com `
+          dotnet swagger tofile --output ../../swagger.json --host https://api.bitwarden.com \
-            ./obj/Docker/publish/Api/Api.dll public
+            ./obj/Azure/publish/Api.dll public
           cd ../..
-        shell: pwsh
         env:
           ASPNETCORE_ENVIRONMENT: Production
           swaggerGen: 'True'
 
-      - name: Upload swagger artifact
+      - name: Upload Swagger artifact
         uses: actions/upload-artifact@ee69f02b3dfdecd58bb31b4d133da38ba6fe3700
         with:
           name: swagger.json
           path: ./swagger.json
 
-      - name: Log out of docker
-        run: docker logout
-
       - name: Upload release assets
         if: github.event_name == 'release'
         run: |
@@ -223,4 +210,3 @@ jobs:
             $RELEASE_TAG_NAME
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-

build.sh

@@ -1,91 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-
-echo ""
-
-if [ $# -gt 1 -a "$1" == "push" ]
-then
-    TAG=$2
-
-    echo "Pushing Bitwarden ($TAG)"
-    echo "========================"
-
-    docker push bitwarden/api:$TAG
-    docker push bitwarden/identity:$TAG
-    docker push bitwarden/server:$TAG
-    docker push bitwarden/attachments:$TAG
-    docker push bitwarden/icons:$TAG
-    docker push bitwarden/notifications:$TAG
-    docker push bitwarden/events:$TAG
-    docker push bitwarden/admin:$TAG
-    docker push bitwarden/nginx:$TAG
-    docker push bitwarden/k8s-proxy:$TAG
-    docker push bitwarden/sso:$TAG
-    docker push bitwarden/portal:$TAG
-    docker push bitwarden/mssql:$TAG
-    docker push bitwarden/setup:$TAG
-elif [ $# -gt 1 -a "$1" == "tag" ]
-then
-    TAG=$2
-
-    echo "Tagging Bitwarden as '$TAG'"
-
-    docker tag bitwarden/api bitwarden/api:$TAG
-    docker tag bitwarden/identity bitwarden/identity:$TAG
-    docker tag bitwarden/server bitwarden/server:$TAG
-    docker tag bitwarden/attachments bitwarden/attachments:$TAG
-    docker tag bitwarden/icons bitwarden/icons:$TAG
-    docker tag bitwarden/notifications bitwarden/notifications:$TAG
-    docker tag bitwarden/events bitwarden/events:$TAG
-    docker tag bitwarden/admin bitwarden/admin:$TAG
-    docker tag bitwarden/nginx bitwarden/nginx:$TAG
-    docker tag bitwarden/nginx bitwarden/k8s-proxy:$TAG
-    docker tag bitwarden/sso bitwarden/sso:$TAG
-    docker tag bitwarden/portal bitwarden/portal:$TAG
-    docker tag bitwarden/mssql bitwarden/mssql:$TAG
-    docker tag bitwarden/setup bitwarden/setup:$TAG
-else
-    echo "Building Bitwarden"
-    echo "=================="
-
-    chmod u+x "$DIR/src/Api/build.sh"
-    "$DIR/src/Api/build.sh"
-
-    chmod u+x "$DIR/src/Identity/build.sh"
-    "$DIR/src/Identity/build.sh"
-
-    chmod u+x "$DIR/util/Server/build.sh"
-    "$DIR/util/Server/build.sh"
-
-    chmod u+x "$DIR/util/Nginx/build.sh"
-    "$DIR/util/Nginx/build.sh"
-
-    chmod u+x "$DIR/util/Attachments/build.sh"
-    "$DIR/util/Attachments/build.sh"
-
-    chmod u+x "$DIR/src/Icons/build.sh"
-    "$DIR/src/Icons/build.sh"
-
-    chmod u+x "$DIR/src/Notifications/build.sh"
-    "$DIR/src/Notifications/build.sh"
-
-    chmod u+x "$DIR/src/Events/build.sh"
-    "$DIR/src/Events/build.sh"
-
-    chmod u+x "$DIR/src/Admin/build.sh"
-    "$DIR/src/Admin/build.sh"
-
-    chmod u+x "$DIR/bitwarden_license/src/Sso/build.sh"
-    "$DIR/bitwarden_license/src/Sso/build.sh"
-
-    chmod u+x "$DIR/bitwarden_license/src/Portal/build.sh"
-    "$DIR/bitwarden_license/src/Portal/build.sh"
-
-    chmod u+x "$DIR/util/MsSql/build.sh"
-    "$DIR/util/MsSql/build.sh"
-
-    chmod u+x "$DIR/util/Setup/build.sh"
-    "$DIR/util/Setup/build.sh"
-fi