From 0faf97331fd104d32f60ba0cb748c3d9b48247b0 Mon Sep 17 00:00:00 2001 From: Vince Grassia <593223+vgrassia@users.noreply.github.com> Date: Thu, 27 May 2021 10:32:36 -0400 Subject: [PATCH] Update Build and Release workflows (#1349) * Fix K8S-Proxy image build and tag * Update build script to remove unused PUSH and TAG features * Update workflows - Remove build.sh from root of repo as it's no longer used - Update workflows to use matrix feature for parallelization * Add Docker image build step and other projects * Remove csproj filenames from steps * Fix typo * Fix directory for k8s-proxy image * Move Docker stub and uploads out of matrix * Change to block-style sequence for service_name matrix * Add missing quote character * Fix working directories of Dotnet steps * Update step name * Add checkout repo step to Upload job * Add 'dotnet tool restore' step * Fix Swagger directory * Fix Swagger build step * Fix Swagger build step (again) --- .github/workflows/build.yml | 178 ++++++++++++++++++++++++++-------- .github/workflows/release.yml | 148 +++++++++++++--------------- build.sh | 91 ----------------- 3 files changed, 207 insertions(+), 210 deletions(-) delete mode 100755 build.sh diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 77897e2eae..22f29a35ba 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -10,12 +10,13 @@ on: jobs: cloc: + name: CLOC runs-on: ubuntu-latest steps: - name: Checkout repo uses: actions/checkout@5a4ac9002d0be2fb38bd78e4b4dbde5606d7042f - - name: Set up cloc + - name: Install cloc run: | sudo apt-get update sudo apt-get -y install cloc @@ -24,12 +25,13 @@ jobs: run: cloc --include-lang C#,SQL,Razor,"Bourne Shell",PowerShell,HTML,CSS,Sass,JavaScript,TypeScript --vcs git testing: + name: Testing runs-on: windows-latest steps: - name: Set up NuGet uses: nuget/setup-nuget@04b0c2b8d1b97922f67eca497d7cf0bf17b8ffe1 with: - nuget-version: 'latest' + nuget-version: '5' - name: Set up MSBuild uses: microsoft/setup-msbuild@c26a08ba26249b81327e26f6ef381897b6a8754d @@ -48,9 +50,6 @@ jobs: npm --version echo "GitHub ref: $GITHUB_REF" echo "GitHub event: $GITHUB_EVENT" - env: - GITHUB_REF: ${{ github.ref }} - GITHUB_EVENT: ${{ github.event_name }} - name: Checkout repo uses: actions/checkout@5a4ac9002d0be2fb38bd78e4b4dbde5606d7042f @@ -68,17 +67,66 @@ jobs: shell: pwsh build: + name: Build runs-on: ubuntu-latest needs: testing + strategy: + fail-fast: false + matrix: + include: + - service_name: Admin + base_path: ./src + dotnet: true + gulp: true + - service_name: Api + base_path: ./src + dotnet: true + - service_name: Attachments + base_path: ./util + - service_name: Billing + base_path: ./src + dotnet: true + - service_name: Events + base_path: ./src + dotnet: true + - service_name: Icons + base_path: ./src + dotnet: true + - service_name: Identity + base_path: ./src + dotnet: true + - service_name: K8S-Proxy + base_path: ./util + - service_name: MsSql + base_path: ./util + - service_name: Nginx + base_path: ./util + - service_name: Notifications + base_path: ./src + dotnet: true + - service_name: Portal + base_path: ./bitwarden_license/src + dotnet: true + gulp: true + - service_name: Server + base_path: ./util + dotnet: true + - service_name: Setup + base_path: ./util + dotnet: true + - service_name: Sso + base_path: ./bitwarden_license/src + dotnet: true + gulp: true steps: + - name: Checkout repo + uses: actions/checkout@5a4ac9002d0be2fb38bd78e4b4dbde5606d7042f + - name: Set up Node uses: actions/setup-node@46071b5c7a2e0c34e49c3cb8a0e792e86e18d5ea with: node-version: '14' - - name: Set up gulp - run: npm i -g gulp - - name: Print environment run: | whoami @@ -89,9 +137,6 @@ jobs: docker --version echo "GitHub ref: $GITHUB_REF" echo "GitHub event: $GITHUB_EVENT" - env: - GITHUB_REF: ${{ github.ref }} - GITHUB_EVENT: ${{ github.event_name }} - name: Login to Azure uses: Azure/login@77f1b2e3fb80c0e8645114159d17008b8a2e475a @@ -108,7 +153,7 @@ jobs: dct-delegate-2-repo-passphrase, dct-delegate-2-key" - - name: Log into docker + - name: Log into Docker if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/rc' run: echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin env: @@ -125,44 +170,99 @@ jobs: DCT_DELEGATION_KEY_ID: "c9bde8ec820701516491e5e03d3a6354e7bd66d05fa3df2b0062f68b116dc59c" DCT_DELEGATE_KEY: ${{ steps.retrieve-secrets.outputs.dct-delegate-2-key }} - - name: Checkout repo - uses: actions/checkout@5a4ac9002d0be2fb38bd78e4b4dbde5606d7042f - - - name: Restore - run: dotnet tool restore - - - name: Build + - name: Set up Gulp + if: ${{ matrix.gulp }} + working-directory: ${{ matrix.base_path }}/${{ matrix.service_name }} run: | - chmod +x ./build.sh - ./build.sh + npm install -g gulp - - name: Tag rc branch + - name: Restore/Clean service + if: ${{ matrix.dotnet }} + working-directory: ${{ matrix.base_path }}/${{ matrix.service_name }} + run: | + echo "Restore" + dotnet restore + echo "Clean" + dotnet clean -c "Release" -o obj/Azure/publish + + - name: Execute Gulp + if: ${{ matrix.gulp }} + working-directory: ${{ matrix.base_path }}/${{ matrix.service_name }} + run: | + npm install + gulp --gulpfile gulpfile.js build + + - name: Publish service + if: ${{ matrix.dotnet }} + working-directory: ${{ matrix.base_path }}/${{ matrix.service_name }} + run: | + echo "Publish" + dotnet publish -c "Release" -o obj/Azure/publish + + cd obj/Azure/publish + zip -r ${{ matrix.service_name }}.zip . + mv ${{ matrix.service_name }}.zip ../../../ + + - name: Build Docker images + if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/rc' + run: | + SERVICE_NAME=$(echo "${{ matrix.service_name }}" | tr '[:upper:]' '[:lower:]') + if [ "$SERVICE_NAME" = "k8s-proxy" ]; then + docker build -f ${{ matrix.base_path }}/Nginx/Dockerfile-k8s -t bitwarden/$SERVICE_NAME ${{ matrix.base_path }}/Nginx + else + docker build -t bitwarden/$SERVICE_NAME ${{ matrix.base_path }}/$SERVICE_NAME + fi + + - name: Tag rc if: github.ref == 'refs/heads/rc' - run: ./build.sh tag rc + run: | + SERVICE_NAME=$(echo "${{ matrix.service_name }}" | tr '[:upper:]' '[:lower:]') + docker tag bitwarden/$SERVICE_NAME bitwarden/$SERVICE_NAME:rc - name: Tag dev if: github.ref == 'refs/heads/master' - run: ./build.sh tag dev + run: | + SERVICE_NAME=$(echo "${{ matrix.service_name }}" | tr '[:upper:]' '[:lower:]') + docker tag bitwarden/$SERVICE_NAME bitwarden/$SERVICE_NAME:dev - - name: List docker images + - name: List Docker images if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/rc' run: docker images - name: Push rc images if: github.ref == 'refs/heads/rc' - run: ./build.sh push rc + run: | + SERVICE_NAME=$(echo "${{ matrix.service_name }}" | tr '[:upper:]' '[:lower:]') + docker push bitwarden/$SERVICE_NAME:rc env: DOCKER_CONTENT_TRUST: 1 DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE: ${{ steps.retrieve-secrets.outputs.dct-delegate-2-repo-passphrase }} - name: Push dev images if: github.ref == 'refs/heads/master' - run: ./build.sh push dev + run: | + SERVICE_NAME=$(echo "${{ matrix.service_name }}" | tr '[:upper:]' '[:lower:]') + docker push bitwarden/$SERVICE_NAME:dev env: DOCKER_CONTENT_TRUST: 1 DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE: ${{ steps.retrieve-secrets.outputs.dct-delegate-2-repo-passphrase }} - - name: Make docker stub + - name: Log out of Docker + if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/rc' + run: docker logout + + upload: + name: Upload + runs-on: ubuntu-latest + needs: build + steps: + - name: Checkout repo + uses: actions/checkout@5a4ac9002d0be2fb38bd78e4b4dbde5606d7042f + + - name: Restore + run: dotnet tool restore + + - name: Make Docker stub run: | STUB_OUTPUT=$(pwd)/docker-stub docker run -i --rm --name setup -v $STUB_OUTPUT:/bitwarden bitwarden/setup:dev \ @@ -173,29 +273,31 @@ jobs: touch $STUB_OUTPUT/env/uid.env cd docker-stub; zip -r ../docker-stub.zip *; cd .. - - name: Upload docker stub artifact + - name: Upload Docker stub artifact uses: actions/upload-artifact@ee69f02b3dfdecd58bb31b4d133da38ba6fe3700 with: name: docker-stub.zip path: ./docker-stub.zip - - name: Build swagger + - name: Build Swagger run: | cd ./src/Api - dotnet swagger tofile --output ../../swagger.json --host https://api.bitwarden.com ` - ./obj/Docker/publish/Api/Api.dll public + echo "Restore" + dotnet restore + echo "Clean" + dotnet clean -c "Release" -o obj/Azure/publish + echo "Publish" + dotnet publish -c "Release" -o obj/Azure/publish + + dotnet swagger tofile --output ../../swagger.json --host https://api.bitwarden.com \ + ./obj/Azure/publish/Api.dll public cd ../.. - shell: pwsh env: ASPNETCORE_ENVIRONMENT: Production swaggerGen: 'True' - - name: Upload swagger artifact + - name: Upload Swagger artifact uses: actions/upload-artifact@ee69f02b3dfdecd58bb31b4d133da38ba6fe3700 with: name: swagger.json path: ./swagger.json - - - name: Log out of docker - if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/rc' - run: docker logout diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 39683051c1..e7f6690275 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -9,6 +9,7 @@ on: jobs: setup: + name: Setup runs-on: ubuntu-latest outputs: release_upload_url: ${{ steps.create_release.outputs.upload_url }} @@ -53,8 +54,27 @@ jobs: prerelease: false release: + name: Release runs-on: ubuntu-latest needs: setup + strategy: + fail-fast: false + matrix: + service_name: + - admin + - api + - attachments + - events + - icons + - identity + - k8s-proxy + - mssql + - nginx + - notifications + - portal + - server + - setup + - sso env: RELEASE_VERSION: ${{ needs.setup.outputs.release_version }} TAG_VERSION: ${{ needs.setup.outputs.tag_version }} @@ -65,9 +85,6 @@ jobs: docker --version echo "GitHub ref: $GITHUB_REF" echo "GitHub event: $GITHUB_EVENT" - env: - GITHUB_REF: ${{ github.ref }} - GITHUB_EVENT: ${{ github.event_name }} - name: Login to Azure uses: Azure/login@77f1b2e3fb80c0e8645114159d17008b8a2e475a @@ -84,7 +101,7 @@ jobs: dct-delegate-2-repo-passphrase, dct-delegate-2-key" - - name: Log into docker + - name: Log into Docker if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/rc' run: echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin env: @@ -92,6 +109,7 @@ jobs: DOCKER_PASSWORD: ${{ steps.retrieve-secrets.outputs.docker-password }} - name: Setup Docker Trust + if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/rc' run: | mkdir -p ~/.docker/trust/private @@ -103,73 +121,46 @@ jobs: - name: Checkout repo uses: actions/checkout@5a4ac9002d0be2fb38bd78e4b4dbde5606d7042f + - name: Pull Docker images + run: | + docker pull bitwarden/${{ matrix.service_name }}:rc + env: + DOCKER_CONTENT_TRUST: 1 + + - name: Re-tag Docker images + run: | + docker tag bitwarden/${{ matrix.service_name }}:rc bitwarden/${{ matrix.service_name }}:latest + docker tag bitwarden/${{ matrix.service_name }}:rc bitwarden/${{ matrix.service_name }}:$RELEASE_VERSION + + - name: List Docker images + run: docker images + + - name: Push Docker images + if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/rc' + run: | + docker push bitwarden/${{ matrix.service_name }}:latest + docker push bitwarden/${{ matrix.service_name }}:$RELEASE_VERSION + env: + DOCKER_CONTENT_TRUST: 1 + DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE: ${{ steps.retrieve-secrets.outputs.dct-delegate-2-repo-passphrase }} + + - name: Log out of Docker + if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/rc' + run: docker logout + + + upload: + name: Upload + runs-on: ubuntu-latest + needs: release + steps: + - name: Checkout repo + uses: actions/checkout@5a4ac9002d0be2fb38bd78e4b4dbde5606d7042f + - name: Restore run: dotnet tool restore - - name: pull docker images - run: | - docker pull bitwarden/api:rc - docker pull bitwarden/identity:rc - docker pull bitwarden/server:rc - docker pull bitwarden/attachments:rc - docker pull bitwarden/icons:rc - docker pull bitwarden/notifications:rc - docker pull bitwarden/events:rc - docker pull bitwarden/admin:rc - docker pull bitwarden/nginx:rc - docker pull bitwarden/nginx:rc - docker pull bitwarden/sso:rc - docker pull bitwarden/portal:rc - docker pull bitwarden/mssql:rc - docker pull bitwarden/setup:rc - env: - DOCKER_CONTENT_TRUST: 1 - - - - name: re-tag docker images - run: | - tags=( latest beta ${RELEASE_VERSION} ) - for TAG in "${tags[@]}" - do - docker tag bitwarden/api:rc bitwarden/api:$TAG - docker tag bitwarden/identity:rc bitwarden/identity:$TAG - docker tag bitwarden/server:rc bitwarden/server:$TAG - docker tag bitwarden/attachments:rc bitwarden/attachments:$TAG - docker tag bitwarden/icons:rc bitwarden/icons:$TAG - docker tag bitwarden/notifications:rc bitwarden/notifications:$TAG - docker tag bitwarden/events:rc bitwarden/events:$TAG - docker tag bitwarden/admin:rc bitwarden/admin:$TAG - docker tag bitwarden/nginx:rc bitwarden/nginx:$TAG - docker tag bitwarden/nginx:rc bitwarden/k8s-proxy:$TAG - docker tag bitwarden/sso:rc bitwarden/sso:$TAG - docker tag bitwarden/portal:rc bitwarden/portal:$TAG - docker tag bitwarden/mssql:rc bitwarden/mssql:$TAG - docker tag bitwarden/setup:rc bitwarden/setup:$TAG - done - - - name: List docker images - run: docker images - - - name: Push beta images - run: ./build.sh push beta - env: - DOCKER_CONTENT_TRUST: 1 - DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE: ${{ steps.retrieve-secrets.outputs.dct-delegate-2-repo-passphrase }} - - - name: Push latest images - run: ./build.sh push latest - env: - DOCKER_CONTENT_TRUST: 1 - DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE: ${{ steps.retrieve-secrets.outputs.dct-delegate-2-repo-passphrase }} - - - name: Push version images - run: ./build.sh push $($env:$RELEASE_VERSION) - shell: pwsh - env: - DOCKER_CONTENT_TRUST: 1 - DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE: ${{ steps.retrieve-secrets.outputs.dct-delegate-2-repo-passphrase }} - - - name: Make docker stub + - name: Make Docker stub run: | STUB_OUTPUT=$(pwd)/docker-stub docker run -i --rm --name setup -v $STUB_OUTPUT:/bitwarden bitwarden/setup:dev \ @@ -180,39 +171,35 @@ jobs: touch $STUB_OUTPUT/env/uid.env cd docker-stub; zip -r ../docker-stub.zip *; cd .. - - name: Upload docker stub artifact + - name: Upload Docker stub artifact uses: actions/upload-artifact@ee69f02b3dfdecd58bb31b4d133da38ba6fe3700 with: name: docker-stub.zip path: ./docker-stub.zip - - name: Build swagger + - name: Build Swagger run: | cd ./src/Api echo "Restore" - dotnet restore "Api.csproj" + dotnet restore echo "Clean" - dotnet clean "Api.csproj" -c "Release" -o "obj/Docker/publish/Api" + dotnet clean -c "Release" -o obj/Azure/publish echo "Publish" - dotnet publish "Api.csproj" -c "Release" -o "obj/Docker/publish/Api" + dotnet publish -c "Release" -o obj/Azure/publish - dotnet swagger tofile --output ../../swagger.json --host https://api.bitwarden.com ` - ./obj/Docker/publish/Api/Api.dll public + dotnet swagger tofile --output ../../swagger.json --host https://api.bitwarden.com \ + ./obj/Azure/publish/Api.dll public cd ../.. - shell: pwsh env: ASPNETCORE_ENVIRONMENT: Production swaggerGen: 'True' - - name: Upload swagger artifact + - name: Upload Swagger artifact uses: actions/upload-artifact@ee69f02b3dfdecd58bb31b4d133da38ba6fe3700 with: name: swagger.json path: ./swagger.json - - name: Log out of docker - run: docker logout - - name: Upload release assets if: github.event_name == 'release' run: | @@ -223,4 +210,3 @@ jobs: $RELEASE_TAG_NAME env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - diff --git a/build.sh b/build.sh deleted file mode 100755 index db563df889..0000000000 --- a/build.sh +++ /dev/null @@ -1,91 +0,0 @@ -#!/usr/bin/env bash -set -e - -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - -echo "" - -if [ $# -gt 1 -a "$1" == "push" ] -then - TAG=$2 - - echo "Pushing Bitwarden ($TAG)" - echo "========================" - - docker push bitwarden/api:$TAG - docker push bitwarden/identity:$TAG - docker push bitwarden/server:$TAG - docker push bitwarden/attachments:$TAG - docker push bitwarden/icons:$TAG - docker push bitwarden/notifications:$TAG - docker push bitwarden/events:$TAG - docker push bitwarden/admin:$TAG - docker push bitwarden/nginx:$TAG - docker push bitwarden/k8s-proxy:$TAG - docker push bitwarden/sso:$TAG - docker push bitwarden/portal:$TAG - docker push bitwarden/mssql:$TAG - docker push bitwarden/setup:$TAG -elif [ $# -gt 1 -a "$1" == "tag" ] -then - TAG=$2 - - echo "Tagging Bitwarden as '$TAG'" - - docker tag bitwarden/api bitwarden/api:$TAG - docker tag bitwarden/identity bitwarden/identity:$TAG - docker tag bitwarden/server bitwarden/server:$TAG - docker tag bitwarden/attachments bitwarden/attachments:$TAG - docker tag bitwarden/icons bitwarden/icons:$TAG - docker tag bitwarden/notifications bitwarden/notifications:$TAG - docker tag bitwarden/events bitwarden/events:$TAG - docker tag bitwarden/admin bitwarden/admin:$TAG - docker tag bitwarden/nginx bitwarden/nginx:$TAG - docker tag bitwarden/nginx bitwarden/k8s-proxy:$TAG - docker tag bitwarden/sso bitwarden/sso:$TAG - docker tag bitwarden/portal bitwarden/portal:$TAG - docker tag bitwarden/mssql bitwarden/mssql:$TAG - docker tag bitwarden/setup bitwarden/setup:$TAG -else - echo "Building Bitwarden" - echo "==================" - - chmod u+x "$DIR/src/Api/build.sh" - "$DIR/src/Api/build.sh" - - chmod u+x "$DIR/src/Identity/build.sh" - "$DIR/src/Identity/build.sh" - - chmod u+x "$DIR/util/Server/build.sh" - "$DIR/util/Server/build.sh" - - chmod u+x "$DIR/util/Nginx/build.sh" - "$DIR/util/Nginx/build.sh" - - chmod u+x "$DIR/util/Attachments/build.sh" - "$DIR/util/Attachments/build.sh" - - chmod u+x "$DIR/src/Icons/build.sh" - "$DIR/src/Icons/build.sh" - - chmod u+x "$DIR/src/Notifications/build.sh" - "$DIR/src/Notifications/build.sh" - - chmod u+x "$DIR/src/Events/build.sh" - "$DIR/src/Events/build.sh" - - chmod u+x "$DIR/src/Admin/build.sh" - "$DIR/src/Admin/build.sh" - - chmod u+x "$DIR/bitwarden_license/src/Sso/build.sh" - "$DIR/bitwarden_license/src/Sso/build.sh" - - chmod u+x "$DIR/bitwarden_license/src/Portal/build.sh" - "$DIR/bitwarden_license/src/Portal/build.sh" - - chmod u+x "$DIR/util/MsSql/build.sh" - "$DIR/util/MsSql/build.sh" - - chmod u+x "$DIR/util/Setup/build.sh" - "$DIR/util/Setup/build.sh" -fi