[DEVOPS-1519] Add transition mode to mssql migrator utility (#3259)

* Add RerunableSqlTableJournal

* Add extension to use rerunable sql table journal

* Use rerunable sql journal

* format

* Enable logging

* FIx

* Disable logging

* Rename to SqlTableJournalExtensions

* Move RerunableSqlTableJournal to Extension class

* Fix usings

* Add rerunable schema

* Format

* Fix typo

* Enable logging in db migrator

* add rerunable column in dbo migrations table migration

* Trying

* Fix journal table name

* Trying to migrate first

* After migration

* Testing

* Add update from rerunable to not rerunable script

* Change name

* Add rerunable option and script folder name

* Add rerunable options and folder

* Fix

* Add transition (aka rerunable) migrations to Setup

* Parse parameters on migrator utility

* Fix sql scripts

* Remove CreateSchemaTableSql as it'll be migrated using migration

* Embed dbScripts_data_migration folder

* Remove testing sql script

* Add optins parsing nuget for msSqlMigratorUtility

* Fix sql journal

* Ran dotnet format

* Comment out index

* ▫️Revert "Comment out index"

This reverts commit df15fa91e05d5b195e130c36e5ae51fc508b2506.

* Disable logging

* Add newline

* Rename rerunable to repeatable

* remove repeatable journal

* Remove migration adding the repeatable column in  dbo.Migrations table

* Add using

* Enable log for testing

* Disable logging in the setup

* Remove unused method

* Add migrator constants

* Use constants in yet another place

* Fix

* Add constant

* Fix

* Fix

util/Migrator/DbMigrator.cs

@@ -2,6 +2,7 @@
 using System.Reflection;
 using Bit.Core;
 using DbUp;
+using DbUp.Helpers;
 using Microsoft.Data.SqlClient;
 using Microsoft.Extensions.Logging;
 
@@ -24,6 +25,8 @@ public class DbMigrator
     }
 
     public bool MigrateMsSqlDatabaseWithRetries(bool enableLogging = true,
+        bool repeatable = false,
+        string folderName = MigratorConstants.DefaultMigrationsFolderName,
         CancellationToken cancellationToken = default(CancellationToken))
     {
         var attempt = 1;
@@ -32,7 +35,7 @@ public class DbMigrator
         {
             try
             {
-                var success = MigrateDatabase(enableLogging, cancellationToken);
+                var success = MigrateDatabase(enableLogging, repeatable, folderName, cancellationToken);
                 return success;
             }
             catch (SqlException ex)
@@ -54,6 +57,8 @@ public class DbMigrator
     }
 
     public bool MigrateDatabase(bool enableLogging = true,
+        bool repeatable = false,
+        string folderName = MigratorConstants.DefaultMigrationsFolderName,
         CancellationToken cancellationToken = default(CancellationToken))
     {
         if (_logger != null)
@@ -98,12 +103,20 @@ public class DbMigrator
         cancellationToken.ThrowIfCancellationRequested();
         var builder = DeployChanges.To
             .SqlDatabase(_connectionString)
-            .JournalToSqlTable("dbo", "Migration")
             .WithScriptsAndCodeEmbeddedInAssembly(Assembly.GetExecutingAssembly(),
-                s => s.Contains($".DbScripts.") && !s.Contains(".Archive."))
+                s => s.Contains($".{folderName}.") && !s.Contains(".Archive."))
             .WithTransaction()
             .WithExecutionTimeout(new TimeSpan(0, 5, 0));
 
+        if (repeatable)
+        {
+            builder.JournalTo(new NullJournal());
+        }
+        else
+        {
+            builder.JournalToSqlTable("dbo", MigratorConstants.SqlTableJournalName);
+        }
+
         if (enableLogging)
         {
             if (_logger != null)

util/Migrator/Migrator.csproj

@@ -2,6 +2,7 @@
 
   <ItemGroup>
     <EmbeddedResource Include="DbScripts\**\*.sql" />
+    <EmbeddedResource Include="DbScripts_data_migration\**\*.sql" />
   </ItemGroup>
 
   <ItemGroup>

util/Migrator/MigratorConstants.cs

@@ -0,0 +1,8 @@
+namespace Bit.Migrator;
+
+public static class MigratorConstants
+{
+    public const string SqlTableJournalName = "Migration";
+    public const string DefaultMigrationsFolderName = "DbScripts";
+    public const string TransitionMigrationsFolderName = "DbScripts_data_migration";
+}

util/Migrator/SqlServerDbMigrator.cs

@@ -70,7 +70,7 @@ public class SqlServerDbMigrator : IDbMigrator
         cancellationToken.ThrowIfCancellationRequested();
         var builder = DeployChanges.To
             .SqlDatabase(_connectionString)
-            .JournalToSqlTable("dbo", "Migration")
+            .JournalToSqlTable("dbo", MigratorConstants.SqlTableJournalName)
             .WithScriptsAndCodeEmbeddedInAssembly(Assembly.GetExecutingAssembly(),
                 s => s.Contains($".DbScripts.") && !s.Contains(".Archive."))
             .WithTransaction()

util/MsSqlMigratorUtility/MsSqlMigratorUtility.csproj

@@ -10,6 +10,7 @@
   </ItemGroup>
 
   <ItemGroup>
+    <PackageReference Include="CommandDotNet" Version="7.0.2" />
     <PackageReference Include="Microsoft.Extensions.Logging" Version="6.0.0" />
     <PackageReference Include="Microsoft.Extensions.Logging.Console" Version="6.0.0" />
   </ItemGroup>

util/MsSqlMigratorUtility/Program.cs

@@ -1,55 +1,51 @@
 using Bit.Migrator;
+using CommandDotNet;
 using Microsoft.Extensions.Logging;
 
 internal class Program
 {
+    private static IDictionary<string, string> Parameters { get; set; }
+
     private static int Main(string[] args)
     {
-        if (args.Length == 0)
+        return new AppRunner<Program>().Run(args);
-        {
-            Console.WriteLine("Please enter a database connection string argument.");
-            WriteUsageToConsole();
-            return 1;
-        }
-
-        if (args.Length == 1 && (args[0] == "--verbose" || args[0] == "-v"))
-        {
-            Console.WriteLine($"Please enter a database connection string argument before {args[0]} option.");
-            WriteUsageToConsole();
-            return 1;
-        }
-
-        var databaseConnectionString = args[0];
-
-        var verbose = false;
-
-        if (args.Length == 2 && (args[1] == "--verbose" || args[1] == "-v"))
-        {
-            verbose = true;
-        }
-
-        var success = MigrateDatabase(databaseConnectionString, verbose);
-
-        if (!success)
-        {
-            return -1;
-        }
-
-        return 0;
     }
 
+    [DefaultCommand]
+    public void Execute(
+        [Operand(Description = "Database connection string")]
+        string databaseConnectionString,
+        [Option('v', "verbose", Description = "Enable verbose output of migrator logs")]
+        bool verbose = false,
+        [Option('r', "repeatable", Description = "Mark scripts as repeatable")]
+        bool repeatable = false,
+        [Option('f', "folder", Description = "Folder name of database scripts")]
+        string folderName = MigratorConstants.DefaultMigrationsFolderName) => MigrateDatabase(databaseConnectionString, verbose, repeatable, folderName);
+
     private static void WriteUsageToConsole()
     {
         Console.WriteLine("Usage: MsSqlMigratorUtility <database-connection-string>");
         Console.WriteLine("Usage: MsSqlMigratorUtility <database-connection-string> -v|--verbose (for verbose output of migrator logs)");
+        Console.WriteLine("Usage: MsSqlMigratorUtility <database-connection-string> -r|--repeatable (for marking scripts as repeatable) -f|--folder <folder-name-in-migrator-project> (for specifying folder name of scripts)");
+        Console.WriteLine("Usage: MsSqlMigratorUtility <database-connection-string> -v|--verbose (for verbose output of migrator logs) -r|--repeatable (for marking scripts as repeatable) -f|--folder <folder-name-in-migrator-project> (for specifying folder name of scripts)");
     }
 
-    private static bool MigrateDatabase(string databaseConnectionString, bool verbose = false, int attempt = 1)
+    private static bool MigrateDatabase(string databaseConnectionString, bool verbose = false, bool repeatable = false, string folderName = "")
     {
         var logger = CreateLogger(verbose);
 
+        logger.LogInformation($"Migrating database with repeatable: {repeatable} and folderName: {folderName}.");
+
         var migrator = new DbMigrator(databaseConnectionString, logger);
-        var success = migrator.MigrateMsSqlDatabaseWithRetries(verbose);
+        bool success = false;
+        if (!string.IsNullOrWhiteSpace(folderName))
+        {
+            success = migrator.MigrateMsSqlDatabaseWithRetries(verbose, repeatable, folderName);
+        }
+        else
+        {
+            success = migrator.MigrateMsSqlDatabaseWithRetries(verbose, repeatable);
+        }
 
         return success;
     }

util/MsSqlMigratorUtility/packages.lock.json

@@ -2,6 +2,15 @@
   "version": 1,
   "dependencies": {
     "net6.0": {
+      "CommandDotNet": {
+        "type": "Direct",
+        "requested": "[7.0.2, )",
+        "resolved": "7.0.2",
+        "contentHash": "sFfqn4T6Ux4AbGnhS+BZvf9iVeP6b9p9bwaMT8nsefVcYH2tC9MHj3AoY+GG0nnBPmFawRqdgud08cBjBdPByQ==",
+        "dependencies": {
+          "Microsoft.CSharp": "4.7.0"
+        }
+      },
       "Microsoft.Extensions.Logging": {
         "type": "Direct",
         "requested": "[6.0.0, )",

util/Setup/Program.cs

@@ -190,7 +190,12 @@ public class Program
         var vaultConnectionString = Helpers.GetValueFromEnvFile("global",
             "globalSettings__sqlServer__connectionString");
         var migrator = new DbMigrator(vaultConnectionString, null);
-        migrator.MigrateMsSqlDatabaseWithRetries(false);
+
+        var log = false;
+
+        migrator.MigrateMsSqlDatabaseWithRetries(log);
+
+        migrator.MigrateMsSqlDatabaseWithRetries(log, true, MigratorConstants.TransitionMigrationsFolderName);
     }
 
     private static bool ValidateInstallation()