From 107e2fc0c379ba1b4ce3b6b834e2b3c19dba161e Mon Sep 17 00:00:00 2001 From: Hinton Date: Thu, 1 May 2025 13:43:17 +0200 Subject: [PATCH] Cleanup and re-organize the code slightly --- .../OrganizationUsersControllerTests.cs | 4 - util/DbSeederUtility/DbSeederUtility.csproj | 4 +- .../GlobalSettingsFactory.cs | 24 +--- util/DbSeederUtility/Program.cs | 23 +++- util/DbSeederUtility/README.md | 115 +--------------- .../ServiceCollectionExtension.cs | 25 ++++ util/Seeder/Commands/GenerateCommand.cs | 46 ------- util/Seeder/README.md | 128 ++---------------- util/Seeder/Seeder.csproj | 17 +-- 9 files changed, 59 insertions(+), 327 deletions(-) rename util/{Seeder/Settings => DbSeederUtility}/GlobalSettingsFactory.cs (74%) create mode 100644 util/DbSeederUtility/ServiceCollectionExtension.cs delete mode 100644 util/Seeder/Commands/GenerateCommand.cs diff --git a/test/Api.IntegrationTest/AdminConsole/Controllers/OrganizationUsersControllerTests.cs b/test/Api.IntegrationTest/AdminConsole/Controllers/OrganizationUsersControllerTests.cs index fa433db038..21c9bb3cb8 100644 --- a/test/Api.IntegrationTest/AdminConsole/Controllers/OrganizationUsersControllerTests.cs +++ b/test/Api.IntegrationTest/AdminConsole/Controllers/OrganizationUsersControllerTests.cs @@ -75,10 +75,6 @@ public class OrganizationUsersControllerTest : IClassFixture>(); - // Assert.NotNull(result?.Data); - // Assert.Equal(600001, result.Data.Count()); } } diff --git a/util/DbSeederUtility/DbSeederUtility.csproj b/util/DbSeederUtility/DbSeederUtility.csproj index 30aec53768..90ac7f22b4 100644 --- a/util/DbSeederUtility/DbSeederUtility.csproj +++ b/util/DbSeederUtility/DbSeederUtility.csproj @@ -17,8 +17,6 @@ - - - \ No newline at end of file + diff --git a/util/Seeder/Settings/GlobalSettingsFactory.cs b/util/DbSeederUtility/GlobalSettingsFactory.cs similarity index 74% rename from util/Seeder/Settings/GlobalSettingsFactory.cs rename to util/DbSeederUtility/GlobalSettingsFactory.cs index 4b593327ca..6971a752fe 100644 --- a/util/Seeder/Settings/GlobalSettingsFactory.cs +++ b/util/DbSeederUtility/GlobalSettingsFactory.cs @@ -1,7 +1,7 @@ using Bit.Core.Settings; using Microsoft.Extensions.Configuration; -namespace Bit.Seeder.Settings; +namespace Bit.DbSeederUtility; public static class GlobalSettingsFactory { @@ -63,25 +63,3 @@ public static class GlobalSettingsFactory return settings; } } - -// Non-static version that can accept command-line arguments -public class GlobalSettingsFactoryWithArgs -{ - public GlobalSettings GlobalSettings { get; } - - public GlobalSettingsFactoryWithArgs(string[] args) - { - GlobalSettings = new GlobalSettings(); - - var config = new ConfigurationBuilder() - .SetBasePath(Directory.GetCurrentDirectory()) - .AddJsonFile("appsettings.json", optional: true, reloadOnChange: true) - .AddJsonFile($"appsettings.{Environment.GetEnvironmentVariable("ASPNETCORE_ENVIRONMENT") ?? "Production"}.json", optional: true, reloadOnChange: true) - .AddUserSecrets("bitwarden-Api") - .AddCommandLine(args) - .AddEnvironmentVariables() - .Build(); - - config.GetSection("globalSettings").Bind(GlobalSettings); - } -} diff --git a/util/DbSeederUtility/Program.cs b/util/DbSeederUtility/Program.cs index 37e7ac2760..df6516ae7b 100644 --- a/util/DbSeederUtility/Program.cs +++ b/util/DbSeederUtility/Program.cs @@ -1,6 +1,7 @@ -using Bit.Seeder.Commands; -using Bit.Seeder.Settings; +using Bit.Infrastructure.EntityFramework.Repositories; +using Bit.Seeder.Recipes; using CommandDotNet; +using Microsoft.Extensions.DependencyInjection; namespace Bit.DbSeederUtility; @@ -16,18 +17,26 @@ public class Program } [Command("organization", Description = "Seed an organization and organization users")] - public int Organization( + public void Organization( [Option('n', "Name", Description = "Name of organization")] string name, - [Option('u', "users", Description = "Number of users to generate")] int users, - [Option('d', "domain", Description = "Email domain for users")] string domain ) { - var generateCommand = new GenerateCommand(); - return generateCommand.Execute(name, users, domain) ? 0 : 1; + // Create service provider with necessary services + var services = new ServiceCollection(); + ServiceCollectionExtension.ConfigureServices(services); + var serviceProvider = services.BuildServiceProvider(); + + // TODO: Can we remove GenerateCommand and provide a RecipeFactory or something. Or wire up DI. + using var scope = serviceProvider.CreateScope(); + var scopedServices = scope.ServiceProvider; + var db = scopedServices.GetRequiredService(); + + var recipe = new OrganizationWithUsersRecipe(db); + recipe.Seed(name, users, domain); } } diff --git a/util/DbSeederUtility/README.md b/util/DbSeederUtility/README.md index e3ab154e39..0eb21ae6c5 100644 --- a/util/DbSeederUtility/README.md +++ b/util/DbSeederUtility/README.md @@ -4,90 +4,8 @@ A command-line utility for generating and managing test data for Bitwarden datab ## Overview -DbSeederUtility is an executable wrapper around the Seeder class library that provides a convenient command-line interface for: - -1. **Generating** test data as JSON files -2. **Loading** test data into a database -3. **Extracting** database data into seed files -4. **Generating and loading** data in a single operation - -## Commands - -The utility provides the following commands: - -### generate - -Generates seed data as JSON files. - -``` -DbSeeder.exe generate --users --ciphers-per-user --seed-name -``` - -Options: -- `-u, --users`: Number of users to generate -- `-c, --ciphers-per-user`: Number of ciphers per user to generate -- `-n, --seed-name`: Name for the seed data files - -Example: -``` -DbSeeder.exe generate --users 10 --ciphers-per-user 5 --seed-name test_data -``` - -### load - -Loads seed data from JSON files into the database. - -``` -DbSeeder.exe load --seed-name [--timestamp ] [--dry-run] -``` - -Options: -- `-n, --seed-name`: Name of the seed data to load -- `-t, --timestamp`: Specific timestamp of the seed data to load (defaults to most recent) -- `-d, --dry-run`: Validate the seed data without actually loading it - -Example: -``` -DbSeeder.exe load --seed-name test_data -``` - -### generate-direct-load - -Generates seed data and loads it directly into the database without creating JSON files. - -``` -DbSeeder.exe generate-direct-load --users --ciphers-per-user --seed-name -``` - -Options: -- `-u, --users`: Number of users to generate -- `-c, --ciphers-per-user`: Number of ciphers per user to generate -- `-n, --seed-name`: Name identifier for this seed operation - -Example: -``` -DbSeeder.exe generate-direct-load --users 3 --ciphers-per-user 5 --seed-name direct_test_data -``` - -### extract - -Extracts data from the database into seed files. - -``` -DbSeeder.exe extract --seed-name -``` - -Options: -- `-n, --seed-name`: Name for the extracted seed - -Example: -``` -DbSeeder.exe extract --seed-name extracted_data -``` - -## Configuration - -DbSeederUtility uses the same configuration as the Seeder library. See the Seeder README for details on configuration options and file structure. +DbSeederUtility is an executable wrapper around the Seeder class library that provides a convenient command-line +interface for executing seed-recipes in your local environment. ## Installation @@ -106,31 +24,12 @@ DbSeeder.exe [options] ## Examples -### Generate and load test data +### Generate and load test organization ```bash -# Generate 10 users, each with 5 ciphers -DbSeeder.exe generate --users 10 --ciphers-per-user 5 --seed-name demo_data - -# Load the generated data -DbSeeder.exe load --seed-name demo_data -``` - -### Extract and reload data - -```bash -# Extract existing data -DbSeeder.exe extract --seed-name production_backup - -# Load the extracted data -DbSeeder.exe load --seed-name production_backup -``` - -### One-step generation and loading - -```bash -# Generate and load in one step -DbSeeder.exe generate-direct-load --users 5 --ciphers-per-user 10 --seed-name quick_test +# Generate an organization called "seeded" with 10000 users using the @large.test email domain. +# Login using "admin@large.test" with password "asdfasdfasdf" +DbSeeder.exe organization -n seeded -u 10000 -d large.test ``` ## Dependencies @@ -138,4 +37,4 @@ DbSeeder.exe generate-direct-load --users 5 --ciphers-per-user 10 --seed-name qu This utility depends on: - The Seeder class library - CommandDotNet for command-line parsing -- .NET 8.0 runtime \ No newline at end of file +- .NET 8.0 runtime diff --git a/util/DbSeederUtility/ServiceCollectionExtension.cs b/util/DbSeederUtility/ServiceCollectionExtension.cs new file mode 100644 index 0000000000..0653bb1801 --- /dev/null +++ b/util/DbSeederUtility/ServiceCollectionExtension.cs @@ -0,0 +1,25 @@ +using Bit.SharedWeb.Utilities; +using Microsoft.AspNetCore.DataProtection; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; + +namespace Bit.DbSeederUtility; + +public static class ServiceCollectionExtension +{ + public static void ConfigureServices(ServiceCollection services) + { + // Load configuration using the GlobalSettingsFactory + var globalSettings = GlobalSettingsFactory.GlobalSettings; + + // Register services + services.AddLogging(builder => builder.AddConsole()); + services.AddSingleton(globalSettings); + + // Add Data Protection services + services.AddDataProtection() + .SetApplicationName("Bitwarden"); + + services.AddDatabaseRepositories(globalSettings); + } +} diff --git a/util/Seeder/Commands/GenerateCommand.cs b/util/Seeder/Commands/GenerateCommand.cs deleted file mode 100644 index eb15b828df..0000000000 --- a/util/Seeder/Commands/GenerateCommand.cs +++ /dev/null @@ -1,46 +0,0 @@ -using Bit.Seeder.Recipes; -using Bit.Seeder.Settings; -using Bit.SharedWeb.Utilities; -using Microsoft.AspNetCore.DataProtection; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; -using DatabaseContext = Bit.Infrastructure.EntityFramework.Repositories.DatabaseContext; - -namespace Bit.Seeder.Commands; - -public class GenerateCommand -{ - public bool Execute(string name, int users, string domain) - { - // Create service provider with necessary services - var services = new ServiceCollection(); - ConfigureServices(services); - var serviceProvider = services.BuildServiceProvider(); - - // TODO: Can we remove GenerateCommand and provide a RecipeFactory or something. Or wire up DI. - using var scope = serviceProvider.CreateScope(); - var scopedServices = scope.ServiceProvider; - var db = scopedServices.GetRequiredService(); - - var recipe = new OrganizationWithUsersRecipe(db); - recipe.Seed(name, users, domain); - - return true; - } - - private void ConfigureServices(ServiceCollection services) - { - // Load configuration using the GlobalSettingsFactory - var globalSettings = GlobalSettingsFactory.GlobalSettings; - - // Register services - services.AddLogging(builder => builder.AddConsole()); - services.AddSingleton(globalSettings); - - // Add Data Protection services - services.AddDataProtection() - .SetApplicationName("Bitwarden"); - - services.AddDatabaseRepositories(globalSettings); - } -} diff --git a/util/Seeder/README.md b/util/Seeder/README.md index 1f7f7b4f30..8597ad6e39 100644 --- a/util/Seeder/README.md +++ b/util/Seeder/README.md @@ -1,130 +1,18 @@ # Bitwarden Database Seeder -A class library for generating, loading, and extracting test data for Bitwarden databases. - -## Overview - -The Seeder library provides functionality to: - -1. **Generate** realistic test data for Bitwarden, including users and ciphers -2. **Load** previously generated test data into the database -3. **Extract** existing data from a database into seed files -4. **Generate and load** data in a single operation +A class library for generating and inserting test data. ## Project Structure The project is organized into these main components: -### Commands +### Factories -- **ExtractCommand** - Extracts existing data from the database into seed files -- **GenerateCommand** - Generates new test data as seed files or directly loads it -- **LoadCommand** - Loads previously generated seed files into the database +Factories are helper classes for creating domain entities and populating them with realistic data. This assist in +decreasing the amount of boilerplate code needed to create test data in recipes. -### Services +### Recipes -- **DatabaseContext** - EF Core DbContext connecting to the configured database -- **DatabaseService** - Provides database operations (save, retrieve, clear data) -- **EncryptionService** - Handles security operations (password hashing, encryption) -- **SeederService** - Core service that generates realistic test data using Bogus - -### Settings - -- **GlobalSettings** - Configuration model for database connections -- **GlobalSettingsFactory** - Loads and caches settings from config sources - -## Usage - -The Seeder library is designed to be used by the DbSeederUtility executable. For direct usage in code: - -```csharp -// Get seeder service from dependency injection -var seederService = serviceProvider.GetRequiredService(); - -// Generate seed data -await seederService.GenerateSeedsAsync( - userCount: 10, - ciphersPerUser: 5, - seedName: "test_data" -); - -// Load seed data -await seederService.LoadSeedsAsync( - seedName: "test_data", - timestamp: null // Use most recent if null -); - -// Extract data -await seederService.ExtractSeedsAsync( - seedName: "extracted_data" -); - -// Generate and load in one step -await seederService.GenerateAndLoadSeedsAsync( - userCount: 10, - ciphersPerUser: 5, - seedName: "direct_load" -); -``` - -## Configuration - -The library uses the following configuration sources (in order of precedence): - -1. Environment variables -2. User secrets (with ID "Bit.Seeder") -3. appsettings.{Environment}.json -4. appsettings.json - -The expected configuration structure is: - -```json -{ - "globalSettings": { - "selfHosted": true, - "databaseProvider": "postgres", - "sqlServer": { - "connectionString": "..." - }, - "postgreSql": { - "connectionString": "..." - }, - "mySql": { - "connectionString": "..." - }, - "sqlite": { - "connectionString": "..." - } - } -} -``` - -## Seed File Structure - -Seed files are organized as follows: - -``` -seeds/ -├── {seed_name}/ -│ └── {timestamp}/ -│ ├── users/ -│ │ └── users.json -│ └── ciphers/ -│ ├── {user_id1}.json -│ ├── {user_id2}.json -│ └── ... -``` - -## Dependencies - -- **EntityFrameworkCore** - Database access -- **Bogus** - Realistic test data generation -- **CommandDotNet** - Used by DbSeederUtility for CLI commands -- **DataProtection** - Used for secure data handling - -## Best Practices - -- Clear the database before loading new seed data -- Use consistent seed names for related operations -- Store sensitive connection strings in user secrets or environment variables -- Use the DbSeederUtility executable for command-line operations \ No newline at end of file +Recipes are pre-defined data sets which can be run to generate and load data into the database. They often allow a allow +for a few arguments to customize the data slightly. Recipes should be kept simple and focused on a single task. Default +to creating more recipes rather than adding complexity to existing ones. diff --git a/util/Seeder/Seeder.csproj b/util/Seeder/Seeder.csproj index 51f58bc10a..a07688f4f8 100644 --- a/util/Seeder/Seeder.csproj +++ b/util/Seeder/Seeder.csproj @@ -13,22 +13,7 @@ - - - - - - - - - - - - - - - - +