mirror of
https://github.com/bitwarden/server.git
synced 2025-05-08 13:12:16 -05:00
Cleanup and re-organize the code slightly
This commit is contained in:
parent
a98eeac602
commit
107e2fc0c3
@ -75,10 +75,6 @@ public class OrganizationUsersControllerTest : IClassFixture<ApiApplicationFacto
|
|||||||
|
|
||||||
stopwatch.Stop();
|
stopwatch.Stop();
|
||||||
_testOutputHelper.WriteLine($"Request duration: {stopwatch.ElapsedMilliseconds} ms");
|
_testOutputHelper.WriteLine($"Request duration: {stopwatch.ElapsedMilliseconds} ms");
|
||||||
|
|
||||||
// var result = await response.Content.ReadFromJsonAsync<ListResponseModel<OrganizationUserUserDetailsResponseModel>>();
|
|
||||||
// Assert.NotNull(result?.Data);
|
|
||||||
// Assert.Equal(600001, result.Data.Count());
|
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -17,8 +17,6 @@
|
|||||||
|
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
<PackageReference Include="CommandDotNet" Version="7.0.4" />
|
<PackageReference Include="CommandDotNet" Version="7.0.4" />
|
||||||
<PackageReference Include="Microsoft.Extensions.Logging" Version="8.0.1" />
|
|
||||||
<PackageReference Include="Microsoft.Extensions.Logging.Console" Version="8.0.1" />
|
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
|
|
||||||
</Project>
|
</Project>
|
||||||
|
@ -1,7 +1,7 @@
|
|||||||
using Bit.Core.Settings;
|
using Bit.Core.Settings;
|
||||||
using Microsoft.Extensions.Configuration;
|
using Microsoft.Extensions.Configuration;
|
||||||
|
|
||||||
namespace Bit.Seeder.Settings;
|
namespace Bit.DbSeederUtility;
|
||||||
|
|
||||||
public static class GlobalSettingsFactory
|
public static class GlobalSettingsFactory
|
||||||
{
|
{
|
||||||
@ -63,25 +63,3 @@ public static class GlobalSettingsFactory
|
|||||||
return settings;
|
return settings;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Non-static version that can accept command-line arguments
|
|
||||||
public class GlobalSettingsFactoryWithArgs
|
|
||||||
{
|
|
||||||
public GlobalSettings GlobalSettings { get; }
|
|
||||||
|
|
||||||
public GlobalSettingsFactoryWithArgs(string[] args)
|
|
||||||
{
|
|
||||||
GlobalSettings = new GlobalSettings();
|
|
||||||
|
|
||||||
var config = new ConfigurationBuilder()
|
|
||||||
.SetBasePath(Directory.GetCurrentDirectory())
|
|
||||||
.AddJsonFile("appsettings.json", optional: true, reloadOnChange: true)
|
|
||||||
.AddJsonFile($"appsettings.{Environment.GetEnvironmentVariable("ASPNETCORE_ENVIRONMENT") ?? "Production"}.json", optional: true, reloadOnChange: true)
|
|
||||||
.AddUserSecrets("bitwarden-Api")
|
|
||||||
.AddCommandLine(args)
|
|
||||||
.AddEnvironmentVariables()
|
|
||||||
.Build();
|
|
||||||
|
|
||||||
config.GetSection("globalSettings").Bind(GlobalSettings);
|
|
||||||
}
|
|
||||||
}
|
|
@ -1,6 +1,7 @@
|
|||||||
using Bit.Seeder.Commands;
|
using Bit.Infrastructure.EntityFramework.Repositories;
|
||||||
using Bit.Seeder.Settings;
|
using Bit.Seeder.Recipes;
|
||||||
using CommandDotNet;
|
using CommandDotNet;
|
||||||
|
using Microsoft.Extensions.DependencyInjection;
|
||||||
|
|
||||||
namespace Bit.DbSeederUtility;
|
namespace Bit.DbSeederUtility;
|
||||||
|
|
||||||
@ -16,18 +17,26 @@ public class Program
|
|||||||
}
|
}
|
||||||
|
|
||||||
[Command("organization", Description = "Seed an organization and organization users")]
|
[Command("organization", Description = "Seed an organization and organization users")]
|
||||||
public int Organization(
|
public void Organization(
|
||||||
[Option('n', "Name", Description = "Name of organization")]
|
[Option('n', "Name", Description = "Name of organization")]
|
||||||
string name,
|
string name,
|
||||||
|
|
||||||
[Option('u', "users", Description = "Number of users to generate")]
|
[Option('u', "users", Description = "Number of users to generate")]
|
||||||
int users,
|
int users,
|
||||||
|
|
||||||
[Option('d', "domain", Description = "Email domain for users")]
|
[Option('d', "domain", Description = "Email domain for users")]
|
||||||
string domain
|
string domain
|
||||||
)
|
)
|
||||||
{
|
{
|
||||||
var generateCommand = new GenerateCommand();
|
// Create service provider with necessary services
|
||||||
return generateCommand.Execute(name, users, domain) ? 0 : 1;
|
var services = new ServiceCollection();
|
||||||
|
ServiceCollectionExtension.ConfigureServices(services);
|
||||||
|
var serviceProvider = services.BuildServiceProvider();
|
||||||
|
|
||||||
|
// TODO: Can we remove GenerateCommand and provide a RecipeFactory or something. Or wire up DI.
|
||||||
|
using var scope = serviceProvider.CreateScope();
|
||||||
|
var scopedServices = scope.ServiceProvider;
|
||||||
|
var db = scopedServices.GetRequiredService<DatabaseContext>();
|
||||||
|
|
||||||
|
var recipe = new OrganizationWithUsersRecipe(db);
|
||||||
|
recipe.Seed(name, users, domain);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -4,90 +4,8 @@ A command-line utility for generating and managing test data for Bitwarden datab
|
|||||||
|
|
||||||
## Overview
|
## Overview
|
||||||
|
|
||||||
DbSeederUtility is an executable wrapper around the Seeder class library that provides a convenient command-line interface for:
|
DbSeederUtility is an executable wrapper around the Seeder class library that provides a convenient command-line
|
||||||
|
interface for executing seed-recipes in your local environment.
|
||||||
1. **Generating** test data as JSON files
|
|
||||||
2. **Loading** test data into a database
|
|
||||||
3. **Extracting** database data into seed files
|
|
||||||
4. **Generating and loading** data in a single operation
|
|
||||||
|
|
||||||
## Commands
|
|
||||||
|
|
||||||
The utility provides the following commands:
|
|
||||||
|
|
||||||
### generate
|
|
||||||
|
|
||||||
Generates seed data as JSON files.
|
|
||||||
|
|
||||||
```
|
|
||||||
DbSeeder.exe generate --users <count> --ciphers-per-user <count> --seed-name <name>
|
|
||||||
```
|
|
||||||
|
|
||||||
Options:
|
|
||||||
- `-u, --users`: Number of users to generate
|
|
||||||
- `-c, --ciphers-per-user`: Number of ciphers per user to generate
|
|
||||||
- `-n, --seed-name`: Name for the seed data files
|
|
||||||
|
|
||||||
Example:
|
|
||||||
```
|
|
||||||
DbSeeder.exe generate --users 10 --ciphers-per-user 5 --seed-name test_data
|
|
||||||
```
|
|
||||||
|
|
||||||
### load
|
|
||||||
|
|
||||||
Loads seed data from JSON files into the database.
|
|
||||||
|
|
||||||
```
|
|
||||||
DbSeeder.exe load --seed-name <name> [--timestamp <timestamp>] [--dry-run]
|
|
||||||
```
|
|
||||||
|
|
||||||
Options:
|
|
||||||
- `-n, --seed-name`: Name of the seed data to load
|
|
||||||
- `-t, --timestamp`: Specific timestamp of the seed data to load (defaults to most recent)
|
|
||||||
- `-d, --dry-run`: Validate the seed data without actually loading it
|
|
||||||
|
|
||||||
Example:
|
|
||||||
```
|
|
||||||
DbSeeder.exe load --seed-name test_data
|
|
||||||
```
|
|
||||||
|
|
||||||
### generate-direct-load
|
|
||||||
|
|
||||||
Generates seed data and loads it directly into the database without creating JSON files.
|
|
||||||
|
|
||||||
```
|
|
||||||
DbSeeder.exe generate-direct-load --users <count> --ciphers-per-user <count> --seed-name <name>
|
|
||||||
```
|
|
||||||
|
|
||||||
Options:
|
|
||||||
- `-u, --users`: Number of users to generate
|
|
||||||
- `-c, --ciphers-per-user`: Number of ciphers per user to generate
|
|
||||||
- `-n, --seed-name`: Name identifier for this seed operation
|
|
||||||
|
|
||||||
Example:
|
|
||||||
```
|
|
||||||
DbSeeder.exe generate-direct-load --users 3 --ciphers-per-user 5 --seed-name direct_test_data
|
|
||||||
```
|
|
||||||
|
|
||||||
### extract
|
|
||||||
|
|
||||||
Extracts data from the database into seed files.
|
|
||||||
|
|
||||||
```
|
|
||||||
DbSeeder.exe extract --seed-name <name>
|
|
||||||
```
|
|
||||||
|
|
||||||
Options:
|
|
||||||
- `-n, --seed-name`: Name for the extracted seed
|
|
||||||
|
|
||||||
Example:
|
|
||||||
```
|
|
||||||
DbSeeder.exe extract --seed-name extracted_data
|
|
||||||
```
|
|
||||||
|
|
||||||
## Configuration
|
|
||||||
|
|
||||||
DbSeederUtility uses the same configuration as the Seeder library. See the Seeder README for details on configuration options and file structure.
|
|
||||||
|
|
||||||
## Installation
|
## Installation
|
||||||
|
|
||||||
@ -106,31 +24,12 @@ DbSeeder.exe <command> [options]
|
|||||||
|
|
||||||
## Examples
|
## Examples
|
||||||
|
|
||||||
### Generate and load test data
|
### Generate and load test organization
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Generate 10 users, each with 5 ciphers
|
# Generate an organization called "seeded" with 10000 users using the @large.test email domain.
|
||||||
DbSeeder.exe generate --users 10 --ciphers-per-user 5 --seed-name demo_data
|
# Login using "admin@large.test" with password "asdfasdfasdf"
|
||||||
|
DbSeeder.exe organization -n seeded -u 10000 -d large.test
|
||||||
# Load the generated data
|
|
||||||
DbSeeder.exe load --seed-name demo_data
|
|
||||||
```
|
|
||||||
|
|
||||||
### Extract and reload data
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Extract existing data
|
|
||||||
DbSeeder.exe extract --seed-name production_backup
|
|
||||||
|
|
||||||
# Load the extracted data
|
|
||||||
DbSeeder.exe load --seed-name production_backup
|
|
||||||
```
|
|
||||||
|
|
||||||
### One-step generation and loading
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Generate and load in one step
|
|
||||||
DbSeeder.exe generate-direct-load --users 5 --ciphers-per-user 10 --seed-name quick_test
|
|
||||||
```
|
```
|
||||||
|
|
||||||
## Dependencies
|
## Dependencies
|
||||||
@ -138,4 +37,4 @@ DbSeeder.exe generate-direct-load --users 5 --ciphers-per-user 10 --seed-name qu
|
|||||||
This utility depends on:
|
This utility depends on:
|
||||||
- The Seeder class library
|
- The Seeder class library
|
||||||
- CommandDotNet for command-line parsing
|
- CommandDotNet for command-line parsing
|
||||||
- .NET 8.0 runtime
|
- .NET 8.0 runtime
|
||||||
|
25
util/DbSeederUtility/ServiceCollectionExtension.cs
Normal file
25
util/DbSeederUtility/ServiceCollectionExtension.cs
Normal file
@ -0,0 +1,25 @@
|
|||||||
|
using Bit.SharedWeb.Utilities;
|
||||||
|
using Microsoft.AspNetCore.DataProtection;
|
||||||
|
using Microsoft.Extensions.DependencyInjection;
|
||||||
|
using Microsoft.Extensions.Logging;
|
||||||
|
|
||||||
|
namespace Bit.DbSeederUtility;
|
||||||
|
|
||||||
|
public static class ServiceCollectionExtension
|
||||||
|
{
|
||||||
|
public static void ConfigureServices(ServiceCollection services)
|
||||||
|
{
|
||||||
|
// Load configuration using the GlobalSettingsFactory
|
||||||
|
var globalSettings = GlobalSettingsFactory.GlobalSettings;
|
||||||
|
|
||||||
|
// Register services
|
||||||
|
services.AddLogging(builder => builder.AddConsole());
|
||||||
|
services.AddSingleton(globalSettings);
|
||||||
|
|
||||||
|
// Add Data Protection services
|
||||||
|
services.AddDataProtection()
|
||||||
|
.SetApplicationName("Bitwarden");
|
||||||
|
|
||||||
|
services.AddDatabaseRepositories(globalSettings);
|
||||||
|
}
|
||||||
|
}
|
@ -1,46 +0,0 @@
|
|||||||
using Bit.Seeder.Recipes;
|
|
||||||
using Bit.Seeder.Settings;
|
|
||||||
using Bit.SharedWeb.Utilities;
|
|
||||||
using Microsoft.AspNetCore.DataProtection;
|
|
||||||
using Microsoft.Extensions.DependencyInjection;
|
|
||||||
using Microsoft.Extensions.Logging;
|
|
||||||
using DatabaseContext = Bit.Infrastructure.EntityFramework.Repositories.DatabaseContext;
|
|
||||||
|
|
||||||
namespace Bit.Seeder.Commands;
|
|
||||||
|
|
||||||
public class GenerateCommand
|
|
||||||
{
|
|
||||||
public bool Execute(string name, int users, string domain)
|
|
||||||
{
|
|
||||||
// Create service provider with necessary services
|
|
||||||
var services = new ServiceCollection();
|
|
||||||
ConfigureServices(services);
|
|
||||||
var serviceProvider = services.BuildServiceProvider();
|
|
||||||
|
|
||||||
// TODO: Can we remove GenerateCommand and provide a RecipeFactory or something. Or wire up DI.
|
|
||||||
using var scope = serviceProvider.CreateScope();
|
|
||||||
var scopedServices = scope.ServiceProvider;
|
|
||||||
var db = scopedServices.GetRequiredService<DatabaseContext>();
|
|
||||||
|
|
||||||
var recipe = new OrganizationWithUsersRecipe(db);
|
|
||||||
recipe.Seed(name, users, domain);
|
|
||||||
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
private void ConfigureServices(ServiceCollection services)
|
|
||||||
{
|
|
||||||
// Load configuration using the GlobalSettingsFactory
|
|
||||||
var globalSettings = GlobalSettingsFactory.GlobalSettings;
|
|
||||||
|
|
||||||
// Register services
|
|
||||||
services.AddLogging(builder => builder.AddConsole());
|
|
||||||
services.AddSingleton(globalSettings);
|
|
||||||
|
|
||||||
// Add Data Protection services
|
|
||||||
services.AddDataProtection()
|
|
||||||
.SetApplicationName("Bitwarden");
|
|
||||||
|
|
||||||
services.AddDatabaseRepositories(globalSettings);
|
|
||||||
}
|
|
||||||
}
|
|
@ -1,130 +1,18 @@
|
|||||||
# Bitwarden Database Seeder
|
# Bitwarden Database Seeder
|
||||||
|
|
||||||
A class library for generating, loading, and extracting test data for Bitwarden databases.
|
A class library for generating and inserting test data.
|
||||||
|
|
||||||
## Overview
|
|
||||||
|
|
||||||
The Seeder library provides functionality to:
|
|
||||||
|
|
||||||
1. **Generate** realistic test data for Bitwarden, including users and ciphers
|
|
||||||
2. **Load** previously generated test data into the database
|
|
||||||
3. **Extract** existing data from a database into seed files
|
|
||||||
4. **Generate and load** data in a single operation
|
|
||||||
|
|
||||||
## Project Structure
|
## Project Structure
|
||||||
|
|
||||||
The project is organized into these main components:
|
The project is organized into these main components:
|
||||||
|
|
||||||
### Commands
|
### Factories
|
||||||
|
|
||||||
- **ExtractCommand** - Extracts existing data from the database into seed files
|
Factories are helper classes for creating domain entities and populating them with realistic data. This assist in
|
||||||
- **GenerateCommand** - Generates new test data as seed files or directly loads it
|
decreasing the amount of boilerplate code needed to create test data in recipes.
|
||||||
- **LoadCommand** - Loads previously generated seed files into the database
|
|
||||||
|
|
||||||
### Services
|
### Recipes
|
||||||
|
|
||||||
- **DatabaseContext** - EF Core DbContext connecting to the configured database
|
Recipes are pre-defined data sets which can be run to generate and load data into the database. They often allow a allow
|
||||||
- **DatabaseService** - Provides database operations (save, retrieve, clear data)
|
for a few arguments to customize the data slightly. Recipes should be kept simple and focused on a single task. Default
|
||||||
- **EncryptionService** - Handles security operations (password hashing, encryption)
|
to creating more recipes rather than adding complexity to existing ones.
|
||||||
- **SeederService** - Core service that generates realistic test data using Bogus
|
|
||||||
|
|
||||||
### Settings
|
|
||||||
|
|
||||||
- **GlobalSettings** - Configuration model for database connections
|
|
||||||
- **GlobalSettingsFactory** - Loads and caches settings from config sources
|
|
||||||
|
|
||||||
## Usage
|
|
||||||
|
|
||||||
The Seeder library is designed to be used by the DbSeederUtility executable. For direct usage in code:
|
|
||||||
|
|
||||||
```csharp
|
|
||||||
// Get seeder service from dependency injection
|
|
||||||
var seederService = serviceProvider.GetRequiredService<ISeederService>();
|
|
||||||
|
|
||||||
// Generate seed data
|
|
||||||
await seederService.GenerateSeedsAsync(
|
|
||||||
userCount: 10,
|
|
||||||
ciphersPerUser: 5,
|
|
||||||
seedName: "test_data"
|
|
||||||
);
|
|
||||||
|
|
||||||
// Load seed data
|
|
||||||
await seederService.LoadSeedsAsync(
|
|
||||||
seedName: "test_data",
|
|
||||||
timestamp: null // Use most recent if null
|
|
||||||
);
|
|
||||||
|
|
||||||
// Extract data
|
|
||||||
await seederService.ExtractSeedsAsync(
|
|
||||||
seedName: "extracted_data"
|
|
||||||
);
|
|
||||||
|
|
||||||
// Generate and load in one step
|
|
||||||
await seederService.GenerateAndLoadSeedsAsync(
|
|
||||||
userCount: 10,
|
|
||||||
ciphersPerUser: 5,
|
|
||||||
seedName: "direct_load"
|
|
||||||
);
|
|
||||||
```
|
|
||||||
|
|
||||||
## Configuration
|
|
||||||
|
|
||||||
The library uses the following configuration sources (in order of precedence):
|
|
||||||
|
|
||||||
1. Environment variables
|
|
||||||
2. User secrets (with ID "Bit.Seeder")
|
|
||||||
3. appsettings.{Environment}.json
|
|
||||||
4. appsettings.json
|
|
||||||
|
|
||||||
The expected configuration structure is:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"globalSettings": {
|
|
||||||
"selfHosted": true,
|
|
||||||
"databaseProvider": "postgres",
|
|
||||||
"sqlServer": {
|
|
||||||
"connectionString": "..."
|
|
||||||
},
|
|
||||||
"postgreSql": {
|
|
||||||
"connectionString": "..."
|
|
||||||
},
|
|
||||||
"mySql": {
|
|
||||||
"connectionString": "..."
|
|
||||||
},
|
|
||||||
"sqlite": {
|
|
||||||
"connectionString": "..."
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
## Seed File Structure
|
|
||||||
|
|
||||||
Seed files are organized as follows:
|
|
||||||
|
|
||||||
```
|
|
||||||
seeds/
|
|
||||||
├── {seed_name}/
|
|
||||||
│ └── {timestamp}/
|
|
||||||
│ ├── users/
|
|
||||||
│ │ └── users.json
|
|
||||||
│ └── ciphers/
|
|
||||||
│ ├── {user_id1}.json
|
|
||||||
│ ├── {user_id2}.json
|
|
||||||
│ └── ...
|
|
||||||
```
|
|
||||||
|
|
||||||
## Dependencies
|
|
||||||
|
|
||||||
- **EntityFrameworkCore** - Database access
|
|
||||||
- **Bogus** - Realistic test data generation
|
|
||||||
- **CommandDotNet** - Used by DbSeederUtility for CLI commands
|
|
||||||
- **DataProtection** - Used for secure data handling
|
|
||||||
|
|
||||||
## Best Practices
|
|
||||||
|
|
||||||
- Clear the database before loading new seed data
|
|
||||||
- Use consistent seed names for related operations
|
|
||||||
- Store sensitive connection strings in user secrets or environment variables
|
|
||||||
- Use the DbSeederUtility executable for command-line operations
|
|
||||||
|
@ -13,22 +13,7 @@
|
|||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
|
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
<PackageReference Include="BCrypt.Net-Next" Version="4.0.3" />
|
<Folder Include="Settings\" />
|
||||||
<PackageReference Include="Bogus" Version="35.4.0" />
|
|
||||||
<PackageReference Include="CommandDotNet" Version="7.0.3" />
|
|
||||||
<PackageReference Include="Microsoft.AspNetCore.DataProtection" Version="8.0.10" />
|
|
||||||
<PackageReference Include="Microsoft.EntityFrameworkCore" Version="8.0.8" />
|
|
||||||
<PackageReference Include="Microsoft.EntityFrameworkCore.SqlServer" Version="8.0.8" />
|
|
||||||
<PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="8.0.8" />
|
|
||||||
<PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="8.0.4" />
|
|
||||||
<PackageReference Include="Microsoft.Extensions.Configuration" Version="8.0.0" />
|
|
||||||
<PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="8.0.0" />
|
|
||||||
<PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="8.0.0" />
|
|
||||||
<PackageReference Include="Microsoft.Extensions.Configuration.UserSecrets" Version="8.0.0" />
|
|
||||||
<PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="8.0.1" />
|
|
||||||
<PackageReference Include="Microsoft.Extensions.Logging" Version="8.0.1" />
|
|
||||||
<PackageReference Include="Microsoft.Extensions.Logging.Console" Version="8.0.0" />
|
|
||||||
<PackageReference Include="Pomelo.EntityFrameworkCore.MySql" Version="8.0.2" />
|
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
|
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
|
Loading…
x
Reference in New Issue
Block a user