diff --git a/src/Core/Jobs/BaseJobsHostedService.cs b/src/Core/Jobs/BaseJobsHostedService.cs index 72758091ac..57b7b60fde 100644 --- a/src/Core/Jobs/BaseJobsHostedService.cs +++ b/src/Core/Jobs/BaseJobsHostedService.cs @@ -15,6 +15,8 @@ namespace Bit.Core.Jobs { public abstract class BaseJobsHostedService : IHostedService, IDisposable { + private const int MaximumJobRetries = 10; + private readonly IServiceProvider _serviceProvider; private readonly ILogger _listenerLogger; protected readonly ILogger _logger; @@ -68,23 +70,42 @@ namespace Bit.Core.Jobs { foreach (var (job, trigger) in Jobs) { - var dupeT = await _scheduler.GetTrigger(trigger.Key); - if (dupeT != null) + for (var retry = 0; retry < MaximumJobRetries; retry++) { - await _scheduler.RescheduleJob(trigger.Key, trigger); + // There's a race condition when starting multiple containers simultaneously, retry until it succeeds.. + try + { + var dupeT = await _scheduler.GetTrigger(trigger.Key); + if (dupeT != null) + { + await _scheduler.RescheduleJob(trigger.Key, trigger); + } + + var jobDetail = JobBuilder.Create(job) + .WithIdentity(job.FullName) + .Build(); + + var dupeJ = await _scheduler.GetJobDetail(jobDetail.Key); + if (dupeJ != null) + { + await _scheduler.DeleteJob(jobDetail.Key); + } + + await _scheduler.ScheduleJob(jobDetail, trigger); + break; + } + catch (Exception e) + { + if (retry == MaximumJobRetries - 1) + { + throw new Exception("Job failed to start after 10 retries."); + } + + _logger.LogWarning($"Exception while trying to schedule job: {job.FullName}, {e}"); + var random = new Random(); + Thread.Sleep(random.Next(50, 250)); + } } - - var jobDetail = JobBuilder.Create(job) - .WithIdentity(job.FullName) - .Build(); - - var dupeJ = await _scheduler.GetJobDetail(jobDetail.Key); - if (dupeJ != null) - { - await _scheduler.DeleteJob(jobDetail.Key); - } - - await _scheduler.ScheduleJob(jobDetail, trigger); } }