Fix billing downgrade recharge and suspension race

src/Exceptionless.Core/Billing/BillingManager.cs

@@ -2,26 +2,37 @@
 using Exceptionless.Core.Models;
 using Exceptionless.Core.Models.Billing;
 using Exceptionless.Core.Repositories;
+using Foundatio.Lock;
 
 namespace Exceptionless.Core.Billing;
 
 public class BillingManager
 {
+    // Cover the expected multi-call Stripe workflow while still allowing crashed holders to expire.
+    private static readonly TimeSpan OrganizationLockDuration = TimeSpan.FromMinutes(5);
+    private static readonly TimeSpan OrganizationLockTimeout = TimeSpan.FromSeconds(5);
+
+    private readonly ILockProvider _lockProvider;
     private readonly IOrganizationRepository _organizationRepository;
     private readonly IProjectRepository _projectRepository;
     private readonly IUserRepository _userRepository;
     private readonly BillingPlans _plans;
     private readonly TimeProvider _timeProvider;
 
-    public BillingManager(IOrganizationRepository organizationRepository, IProjectRepository projectRepository, IUserRepository userRepository, BillingPlans plans, TimeProvider timeProvider)
+    public BillingManager(ILockProvider lockProvider, IOrganizationRepository organizationRepository, IProjectRepository projectRepository,
+        IUserRepository userRepository, BillingPlans plans, TimeProvider timeProvider)
     {
+        _lockProvider = lockProvider;
         _organizationRepository = organizationRepository;
         _projectRepository = projectRepository;
         _userRepository = userRepository;
         _plans = plans;
         _timeProvider = timeProvider;
     }
 
+    public Task<ILock> AcquireOrganizationLockAsync(string organizationId)
+        => _lockProvider.AcquireAsync(GetOrganizationLockKey(organizationId), OrganizationLockDuration, OrganizationLockTimeout);
+
     public async Task<bool> CanAddOrganizationAsync(User? user)
     {
         if (user is null)
@@ -94,6 +105,9 @@ public async Task<ChangePlanResult> CanDownGradeAsync(Organization organization,
         return _plans.Plans.Where(p => p.RetentionDays > retentionDays && p.Price > 0).OrderBy(p => p.RetentionDays).ThenBy(p => p.Price).FirstOrDefault();
     }
 
+    public Task<ILock?> TryAcquireOrganizationLockAsync(string organizationId)
+        => _lockProvider.TryAcquireAsync(GetOrganizationLockKey(organizationId), OrganizationLockDuration, OrganizationLockTimeout);
+
     public void ApplyBillingPlan(Organization organization, BillingPlan plan, User? user = null, bool updateBillingPrice = true)
     {
         organization.PlanId = plan.Id;
@@ -122,4 +136,6 @@ public void ApplyBonus(Organization organization, int bonusEvents, DateTime? exp
         organization.BonusExpiration = expires;
         organization.GetCurrentUsage(_timeProvider).Limit = organization.GetMaxEventsPerMonthWithBonus(_timeProvider);
     }
+
+    private static string GetOrganizationLockKey(string organizationId) => $"billing:{organizationId}";
 }

src/Exceptionless.Core/Billing/BillingPlans.cs

@@ -4,11 +4,13 @@ namespace Exceptionless.Core.Billing;
 
 public class BillingPlans
 {
+    public const string FreePlanId = "EX_FREE";
+
     public BillingPlans(AppOptions options)
     {
         FreePlan = new BillingPlan
         {
-            Id = "EX_FREE",
+            Id = FreePlanId,
             Name = "Free",
             Description = "Free",
             Price = 0,

src/Exceptionless.Core/Billing/StripeEventHandler.cs

@@ -11,17 +11,21 @@ namespace Exceptionless.Core.Billing;
 public class StripeEventHandler
 {
     private readonly ILogger _logger;
+    private readonly BillingManager _billingManager;
     private readonly IOrganizationRepository _organizationRepository;
     private readonly IUserRepository _userRepository;
+    private readonly IStripeBillingClient _stripeBillingClient;
     private readonly IMailer _mailer;
     private readonly TimeProvider _timeProvider;
 
-    public StripeEventHandler(IOrganizationRepository organizationRepository, IUserRepository userRepository, IMailer mailer,
-        TimeProvider timeProvider, ILogger<StripeEventHandler> logger)
+    public StripeEventHandler(IOrganizationRepository organizationRepository, IUserRepository userRepository, IStripeBillingClient stripeBillingClient,
+        BillingManager billingManager, IMailer mailer, TimeProvider timeProvider, ILogger<StripeEventHandler> logger)
     {
         _logger = logger;
+        _billingManager = billingManager;
         _organizationRepository = organizationRepository;
         _userRepository = userRepository;
+        _stripeBillingClient = stripeBillingClient;
         _mailer = mailer;
         _timeProvider = timeProvider;
     }
@@ -60,14 +64,19 @@ public async Task HandleEventAsync(Stripe.Event stripeEvent)
 
     private async Task SubscriptionUpdatedAsync(Subscription sub)
     {
-        var org = await _organizationRepository.GetByStripeCustomerIdAsync(sub.CustomerId);
-        if (org is null)
+        var organization = await _organizationRepository.GetByStripeCustomerIdAsync(sub.CustomerId);
+        if (organization is null)
         {
             _logger.LogError("Unknown customer id in updated subscription: {CustomerId}", sub.CustomerId);
             return;
         }
 
-        _logger.LogInformation("Stripe subscription updated. Customer: {CustomerId} Org: {Organization} Org Name: {OrganizationName}", sub.CustomerId, org.Id, org.Name);
+        await using var billingLock = await _billingManager.AcquireOrganizationLockAsync(organization.Id);
+        organization = await _organizationRepository.GetByIdAsync(organization.Id);
+        if (organization is null)
+            return;
+
+        _logger.LogInformation("Stripe subscription updated. Customer: {CustomerId} Org: {Organization} Org Name: {OrganizationName}", sub.CustomerId, organization.Id, organization.Name);
 
         BillingStatus? status = null;
         switch (sub.Status)
@@ -99,49 +108,82 @@ private async Task SubscriptionUpdatedAsync(Subscription sub)
             }
         }
 
-        if (!status.HasValue || status.Value == org.BillingStatus)
+        if (!status.HasValue || status.Value == organization.BillingStatus)
+            return;
+
+        if (status.Value == BillingStatus.Canceled && await ShouldIgnoreCancellationAsync(organization, sub))
             return;
 
         var utcNow = _timeProvider.GetUtcNow().UtcDateTime;
-        org.BillingStatus = status.Value;
-        org.BillingChangeDate = utcNow;
+        organization.BillingStatus = status.Value;
+        organization.BillingChangeDate = utcNow;
         if (status.Value == BillingStatus.Unpaid || status.Value == BillingStatus.Canceled)
         {
-            org.IsSuspended = true;
-            org.SuspensionDate = utcNow;
-            org.SuspensionCode = SuspensionCode.Billing;
-            org.SuspensionNotes = $"Stripe subscription status changed to \"{status.Value}\".";
-            org.SuspendedByUserId = "Stripe";
+            organization.IsSuspended = true;
+            organization.SuspensionDate = utcNow;
+            organization.SuspensionCode = SuspensionCode.Billing;
+            organization.SuspensionNotes = $"Stripe subscription status changed to \"{status.Value}\".";
+            organization.SuspendedByUserId = SystemIds.UserId;
         }
         else if (status.Value == BillingStatus.Active || status.Value == BillingStatus.Trialing)
         {
-            org.RemoveSuspension();
+            organization.RemoveSuspension();
         }
 
-        await _organizationRepository.SaveAsync(org, o => o.Cache().Originals());
+        await _organizationRepository.SaveAsync(organization, o => o.Cache().Originals());
     }
 
     private async Task SubscriptionDeletedAsync(Subscription sub)
     {
-        var org = await _organizationRepository.GetByStripeCustomerIdAsync(sub.CustomerId);
-        if (org is null)
+        var organization = await _organizationRepository.GetByStripeCustomerIdAsync(sub.CustomerId);
+        if (organization is null)
         {
             _logger.LogError("Unknown customer id in deleted subscription: {CustomerId}", sub.CustomerId);
             return;
         }
 
-        _logger.LogInformation("Stripe subscription deleted. Customer: {CustomerId} Org: {Organization} Org Name: {OrganizationName}", sub.CustomerId, org.Id, org.Name);
+        await using var billingLock = await _billingManager.AcquireOrganizationLockAsync(organization.Id);
+        organization = await _organizationRepository.GetByIdAsync(organization.Id);
+        if (organization is null)
+            return;
+
+        _logger.LogInformation("Stripe subscription deleted. Customer: {CustomerId} Org: {Organization} Org Name: {OrganizationName}", sub.CustomerId, organization.Id, organization.Name);
+
+        if (await ShouldIgnoreCancellationAsync(organization, sub))
+            return;
 
         var utcNow = _timeProvider.GetUtcNow().UtcDateTime;
-        org.BillingChangeDate = utcNow;
-        org.BillingStatus = BillingStatus.Canceled;
-        org.IsSuspended = true;
-        org.SuspensionDate = utcNow;
-        org.SuspensionCode = SuspensionCode.Billing;
-        org.SuspensionNotes = "Stripe subscription deleted.";
-        org.SuspendedByUserId = "Stripe";
-
-        await _organizationRepository.SaveAsync(org, o => o.Cache().Originals());
+        organization.BillingChangeDate = utcNow;
+        organization.BillingStatus = BillingStatus.Canceled;
+        organization.IsSuspended = true;
+        organization.SuspensionDate = utcNow;
+        organization.SuspensionCode = SuspensionCode.Billing;
+        organization.SuspensionNotes = "Stripe subscription deleted.";
+        organization.SuspendedByUserId = SystemIds.UserId;
+
+        await _organizationRepository.SaveAsync(organization, o => o.Cache().Originals());
+    }
+
+    private async Task<bool> ShouldIgnoreCancellationAsync(Organization organization, Subscription canceledSubscription)
+    {
+        if (String.Equals(organization.PlanId, BillingPlans.FreePlanId, StringComparison.OrdinalIgnoreCase))
+        {
+            _logger.LogInformation("Ignoring canceled Stripe subscription {SubscriptionId} for free organization {Organization}", canceledSubscription.Id, organization.Id);
+            return true;
+        }
+
+        var subscriptions = await _stripeBillingClient.ListSubscriptionsAsync(new SubscriptionListOptions { Customer = canceledSubscription.CustomerId });
+        var replacement = subscriptions.FirstOrDefault(s => !s.CanceledAt.HasValue && !String.Equals(s.Id, canceledSubscription.Id, StringComparison.Ordinal));
+        if (replacement is null)
+            return false;
+
+        _logger.LogInformation(
+            "Ignoring canceled Stripe subscription {SubscriptionId} for organization {Organization} because replacement subscription {ReplacementSubscriptionId} exists",
+            canceledSubscription.Id,
+            organization.Id,
+            replacement.Id
+        );
+        return true;
     }
 
     private async Task InvoicePaymentSucceededAsync(Invoice invoice)

src/Exceptionless.Core/Models/OAuthApplication.cs

@@ -7,7 +7,7 @@ namespace Exceptionless.Core.Models;
 
 public class OAuthApplication : IIdentity, IHaveDates, IValidatableObject
 {
-    public const string SystemUserId = "000000000000000000000001";
+    public const string SystemUserId = SystemIds.UserId;
 
     [ObjectId]
     public string Id { get; set; } = null!;

src/Exceptionless.Core/Models/SystemIds.cs

@@ -0,0 +1,7 @@
+namespace Exceptionless.Core.Models;
+
+public static class SystemIds
+{
+    public const string OrganizationId = "000000000000000000000001";
+    public const string UserId = "000000000000000000000001";
+}

src/Exceptionless.Core/Seed/PredefinedSavedViewsDataSeed.cs

@@ -10,8 +10,8 @@ namespace Exceptionless.Core.Seed;
 
 public class PredefinedSavedViewsDataSeed : IDataSeed
 {
-    public const string SystemOrganizationId = "000000000000000000000001";
-    public const string SystemUserId = "000000000000000000000001";
+    public const string SystemOrganizationId = SystemIds.OrganizationId;
+    public const string SystemUserId = SystemIds.UserId;
     public const string SeedFileName = "predefined-saved-views.json";
 
     private static readonly JsonSerializerOptions JsonOptions = new()

src/Exceptionless.Web/Api/Handlers/OrganizationHandler.cs

@@ -361,6 +361,10 @@ public async Task<Result<ChangePlanResult>> Handle(ChangeOrganizationPlan messag
         if (!options.StripeOptions.EnableBilling)
             return Result.NotFound("Organization not found.");
 
+        await using var billingLock = await billingManager.TryAcquireOrganizationLockAsync(message.Id);
+        if (billingLock is null)
+            return ChangePlanResult.FailWithMessage("A billing change is already in progress. Please try again.");
+
         var organization = await GetModelAsync(message.Id, useCache: false);
         if (organization is null)
             return Result.NotFound("Organization not found.");
@@ -398,7 +402,14 @@ public async Task<Result<ChangePlanResult>> Handle(ChangeOrganizationPlan messag
                 {
                     var subs = await stripeBillingClient.ListSubscriptionsAsync(new SubscriptionListOptions { Customer = organization.StripeCustomerId });
                     foreach (var sub in subs.Where(s => !s.CanceledAt.HasValue))
-                        await stripeBillingClient.CancelSubscriptionAsync(sub.Id, new SubscriptionCancelOptions());
+                    {
+                        // Finalize proration now so unused paid time becomes customer credit before a later re-upgrade.
+                        await stripeBillingClient.CancelSubscriptionAsync(sub.Id, new SubscriptionCancelOptions
+                        {
+                            InvoiceNow = true,
+                            Prorate = true
+                        });
+                    }
                 }
 
                 organization.BillingStatus = BillingStatus.Trialing;
@@ -433,7 +444,8 @@ public async Task<Result<ChangePlanResult>> Handle(ChangeOrganizationPlan messag
                 var customer = await stripeBillingClient.CreateCustomerAsync(createCustomer);
                 organization.StripeCustomerId = customer.Id;
                 organization.CardLast4 = model.Last4;
-                await repository.SaveAsync(organization, o => o.Cache());
+                // Make the customer id searchable before Stripe can deliver subscription webhooks.
+                await repository.SaveAsync(organization, o => o.Cache().ImmediateConsistency());
 
                 var subscriptionOptions = new SubscriptionCreateOptions
                 {