Add support for international domain names (IDN) in email addresses (#1512)

* Adjust email address checking to handle unicode

* ASCII only in local part
* allow unicode in second-level and top-level domain

* Add PunyEncoding/Decoding methods and tests

* Use PunyEncoding for outbound email recipients

* Use MailKit for punycode, handle edge cases

* Punyencode all email addresses in mailServices

* Remove punyencoding from HandlebarsMailService

* Add to punyencoding tests

* Use more inclusive e-mail error

* Fix comment wording

* Apply StrictEmail checking to emergency access invite

* Remove punyDecode helper

src/Core/Models/Api/Request/EmergencyAccessRequstModels.cs

@@ -1,4 +1,5 @@
 using System.ComponentModel.DataAnnotations;
+using Bit.Core.Utilities;
 using Bit.Core.Models.Table;
 
 namespace Bit.Core.Models.Api.Request
@@ -6,7 +7,7 @@ namespace Bit.Core.Models.Api.Request
     public class EmergencyAccessInviteRequestModel
     {
         [Required]
-        [EmailAddress]
+        [StrictEmailAddress]
         [StringLength(256)]
         public string Email { get; set; }
         [Required]

src/Core/Services/Implementations/AmazonSesMailDeliveryService.cs

@@ -3,6 +3,7 @@ using System.Collections.Generic;
 using System.Threading.Tasks;
 using Bit.Core.Models.Mail;
 using Bit.Core.Settings;
+using Bit.Core.Utilities;
 using System.Linq;
 using Amazon.SimpleEmail;
 using Amazon;
@@ -54,11 +55,13 @@ namespace Bit.Core.Services
                 throw new ArgumentNullException(nameof(globalSettings.Amazon.Region));
             }
 
+            var replyToEmail = CoreHelpers.PunyEncode(globalSettings.Mail.ReplyToEmail);
+
             _globalSettings = globalSettings;
             _hostingEnvironment = hostingEnvironment;
             _logger = logger;
             _client = amazonSimpleEmailService;
-            _source = $"\"{globalSettings.SiteName}\" <{globalSettings.Mail.ReplyToEmail}>";
+            _source = $"\"{globalSettings.SiteName}\" <{replyToEmail}>";
             _senderTag = $"Server_{globalSettings.ProjectName?.Replace(' ', '_')}";
             if (!string.IsNullOrWhiteSpace(_globalSettings.Mail.AmazonConfigSetName))
             {
@@ -79,7 +82,9 @@ namespace Bit.Core.Services
                 Source = _source,
                 Destination = new Destination
                 {
-                    ToAddresses = message.ToEmails.ToList()
+                    ToAddresses = message.ToEmails
+                        .Select(email => CoreHelpers.PunyEncode(email))
+                        .ToList()
                 },
                 Message = new Message
                 {
@@ -107,7 +112,9 @@ namespace Bit.Core.Services
 
             if (message.BccEmails?.Any() ?? false)
             {
-                request.Destination.BccAddresses = message.BccEmails.ToList();
+                request.Destination.BccAddresses = message.BccEmails
+                    .Select(email => CoreHelpers.PunyEncode(email))
+                    .ToList();
             }
 
             if (!string.IsNullOrWhiteSpace(message.Category))

src/Core/Services/Implementations/MailKitSmtpMailDeliveryService.cs

@@ -13,6 +13,7 @@ namespace Bit.Core.Services
         private readonly GlobalSettings _globalSettings;
         private readonly ILogger<MailKitSmtpMailDeliveryService> _logger;
         private readonly string _replyDomain;
+        private readonly string _replyEmail;
 
         public MailKitSmtpMailDeliveryService(
             GlobalSettings globalSettings,
@@ -22,9 +23,12 @@ namespace Bit.Core.Services
             {
                 throw new ArgumentNullException(nameof(globalSettings.Mail.Smtp.Host));
             }
-            if (globalSettings.Mail?.ReplyToEmail?.Contains("@") ?? false)
+
+            _replyEmail = CoreHelpers.PunyEncode(globalSettings.Mail?.ReplyToEmail);
+
+            if (_replyEmail.Contains("@"))
             {
-                _replyDomain = globalSettings.Mail.ReplyToEmail.Split('@')[1];
+                _replyDomain = _replyEmail.Split('@')[1];
             }
 
             _globalSettings = globalSettings;
@@ -34,7 +38,7 @@ namespace Bit.Core.Services
         public async Task SendEmailAsync(Models.Mail.MailMessage message)
         {
             var mimeMessage = new MimeMessage();
-            mimeMessage.From.Add(new MailboxAddress(_globalSettings.SiteName, _globalSettings.Mail.ReplyToEmail));
+            mimeMessage.From.Add(new MailboxAddress(_globalSettings.SiteName, _replyEmail));
             mimeMessage.Subject = message.Subject;
             if (!string.IsNullOrWhiteSpace(_replyDomain))
             {
@@ -43,14 +47,16 @@ namespace Bit.Core.Services
 
             foreach (var address in message.ToEmails)
             {
-                mimeMessage.To.Add(MailboxAddress.Parse(address));
+                var punyencoded = CoreHelpers.PunyEncode(address);
+                mimeMessage.To.Add(MailboxAddress.Parse(punyencoded));
             }
 
             if (message.BccEmails != null)
             {
                 foreach (var address in message.BccEmails)
                 {
-                    mimeMessage.Bcc.Add(MailboxAddress.Parse(address));
+                    var punyencoded = CoreHelpers.PunyEncode(address);
+                    mimeMessage.Bcc.Add(MailboxAddress.Parse(punyencoded));
                 }
             }
 

src/Core/Services/Implementations/PostalMailDeliveryService.cs

@@ -7,6 +7,7 @@ using System.Collections.Generic;
 using Newtonsoft.Json;
 using Microsoft.AspNetCore.Hosting;
 using System.Text;
+using Bit.Core.Utilities;
 
 namespace Bit.Core.Services
 {
@@ -25,13 +26,16 @@ namespace Bit.Core.Services
             IWebHostEnvironment hostingEnvironment,
             IHttpClientFactory clientFactory)
         {
+            var postalDomain = CoreHelpers.PunyEncode(globalSettings.Mail.PostalDomain);
+            var replyToEmail = CoreHelpers.PunyEncode(globalSettings.Mail.ReplyToEmail);
+
             _globalSettings = globalSettings;
             _logger = logger;
             _clientFactory = clientFactory;
             _baseTag = $"Env_{hostingEnvironment.EnvironmentName}-" +
                 $"Server_{globalSettings.ProjectName?.Replace(' ', '_')}";
-            _from = $"\"{globalSettings.SiteName}\" <no-reply@{_globalSettings.Mail.PostalDomain}>";
+            _from = $"\"{globalSettings.SiteName}\" <no-reply@{postalDomain}>";
-            _reply = $"\"{globalSettings.SiteName}\" <{globalSettings.Mail.ReplyToEmail}>";
+            _reply = $"\"{globalSettings.SiteName}\" <{replyToEmail}>";
         }
 
         public async Task SendEmailAsync(Models.Mail.MailMessage message)
@@ -50,7 +54,7 @@ namespace Bit.Core.Services
             };
             foreach (var address in message.ToEmails)
             {
-                request.to.Add(address);
+                request.to.Add(CoreHelpers.PunyEncode(address));
             }
 
             if (message.BccEmails != null)
@@ -58,7 +62,7 @@ namespace Bit.Core.Services
                 request.bcc = new List<string>();
                 foreach (var address in message.BccEmails)
                 {
-                    request.bcc.Add(address);
+                    request.bcc.Add(CoreHelpers.PunyEncode(address));
                 }
             }
 

src/Core/Utilities/CoreHelpers.cs

@@ -27,6 +27,7 @@ using Bit.Core.Enums.Provider;
 using Azure.Storage.Queues;
 using Azure.Storage.Queues.Models;
 using System.Threading;
+using MimeKit;
 
 namespace Bit.Core.Utilities
 {
@@ -488,6 +489,31 @@ namespace Bit.Core.Utilities
             return Convert.FromBase64String(output);
         }
 
+        public static string PunyEncode(string text)
+        {
+            if (text == "")
+            {
+                return "";
+            }
+
+            if (text == null)
+            {
+                return null;
+            }
+
+            if (!text.Contains("@"))
+            {
+                // Assume domain name or non-email address
+                var idn = new IdnMapping();
+                return idn.GetAscii(text);
+            }
+            else
+            {
+                // Assume email address
+                return MailboxAddress.EncodeAddrspec(text);
+            }
+        }
+
         public static string FormatLicenseSignatureValue(object val)
         {
             if (val == null)

src/Core/Utilities/StrictEmailAddressAttribute.cs

@@ -7,7 +7,7 @@ namespace Bit.Core.Utilities
     public class StrictEmailAddressAttribute : ValidationAttribute
     {
         public StrictEmailAddressAttribute()
-            : base("The {0} field is not a valid e-mail address.")
+            : base("The {0} field is not a supported e-mail address format.")
         {}
         
         public override bool IsValid(object value)
@@ -31,7 +31,18 @@ namespace Bit.Core.Utilities
                 return false;
             }
 
-            if (!Regex.IsMatch(emailAddress, @"@.+\.[A-Za-z0-9]+$"))
+            /**
+            The regex below is intended to catch edge cases that are not handled by the general parsing check above.
+            This enforces the following rules:
+            * Requires ASCII only in the local-part (code points 0-127)
+            * Requires an @ symbol
+            * Allows any char in second-level domain name, including unicode and symbols
+            * Requires at least one period (.) separating SLD from TLD
+            * Must end in a letter (including unicode)
+            See the unit tests for examples of what is allowed.
+            **/
+            var emailFormat = @"[\x00-\x7F]+@.+\.\p{L}+$";
+            if (!Regex.IsMatch(emailAddress, emailFormat))
             {
                 return false;
             }

test/Core.Test/Utilities/CoreHelpersTests.cs

@@ -3,6 +3,7 @@ using System.Collections.Generic;
 using System.Linq;
 using Bit.Core.Utilities;
 using Xunit;
+using MimeKit;
 
 namespace Bit.Core.Test.Utilities
 {
@@ -216,5 +217,20 @@ namespace Bit.Core.Test.Utilities
             // Assert
             Assert.Equal(startingUri, newUri.ToString());
         }
+
+        [Theory]
+        [InlineData("bücher.com", "xn--bcher-kva.com")]
+        [InlineData("bücher.cömé", "xn--bcher-kva.xn--cm-cja4c")]
+        [InlineData("hello@bücher.com", "hello@xn--bcher-kva.com")]
+        [InlineData("hello@world.cömé", "hello@world.xn--cm-cja4c")]
+        [InlineData("hello@bücher.cömé", "hello@xn--bcher-kva.xn--cm-cja4c")]
+        [InlineData("ascii.com", "ascii.com")]
+        [InlineData("", "")]
+        [InlineData(null, null)]
+        public void PunyEncode_Success(string text, string expected)
+        {
+            var actual = CoreHelpers.PunyEncode(text);
+            Assert.Equal(expected, actual);
+        }
     }
 }

test/Core.Test/Utilities/StrictEmailAddressAttributeTests.cs

@@ -10,6 +10,8 @@ namespace Bit.Core.Test.Utilities
         [InlineData("hello@world.planet.com")]  // subdomain
         [InlineData("hello+1@world.com")]       // alias
         [InlineData("hello.there@world.com")]   // period in local-part
+        [InlineData("hello@wörldé.com")]        // unicode domain
+        [InlineData("hello@world.cömé")]        // unicode top-level domain
         public void IsValid_ReturnsTrueWhenValid(string email)
         {
             var sut = new StrictEmailAddressAttribute();
@@ -43,6 +45,7 @@ namespace Bit.Core.Test.Utilities
         [InlineData("hellothere@.worldcom")]                // domain beginning with dot
         [InlineData("hellothere@worldcom.")]                // domain ending in dot
         [InlineData("hellothere@world.com-")]               // domain ending in hyphen
+        [InlineData("héllö@world.com")]                     // unicode in local-part
         public void IsValid_ReturnsFalseWhenInvalid(string email)
         {
             var sut = new StrictEmailAddressAttribute();