diff --git a/src/Api/Api.csproj b/src/Api/Api.csproj index ea437a497d..ef1ae65c7e 100644 --- a/src/Api/Api.csproj +++ b/src/Api/Api.csproj @@ -26,6 +26,7 @@ + diff --git a/src/Api/Controllers/SendsController.cs b/src/Api/Controllers/SendsController.cs index 38fd1214d6..6cb01154f0 100644 --- a/src/Api/Controllers/SendsController.cs +++ b/src/Api/Controllers/SendsController.cs @@ -7,11 +7,17 @@ using Microsoft.AspNetCore.Authorization; using Bit.Core.Models.Api; using Bit.Core.Exceptions; using Bit.Core.Services; -using Bit.Api.Utilities; -using Bit.Core.Models.Table; using Bit.Core.Utilities; using Bit.Core.Settings; using Bit.Core.Models.Api.Response; +using Bit.Core.Enums; +using Microsoft.Azure.EventGrid.Models; +using Bit.Api.Utilities; +using System.Collections.Generic; +using Bit.Core.Models.Table; +using Newtonsoft.Json; +using Bit.Core.Models.Data; +using Microsoft.Extensions.Logging; namespace Bit.Api.Controllers { @@ -23,6 +29,7 @@ namespace Bit.Api.Controllers private readonly IUserService _userService; private readonly ISendService _sendService; private readonly ISendFileStorageService _sendFileStorageService; + private readonly ILogger _logger; private readonly GlobalSettings _globalSettings; public SendsController( @@ -30,12 +37,14 @@ namespace Bit.Api.Controllers IUserService userService, ISendService sendService, ISendFileStorageService sendFileStorageService, + ILogger logger, GlobalSettings globalSettings) { _sendRepository = sendRepository; _userService = userService; _sendService = sendService; _sendFileStorageService = sendFileStorageService; + _logger = logger; _globalSettings = globalSettings; } @@ -160,12 +169,113 @@ namespace Bit.Api.Controllers var userId = _userService.GetProperUserId(User).Value; var (madeSend, madeData) = model.ToSend(userId, fileName, _sendService); send = madeSend; - await _sendService.CreateSendAsync(send, madeData, stream, model.FileLength.GetValueOrDefault(0)); + await _sendService.SaveFileSendAsync(send, madeData, model.FileLength.GetValueOrDefault(0)); + await _sendService.UploadFileToExistingSendAsync(stream, send); }); return new SendResponseModel(send, _globalSettings); } + + [HttpPost("file/v2")] + public async Task PostFile([FromBody] SendRequestModel model) + { + if (model.Type != SendType.File) + { + throw new BadRequestException("Invalid content."); + } + + if (!model.FileLength.HasValue) + { + throw new BadRequestException("Invalid content. File size hint is required."); + } + + var userId = _userService.GetProperUserId(User).Value; + var (send, data) = model.ToSend(userId, model.File.FileName, _sendService); + var uploadUrl = await _sendService.SaveFileSendAsync(send, data, model.FileLength.Value); + return new SendFileUploadDataResponseModel + { + Url = uploadUrl, + FileUploadType = _sendFileStorageService.FileUploadType, + SendResponse = new SendResponseModel(send, _globalSettings) + }; + } + + [HttpGet("{id}/file/{fileId}")] + public async Task RenewFileUpload(string id, string fileId) + { + var userId = _userService.GetProperUserId(User).Value; + var sendId = new Guid(id); + var send = await _sendRepository.GetByIdAsync(sendId); + var fileData = JsonConvert.DeserializeObject(send?.Data); + + if (send == null || send.Type != SendType.File || (send.UserId.HasValue && send.UserId.Value != userId) || + !send.UserId.HasValue || fileData.Id != fileId || fileData.Validated) + { + // Not found if Send isn't found, user doesn't have access, request is faulty, + // or we've already validated the file. This last is to emulate create-only blob permissions for Azure + throw new NotFoundException(); + } + + return new SendFileUploadDataResponseModel + { + Url = await _sendFileStorageService.GetSendFileUploadUrlAsync(send, fileId), + FileUploadType = _sendFileStorageService.FileUploadType, + SendResponse = new SendResponseModel(send, _globalSettings), + }; + } + + [HttpPost("{id}/file/{fileId}")] + [DisableFormValueModelBinding] + public async Task PostFileForExistingSend(string id, string fileId) + { + if (!Request?.ContentType.Contains("multipart/") ?? true) + { + throw new BadRequestException("Invalid content."); + } + + if (Request.ContentLength > 105906176 && !_globalSettings.SelfHosted) // 101 MB, give em' 1 extra MB for cushion + { + throw new BadRequestException("Max file size for direct upload is 100 MB."); + } + + var send = await _sendRepository.GetByIdAsync(new Guid(id)); + await Request.GetSendFileAsync(async (stream) => + { + await _sendService.UploadFileToExistingSendAsync(stream, send); + }); + } + + [AllowAnonymous] + [HttpPost("file/validate/azure")] + public async Task AzureValidateFile() + { + return await ApiHelpers.HandleAzureEvents(Request, new Dictionary> + { + { + "Microsoft.Storage.BlobCreated", async (eventGridEvent) => + { + try + { + var blobName = eventGridEvent.Subject.Split($"{AzureSendFileStorageService.FilesContainerName}/blobs/")[1]; + var sendId = AzureSendFileStorageService.SendIdFromBlobName(blobName); + var send = await _sendRepository.GetByIdAsync(new Guid(sendId)); + if (send == null) + { + return; + } + await _sendService.ValidateSendFile(send); + } + catch (Exception e) + { + _logger.LogError(e, $"Uncaught exception occurred while handling event grid event: {JsonConvert.SerializeObject(eventGridEvent)}"); + return; + } + } + } + }); + } + [HttpPut("{id}")] public async Task Put(string id, [FromBody] SendRequestModel model) { diff --git a/src/Api/Utilities/ApiHelpers.cs b/src/Api/Utilities/ApiHelpers.cs index 8aef098b52..20ed178762 100644 --- a/src/Api/Utilities/ApiHelpers.cs +++ b/src/Api/Utilities/ApiHelpers.cs @@ -1,5 +1,10 @@ using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Azure.EventGrid; +using Microsoft.Azure.EventGrid.Models; using Newtonsoft.Json; +using System; +using System.Collections.Generic; using System.IO; using System.Threading.Tasks; @@ -29,5 +34,47 @@ namespace Bit.Api.Utilities return obj; } + + /// + /// Validates Azure event subscription and calls the appropriate event handler. Responds HttpOk. + /// + /// HttpRequest received from Azure + /// Dictionary of eventType strings and their associated handlers. + /// OkObjectResult + /// Reference https://docs.microsoft.com/en-us/azure/event-grid/receive-events + public async static Task HandleAzureEvents(HttpRequest request, + Dictionary> eventTypeHandlers) + { + var response = string.Empty; + var requestContent = await new StreamReader(request.Body).ReadToEndAsync(); + if (string.IsNullOrWhiteSpace(requestContent)) + { + return new OkObjectResult(response); + } + + var eventGridSubscriber = new EventGridSubscriber(); + var eventGridEvents = eventGridSubscriber.DeserializeEventGridEvents(requestContent); + + foreach (var eventGridEvent in eventGridEvents) + { + if (eventGridEvent.Data is SubscriptionValidationEventData eventData) + { + // Might want to enable additional validation: subject, topic etc. + + var responseData = new SubscriptionValidationResponse() + { + ValidationResponse = eventData.ValidationCode + }; + + return new OkObjectResult(responseData); + } + else if (eventTypeHandlers.ContainsKey(eventGridEvent.EventType)) + { + await eventTypeHandlers[eventGridEvent.EventType](eventGridEvent); + } + } + + return new OkObjectResult(response); + } } } diff --git a/src/Api/Utilities/MultipartFormDataHelper.cs b/src/Api/Utilities/MultipartFormDataHelper.cs index 03ed0f1ae7..01c4e35823 100644 --- a/src/Api/Utilities/MultipartFormDataHelper.cs +++ b/src/Api/Utilities/MultipartFormDataHelper.cs @@ -108,6 +108,27 @@ namespace Bit.Api.Utilities } } + public static async Task GetSendFileAsync(this HttpRequest request, Func callback) + { + var boundary = GetBoundary(MediaTypeHeaderValue.Parse(request.ContentType), + _defaultFormOptions.MultipartBoundaryLengthLimit); + var reader = new MultipartReader(boundary, request.Body); + + var dataSection = await reader.ReadNextSectionAsync(); + if (dataSection != null) + { + if (ContentDispositionHeaderValue.TryParse(dataSection.ContentDisposition, out var dataContent) + && HasFileContentDisposition(dataContent)) + { + using (dataSection.Body) + { + await callback(dataSection.Body); + } + } + dataSection = null; + } + } + private static string GetBoundary(MediaTypeHeaderValue contentType, int lengthLimit) { diff --git a/src/Core/Enums/FileUploadType.cs b/src/Core/Enums/FileUploadType.cs new file mode 100644 index 0000000000..dc50eb6696 --- /dev/null +++ b/src/Core/Enums/FileUploadType.cs @@ -0,0 +1,8 @@ +namespace Bit.Core.Enums +{ + public enum FileUploadType + { + Direct = 0, + Azure = 1, + } +} diff --git a/src/Core/Models/Api/Request/SendRequestModel.cs b/src/Core/Models/Api/Request/SendRequestModel.cs index a0cca1a1fc..d64faef176 100644 --- a/src/Core/Models/Api/Request/SendRequestModel.cs +++ b/src/Core/Models/Api/Request/SendRequestModel.cs @@ -13,7 +13,7 @@ namespace Bit.Core.Models.Api public class SendRequestModel { public SendType Type { get; set; } - public long? FileLength { get; set; } + public long? FileLength { get; set; } = null; [EncryptedString] [EncryptedStringLength(1000)] public string Name { get; set; } diff --git a/src/Core/Models/Api/Response/SendFileUploadDataResponseModel.cs b/src/Core/Models/Api/Response/SendFileUploadDataResponseModel.cs new file mode 100644 index 0000000000..aded0d7142 --- /dev/null +++ b/src/Core/Models/Api/Response/SendFileUploadDataResponseModel.cs @@ -0,0 +1,14 @@ +using Bit.Core.Enums; + +namespace Bit.Core.Models.Api.Response +{ + public class SendFileUploadDataResponseModel : ResponseModel + { + public SendFileUploadDataResponseModel() : base("send-fileUpload") { } + + public string Url { get; set; } + public FileUploadType FileUploadType { get; set; } + public SendResponseModel SendResponse { get; set; } + + } +} diff --git a/src/Core/Models/Data/SendFileData.cs b/src/Core/Models/Data/SendFileData.cs index 6227cf4bbe..0053aa2123 100644 --- a/src/Core/Models/Data/SendFileData.cs +++ b/src/Core/Models/Data/SendFileData.cs @@ -33,5 +33,6 @@ namespace Bit.Core.Models.Data public string Id { get; set; } public string FileName { get; set; } + public bool Validated { get; set; } = true; } } diff --git a/src/Core/Services/ISendService.cs b/src/Core/Services/ISendService.cs index 5fb354cf2c..ae4162a7b7 100644 --- a/src/Core/Services/ISendService.cs +++ b/src/Core/Services/ISendService.cs @@ -10,9 +10,11 @@ namespace Bit.Core.Services { Task DeleteSendAsync(Send send); Task SaveSendAsync(Send send); - Task CreateSendAsync(Send send, SendFileData data, Stream stream, long requestLength); + Task SaveFileSendAsync(Send send, SendFileData data, long fileLength); + Task UploadFileToExistingSendAsync(Stream stream, Send send); Task<(Send, bool, bool)> AccessAsync(Guid sendId, string password); string HashPassword(string password); Task<(string, bool, bool)> GetSendFileDownloadUrlAsync(Send send, string fileId, string password); + Task ValidateSendFile(Send send); } } diff --git a/src/Core/Services/ISendStorageService.cs b/src/Core/Services/ISendStorageService.cs index ad65714c44..474e856e2c 100644 --- a/src/Core/Services/ISendStorageService.cs +++ b/src/Core/Services/ISendStorageService.cs @@ -1,4 +1,5 @@ -using Bit.Core.Models.Table; +using Bit.Core.Enums; +using Bit.Core.Models.Table; using System; using System.IO; using System.Threading.Tasks; @@ -7,10 +8,13 @@ namespace Bit.Core.Services { public interface ISendFileStorageService { + FileUploadType FileUploadType { get; } Task UploadNewFileAsync(Stream stream, Send send, string fileId); Task DeleteFileAsync(Send send, string fileId); Task DeleteFilesForOrganizationAsync(Guid organizationId); Task DeleteFilesForUserAsync(Guid userId); Task GetSendFileDownloadUrlAsync(Send send, string fileId); + Task GetSendFileUploadUrlAsync(Send send, string fileId); + Task<(bool, long?)> ValidateFileAsync(Send send, string fileId, long expectedFileSize, long leeway); } } diff --git a/src/Core/Services/Implementations/AzureSendFileStorageService.cs b/src/Core/Services/Implementations/AzureSendFileStorageService.cs index 1c33617e0b..62f998d72c 100644 --- a/src/Core/Services/Implementations/AzureSendFileStorageService.cs +++ b/src/Core/Services/Implementations/AzureSendFileStorageService.cs @@ -5,6 +5,7 @@ using System.IO; using System; using Bit.Core.Models.Table; using Bit.Core.Settings; +using Bit.Core.Enums; namespace Bit.Core.Services { @@ -15,6 +16,8 @@ namespace Bit.Core.Services private readonly CloudBlobClient _blobClient; private CloudBlobContainer _sendFilesContainer; + public FileUploadType FileUploadType => FileUploadType.Azure; + public static string SendIdFromBlobName(string blobName) => blobName.Split('/')[0]; public static string BlobName(Send send, string fileId) => $"{send.Id}/{fileId}"; @@ -71,6 +74,54 @@ namespace Bit.Core.Services return blob.Uri + blob.GetSharedAccessSignature(accessPolicy); } + public async Task GetSendFileUploadUrlAsync(Send send, string fileId) + { + await InitAsync(); + var blob = _sendFilesContainer.GetBlockBlobReference(BlobName(send, fileId)); + + var accessPolicy = new SharedAccessBlobPolicy() + { + SharedAccessExpiryTime = DateTime.UtcNow.Add(_downloadLinkLiveTime), + Permissions = SharedAccessBlobPermissions.Create | SharedAccessBlobPermissions.Write, + }; + + return blob.Uri + blob.GetSharedAccessSignature(accessPolicy); + } + + public async Task<(bool, long?)> ValidateFileAsync(Send send, string fileId, long expectedFileSize, long leeway) + { + await InitAsync(); + + var blob = _sendFilesContainer.GetBlockBlobReference(BlobName(send, fileId)); + + if (!blob.Exists()) + { + return (false, null); + } + + blob.FetchAttributes(); + + if (send.UserId.HasValue) + { + blob.Metadata["userId"] = send.UserId.Value.ToString(); + } + else + { + blob.Metadata["organizationId"] = send.OrganizationId.Value.ToString(); + } + blob.Properties.ContentDisposition = $"attachment; filename=\"{fileId}\""; + blob.SetMetadata(); + blob.SetProperties(); + + var length = blob.Properties.Length; + if (length < expectedFileSize - leeway || length > expectedFileSize + leeway) + { + return (false, length); + } + + return (true, length); + } + private async Task InitAsync() { if (_sendFilesContainer == null) diff --git a/src/Core/Services/Implementations/LocalSendStorageService.cs b/src/Core/Services/Implementations/LocalSendStorageService.cs index 5406ab73c2..26339f343c 100644 --- a/src/Core/Services/Implementations/LocalSendStorageService.cs +++ b/src/Core/Services/Implementations/LocalSendStorageService.cs @@ -4,6 +4,7 @@ using System; using Bit.Core.Models.Table; using Bit.Core.Settings; using System.Linq; +using Bit.Core.Enums; namespace Bit.Core.Services { @@ -14,6 +15,7 @@ namespace Bit.Core.Services private string RelativeFilePath(Send send, string fileID) => $"{send.Id}/{fileID}"; private string FilePath(Send send, string fileID) => $"{_baseDirPath}/{RelativeFilePath(send, fileID)}"; + public FileUploadType FileUploadType => FileUploadType.Direct; public LocalSendStorageService( GlobalSettings globalSettings) @@ -83,5 +85,26 @@ namespace Bit.Core.Services return Task.FromResult(0); } + + public Task GetSendFileUploadUrlAsync(Send send, string fileId) + => Task.FromResult($"/sends/{send.Id}/file/{fileId}"); + + public Task<(bool, long?)> ValidateFileAsync(Send send, string fileId, long expectedFileSize, long leeway) + { + long? length = null; + var path = FilePath(send, fileId); + if (!File.Exists(path)) + { + return Task.FromResult((false, length)); + } + + length = new FileInfo(path).Length; + if (expectedFileSize < length - leeway || expectedFileSize > length + leeway) + { + return Task.FromResult((false, length)); + } + + return Task.FromResult((true, length)); + } } } diff --git a/src/Core/Services/Implementations/SendService.cs b/src/Core/Services/Implementations/SendService.cs index d315772586..945efa8e00 100644 --- a/src/Core/Services/Implementations/SendService.cs +++ b/src/Core/Services/Implementations/SendService.cs @@ -28,6 +28,7 @@ namespace Bit.Core.Services private readonly IReferenceEventService _referenceEventService; private readonly GlobalSettings _globalSettings; private readonly ICurrentContext _currentContext; + private const long _fileSizeLeeway = 1024L * 1024L; // 1MB public SendService( ISendRepository sendRepository, @@ -74,51 +75,21 @@ namespace Bit.Core.Services } } - public async Task CreateSendAsync(Send send, SendFileData data, Stream stream, long requestLength) + public async Task SaveFileSendAsync(Send send, SendFileData data, long fileLength) { if (send.Type != SendType.File) { throw new BadRequestException("Send is not of type \"file\"."); } - if (requestLength < 1) + if (fileLength < 1) { throw new BadRequestException("No file data."); } - var storageBytesRemaining = 0L; - if (send.UserId.HasValue) - { - var user = await _userRepository.GetByIdAsync(send.UserId.Value); - if (!(await _userService.CanAccessPremium(user))) - { - throw new BadRequestException("You must have premium status to use file sends."); - } + var storageBytesRemaining = await StorageRemainingForSendAsync(send); - if (user.Premium) - { - storageBytesRemaining = user.StorageBytesRemaining(); - } - else - { - // Users that get access to file storage/premium from their organization get the default - // 1 GB max storage. - storageBytesRemaining = user.StorageBytesRemaining( - _globalSettings.SelfHosted ? (short)10240 : (short)1); - } - } - else if (send.OrganizationId.HasValue) - { - var org = await _organizationRepository.GetByIdAsync(send.OrganizationId.Value); - if (!org.MaxStorageGb.HasValue) - { - throw new BadRequestException("This organization cannot use file sends."); - } - - storageBytesRemaining = org.StorageBytesRemaining(); - } - - if (storageBytesRemaining < requestLength) + if (storageBytesRemaining < fileLength) { throw new BadRequestException("Not enough storage available."); } @@ -128,24 +99,12 @@ namespace Bit.Core.Services try { data.Id = fileId; + data.Size = fileLength; + data.Validated = false; send.Data = JsonConvert.SerializeObject(data, new JsonSerializerSettings { NullValueHandling = NullValueHandling.Ignore }); await SaveSendAsync(send); - await _sendFileStorageService.UploadNewFileAsync(stream, send, fileId); - // Need to save length of stream since that isn't available until it is read - if (stream.Length <= requestLength) - { - data.Size = stream.Length; - send.Data = JsonConvert.SerializeObject(data, - new JsonSerializerSettings { NullValueHandling = NullValueHandling.Ignore }); - await SaveSendAsync(send); - } - else - { - await DeleteSendAsync(send); - throw new BadRequestException("Content-Length header is smaller than file received."); - } - + return await _sendFileStorageService.GetSendFileUploadUrlAsync(send, fileId); } catch { @@ -155,6 +114,53 @@ namespace Bit.Core.Services } } + public async Task UploadFileToExistingSendAsync(Stream stream, Send send) + { + if (send?.Data == null) + { + throw new BadRequestException("Send does not have file data"); + } + + if (send.Type != SendType.File) + { + throw new BadRequestException("Not a File Type Send."); + } + + var data = JsonConvert.DeserializeObject(send.Data); + + await _sendFileStorageService.UploadNewFileAsync(stream, send, data.Id); + + if (!await ValidateSendFile(send)) + { + throw new BadRequestException("File received does not match expected file length."); + } + } + + public async Task ValidateSendFile(Send send) + { + var fileData = JsonConvert.DeserializeObject(send.Data); + + var (valid, realSize) = await _sendFileStorageService.ValidateFileAsync(send, fileData.Id, fileData.Size, _fileSizeLeeway); + + if (!valid) + { + // File reported differs in size from that promised. Must be a rogue client. Delete Send + await DeleteSendAsync(send); + } + + // Update Send data if necessary + if (realSize != fileData.Size) + { + fileData.Size = realSize.Value; + } + fileData.Validated = true; + send.Data = JsonConvert.SerializeObject(fileData, + new JsonSerializerSettings { NullValueHandling = NullValueHandling.Ignore }); + await SaveSendAsync(send); + + return valid; + } + public async Task DeleteSendAsync(Send send) { await _sendRepository.DeleteAsync(send); @@ -281,5 +287,42 @@ namespace Bit.Core.Services } } } + + private async Task StorageRemainingForSendAsync(Send send) + { + var storageBytesRemaining = 0L; + if (send.UserId.HasValue) + { + var user = await _userRepository.GetByIdAsync(send.UserId.Value); + if (!await _userService.CanAccessPremium(user)) + { + throw new BadRequestException("You must have premium status to use file sends."); + } + + if (user.Premium) + { + storageBytesRemaining = user.StorageBytesRemaining(); + } + else + { + // Users that get access to file storage/premium from their organization get the default + // 1 GB max storage. + storageBytesRemaining = user.StorageBytesRemaining( + _globalSettings.SelfHosted ? (short)10240 : (short)1); + } + } + else if (send.OrganizationId.HasValue) + { + var org = await _organizationRepository.GetByIdAsync(send.OrganizationId.Value); + if (!org.MaxStorageGb.HasValue) + { + throw new BadRequestException("This organization cannot use file sends."); + } + + storageBytesRemaining = org.StorageBytesRemaining(); + } + + return storageBytesRemaining; + } } } diff --git a/src/Core/Services/NoopImplementations/NoopSendFileStorageService.cs b/src/Core/Services/NoopImplementations/NoopSendFileStorageService.cs index 7eac969564..819e42c886 100644 --- a/src/Core/Services/NoopImplementations/NoopSendFileStorageService.cs +++ b/src/Core/Services/NoopImplementations/NoopSendFileStorageService.cs @@ -2,11 +2,14 @@ using System.IO; using System; using Bit.Core.Models.Table; +using Bit.Core.Enums; namespace Bit.Core.Services { public class NoopSendFileStorageService : ISendFileStorageService { + public FileUploadType FileUploadType => FileUploadType.Direct; + public Task UploadNewFileAsync(Stream stream, Send send, string attachmentId) { return Task.FromResult(0); @@ -31,5 +34,15 @@ namespace Bit.Core.Services { return Task.FromResult((string)null); } + + public Task GetSendFileUploadUrlAsync(Send send, string fileId) + { + return Task.FromResult((string)null); + } + + public Task<(bool, long?)> ValidateFileAsync(Send send, string fileId, long expectedFileSize, long leeway) + { + return Task.FromResult((false, default(long?))); + } } }