nhyatt/bitwarden
1
0
Fork 0
mirror of https://github.com/bitwarden/server.git synced 2025-06-30 15:42:48 -05:00
Code Activity

Direct upload to Azure/Local (#1188)

Browse Source 
* Direct upload to azure

To validate file sizes in the event of a rogue client, Azure event webhooks
will be hooked up to AzureValidateFile.
Sends outside of a grace size will be deleted as non-compliant.

TODO: LocalSendFileStorageService direct upload method/endpoint.

* Quick respond to no-body event calls

These shouldn't happen, but might if some errant get requests occur

* Event Grid only POSTS to webhook

* Enable local storage direct file upload

* Increase file size difference leeway

* Upload through service

* Fix LocalFileSendStorage

It turns out that multipartHttpStreams do not have a length
until read. this causes all long files to be "invalid". We need to
write the entire stream, then validate length, just like Azure.

the difference is, We can return an exception to local storage
admonishing the client for lying

* Update src/Api/Utilities/ApiHelpers.cs

Co-authored-by: Chad Scharf <3904944+cscharf@users.noreply.github.com>

* Do not delete directory if it has files

* Allow large uploads for self hosted instances

* Fix formatting

* Re-verfiy access and increment access count on download of Send File

* Update src/Core/Services/Implementations/SendService.cs

Co-authored-by: Chad Scharf <3904944+cscharf@users.noreply.github.com>

* Add back in original Send upload

* Update size and mark as validated upon Send file validation

* Log azure file validation errors

* Lint fix

Co-authored-by: Chad Scharf <3904944+cscharf@users.noreply.github.com>
This commit is contained in:
Matt Gibson
2021-03-21 23:01:19 -05:00
committed by GitHub
parent fd42b227b3
commit 989d4df599
14 changed files with 393 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files

51
src/Core/Services/Implementations/AzureSendFileStorageService.cs
View File

@ -5,6 +5,7 @@ using System.IO;
using System;
using Bit.Core.Models.Table;
using Bit.Core.Settings;
using Bit.Core.Enums;
namespace Bit.Core.Services
{
@ -15,6 +16,8 @@ namespace Bit.Core.Services
private readonly CloudBlobClient _blobClient;
private CloudBlobContainer _sendFilesContainer;
public FileUploadType FileUploadType => FileUploadType.Azure;
public static string SendIdFromBlobName(string blobName) => blobName.Split('/')[0];
public static string BlobName(Send send, string fileId) => $"{send.Id}/{fileId}";
@ -71,6 +74,54 @@ namespace Bit.Core.Services
return blob.Uri + blob.GetSharedAccessSignature(accessPolicy);
}
public async Task<string> GetSendFileUploadUrlAsync(Send send, string fileId)
{
await InitAsync();
var blob = _sendFilesContainer.GetBlockBlobReference(BlobName(send, fileId));
var accessPolicy = new SharedAccessBlobPolicy()
{
SharedAccessExpiryTime = DateTime.UtcNow.Add(_downloadLinkLiveTime),
Permissions = SharedAccessBlobPermissions.Create | SharedAccessBlobPermissions.Write,
};
return blob.Uri + blob.GetSharedAccessSignature(accessPolicy);
}
public async Task<(bool, long?)> ValidateFileAsync(Send send, string fileId, long expectedFileSize, long leeway)
{
await InitAsync();
var blob = _sendFilesContainer.GetBlockBlobReference(BlobName(send, fileId));
if (!blob.Exists())
{
return (false, null);
}
blob.FetchAttributes();
if (send.UserId.HasValue)
{
blob.Metadata["userId"] = send.UserId.Value.ToString();
}
else
{
blob.Metadata["organizationId"] = send.OrganizationId.Value.ToString();
}
blob.Properties.ContentDisposition = $"attachment; filename=\"{fileId}\"";
blob.SetMetadata();
blob.SetProperties();
var length = blob.Properties.Length;
if (length < expectedFileSize - leeway || length > expectedFileSize + leeway)
{
return (false, length);
}
return (true, length);
}
private async Task InitAsync()
{
if (_sendFilesContainer == null)

23
src/Core/Services/Implementations/LocalSendStorageService.cs
View File

@ -4,6 +4,7 @@ using System;
using Bit.Core.Models.Table;
using Bit.Core.Settings;
using System.Linq;
using Bit.Core.Enums;
namespace Bit.Core.Services
{
@ -14,6 +15,7 @@ namespace Bit.Core.Services
private string RelativeFilePath(Send send, string fileID) => $"{send.Id}/{fileID}";
private string FilePath(Send send, string fileID) => $"{_baseDirPath}/{RelativeFilePath(send, fileID)}";
public FileUploadType FileUploadType => FileUploadType.Direct;
public LocalSendStorageService(
GlobalSettings globalSettings)
@ -83,5 +85,26 @@ namespace Bit.Core.Services
return Task.FromResult(0);
}
public Task<string> GetSendFileUploadUrlAsync(Send send, string fileId)
=> Task.FromResult($"/sends/{send.Id}/file/{fileId}");
public Task<(bool, long?)> ValidateFileAsync(Send send, string fileId, long expectedFileSize, long leeway)
{
long? length = null;
var path = FilePath(send, fileId);
if (!File.Exists(path))
{
return Task.FromResult((false, length));
}
length = new FileInfo(path).Length;
if (expectedFileSize < length - leeway || expectedFileSize > length + leeway)
{
return Task.FromResult((false, length));
}
return Task.FromResult((true, length));
}
}
}

141
src/Core/Services/Implementations/SendService.cs
View File

@ -28,6 +28,7 @@ namespace Bit.Core.Services
private readonly IReferenceEventService _referenceEventService;
private readonly GlobalSettings _globalSettings;
private readonly ICurrentContext _currentContext;
private const long _fileSizeLeeway = 1024L * 1024L; // 1MB
public SendService(
ISendRepository sendRepository,
@ -74,51 +75,21 @@ namespace Bit.Core.Services
}
}
public async Task CreateSendAsync(Send send, SendFileData data, Stream stream, long requestLength)
public async Task<string> SaveFileSendAsync(Send send, SendFileData data, long fileLength)
{
if (send.Type != SendType.File)
{
throw new BadRequestException("Send is not of type \"file\".");
}
if (requestLength < 1)
if (fileLength < 1)
{
throw new BadRequestException("No file data.");
}
var storageBytesRemaining = 0L;
if (send.UserId.HasValue)
{
var user = await _userRepository.GetByIdAsync(send.UserId.Value);
if (!(await _userService.CanAccessPremium(user)))
{
throw new BadRequestException("You must have premium status to use file sends.");
}
var storageBytesRemaining = await StorageRemainingForSendAsync(send);
if (user.Premium)
{
storageBytesRemaining = user.StorageBytesRemaining();
}
else
{
// Users that get access to file storage/premium from their organization get the default
// 1 GB max storage.
storageBytesRemaining = user.StorageBytesRemaining(
_globalSettings.SelfHosted ? (short)10240 : (short)1);
}
}
else if (send.OrganizationId.HasValue)
{
var org = await _organizationRepository.GetByIdAsync(send.OrganizationId.Value);
if (!org.MaxStorageGb.HasValue)
{
throw new BadRequestException("This organization cannot use file sends.");
}
storageBytesRemaining = org.StorageBytesRemaining();
}
if (storageBytesRemaining < requestLength)
if (storageBytesRemaining < fileLength)
{
throw new BadRequestException("Not enough storage available.");
}
@ -128,24 +99,12 @@ namespace Bit.Core.Services
try
{
data.Id = fileId;
data.Size = fileLength;
data.Validated = false;
send.Data = JsonConvert.SerializeObject(data,
new JsonSerializerSettings { NullValueHandling = NullValueHandling.Ignore });
await SaveSendAsync(send);
await _sendFileStorageService.UploadNewFileAsync(stream, send, fileId);
// Need to save length of stream since that isn't available until it is read
if (stream.Length <= requestLength)
{
data.Size = stream.Length;
send.Data = JsonConvert.SerializeObject(data,
new JsonSerializerSettings { NullValueHandling = NullValueHandling.Ignore });
await SaveSendAsync(send);
}
else
{
await DeleteSendAsync(send);
throw new BadRequestException("Content-Length header is smaller than file received.");
}
return await _sendFileStorageService.GetSendFileUploadUrlAsync(send, fileId);
}
catch
{
@ -155,6 +114,53 @@ namespace Bit.Core.Services
}
}
public async Task UploadFileToExistingSendAsync(Stream stream, Send send)
{
if (send?.Data == null)
{
throw new BadRequestException("Send does not have file data");
}
if (send.Type != SendType.File)
{
throw new BadRequestException("Not a File Type Send.");
}
var data = JsonConvert.DeserializeObject<SendFileData>(send.Data);
await _sendFileStorageService.UploadNewFileAsync(stream, send, data.Id);
if (!await ValidateSendFile(send))
{
throw new BadRequestException("File received does not match expected file length.");
}
}
public async Task<bool> ValidateSendFile(Send send)
{
var fileData = JsonConvert.DeserializeObject<SendFileData>(send.Data);
var (valid, realSize) = await _sendFileStorageService.ValidateFileAsync(send, fileData.Id, fileData.Size, _fileSizeLeeway);
if (!valid)
{
// File reported differs in size from that promised. Must be a rogue client. Delete Send
await DeleteSendAsync(send);
}
// Update Send data if necessary
if (realSize != fileData.Size)
{
fileData.Size = realSize.Value;
}
fileData.Validated = true;
send.Data = JsonConvert.SerializeObject(fileData,
new JsonSerializerSettings { NullValueHandling = NullValueHandling.Ignore });
await SaveSendAsync(send);
return valid;
}
public async Task DeleteSendAsync(Send send)
{
await _sendRepository.DeleteAsync(send);
@ -281,5 +287,42 @@ namespace Bit.Core.Services
}
}
}
private async Task<long> StorageRemainingForSendAsync(Send send)
{
var storageBytesRemaining = 0L;
if (send.UserId.HasValue)
{
var user = await _userRepository.GetByIdAsync(send.UserId.Value);
if (!await _userService.CanAccessPremium(user))
{
throw new BadRequestException("You must have premium status to use file sends.");
}
if (user.Premium)
{
storageBytesRemaining = user.StorageBytesRemaining();
}
else
{
// Users that get access to file storage/premium from their organization get the default
// 1 GB max storage.
storageBytesRemaining = user.StorageBytesRemaining(
_globalSettings.SelfHosted ? (short)10240 : (short)1);
}
}
else if (send.OrganizationId.HasValue)
{
var org = await _organizationRepository.GetByIdAsync(send.OrganizationId.Value);
if (!org.MaxStorageGb.HasValue)
{
throw new BadRequestException("This organization cannot use file sends.");
}
storageBytesRemaining = org.StorageBytesRemaining();
}
return storageBytesRemaining;
}
}
}