Skip to content

Add operation to read and store data from definition blob store instead of db #82

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

Add operation to read and store data from definition blob store instead of db #82

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
58 changes: 50 additions & 8 deletions tools/blobstorage-backupdata/BackupJob.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,8 @@ namespace BackupData;

internal sealed class BackupJob
{
private readonly BlobContainerClient BlobContainerClient;
private readonly BlobContainerClient SnapshotBlobContainerClient;
private readonly BlobContainerClient DefinitionBlobContainerClient;
private readonly IMongoClient MongoClient;
private readonly DateTime Now;
private readonly ILogger Logger;
Expand All @@ -25,9 +26,10 @@ internal sealed class BackupJob
private const string UpdatedFieldName = "_meta.updated";
private const string MetaFieldName = "_meta";

public BackupJob(BlobContainerClient blobContainerClient, IMongoClient mongoClient, DateTime now, ILoggerFactory loggerFactory, IFilterRenderer filterRenderer)
public BackupJob(BlobContainerClient snapshotBlobContainerClient, BlobContainerClient definitionBlobContainerClient, IMongoClient mongoClient, DateTime now, ILoggerFactory loggerFactory, IFilterRenderer filterRenderer)
{
BlobContainerClient = blobContainerClient;
SnapshotBlobContainerClient = snapshotBlobContainerClient;
DefinitionBlobContainerClient = definitionBlobContainerClient;
MongoClient = mongoClient;
Now = now;
Logger = loggerFactory.CreateLogger(nameof(BackupJob));
Expand Down Expand Up @@ -109,7 +111,15 @@ await Parallel.ForEachAsync(cursor.Current, async (document, _) =>
{
throw new Exception("Blob name is null or empty.");
}
await UploadString(jObject.ToString(), blobName);
var definitionCoordinates = jObject.ConstructBlobUrl();
if (string.IsNullOrWhiteSpace(definitionCoordinates))
{
throw new Exception("Definition blob name is null or empty.");
}
// Get the data from the definition blob store
string definitionBlobData = await GetDefinition(definitionCoordinates);

await UploadString(definitionBlobData, blobName);
AddChangesToIndex(changesIndex, jObject, blobName);
}
catch (Exception e)
Expand Down Expand Up @@ -163,13 +173,32 @@ await UploadString(
"changes/index");
}

private async Task<string> ReadFromBlob(
BlobContainerClient blobContainerClient, string blobName)
{
try
{
var blobClient = blobContainerClient.GetBlobClient(blobName);
var response = await blobClient.DownloadContentAsync();
var content = response.Value.Content.ToString();
Logger.LogInformation("Content of the blob: {blobName}, content: {content}", blobName, content);

return content;
}
catch (Exception e)
{
Logger.LogError("Failed to read from definition blob: {blobName}, error message: {exceptionMessage}", blobName, e.Message);
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This function is used to read definition and index. so may want to use "Failed to read from blob" in error message.

throw;
}
}

private async Task UploadString(
string blobContent,
string blobName)
{
try
{
var blobClient = BlobContainerClient.GetBlobClient(blobName);
var blobClient = SnapshotBlobContainerClient.GetBlobClient(blobName);
using var stream = new MemoryStream(Encoding.UTF8.GetBytes(blobContent));
await blobClient.UploadAsync(stream, overwrite: true);
}
Expand All @@ -180,13 +209,26 @@ private async Task UploadString(
}
}

internal async Task<string> GetDefinition(string coordinatePath)
{
try
{
var data = await ReadFromBlob(DefinitionBlobContainerClient, coordinatePath);
return data;
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Based on Readme, published changed definition json is definition minus files. The definition blob storage content contains the entire definition (with files). so probably need to remove the "files" section here.

}
catch (Exception e)
{
Logger.LogError("Failed to get definition from blob storage, exception: {exceptionMessage}", e.Message);
throw;
}
}

internal async Task<string[]> GetIndex()
{
try
{
var blobClient = BlobContainerClient.GetBlobClient("changes/index");
return (await blobClient.DownloadContentAsync()).Value.Content.ToString()
.Split(Environment.NewLine, StringSplitOptions.RemoveEmptyEntries);
var data = await ReadFromBlob(SnapshotBlobContainerClient, "changes/index");
return data.Split(Environment.NewLine, StringSplitOptions.RemoveEmptyEntries);
}
catch (Exception e)
{
Expand Down
Loading