I have some files in Azure Blob and after processing the blobs, I am moving the files to another container.
The file I am getting in the destination folder is junk. What I am doing wrong?
The source file is under the ‘.log.gz‘ extension.
[Function("Function1")]
public async Task Run([TimerTrigger("0 */5 * * * *", RunOnStartup = true)] TimerInfo myTimer)
{
//Reading the config file. For local it is local.settings.json
var config = new ConfigurationBuilder()
.AddJsonFile("local.settings.json", optional: true, reloadOnChange: true)
.AddEnvironmentVariables()
.Build();
string blbobConnectionString = config["AzureWebJobsStorage"];
BlobContainerClient blobContainerClient = new BlobContainerClient(blbobConnectionString, "abc");
BlobContainerClient destinationBlobContainerClient = new BlobContainerClient(blbobConnectionString, "xyz");
var blobs = blobContainerClient.GetBlobsAsync();
await foreach (var blob in blobs)
{
_logger.LogInformation($"Blob Name: {blob.Name}");
BlobClient blobClient = blobContainerClient.GetBlobClient(blob.Name);
using (Stream blobStream = await blobClient.OpenReadAsync())
using (GZipStream decompressionStream = new GZipStream(blobStream, CompressionMode.Decompress))
using (StreamReader reader = new StreamReader(decompressionStream, Encoding.Default))
{
string line;
while ((line = await reader.ReadLineAsync()) != null)
{
// Process the line here
Console.WriteLine(line);
}
BlobClient destinationBlobClient = destinationBlobContainerClient.GetBlobClient(blob.Name);
blobStream.Position = 0;
await destinationBlobClient.UploadAsync(blobStream);
}
blobClient.DeleteAsync(DeleteSnapshotsOption.None);
}
}
Edit 1:
I re-read the blob for uploading but still get the same error. Following is the updated code.
[Function("Function1")]
public async Task Run([TimerTrigger("0 */5 * * * *", RunOnStartup = true)] TimerInfo myTimer)
{
_logger.LogInformation($"C# Timer trigger function executed at: {DateTime.Now}");
if (myTimer.ScheduleStatus is not null)
{
_logger.LogInformation($"Next timer schedule at: {myTimer.ScheduleStatus.Next}");
}
//Reading the config file. For local it is local.settings.json
var config = new ConfigurationBuilder()
.AddJsonFile("local.settings.json", optional: true, reloadOnChange: true)
.AddEnvironmentVariables()
.Build();
string blbobConnectionString = config["AzureWebJobsStorage"];
BlobContainerClient blobContainerClient = new BlobContainerClient(blbobConnectionString, "abc");
BlobContainerClient destinationBlobContainerClient = new BlobContainerClient(blbobConnectionString, "xyz");
var blobs = blobContainerClient.GetBlobsAsync();
await foreach (var blob in blobs)
{
_logger.LogInformation($"Blob Name: {blob.Name}");
BlobClient blobClient = blobContainerClient.GetBlobClient(blob.Name);
using (Stream blobStream = await blobClient.OpenReadAsync())
using (GZipStream decompressionStream = new GZipStream(blobStream, CompressionMode.Decompress))
using (StreamReader reader = new StreamReader(decompressionStream, Encoding.Default))
{
string line;
while ((line = await reader.ReadLineAsync()) != null)
{
// Process the line here
Console.WriteLine(line);
JObject json = JObject.Parse(line);
string dateOfEvent = System.Convert.ToString( json["Datetime"]);
string datePattern = "M/d/yyyy h:m:s tt";
DateTime parsedDate;
if (DateTime.TryParseExact(dateOfEvent, datePattern, null, DateTimeStyles.None, out parsedDate))
Console.WriteLine("Parsed date: "+ parsedDate);
json["Datetime"] = parsedDate;
line = json.ToString();
Console.WriteLine(System.DateTime.UtcNow.ToString("YYYY-MM-DDThh:mm:ssZ"));
dateOfEvent = parsedDate.ToString("yyyy-MM-ddTHH:mm:ssZ", CultureInfo.InvariantCulture);
var datestring = DateTime.UtcNow.ToString("r");
var jsonBytes = Encoding.UTF8.GetBytes(line);
string stringToHash = "POSTn" + jsonBytes.Length + "napplication/jsonn" + "x-ms-date:" + datestring + "n/api/logs";
string hashedString = BuildSignature(stringToHash, sharedKey);
string signature = "SharedKey " + customerId + ":" + hashedString;
PostData(signature, datestring, line);
}
}
BlobClient destinationBlobClient = destinationBlobContainerClient.GetBlobClient(blob.Name);
//blobStream.Position = 0;
await destinationBlobClient.UploadAsync(await blobClient.OpenReadAsync());
blobClient.DeleteAsync(DeleteSnapshotsOption.None);
}
}
5
Data getting corrupted on Upload in Azure Blob
I agree with Gaurav Mantri’s comment, To copy a blob from one container to another container you can use the StartCopyFromUriAsync() method with Azure .Net SDK.
Code:
string blobConnectionString = config["AzureWebJobsStorage"];
BlobContainerClient sourceBlobContainerClient = new BlobContainerClient(blobConnectionString, "abc");
BlobContainerClient destinationBlobContainerClient = new BlobContainerClient(blobConnectionString, "xyz");
var blobs = sourceBlobContainerClient.GetBlobsAsync();
await foreach (var blob in blobs)
{
Console.WriteLine($"Copying Blob: {blob.Name}");
BlobClient sourceBlobClient = sourceBlobContainerClient.GetBlobClient(blob.Name);
BlobClient destinationBlobClient = destinationBlobContainerClient.GetBlobClient(blob.Name);
Uri sourceBlobUri = sourceBlobClient.Uri;
CopyFromUriOperation copyOperation = await destinationBlobClient.StartCopyFromUriAsync(sourceBlobUri);
await copyOperation.WaitForCompletionAsync();
BlobProperties properties = await destinationBlobClient.GetPropertiesAsync();
Console.WriteLine($"Blob: {blob.Name} - Copy Status: {properties.CopyStatus}");
await sourceBlobClient.DeleteAsync();
}
Console.WriteLine("Blob Copy Process Completed.");
The above code will copy blobs from one container to another container and
delete the source blobs.
Output:
Blob Copy Process Started...
Copying Blob: sample_log.gz
Blob: sample_log.gz - Copy Status: Success
Blob Copy Process Completed.
Reference:
Copy a blob with asynchronous scheduling using .NET – Azure Storage | Microsoft Learn