Merge pull request #956 from Project-MONAI/AI-275

Ai 275

Author: neildsouth

src/TaskManager/TaskManager/appsettings.json

@@ -108,7 +108,8 @@
         "deadLetterExchange": "monaideploy-dead-letter",
         "exportRequestQueue": "export_tasks",
         "deliveryLimit": 3,
-        "requeueDelay": 30
+        "requeueDelay": 3,
+        "prefetchCount": "5"
       }
     },
     "storage": {

src/WorkflowManager/Common/Services/PayloadService.cs

@@ -14,7 +14,6 @@
  * limitations under the License.
  */
 
-using Ardalis.GuardClauses;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
 using Monai.Deploy.Messaging.Events;
@@ -83,6 +82,7 @@ public PayloadService(
                 }
 
                 var patientDetails = await _dicomService.GetPayloadPatientDetailsAsync(eventPayload.PayloadId.ToString(), eventPayload.Bucket);
+                var dict = await _dicomService.GetMetaData(eventPayload.PayloadId.ToString(), eventPayload.Bucket).ConfigureAwait(false);
 
                 var payload = new Payload
                 {
@@ -96,7 +96,8 @@ public PayloadService(
                     Timestamp = eventPayload.Timestamp,
                     PatientDetails = patientDetails,
                     PayloadDeleted = PayloadDeleted.No,
-                    Expires = await GetExpiry(DateTime.UtcNow, eventPayload.WorkflowInstanceId)
+                    Expires = await GetExpiry(DateTime.UtcNow, eventPayload.WorkflowInstanceId),
+                    SeriesInstanceUid = _dicomService.GetSeriesInstanceUID(dict)
                 };
 
                 if (await _payloadRepository.CreateAsync(payload))
@@ -197,13 +198,7 @@ public async Task<bool> DeletePayloadFromStorageAsync(string payloadId)
         {
             ArgumentNullException.ThrowIfNullOrWhiteSpace(payloadId, nameof(payloadId));
 
-            var payload = await GetByIdAsync(payloadId);
-
-            if (payload is null)
-            {
-                throw new MonaiNotFoundException($"Payload with ID: {payloadId} not found");
-            }
-
+            var payload = await GetByIdAsync(payloadId) ?? throw new MonaiNotFoundException($"Payload with ID: {payloadId} not found");
             if (payload.PayloadDeleted == PayloadDeleted.InProgress || payload.PayloadDeleted == PayloadDeleted.Yes)
             {
                 throw new MonaiBadRequestException($"Deletion of files for payload ID: {payloadId} already in progress or already deleted");

src/WorkflowManager/Contracts/Migrations/M001_Payload_addVerion.cs

@@ -21,7 +21,7 @@ namespace Monai.Deploy.WorkflowManager.Common.Contracts.Migrations
 {
     public class M001_Payload_addVerion : DocumentMigration<Payload>
     {
-        public M001_Payload_addVerion() : base("1.0.0") { }
+        public M001_Payload_addVerion() : base("1.0.1") { }
 
         public override void Up(BsonDocument document)
         {

src/WorkflowManager/Contracts/Migrations/M002_Payload_addPayloadDeleted.cs

@@ -21,7 +21,7 @@ namespace Monai.Deploy.WorkflowManager.Common.Contracts.Migrations
 {
     public class M002_Payload_addPayloadDeleted : DocumentMigration<Payload>
     {
-        public M002_Payload_addPayloadDeleted() : base("1.0.1") { }
+        public M002_Payload_addPayloadDeleted() : base("1.0.2") { }
 
         public override void Up(BsonDocument document)
         {

src/WorkflowManager/Contracts/Migrations/M005_Payload_seriesUid.cs

@@ -0,0 +1,42 @@
+// 
+// Copyright 2023 Guy’s and St Thomas’ NHS Foundation Trust
+// 
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// 
+// http://www.apache.org/licenses/LICENSE-2.0
+// 
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+using Monai.Deploy.WorkflowManager.Common.Contracts.Models;
+using Mongo.Migration.Migrations.Document;
+using MongoDB.Bson;
+
+namespace Monai.Deploy.WorkflowManager.Common.Contracts.Migrations
+{
+    public class M005_Payload_seriesUid : DocumentMigration<Payload>
+    {
+        public M005_Payload_seriesUid() : base("1.0.5") { }
+
+        public override void Up(BsonDocument document)
+        {
+            document.Add("SeriesInstanceUid", BsonNull.Create(null).ToJson(), true);
+        }
+
+        public override void Down(BsonDocument document)
+        {
+            try
+            {
+                document.Remove("SeriesInstanceUid");
+            }
+            catch
+            {  // can ignore we dont want failures stopping startup !
+            }
+        }
+    }
+}

src/WorkflowManager/Contracts/Models/Payload.cs

@@ -27,11 +27,11 @@
 
 namespace Monai.Deploy.WorkflowManager.Common.Contracts.Models
 {
-    [CollectionLocation("Payloads"), RuntimeVersion("1.0.4")]
+    [CollectionLocation("Payloads"), RuntimeVersion("1.0.5")]
     public class Payload : IDocument
     {
         [JsonConverter(typeof(DocumentVersionConvert)), BsonSerializer(typeof(DocumentVersionConverBson))]
-        public DocumentVersion Version { get; set; } = new DocumentVersion(1, 0, 4);
+        public DocumentVersion Version { get; set; } = new DocumentVersion(1, 0, 5);
 
         [JsonProperty(PropertyName = "id")]
         public string Id { get; set; } = string.Empty;
@@ -71,6 +71,9 @@ public class Payload : IDocument
         [JsonProperty(PropertyName = "expires")]
         public DateTime? Expires { get; set; }
 
+        [JsonProperty(PropertyName = "series_instance_uid")]
+        public string? SeriesInstanceUid { get; set; }
+
     }
 
     public enum PayloadDeleted

src/WorkflowManager/Contracts/Models/PayloadDto.cs

@@ -36,6 +36,7 @@ public PayloadDto(Payload payload)
             Files = payload.Files;
             PatientDetails = payload.PatientDetails;
             PayloadDeleted = payload.PayloadDeleted;
+            SeriesInstanceUid = payload.SeriesInstanceUid;
         }
 
         [JsonProperty(PropertyName = "payload_status")]

src/WorkflowManager/Logging/Log.200000.Workflow.cs

@@ -111,5 +111,8 @@ public static partial class Log
 
         [LoggerMessage(EventId = 210018, Level = LogLevel.Error, Message = "ExportList or Artifacts are empty! workflowInstanceId {workflowInstanceId} TaskId {taskId}")]
         public static partial void ExportListOrArtifactsAreEmpty(this ILogger logger, string taskId, string workflowInstanceId);
+
+        [LoggerMessage(EventId = 210019, Level = LogLevel.Error, Message = "Task is missing required input artifacts {taskId} Artifacts {ArtifactsJson}")]
+        public static partial void TaskIsMissingRequiredInputArtifacts(this ILogger logger, string taskId, string ArtifactsJson);
     }
 }

src/WorkflowManager/Logging/Log.600000.Dicom.cs

@@ -40,5 +40,11 @@ public static partial class Log
 
         [LoggerMessage(EventId = 600006, Level = LogLevel.Debug, Message = "Dicom export marked as failed with {fileStatusCount} files marked as exported.")]
         public static partial void DicomExportFailed(this ILogger logger, string fileStatusCount);
+
+        [LoggerMessage(EventId = 600007, Level = LogLevel.Error, Message = "Failed to get DICOM metadata from bucket {bucketId}. Payload: {payloadId}")]
+        public static partial void FailedToGetDicomMetadataFromBucket(this ILogger logger, string payloadId, string bucketId, Exception ex);
+
+        [LoggerMessage(EventId = 600008, Level = LogLevel.Error, Message = "Failed to get DICOM tag {dicomTag} from dictionary")]
+        public static partial void FailedToGetDicomTagFromDictoionary(this ILogger logger, string dicomTag, Exception ex);
     }
 }

src/WorkflowManager/Logging/Log.700000.Artifact.cs

@@ -57,6 +57,15 @@ public static partial class Log
         [LoggerMessage(EventId = 700011, Level = LogLevel.Debug, Message = "adding files to workflowInstance {workflowInstanceId} :Task {taskId} :  {artifactList}")]
         public static partial void AddingFilesToWorkflowInstance(this ILogger logger, string workflowInstanceId, string taskId, string artifactList);
 
+        [LoggerMessage(EventId = 700012, Level = LogLevel.Error, Message = "Error finding Task :{taskId}")]
+        public static partial void ErrorFindingTask(this ILogger logger, string taskId);
+
+        [LoggerMessage(EventId = 700013, Level = LogLevel.Error, Message = "Error finding Task :{taskId} or previousTask {previousTask}")]
+        public static partial void ErrorFindingTaskOrPrevious(this ILogger logger, string taskId, string previousTask);
+
+        [LoggerMessage(EventId = 700014, Level = LogLevel.Warning, Message = "Error Task :{taskId} cant be trigger as it has missing artifacts {missingtypesJson}")]
+        public static partial void ErrorTaskMissingArtifacts(this ILogger logger, string taskId, string missingtypesJson);
+
 
     }
 }