Fix handling MEMORY and DIV BY 0 errors for C600

Summary: Ref T73341

Reviewers: #tensorflow, #framework_ip_review_-_any_oss_or_third-party_code_use_has_been_approved, kamil.andrzejewski

Reviewed By: #tensorflow, #framework_ip_review_-_any_oss_or_third-party_code_use_has_been_approved, kamil.andrzejewski

Maniphest Tasks: T73341

Differential Revision: https://phabricator.sourcevertex.net/D81663

Author: dsciebu

tensorflow/compiler/plugin/poplar/kernels/application_runtime/application_runtime.cc

@@ -728,6 +728,8 @@ class EngineResource {
         tensorflow::ThreadOptions(), engine_name_ + "_execute_thread", [this] {
           Status runtime_status = Status::OK();
           bool reset_engine = false;
+          static constexpr int num_of_reset_retries = 2;
+          int reset_engine_retries_left = num_of_reset_retries;
           while (!communication_manager_.Exiting()) {
             {
               // Prevent any new requests from being inserted whilst the engine
@@ -741,17 +743,19 @@ class EngineResource {
                 communication_manager_.Abort(runtime_status);
                 // If the exception raised only requires an engine reset, then
                 // continue, otherwise we can't recover.
-                if (!reset_engine) {
+                if (!reset_engine || reset_engine_retries_left <= 0) {
                   communication_manager_.InitiateExit();
                   return;
                 }
               }
 
               runtime_status = StartEngineAndConnectStreams(reset_engine);
               if (!runtime_status.ok()) {
+                --reset_engine_retries_left;
                 continue;
               }
 
+              reset_engine_retries_left = num_of_reset_retries;
               reset_engine = false;
               runtime_status = Status::OK();
             }