pool:fix outofspace error sending pool into diasbled mode

Motivation

we are seeing the this error wen there is no space on the pool left, and pool goes into disabled mode,

AB56598C55] WRITE failed : IOError
AB56598C55] Transfer failed due to a disk error: CacheException(rc=204;msg=Disk I/O Error )
AB56598C55] Pool mode changed to disabled(fetch,store,stage,p2p-client,p2p-server): Pool disabled: Disk I/O Error
AB56598C55] Pool: dcache-xfel487-01, fault occurred in transfer: Disk I/O Error . Pool disabled: , cause: CacheException(rc=204;msg=Disk I/O Error )

however we want it to go into READ-ONLY mode and files could stay accesable.

the privous patch (https://rb.dcache.org/r/14357/diff/3/#index_header) did not fix the issue.

so the new changes are still a try to fix the issue.

Acked-by: Dmitry Litvintsev
Target: master. 10.2, 10.1, 10.0, 9.2
Require-book: no
Require-notes: yes
Patch: https://rb.dcache.org/r/14368/

Author: mksahakyan

modules/dcache-dcap/src/main/java/org/dcache/pool/movers/DCapProtocol_3_nio.java

@@ -367,7 +367,12 @@ public void runIO(FileAttributes fileAttributes,
                                 String errmsg = "WRITE failed : " + (ioException == null ? "IOError"
                                       : Exceptions.messageOrClassName(ioException));
                                 int rc;
-                                if (ioException instanceof OutOfDiskException) {
+                                // TODO: To be checked (in production) if the error is an IOException;
+                                //  we can check it only by the error message.
+                                if (ioException instanceof OutOfDiskException ||
+                                      (ioException != null && ioException.getCause().getMessage()
+                                            .contains("No space available."))
+                                ) {
                                     _log.debug(errmsg);
                                     rc = CacheException.RESOURCE;
                                 } else {
@@ -535,7 +540,11 @@ public void runIO(FileAttributes fileAttributes,
                                       "SEEK_AND_WRITE failed : " + (ioException == null ? "IOError"
                                             : Exceptions.messageOrClassName(ioException));
                                 int rc;
-                                if (ioException instanceof OutOfDiskException) {
+                                // TODO: To be checked (in production) if the error is an IOException;
+                                //  we can check it only by the error message.
+                                if (ioException instanceof OutOfDiskException ||
+                                      (ioException != null && ioException.getCause().getMessage()
+                                            .contains("No space available."))) {
                                     _log.debug(errmsg);
                                     rc = CacheException.RESOURCE;
                                 } else {
@@ -678,10 +687,19 @@ public void runIO(FileAttributes fileAttributes,
                 if (ioException instanceof OutOfDiskException) {
                     throw ioException;
                 } else {
-                    throw new
-                          DiskErrorCacheException(
-                          "Disk I/O Error " +
-                                (ioException != null ? ioException.toString() : ""));
+                    //TODO since the OutOfDiskException is not  catched the error ist IOexception
+                    // this is a workaround
+                    if (ioException != null && ioException.getCause().getMessage().contains("No space available.")) {
+                        throw new
+                              OutOfDiskException(
+                              "No space available." +
+                                    ioException.toString());
+                    } else {
+                        throw new
+                              DiskErrorCacheException(
+                              "Disk I/O Error " +
+                                    (ioException != null ? ioException.toString() : ""));
+                    }
                 }
             } else {
                 if (ioException != null && !(ioException instanceof EOFException)) {
@@ -922,6 +940,7 @@ private void doTheWrite(RepositoryChannel fileChannel,
                         // clear interrupted state
                         Thread.interrupted();
                         throw new InterruptedException(ee.getMessage());
+                        // TODO: In the case of a "no space left" error message, this will not work.
                     } catch (OutOfDiskException e) {
                         _io_ok = false;
                         ioException = e;

modules/dcache/src/main/java/org/dcache/pool/classic/MoverRequestScheduler.java

@@ -41,6 +41,7 @@
 import org.dcache.pool.movers.Mover;
 import org.dcache.pool.movers.json.MoverData;
 import org.dcache.pool.repository.FileStore;
+import org.dcache.pool.repository.OutOfDiskException;
 import org.dcache.util.AdjustableSemaphore;
 import org.dcache.util.IoPrioritizable;
 import org.dcache.util.IoPriority;
@@ -527,6 +528,13 @@ public void failed(Throwable exc, Void attachment) {
                               FaultEvent faultEvent = new FaultEvent("transfer",
                                       faultAction, exc.getMessage(), exc);
                               _faultListeners.forEach(l -> l.faultOccurred(faultEvent));
+                          } else if (exc instanceof OutOfDiskException) {
+                              FaultEvent faultEvent = new FaultEvent(
+                                    "post-processing",
+                                    FaultAction.READONLY,
+                                    exc.getMessage(), exc);
+                              _faultListeners.forEach(
+                                    l -> l.faultOccurred(faultEvent));
                           }
                           postprocess();
                       }
@@ -558,6 +566,13 @@ public void failed(Throwable exc, Void attachment) {
                                                       exc.getMessage(), exc);
                                                 _faultListeners.forEach(
                                                       l -> l.faultOccurred(faultEvent));
+                                            } else if (exc instanceof OutOfDiskException) {
+                                                FaultEvent faultEvent = new FaultEvent(
+                                                      "post-processing",
+                                                      FaultAction.READONLY,
+                                                      exc.getMessage(), exc);
+                                                _faultListeners.forEach(
+                                                      l -> l.faultOccurred(faultEvent));
                                             }
                                             release();
                                         }