HDDS-14927. Add Quasi-Closed Container Tracking in Recon.

hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconControllerModule.java

@@ -43,6 +43,7 @@
 import org.apache.hadoop.ozone.om.protocolPB.OzoneManagerProtocolClientSideTranslatorPB;
 import org.apache.hadoop.ozone.recon.heatmap.HeatMapServiceImpl;
 import org.apache.hadoop.ozone.recon.persistence.ContainerHealthSchemaManager;
+import org.apache.hadoop.ozone.recon.persistence.QuasiClosedContainerSchemaManager;
 import org.apache.hadoop.ozone.recon.persistence.DataSourceConfiguration;
 import org.apache.hadoop.ozone.recon.persistence.JooqPersistenceModule;
 import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager;
@@ -110,6 +111,7 @@ protected void configure() {
     bind(OMMetadataManager.class).to(ReconOmMetadataManagerImpl.class);
 
     bind(ContainerHealthSchemaManager.class).in(Singleton.class);
+    bind(QuasiClosedContainerSchemaManager.class).in(Singleton.class);
     bind(ReconContainerMetadataManager.class)
         .to(ReconContainerMetadataManagerImpl.class).in(Singleton.class);
     bind(ReconFileMetadataManager.class)

hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/ContainerEndpoint.java

@@ -72,10 +72,14 @@
 import org.apache.hadoop.ozone.recon.api.types.KeysResponse;
 import org.apache.hadoop.ozone.recon.api.types.MissingContainerMetadata;
 import org.apache.hadoop.ozone.recon.api.types.MissingContainersResponse;
+import org.apache.hadoop.ozone.recon.api.types.QuasiClosedContainerMetadata;
+import org.apache.hadoop.ozone.recon.api.types.QuasiClosedContainersResponse;
 import org.apache.hadoop.ozone.recon.api.types.UnhealthyContainerMetadata;
 import org.apache.hadoop.ozone.recon.api.types.UnhealthyContainersResponse;
 import org.apache.hadoop.ozone.recon.api.types.UnhealthyContainersSummary;
 import org.apache.hadoop.ozone.recon.persistence.ContainerHealthSchemaManager;
+import org.apache.hadoop.ozone.recon.persistence.QuasiClosedContainerSchemaManager;
+import org.apache.hadoop.ozone.recon.persistence.QuasiClosedContainerSchemaManager.QuasiClosedContainerRecord;
 import org.apache.hadoop.ozone.recon.persistence.ContainerHistory;
 import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager;
 import org.apache.hadoop.ozone.recon.scm.ReconContainerManager;
@@ -102,6 +106,7 @@ public class ContainerEndpoint {
   private final ReconContainerManager containerManager;
   private final PipelineManager pipelineManager;
   private final ContainerHealthSchemaManager containerHealthSchemaManager;
+  private final QuasiClosedContainerSchemaManager quasiClosedSchemaManager;
   private final ReconNamespaceSummaryManager reconNamespaceSummaryManager;
   private final OzoneStorageContainerManager reconSCM;
   private static final Logger LOG =
@@ -143,13 +148,15 @@ public static DataFilter fromValue(String value) {
   @Inject
   public ContainerEndpoint(OzoneStorageContainerManager reconSCM,
                            ContainerHealthSchemaManager containerHealthSchemaManager,
+                           QuasiClosedContainerSchemaManager quasiClosedSchemaManager,
                            ReconNamespaceSummaryManager reconNamespaceSummaryManager,
                            ReconContainerMetadataManager reconContainerMetadataManager,
                            ReconOMMetadataManager omMetadataManager) {
     this.containerManager =
         (ReconContainerManager) reconSCM.getContainerManager();
     this.pipelineManager = reconSCM.getPipelineManager();
     this.containerHealthSchemaManager = containerHealthSchemaManager;
+    this.quasiClosedSchemaManager = quasiClosedSchemaManager;
     this.reconNamespaceSummaryManager = reconNamespaceSummaryManager;
     this.reconSCM = reconSCM;
     this.reconContainerMetadataManager = reconContainerMetadataManager;
@@ -812,4 +819,93 @@ public Response getOmContainersDeletedInSCM(
     response.put("containerDiscrepancyInfo", containerDiscrepancyInfoList);
     return Response.ok(response).build();
   }
+
+  // ─────────────────────────────────────────────────────────────────────────
+  // QUASI-CLOSED CONTAINER ENDPOINTS
+  // ─────────────────────────────────────────────────────────────────────────
+
+  /**
+   * Returns all containers currently in QUASI_CLOSED lifecycle state with
+   * cursor-based forward pagination.
+   *
+   * <p>QUASI_CLOSED containers are those locally closed by a datanode
+   * (e.g. due to a pipeline failure) but not yet force-closed to CLOSED by SCM.
+   * Long-running QUASI_CLOSED containers may indicate stalled pipeline finalization.
+   *
+   * @param limit          maximum containers to return (default: 1000)
+   * @param minContainerId cursor — return containers with ID &gt; minContainerId
+   * @return paginated list of quasi-closed containers with replica history
+   */
+  @GET
+  @Path("/quasiClosed")
+  public Response getQuasiClosedContainers(
+      @DefaultValue(DEFAULT_FETCH_COUNT) @QueryParam(RECON_QUERY_LIMIT) int limit,
+      @DefaultValue(PREV_CONTAINER_ID_DEFAULT_VALUE)
+      @QueryParam(RECON_QUERY_PREVKEY) long minContainerId) {
+    return buildQuasiClosedResponse(
+        quasiClosedSchemaManager.getQuasiClosedContainers(minContainerId, limit));
+  }
+
+  /**
+   * Returns QUASI_CLOSED containers belonging to a specific pipeline.
+   * Useful for diagnosing which pipelines have the most stuck containers.
+   *
+   * @param pipelineId     pipeline UUID to filter by
+   * @param limit          maximum containers to return
+   * @param minContainerId forward-pagination cursor
+   * @return paginated list of quasi-closed containers for the given pipeline
+   */
+  @GET
+  @Path("/quasiClosed/pipeline/{pipelineId}")
+  public Response getQuasiClosedContainersByPipeline(
+      @PathParam("pipelineId") String pipelineId,
+      @DefaultValue(DEFAULT_FETCH_COUNT) @QueryParam(RECON_QUERY_LIMIT) int limit,
+      @DefaultValue(PREV_CONTAINER_ID_DEFAULT_VALUE)
+      @QueryParam(RECON_QUERY_PREVKEY) long minContainerId) {
+    if (pipelineId == null || pipelineId.isEmpty()) {
+      throw new WebApplicationException("pipelineId must not be empty",
+          Response.Status.BAD_REQUEST);
+    }
+    return buildQuasiClosedResponse(
+        quasiClosedSchemaManager.getByPipeline(pipelineId, minContainerId, limit));
+  }
+
+  /**
+   * Returns summary count of QUASI_CLOSED containers.
+   */
+  @GET
+  @Path("/quasiClosed/summary")
+  public Response getQuasiClosedSummary() {
+    long count = quasiClosedSchemaManager.getCount();
+    Map<String, Long> summary = new HashMap<>();
+    summary.put("quasiClosedCount", count);
+    return Response.ok(summary).build();
+  }
+
+  private Response buildQuasiClosedResponse(List<QuasiClosedContainerRecord> records) {
+    List<QuasiClosedContainerMetadata> metaList = new ArrayList<>();
+    long lastKey = 0L;
+    for (QuasiClosedContainerRecord rec : records) {
+      long containerId = rec.getContainerId();
+      List<ContainerHistory> replicas =
+          containerManager.getAllContainerHistory(containerId);
+      metaList.add(new QuasiClosedContainerMetadata(
+          containerId,
+          rec.getPipelineId(),
+          rec.getDatanodeCount(),
+          rec.getKeyCount(),
+          rec.getDataSize(),
+          rec.getReplicationType(),
+          rec.getReplicationFactor(),
+          rec.getStateEnterTime(),
+          rec.getFirstSeenTime(),
+          rec.getLastScanTime(),
+          replicas));
+      lastKey = Math.max(lastKey, containerId);
+    }
+    long totalCount = quasiClosedSchemaManager.getCount();
+    QuasiClosedContainersResponse response =
+        new QuasiClosedContainersResponse(totalCount, metaList, lastKey);
+    return Response.ok(response).build();
+  }
 }

hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/QuasiClosedContainerMetadata.java

@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.recon.api.types;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import java.util.List;
+import org.apache.hadoop.ozone.recon.persistence.ContainerHistory;
+
+/**
+ * JSON response DTO for a single QUASI_CLOSED container.
+ */
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class QuasiClosedContainerMetadata {
+
+  private long containerID;
+  private String pipelineID;
+  private int datanodeCount;
+  private long keyCount;
+  private long dataSize;
+  private String replicationType;
+  private int replicationFactor;
+  /** When the container entered QUASI_CLOSED state according to SCM. */
+  private long stateEnterTime;
+  /** When Recon first observed this container as QUASI_CLOSED. */
+  private long firstSeenTime;
+  /** When Recon last confirmed this container is still QUASI_CLOSED. */
+  private long lastScanTime;
+  /** Current datanode replicas — reuses the existing ContainerHistory type. */
+  private List<ContainerHistory> replicas;
+
+  public QuasiClosedContainerMetadata() {
+  }
+
+  public QuasiClosedContainerMetadata(
+      long containerID, String pipelineID, int datanodeCount,
+      long keyCount, long dataSize, String replicationType,
+      int replicationFactor, long stateEnterTime,
+      long firstSeenTime, long lastScanTime,
+      List<ContainerHistory> replicas) {
+    this.containerID = containerID;
+    this.pipelineID = pipelineID;
+    this.datanodeCount = datanodeCount;
+    this.keyCount = keyCount;
+    this.dataSize = dataSize;
+    this.replicationType = replicationType;
+    this.replicationFactor = replicationFactor;
+    this.stateEnterTime = stateEnterTime;
+    this.firstSeenTime = firstSeenTime;
+    this.lastScanTime = lastScanTime;
+    this.replicas = replicas;
+  }
+
+  public long getContainerID() { return containerID; }
+  public void setContainerID(long containerID) { this.containerID = containerID; }
+
+  public String getPipelineID() { return pipelineID; }
+  public void setPipelineID(String pipelineID) { this.pipelineID = pipelineID; }
+
+  public int getDatanodeCount() { return datanodeCount; }
+  public void setDatanodeCount(int datanodeCount) { this.datanodeCount = datanodeCount; }
+
+  public long getKeyCount() { return keyCount; }
+  public void setKeyCount(long keyCount) { this.keyCount = keyCount; }
+
+  public long getDataSize() { return dataSize; }
+  public void setDataSize(long dataSize) { this.dataSize = dataSize; }
+
+  public String getReplicationType() { return replicationType; }
+  public void setReplicationType(String replicationType) { this.replicationType = replicationType; }
+
+  public int getReplicationFactor() { return replicationFactor; }
+  public void setReplicationFactor(int replicationFactor) { this.replicationFactor = replicationFactor; }
+
+  public long getStateEnterTime() { return stateEnterTime; }
+  public void setStateEnterTime(long stateEnterTime) { this.stateEnterTime = stateEnterTime; }
+
+  public long getFirstSeenTime() { return firstSeenTime; }
+  public void setFirstSeenTime(long firstSeenTime) { this.firstSeenTime = firstSeenTime; }
+
+  public long getLastScanTime() { return lastScanTime; }
+  public void setLastScanTime(long lastScanTime) { this.lastScanTime = lastScanTime; }
+
+  public List<ContainerHistory> getReplicas() { return replicas; }
+  public void setReplicas(List<ContainerHistory> replicas) { this.replicas = replicas; }
+}

hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/QuasiClosedContainersResponse.java

@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.recon.api.types;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Wrapper response for the quasi-closed containers API.
+ */
+public class QuasiClosedContainersResponse {
+
+  private long totalCount;
+  private List<QuasiClosedContainerMetadata> containers;
+  /** The highest container ID returned — for forward-pagination cursor. */
+  private long lastKey;
+
+  public QuasiClosedContainersResponse() {
+    this.containers = new ArrayList<>();
+  }
+
+  public QuasiClosedContainersResponse(
+      long totalCount, List<QuasiClosedContainerMetadata> containers, long lastKey) {
+    this.totalCount = totalCount;
+    this.containers = containers;
+    this.lastKey = lastKey;
+  }
+
+  public long getTotalCount() { return totalCount; }
+  public void setTotalCount(long totalCount) { this.totalCount = totalCount; }
+
+  public List<QuasiClosedContainerMetadata> getContainers() { return containers; }
+  public void setContainers(List<QuasiClosedContainerMetadata> containers) {
+    this.containers = containers;
+  }
+
+  public long getLastKey() { return lastKey; }
+  public void setLastKey(long lastKey) { this.lastKey = lastKey; }
+}