Use OSHI for more accurate memory metrics

concourse-server/build.gradle

@@ -46,6 +46,7 @@ dependencies {
     compile group:'com.cinchapi', name: 'off-heap-memory', version: '1.1.0-SNAPSHOT', changing:true
     compile group:'com.cinchapi', name: 'configctl-lib', version: '1.2.0', changing:true
     compile 'it.unimi.dsi:fastutil-core:8.5.13'
+    compile 'com.github.oshi:oshi-core:6.8.0'
 
     testCompile 'com.carrotsearch:junit-benchmarks:0.7.2'
     testCompile 'io.takari.junit:takari-cpsuite:1.2.7'

concourse-server/src/main/java/com/cinchapi/concourse/server/Telemetry.java

@@ -0,0 +1,160 @@
+/*
+ * Copyright (c) 2013-2025 Cinchapi Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.cinchapi.concourse.server;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+
+import oshi.SystemInfo;
+
+/**
+ * Interface for various system metrics and information.
+ *
+ * @author Jeff Nelson
+ */
+public final class Telemetry { 
+
+    /**
+     * Returns the {@link Telemetry} instance.
+     * 
+     * @return the Telemetry instance
+     */
+    public static Telemetry get() {
+        return INSTANCE;
+    }
+
+    /**
+     * The singleton instance of the Telemetry class.
+     */
+    private final static Telemetry INSTANCE = new Telemetry();
+
+    /**
+     * OSHI handle.
+     */
+    private final SystemInfo oshi;
+
+    /**
+     * Construct a new instance.
+     */
+    private Telemetry() {
+        this.oshi = new SystemInfo();
+    }
+
+    /**
+     * Returns the amount of heap memory available to the JVM.
+     * 
+     * @return the available heap memory in bytes
+     */
+    public long availableHeapMemory() {
+        return Runtime.getRuntime().freeMemory();
+    }
+
+    /**
+     * Returns the amount of available physical memory on the system.
+     * 
+     * @return the available physical memory in bytes
+     */
+    public long availableSystemMemory() {
+        return oshi.getHardware().getMemory().getAvailable();
+    }
+
+    /**
+     * Determines if the current process is running in a containerized
+     * environment.
+     * 
+     * @return true if running in a container, false otherwise
+     */
+    public boolean isContainerized() {
+        if(oshi.getOperatingSystem().getVersionInfo().getBuildNumber()
+                .toLowerCase().contains("container")) {
+            return true;
+        }
+        if(System.getenv().containsKey("KUBERNETES_SERVICE_HOST")) {
+            return true;
+        }
+        if(System.getenv().containsKey("DOCKER_CONTAINER")) {
+            return true;
+        }
+        if(new File("/.dockerenv").exists()) {
+            // Check for the very common /.dockerenv marker
+            return true;
+        }
+        try (BufferedReader reader = new BufferedReader(
+                new FileReader("/proc/1/cgroup"))) {
+            String line;
+            while ((line = reader.readLine()) != null) {
+                // Kubernetes pods, Docker, LXC, etc tend to have recognizable
+                // paths
+                if(line.contains("docker") || line.contains("kubepods")
+                        || line.contains("containerd")
+                        || line.contains("lxc")) {
+                    return true;
+                }
+            }
+        }
+        catch (IOException e) {}
+        return false;
+
+    }
+
+    /**
+     * Determines if swap memory is enabled on the system.
+     * 
+     * @return true if swap is enabled, false otherwise
+     */
+    public boolean isSwapEnabled() {
+        return oshi.getHardware().getMemory().getVirtualMemory()
+                .getSwapTotal() > 0;
+    }
+
+    /**
+     * Returns the underlying system information object.
+     * 
+     * @return the SystemInfo instance
+     */
+    public SystemInfo system() {
+        return oshi;
+    }
+
+    /**
+     * Returns the amount of heap memory reserved by the JVM.
+     * 
+     * @return the reserved heap memory in bytes
+     */
+    public long totalHeapMemory() {
+        return Runtime.getRuntime().totalMemory();
+    }
+
+    /**
+     * Returns the total physical memory available on the system.
+     * 
+     * @return the total physical memory in bytes
+     */
+    public long totalMemory() {
+        return oshi.getHardware().getMemory().getTotal();
+    }
+
+    /**
+     * Returns the total amount of virtual memory (swap space) on the system.
+     * 
+     * @return the total virtual memory in bytes
+     */
+    public long virtualMemory() {
+        return oshi.getHardware().getMemory().getVirtualMemory().getSwapTotal();
+    }
+}

concourse-server/src/main/java/com/cinchapi/concourse/server/storage/db/search/SubstringDeduplicator.java

@@ -17,7 +17,6 @@
 
 import java.io.Closeable;
 import java.io.IOException;
-import java.lang.management.ManagementFactory;
 import java.nio.ByteBuffer;
 import java.nio.file.Path;
 import java.util.AbstractSet;
@@ -44,6 +43,7 @@
 import com.cinchapi.common.logging.Logging;
 import com.cinchapi.common.reflect.Reflection;
 import com.cinchapi.concourse.annotate.Experimental;
+import com.cinchapi.concourse.server.Telemetry;
 import com.cinchapi.concourse.server.model.Text;
 import com.cinchapi.concourse.server.storage.db.kernel.CorpusChunk.SearchTermMetrics;
 import com.cinchapi.concourse.util.FileOps;
@@ -107,16 +107,20 @@ public static SubstringDeduplicator create(char[] term,
         int expectedInsertions = metrics.upperBoundOfPossibleSubstrings();
         long estimatedMemoryRequired = (long) metrics.averageSubstringLength()
                 * (long) expectedInsertions;
-        long availableDirectMemory = availableDirectMemory();
-        long freeHeapMemory = Runtime.getRuntime().freeMemory();
+
+        Telemetry telemetry = Telemetry.get();
+        long availableOffHeapMemory = telemetry.availableSystemMemory();
+        long availableHeapMemory = telemetry.availableHeapMemory();
+        boolean swapIsEnabled = telemetry.isSwapEnabled();
+
         Logger.info("The search indexer has encountered a large term that "
                 + "may require a lot of memory to process. The term has "
                 + "up to {} search indexes{}{}", expectedInsertions,
                 System.lineSeparator(), summarizeMemory(estimatedMemoryRequired,
-                        availableDirectMemory, freeHeapMemory));
+                        availableOffHeapMemory, availableHeapMemory));
 
         SubstringDeduplicator deduplicator;
-        if(availableDirectMemory > estimatedMemoryRequired) {
+        if(availableOffHeapMemory > estimatedMemoryRequired || swapIsEnabled) {
             try {
                 Logger.info("Attempting to use off-heap memory to deduplicate "
                         + "the search indexes");
@@ -189,23 +193,6 @@ static SubstringDeduplicator testCreateChronicleBacked(char[] term) {
         return new ChronicleBackedSubstringDeduplicator(term, metrics);
     }
 
-    /**
-     * If possible, return the amount of "direct" memory that is available for
-     * off-heap storage.
-     * 
-     * @return the number of direct memory bytes that are available
-     */
-    @SuppressWarnings("restriction")
-    private static long availableDirectMemory() {
-        try {
-            com.sun.management.OperatingSystemMXBean osBean = (com.sun.management.OperatingSystemMXBean) ManagementFactory
-                    .getOperatingSystemMXBean();
-            return osBean.getFreePhysicalMemorySize();
-        }
-        catch (Exception e) {}
-        return 0;
-    }
-
     /**
      * Formats a byte count into a human-readable string representation.
      *