From 2575cfcfd90942f9b381aee1c03c77a5193c4d85 Mon Sep 17 00:00:00 2001 From: Jeff Nelson Date: Sun, 4 May 2025 06:10:15 -0400 Subject: [PATCH 1/2] Use OSHI for more accurate memory metrics --- concourse-server/build.gradle | 1 + .../cinchapi/concourse/server/Telemetry.java | 160 ++++++++++++++++++ .../db/search/SubstringDeduplicator.java | 31 +--- 3 files changed, 170 insertions(+), 22 deletions(-) create mode 100644 concourse-server/src/main/java/com/cinchapi/concourse/server/Telemetry.java diff --git a/concourse-server/build.gradle b/concourse-server/build.gradle index 11fc37a30..692acaca4 100644 --- a/concourse-server/build.gradle +++ b/concourse-server/build.gradle @@ -46,6 +46,7 @@ dependencies { compile group:'com.cinchapi', name: 'off-heap-memory', version: '1.1.0-SNAPSHOT', changing:true compile group:'com.cinchapi', name: 'configctl-lib', version: '1.2.0', changing:true compile 'it.unimi.dsi:fastutil-core:8.5.13' + compile 'com.github.oshi:oshi-core:6.8.0' testCompile 'com.carrotsearch:junit-benchmarks:0.7.2' testCompile 'io.takari.junit:takari-cpsuite:1.2.7' diff --git a/concourse-server/src/main/java/com/cinchapi/concourse/server/Telemetry.java b/concourse-server/src/main/java/com/cinchapi/concourse/server/Telemetry.java new file mode 100644 index 000000000..dbf33003b --- /dev/null +++ b/concourse-server/src/main/java/com/cinchapi/concourse/server/Telemetry.java @@ -0,0 +1,160 @@ +/* + * Copyright (c) 2013-2025 Cinchapi Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.cinchapi.concourse.server; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; + +import oshi.SystemInfo; + +/** + * Interface for various system metrics and information. + * + * @author Jeff Nelson + */ +public final class Telemetry { + + /** + * Returns the {@link Telemetry} instance. + * + * @return the Telemetry instance + */ + public static Telemetry get() { + return INSTANCE; + } + + /** + * The singleton instance of the Telemetry class. + */ + private final static Telemetry INSTANCE = new Telemetry(); + + /** + * OSHI handle. + */ + private final SystemInfo oshi; + + /** + * Construct a new instance. + */ + private Telemetry() { + this.oshi = new SystemInfo(); + } + + /** + * Returns the amount of heap memory available to the JVM. + * + * @return the available heap memory in bytes + */ + public long availableHeapMemory() { + return Runtime.getRuntime().freeMemory(); + } + + /** + * Returns the amount of available physical memory on the system. + * + * @return the available physical memory in bytes + */ + public long availableSystemMemory() { + return oshi.getHardware().getMemory().getAvailable(); + } + + /** + * Determines if the current process is running in a containerized + * environment. + * + * @return true if running in a container, false otherwise + */ + public boolean isContainerized() { + if(oshi.getOperatingSystem().getVersionInfo().getBuildNumber() + .toLowerCase().contains("container")) { + return true; + } + if(System.getenv().containsKey("KUBERNETES_SERVICE_HOST")) { + return true; + } + if(System.getenv().containsKey("DOCKER_CONTAINER")) { + return true; + } + if(new File("/.dockerenv").exists()) { + // Check for the very common /.dockerenv marker + return true; + } + try (BufferedReader reader = new BufferedReader( + new FileReader("/proc/1/cgroup"))) { + String line; + while ((line = reader.readLine()) != null) { + // Kubernetes pods, Docker, LXC, etc tend to have recognizable + // paths + if(line.contains("docker") || line.contains("kubepods") + || line.contains("containerd") + || line.contains("lxc")) { + return true; + } + } + } + catch (IOException e) {} + return false; + + } + + /** + * Determines if swap memory is enabled on the system. + * + * @return true if swap is enabled, false otherwise + */ + public boolean isSwapEnabled() { + return oshi.getHardware().getMemory().getVirtualMemory() + .getSwapTotal() > 0; + } + + /** + * Returns the underlying system information object. + * + * @return the SystemInfo instance + */ + public SystemInfo system() { + return oshi; + } + + /** + * Returns the amount of heap memory reserved by the JVM. + * + * @return the reserved heap memory in bytes + */ + public long totalHeapMemory() { + return Runtime.getRuntime().totalMemory(); + } + + /** + * Returns the total physical memory available on the system. + * + * @return the total physical memory in bytes + */ + public long totalMemory() { + return oshi.getHardware().getMemory().getTotal(); + } + + /** + * Returns the total amount of virtual memory (swap space) on the system. + * + * @return the total virtual memory in bytes + */ + public long virtualMemory() { + return oshi.getHardware().getMemory().getVirtualMemory().getSwapTotal(); + } +} diff --git a/concourse-server/src/main/java/com/cinchapi/concourse/server/storage/db/search/SubstringDeduplicator.java b/concourse-server/src/main/java/com/cinchapi/concourse/server/storage/db/search/SubstringDeduplicator.java index a85fdaacd..f3263453b 100644 --- a/concourse-server/src/main/java/com/cinchapi/concourse/server/storage/db/search/SubstringDeduplicator.java +++ b/concourse-server/src/main/java/com/cinchapi/concourse/server/storage/db/search/SubstringDeduplicator.java @@ -17,7 +17,6 @@ import java.io.Closeable; import java.io.IOException; -import java.lang.management.ManagementFactory; import java.nio.ByteBuffer; import java.nio.file.Path; import java.util.AbstractSet; @@ -44,6 +43,7 @@ import com.cinchapi.common.logging.Logging; import com.cinchapi.common.reflect.Reflection; import com.cinchapi.concourse.annotate.Experimental; +import com.cinchapi.concourse.server.Telemetry; import com.cinchapi.concourse.server.model.Text; import com.cinchapi.concourse.server.storage.db.kernel.CorpusChunk.SearchTermMetrics; import com.cinchapi.concourse.util.FileOps; @@ -107,16 +107,20 @@ public static SubstringDeduplicator create(char[] term, int expectedInsertions = metrics.upperBoundOfPossibleSubstrings(); long estimatedMemoryRequired = (long) metrics.averageSubstringLength() * (long) expectedInsertions; - long availableDirectMemory = availableDirectMemory(); - long freeHeapMemory = Runtime.getRuntime().freeMemory(); + + Telemetry telemetry = Telemetry.get(); + long availableOffHeapMemory = telemetry.availableSystemMemory(); + long availableHeapMemory = telemetry.availableHeapMemory(); + boolean swapIsEnabled = telemetry.isSwapEnabled(); + Logger.info("The search indexer has encountered a large term that " + "may require a lot of memory to process. The term has " + "up to {} search indexes{}{}", expectedInsertions, System.lineSeparator(), summarizeMemory(estimatedMemoryRequired, - availableDirectMemory, freeHeapMemory)); + availableOffHeapMemory, availableHeapMemory)); SubstringDeduplicator deduplicator; - if(availableDirectMemory > estimatedMemoryRequired) { + if(availableOffHeapMemory > estimatedMemoryRequired || swapIsEnabled) { try { Logger.info("Attempting to use off-heap memory to deduplicate " + "the search indexes"); @@ -189,23 +193,6 @@ static SubstringDeduplicator testCreateChronicleBacked(char[] term) { return new ChronicleBackedSubstringDeduplicator(term, metrics); } - /** - * If possible, return the amount of "direct" memory that is available for - * off-heap storage. - * - * @return the number of direct memory bytes that are available - */ - @SuppressWarnings("restriction") - private static long availableDirectMemory() { - try { - com.sun.management.OperatingSystemMXBean osBean = (com.sun.management.OperatingSystemMXBean) ManagementFactory - .getOperatingSystemMXBean(); - return osBean.getFreePhysicalMemorySize(); - } - catch (Exception e) {} - return 0; - } - /** * Formats a byte count into a human-readable string representation. * From 4834d03dcd075ae2a01ccdc77a42e3f20098a86f Mon Sep 17 00:00:00 2001 From: Jeff Nelson Date: Sun, 4 May 2025 06:24:54 -0400 Subject: [PATCH 2/2] sanity check --- .../src/main/java/com/cinchapi/concourse/server/Telemetry.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/concourse-server/src/main/java/com/cinchapi/concourse/server/Telemetry.java b/concourse-server/src/main/java/com/cinchapi/concourse/server/Telemetry.java index dbf33003b..4b194fd13 100644 --- a/concourse-server/src/main/java/com/cinchapi/concourse/server/Telemetry.java +++ b/concourse-server/src/main/java/com/cinchapi/concourse/server/Telemetry.java @@ -27,7 +27,7 @@ * * @author Jeff Nelson */ -public final class Telemetry { +public final class Telemetry { /** * Returns the {@link Telemetry} instance.