diff --git a/java/common/src/main/java/org/apache/tsfile/utils/RamUsageEstimator.java b/java/common/src/main/java/org/apache/tsfile/utils/RamUsageEstimator.java index af7a8cff4..29b9d085e 100644 --- a/java/common/src/main/java/org/apache/tsfile/utils/RamUsageEstimator.java +++ b/java/common/src/main/java/org/apache/tsfile/utils/RamUsageEstimator.java @@ -27,6 +27,7 @@ import java.security.PrivilegedAction; import java.text.DecimalFormat; import java.text.DecimalFormatSymbols; +import java.time.LocalDate; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -35,6 +36,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; @@ -99,6 +101,11 @@ private RamUsageEstimator() {} /** Sizes of primitive classes. */ public static final Map, Integer> primitiveSizes; + public static final long LOCAL_DATE_ARRAY_SIZE; + public static final long LOCAL_DATE_SIZE; + public static final long BIT_MAP_SIZE; + public static final long SIZE_OF_ARRAYLIST; + static { Map, Integer> primitiveSizesMap = new IdentityHashMap<>(); primitiveSizesMap.put(boolean.class, 1); @@ -111,6 +118,11 @@ private RamUsageEstimator() {} primitiveSizesMap.put(long.class, Long.BYTES); primitiveSizes = Collections.unmodifiableMap(primitiveSizesMap); + + LOCAL_DATE_ARRAY_SIZE = RamUsageEstimator.shallowSizeOf(LocalDate[].class); + LOCAL_DATE_SIZE = RamUsageEstimator.shallowSizeOf(LocalDate.class); + BIT_MAP_SIZE = RamUsageEstimator.shallowSizeOfInstance(BitMap.class); + SIZE_OF_ARRAYLIST = RamUsageEstimator.shallowSizeOfInstance(ArrayList.class); } /** JVMs typically cache small longs. This tries to find out what the range is. */ @@ -654,4 +666,68 @@ public static long sizeOfDoubleArray(int length) { public static long sizeOfObjectArray(int length) { return alignObjectSize(NUM_BYTES_ARRAY_HEADER + (long) NUM_BYTES_OBJECT_REF * length); } + + private static long sizeOf(final Binary binary) { + return Objects.nonNull(binary) ? binary.ramBytesUsed() : 0L; + } + + public static long sizeOf(final Binary[] binaries) { + if (binaries == null) { + return 0L; + } + + long size = 0L; + for (Binary binary : binaries) { + size += sizeOf(binary); + } + + return size + RamUsageEstimator.shallowSizeOf(binaries); + } + + public static long sizeOfStringArray(final String[] values) { + return Objects.nonNull(values) ? RamUsageEstimator.sizeOf(values) : 0L; + } + + public static long sizeOf(BitMap[] bitMaps) { + if (bitMaps == null) { + return 0L; + } + long size = + RamUsageEstimator.alignObjectSize( + NUM_BYTES_ARRAY_HEADER + (long) NUM_BYTES_OBJECT_REF * bitMaps.length); + for (BitMap bitMap : bitMaps) { + size += sizeOf(bitMap); + } + return size; + } + + private static long sizeOf(final BitMap bitMap) { + if (bitMap == null) { + return 0L; + } + long size = BIT_MAP_SIZE; + + size += + RamUsageEstimator.alignObjectSize(NUM_BYTES_ARRAY_HEADER + bitMap.getByteArray().length); + return size; + } + + public static long sizeOf(final LocalDate[] input) { + if (Objects.isNull(input)) { + return 0; + } + long size = + RamUsageEstimator.alignObjectSize( + LOCAL_DATE_ARRAY_SIZE + (long) input.length * NUM_BYTES_OBJECT_REF); + for (final LocalDate date : input) { + if (Objects.nonNull(date)) { + size += LOCAL_DATE_SIZE; + } + } + return size; + } + + public static long shallowSizeOfList(final List input) { + return alignObjectSize(SIZE_OF_ARRAYLIST + (long) input.size() * NUM_BYTES_OBJECT_REF); + } } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java b/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java index 9bd8d88f2..27ed30a60 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java @@ -21,11 +21,13 @@ import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.Accountable; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BitMap; import org.apache.tsfile.utils.BytesUtils; import org.apache.tsfile.utils.DateUtils; import org.apache.tsfile.utils.PublicBAOS; +import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.ReadWriteIOUtils; import org.apache.tsfile.write.UnSupportedDataTypeException; import org.apache.tsfile.write.schema.MeasurementSchema; @@ -50,8 +52,8 @@ * *

Notice: The tablet should not have empty cell, please use BitMap to denote null value */ -public class Tablet { - +public class Tablet implements Accountable { + private static final long TABLET_SIZE = RamUsageEstimator.shallowSizeOfInstance(Tablet.class); private static final int DEFAULT_SIZE = 1024; private static final String NOT_SUPPORT_DATATYPE = "Data type %s is not supported."; @@ -862,4 +864,66 @@ private boolean isBitMapsEqual(BitMap[] thisBitMaps, BitMap[] thatBitMaps, int c } return true; } + + @Override + public long ramBytesUsed() { + long totalSizeInBytes = + TABLET_SIZE + + RamUsageEstimator.sizeOf(deviceId) + + RamUsageEstimator.sizeOf(timestamps) + + RamUsageEstimator.sizeOf(bitMaps) + + RamUsageEstimator.sizeOfMap(measurementIndex); + + // values + final List timeSeries = schemas; + + if (timeSeries != null) { + totalSizeInBytes += RamUsageEstimator.shallowSizeOfList(timeSeries); + for (int column = 0; column < timeSeries.size(); column++) { + final MeasurementSchema measurementSchema = timeSeries.get(column); + if (measurementSchema == null) { + continue; + } + // Measurement schema size + totalSizeInBytes += 75; + + final TSDataType tsDataType = measurementSchema.getType(); + if (tsDataType == null) { + continue; + } + + if (values == null || values.length <= column) { + continue; + } + switch (tsDataType) { + case INT64: + case TIMESTAMP: + totalSizeInBytes += RamUsageEstimator.sizeOf((long[]) values[column]); + break; + case DATE: + totalSizeInBytes += RamUsageEstimator.sizeOf((LocalDate[]) values[column]); + break; + case INT32: + totalSizeInBytes += RamUsageEstimator.sizeOf((int[]) values[column]); + break; + case DOUBLE: + totalSizeInBytes += RamUsageEstimator.sizeOf((double[]) values[column]); + break; + case FLOAT: + totalSizeInBytes += RamUsageEstimator.sizeOf((float[]) values[column]); + break; + case BOOLEAN: + totalSizeInBytes += RamUsageEstimator.sizeOf((boolean[]) values[column]); + break; + case STRING: + case TEXT: + case BLOB: + totalSizeInBytes += RamUsageEstimator.sizeOf((Binary[]) values[column]); + break; + } + } + } + + return totalSizeInBytes; + } }