Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -535,6 +535,7 @@ public void testInformationSchema() throws SQLException {
"functions,INF,",
"keywords,INF,",
"nodes,INF,",
"pipe_memory,INF,",
"pipe_plugins,INF,",
"pipes,INF,",
"queries,INF,",
Expand Down Expand Up @@ -616,6 +617,11 @@ public void testInformationSchema() throws SQLException {
"estimated_remaining_seconds,DOUBLE,ATTRIBUTE,",
"is_degraded,BOOLEAN,ATTRIBUTE,",
"recent_failures,STRING,ATTRIBUTE,")));
TestUtils.assertResultSetEqual(
statement.executeQuery("desc pipe_memory"),
"ColumnName,DataType,Category,",
new HashSet<>(
Arrays.asList("name,STRING,TAG,", "memory_usage_in_bytes,INT64,ATTRIBUTE,")));
TestUtils.assertResultSetEqual(
statement.executeQuery("desc pipe_plugins"),
"ColumnName,DataType,Category,",
Expand Down Expand Up @@ -731,6 +737,9 @@ public void testInformationSchema() throws SQLException {
Assert.assertThrows(SQLException.class, () -> statement.execute("select * from data_nodes"));
Assert.assertThrows(
SQLException.class, () -> statement.executeQuery("select * from pipe_plugins"));
Assert.assertThrows(
SQLException.class, () -> statement.executeQuery("select * from pipe_memory"));
Assert.assertThrows(SQLException.class, () -> statement.executeQuery("SHOW PIPE MEMORY"));
Assert.assertThrows(
SQLException.class, () -> statement.executeQuery("select * from table_disk_usage"));

Expand Down Expand Up @@ -765,6 +774,32 @@ public void testInformationSchema() throws SQLException {
// Test table query
statement.execute("use information_schema");

try (final ResultSet resultSet = statement.executeQuery("SHOW PIPE MEMORY")) {
final ResultSetMetaData metaData = resultSet.getMetaData();
assertEquals(2, metaData.getColumnCount());
assertEquals("name", metaData.getColumnName(1));
assertEquals("memory_usage_in_bytes", metaData.getColumnName(2));
boolean hasFloatingMemory = false;
while (resultSet.next()) {
if ("FloatingMemory".equals(resultSet.getString(1))) {
assertTrue(resultSet.getLong(2) >= 0);
hasFloatingMemory = true;
}
}
assertTrue(hasFloatingMemory);
}
try (final ResultSet resultSet =
statement.executeQuery("select * from information_schema.pipe_memory")) {
boolean hasFloatingMemory = false;
while (resultSet.next()) {
if ("FloatingMemory".equals(resultSet.getString(1))) {
assertTrue(resultSet.getLong(2) >= 0);
hasFloatingMemory = true;
}
}
assertTrue(hasFloatingMemory);
}

statement.execute("create database test");
statement.execute(
"create table test.test (a tag, b attribute, c int32 comment 'turbine') comment 'test'");
Expand Down Expand Up @@ -813,6 +848,7 @@ public void testInformationSchema() throws SQLException {
"information_schema,columns,INF,USING,null,SYSTEM VIEW,false,",
"information_schema,queries,INF,USING,null,SYSTEM VIEW,false,",
"information_schema,regions,INF,USING,null,SYSTEM VIEW,false,",
"information_schema,pipe_memory,INF,USING,null,SYSTEM VIEW,false,",
"information_schema,topics,INF,USING,null,SYSTEM VIEW,false,",
"information_schema,pipe_plugins,INF,USING,null,SYSTEM VIEW,false,",
"information_schema,pipes,INF,USING,null,SYSTEM VIEW,false,",
Expand All @@ -834,7 +870,7 @@ public void testInformationSchema() throws SQLException {
TestUtils.assertResultSetEqual(
statement.executeQuery("count devices from tables where status = 'USING'"),
"count(devices),",
Collections.singleton("23,"));
Collections.singleton("24,"));
TestUtils.assertResultSetEqual(
statement.executeQuery(
"select * from columns where table_name = 'queries' or database = 'test'"),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,9 @@ private void initLoggerPeriodicalLogReducer() {
if (pipeLogReducerMemoryBlock == null) {
pipeLogReducerMemoryBlock =
PipeDataNodeResourceManager.memory()
.tryAllocate(PipeConfig.getInstance().getPipeLoggerCacheMaxSizeInBytes());
.tryAllocate(
PipeDataNodeRuntimeAgent.class.getSimpleName() + "#logger",
PipeConfig.getInstance().getPipeLoggerCacheMaxSizeInBytes());
}

LoggerPeriodicalLogReducer.setMemoryResizeFunction(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,8 @@ public PipeStatementInsertionEvent(
this.statement = statement;
// Allocate empty memory block, will be resized later.
this.allocatedMemoryBlock =
PipeDataNodeResourceManager.memory().forceAllocateForTabletWithRetry(0);
PipeDataNodeResourceManager.memory()
.forceAllocateForTabletWithRetry(PipeStatementInsertionEvent.class.getSimpleName(), 0);
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -498,7 +498,9 @@ public synchronized List<Tablet> convertToTablets() {
allocatedMemoryBlock.compareAndSet(
null,
PipeDataNodeResourceManager.memory()
.forceAllocateForTabletWithRetry(tabletMemoryUsageInBytes));
.forceAllocateForTabletWithRetry(
PipeInsertNodeTabletInsertionEvent.class.getSimpleName(),
tabletMemoryUsageInBytes));
}
return tablets;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,8 @@ private PipeRawTabletInsertionEvent(

// Allocate empty memory block, will be resized later.
this.allocatedMemoryBlock =
PipeDataNodeResourceManager.memory().forceAllocateForTabletWithRetry(0);
PipeDataNodeResourceManager.memory()
.forceAllocateForTabletWithRetry(PipeRawTabletInsertionEvent.class.getSimpleName(), 0);

if (needToReport) {
addOnCommittedHook(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,9 @@ protected TsFileInsertionEventParser(
this.sourceEvent = sourceEvent;
this.memoryManager = memoryManager;

this.allocatedMemoryBlockForTablet = memoryManager.forceAllocateForTabletWithRetry(0);
this.allocatedMemoryBlockForTablet =
memoryManager.forceAllocateForTabletWithRetry(
TsFileInsertionEventParser.class.getSimpleName() + "#tablet", 0);

LOGGER.debug(
DataNodePipeMessages.TSFILE_HAS_INITIALIZED_PIPENAME_CREATION_TIME_PATTERN,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,9 +25,10 @@
/** Allocates parser working memory from the pool owned by the caller. */
public interface TsFileInsertionEventParserMemoryManager {

TsFileInsertionEventParserMemoryBlock forceAllocateForTabletWithRetry(long sizeInBytes);
TsFileInsertionEventParserMemoryBlock forceAllocateForTabletWithRetry(
String name, long sizeInBytes);

TsFileInsertionEventParserMemoryBlock forceAllocate(long sizeInBytes);
TsFileInsertionEventParserMemoryBlock forceAllocate(String name, long sizeInBytes);

static TsFileInsertionEventParserMemoryManager pipe() {
return PipeHolder.INSTANCE;
Expand All @@ -38,14 +39,17 @@ final class PipeHolder {
new TsFileInsertionEventParserMemoryManager() {
@Override
public TsFileInsertionEventParserMemoryBlock forceAllocateForTabletWithRetry(
final long sizeInBytes) {
final String name, final long sizeInBytes) {
return new PipeBlock(
PipeDataNodeResourceManager.memory().forceAllocateForTabletWithRetry(sizeInBytes));
PipeDataNodeResourceManager.memory()
.forceAllocateForTabletWithRetry(name, sizeInBytes));
}

@Override
public TsFileInsertionEventParserMemoryBlock forceAllocate(final long sizeInBytes) {
return new PipeBlock(PipeDataNodeResourceManager.memory().forceAllocate(sizeInBytes));
public TsFileInsertionEventParserMemoryBlock forceAllocate(
final String name, final long sizeInBytes) {
return new PipeBlock(
PipeDataNodeResourceManager.memory().forceAllocate(name, sizeInBytes));
}
};
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -221,7 +221,9 @@ public TsFileInsertionEventQueryParser(
? ModsOperationUtil.loadModificationsFromTsFile(tsFile)
: PatternTreeMapFactory.getModsPatternTreeMap();
allocatedMemoryBlockForModifications =
memoryManager.forceAllocateForTabletWithRetry(currentModifications.ramBytesUsed());
memoryManager.forceAllocateForTabletWithRetry(
TsFileInsertionEventQueryParser.class.getSimpleName() + "#modifications",
currentModifications.ramBytesUsed());

final PipeTsFileResourceManager tsFileResourceManager = PipeDataNodeResourceManager.tsfile();
final Map<IDeviceID, List<String>> deviceMeasurementsMap;
Expand Down Expand Up @@ -282,7 +284,10 @@ public TsFileInsertionEventQueryParser(
memoryRequiredInBytes +=
PipeMemoryWeightUtil.memoryOfIDeviceID2StrList(deviceMeasurementsMap);
}
allocatedMemoryBlock = memoryManager.forceAllocate(memoryRequiredInBytes);
allocatedMemoryBlock =
memoryManager.forceAllocate(
TsFileInsertionEventQueryParser.class.getSimpleName() + "#metadata",
memoryRequiredInBytes);

final Iterator<Map.Entry<IDeviceID, List<String>>> iterator =
deviceMeasurementsMap.entrySet().iterator();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -181,16 +181,22 @@ public TsFileInsertionEventScanParser(
this.endTime = endTime;
filter = Objects.nonNull(timeFilterExpression) ? timeFilterExpression.getFilter() : null;

this.allocatedMemoryBlockForBatchData = memoryManager.forceAllocateForTabletWithRetry(0);
this.allocatedMemoryBlockForChunk = memoryManager.forceAllocateForTabletWithRetry(0);
this.allocatedMemoryBlockForBatchData =
memoryManager.forceAllocateForTabletWithRetry(
TsFileInsertionEventScanParser.class.getSimpleName() + "#batchData", 0);
this.allocatedMemoryBlockForChunk =
memoryManager.forceAllocateForTabletWithRetry(
TsFileInsertionEventScanParser.class.getSimpleName() + "#chunk", 0);

try {
currentModifications =
isWithMod
? ModsOperationUtil.loadModificationsFromTsFile(tsFile)
: PatternTreeMapFactory.getModsPatternTreeMap();
allocatedMemoryBlockForModifications =
memoryManager.forceAllocateForTabletWithRetry(currentModifications.ramBytesUsed());
memoryManager.forceAllocateForTabletWithRetry(
TsFileInsertionEventScanParser.class.getSimpleName() + "#modifications",
currentModifications.ramBytesUsed());

tsFileSequenceReader = createTsFileSequenceReader(tsFile, !currentModifications.isEmpty());
tsFileSequenceReader.position((long) TSFileConfig.MAGIC_STRING.getBytes().length + 1);
Expand Down Expand Up @@ -257,7 +263,9 @@ private TsFileSequenceReader createTsFileSequenceReader(
}

allocatedMemoryBlockForTsFileInput =
memoryManager.forceAllocateForTabletWithRetry(TS_FILE_INPUT_BUFFER_SIZE_IN_BYTES);
memoryManager.forceAllocateForTabletWithRetry(
TsFileInsertionEventScanParser.class.getSimpleName() + "#tsFileInput",
TS_FILE_INPUT_BUFFER_SIZE_IN_BYTES);
return new TsFileSequenceReader(
new BufferedTsFileInput(tsFile.toPath(), TS_FILE_INPUT_BUFFER_SIZE_IN_BYTES),
false,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -119,11 +119,21 @@ public TsFileInsertionEventTableParser(
? ModsOperationUtil.loadModificationsFromTsFile(tsFile)
: PatternTreeMapFactory.getModsPatternTreeMap();
allocatedMemoryBlockForModifications =
memoryManager.forceAllocateForTabletWithRetry(currentModifications.ramBytesUsed());
this.allocatedMemoryBlockForChunk = memoryManager.forceAllocateForTabletWithRetry(0);
this.allocatedMemoryBlockForBatchData = memoryManager.forceAllocateForTabletWithRetry(0);
this.allocatedMemoryBlockForChunkMeta = memoryManager.forceAllocateForTabletWithRetry(0);
this.allocatedMemoryBlockForTableSchemas = memoryManager.forceAllocateForTabletWithRetry(0);
memoryManager.forceAllocateForTabletWithRetry(
TsFileInsertionEventTableParser.class.getSimpleName() + "#modifications",
currentModifications.ramBytesUsed());
this.allocatedMemoryBlockForChunk =
memoryManager.forceAllocateForTabletWithRetry(
TsFileInsertionEventTableParser.class.getSimpleName() + "#chunk", 0);
this.allocatedMemoryBlockForBatchData =
memoryManager.forceAllocateForTabletWithRetry(
TsFileInsertionEventTableParser.class.getSimpleName() + "#batchData", 0);
this.allocatedMemoryBlockForChunkMeta =
memoryManager.forceAllocateForTabletWithRetry(
TsFileInsertionEventTableParser.class.getSimpleName() + "#chunkMetadata", 0);
this.allocatedMemoryBlockForTableSchemas =
memoryManager.forceAllocateForTabletWithRetry(
TsFileInsertionEventTableParser.class.getSimpleName() + "#tableSchemas", 0);

this.startTime = startTime;
this.endTime = endTime;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,9 @@ public abstract class PartialPathLastObjectCache<T> implements AutoCloseable {
private final Cache<String, T> partialPath2ObjectCache;

protected PartialPathLastObjectCache(final long memoryLimitInBytes) {
allocatedMemoryBlock = PipeDataNodeResourceManager.memory().tryAllocate(memoryLimitInBytes);
allocatedMemoryBlock =
PipeDataNodeResourceManager.memory()
.tryAllocate(PartialPathLastObjectCache.class.getSimpleName(), memoryLimitInBytes);

// Currently disable the metric here because it's not a constant cache and the number may
// fluctuate. In the future all the "processorCache"s may be recorded in single metric entry
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1109,7 +1109,9 @@ private void closeMemoryBlock(final PipeMemoryBlock memoryBlock) {
private PipeMemoryBlock tryAllocateReceiverMemory(final long requestedMemorySizeInBytes)
throws PipeRuntimeOutOfMemoryCriticalException {
return PipeDataNodeResourceManager.memory()
.forceAllocate(Math.max(requestedMemorySizeInBytes, 0));
.forceAllocate(
IoTDBDataNodeReceiver.class.getSimpleName() + "#request",
Math.max(requestedMemorySizeInBytes, 0));
}

@Override
Expand Down Expand Up @@ -1189,6 +1191,7 @@ private TSStatus executeStatementAndClassifyExceptions(
allocatedMemoryBlock =
PipeDataNodeResourceManager.memory()
.forceAllocate(
IoTDBDataNodeReceiver.class.getSimpleName() + "#statement",
(long) (estimatedMemory * pipeReceiverActualToEstimatedMemoryRatio));
break;
} catch (final PipeRuntimeOutOfMemoryCriticalException e) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,8 @@

public abstract class PipeFixedMemoryBlock extends PipeMemoryBlock {

public PipeFixedMemoryBlock(long memoryUsageInBytes) {
super(memoryUsageInBytes);
public PipeFixedMemoryBlock(final String name, final long memoryUsageInBytes) {
super(name, memoryUsageInBytes);
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,8 @@ public class PipeMemoryBlock implements AutoCloseable {

private final ReentrantLock lock = new ReentrantLock();

private final String name;

private final AtomicLong memoryUsageInBytes = new AtomicLong(0);

private final AtomicReference<LongUnaryOperator> shrinkMethod = new AtomicReference<>();
Expand All @@ -50,10 +52,15 @@ public class PipeMemoryBlock implements AutoCloseable {

private volatile boolean isReleased = false;

public PipeMemoryBlock(final long memoryUsageInBytes) {
public PipeMemoryBlock(final String name, final long memoryUsageInBytes) {
this.name = Objects.requireNonNull(name);
this.memoryUsageInBytes.set(memoryUsageInBytes);
}

public String getName() {
return name;
}

public long getMemoryUsageInBytes() {
return memoryUsageInBytes.get();
}
Expand Down Expand Up @@ -165,7 +172,10 @@ void markAsReleased() {
@Override
public String toString() {
return "PipeMemoryBlock{"
+ "usedMemoryInBytes="
+ "name='"
+ name
+ '\''
+ ", usedMemoryInBytes="
+ memoryUsageInBytes.get()
+ ", isReleased="
+ isReleased
Expand Down
Loading
Loading