From 41acb04333847cdb32fd048a14326bde55b85f6f Mon Sep 17 00:00:00 2001 From: JophieQu Date: Thu, 18 Sep 2025 11:11:17 +0800 Subject: [PATCH 1/7] feat: support pprof feature --- .../command/CommandDeserializer.java | 2 + .../component/command/PprofTaskCommand.java | 82 ++++++ oap-server/server-core/pom.xml | 5 + .../oap/server/core/CoreModule.java | 10 + .../oap/server/core/CoreModuleProvider.java | 9 + .../server/core/cache/CacheUpdateTimer.java | 48 ++++ .../oap/server/core/cache/PprofTaskCache.java | 93 +++++++ .../server/core/command/CommandService.java | 15 ++ .../profiling/pprof/PprofMutationService.java | 147 +++++++++++ .../profiling/pprof/PprofQueryService.java | 122 +++++++++ .../storage/PprofProfilingDataDispatcher.java | 41 +++ .../storage/PprofProfilingDataRecord.java | 105 ++++++++ .../pprof/storage/PprofTaskLogRecord.java | 99 ++++++++ .../pprof/storage/PprofTaskRecord.java | 116 +++++++++ .../oap/server/core/query/PprofTaskLog.java | 44 ++++ .../query/input/PprofAnalyzationRequest.java | 31 +++ .../query/input/PprofTaskCreationRequest.java | 35 +++ .../query/input/PprofTaskListRequest.java | 12 + .../core/query/type/PprofAnalyzation.java | 28 +++ .../core/query/type/PprofEventType.java | 42 ++++ .../core/query/type/PprofStackElement.java | 38 +++ .../core/query/type/PprofStackTree.java | 62 +++++ .../oap/server/core/query/type/PprofTask.java | 44 ++++ .../query/type/PprofTaskCreationResult.java | 24 ++ .../query/type/PprofTaskCreationType.java | 8 + .../core/query/type/PprofTaskListResult.java | 32 +++ .../query/type/PprofTaskLogOperationType.java | 54 ++++ .../core/query/type/PprofTaskProgress.java | 31 +++ .../core/source/DefaultScopeDefine.java | 3 + .../core/source/PprofProfilingData.java | 50 ++++ .../server/core/storage/StorageModule.java | 6 + .../profiling/pprof/IPprofDataQueryDAO.java | 36 +++ .../pprof/IPprofTaskLogQueryDAO.java | 14 ++ .../profiling/pprof/IPprofTaskQueryDAO.java | 47 ++++ .../library-pprof-parser/pom.xml | 86 +++++++ .../pprof/parser/PprofMergeBuilder.java | 84 +++++++ .../library/pprof/parser/PprofParser.java | 53 ++++ .../server/library/pprof/type/FrameTree.java | 38 +++ .../library/pprof/type/FrameTreeBuilder.java | 102 ++++++++ .../src/main/proto/profile.proto | 233 ++++++++++++++++++ .../server/library/util/CollectionUtils.java | 4 + oap-server/server-library/pom.xml | 1 + .../query/graphql/GraphQLQueryProvider.java | 6 +- .../query/graphql/resolver/PprofMutation.java | 55 +++++ .../query/graphql/resolver/PprofQuery.java | 88 +++++++ oap-server/server-receiver-plugin/pom.xml | 1 + .../receiver-proto/pom.xml | 8 + .../skywalking-pprof-receiver-plugin/pom.xml | 53 ++++ .../receiver/pprof/module/PprofModule.java | 34 +++ .../pprof/module/PprofModuleConfig.java | 46 ++++ .../pprof/provider/PprofModuleProvider.java | 88 +++++++ .../provider/handler/PprofServiceHandler.java | 130 ++++++++++ .../PprofByteBufCollectionObserver.java | 158 ++++++++++++ .../stream/PprofCollectionMetaData.java | 36 +++ .../stream/PprofFileCollectionObserver.java | 170 +++++++++++++ ...ing.oap.server.library.module.ModuleDefine | 19 ++ ...g.oap.server.library.module.ModuleProvider | 19 ++ .../skywalking-sharing-server-plugin/pom.xml | 8 + oap-server/server-starter/pom.xml | 5 + .../src/main/resources/application.yml | 16 +- .../banyandb/BanyanDBStorageConfig.java | 2 + .../banyandb/BanyanDBStorageProvider.java | 17 ++ .../stream/BanyanDBPprofDataQueryDAO.java | 59 +++++ .../stream/BanyanDBPprofTaskLogQueryDAO.java | 82 ++++++ .../stream/BanyanDBPprofTaskQueryDAO.java | 147 +++++++++++ .../StorageModuleElasticsearchProvider.java | 12 + .../query/PprofTaskLogQueryEsDAO.java | 84 +++++++ .../query/PprofTaskQueryEsDAO.java | 147 +++++++++++ 68 files changed, 3624 insertions(+), 2 deletions(-) create mode 100644 apm-protocol/apm-network/src/main/java/org/apache/skywalking/oap/server/network/trace/component/command/PprofTaskCommand.java create mode 100644 oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/cache/PprofTaskCache.java create mode 100644 oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/PprofMutationService.java create mode 100644 oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/PprofQueryService.java create mode 100644 oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofProfilingDataDispatcher.java create mode 100644 oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofProfilingDataRecord.java create mode 100644 oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofTaskLogRecord.java create mode 100644 oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofTaskRecord.java create mode 100644 oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/PprofTaskLog.java create mode 100644 oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/PprofAnalyzationRequest.java create mode 100644 oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/PprofTaskCreationRequest.java create mode 100644 oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/PprofTaskListRequest.java create mode 100644 oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofAnalyzation.java create mode 100644 oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofEventType.java create mode 100644 oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofStackElement.java create mode 100644 oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofStackTree.java create mode 100644 oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTask.java create mode 100644 oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskCreationResult.java create mode 100644 oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskCreationType.java create mode 100644 oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskListResult.java create mode 100644 oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskLogOperationType.java create mode 100644 oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskProgress.java create mode 100644 oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/PprofProfilingData.java create mode 100644 oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/profiling/pprof/IPprofDataQueryDAO.java create mode 100644 oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/profiling/pprof/IPprofTaskLogQueryDAO.java create mode 100644 oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/profiling/pprof/IPprofTaskQueryDAO.java create mode 100755 oap-server/server-library/library-pprof-parser/pom.xml create mode 100644 oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/parser/PprofMergeBuilder.java create mode 100644 oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/parser/PprofParser.java create mode 100755 oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/FrameTree.java create mode 100644 oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/FrameTreeBuilder.java create mode 100755 oap-server/server-library/library-pprof-parser/src/main/proto/profile.proto create mode 100644 oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/PprofMutation.java create mode 100644 oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/PprofQuery.java create mode 100644 oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/pom.xml create mode 100644 oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/module/PprofModule.java create mode 100644 oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/module/PprofModuleConfig.java create mode 100644 oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/PprofModuleProvider.java create mode 100644 oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/PprofServiceHandler.java create mode 100644 oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/stream/PprofByteBufCollectionObserver.java create mode 100644 oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/stream/PprofCollectionMetaData.java create mode 100644 oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/stream/PprofFileCollectionObserver.java create mode 100644 oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleDefine create mode 100644 oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleProvider create mode 100644 oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBPprofDataQueryDAO.java create mode 100644 oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBPprofTaskLogQueryDAO.java create mode 100644 oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBPprofTaskQueryDAO.java create mode 100644 oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofTaskLogQueryEsDAO.java create mode 100644 oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofTaskQueryEsDAO.java diff --git a/apm-protocol/apm-network/src/main/java/org/apache/skywalking/oap/server/network/trace/component/command/CommandDeserializer.java b/apm-protocol/apm-network/src/main/java/org/apache/skywalking/oap/server/network/trace/component/command/CommandDeserializer.java index 6323e6906a12..4bd94b8bc0e0 100644 --- a/apm-protocol/apm-network/src/main/java/org/apache/skywalking/oap/server/network/trace/component/command/CommandDeserializer.java +++ b/apm-protocol/apm-network/src/main/java/org/apache/skywalking/oap/server/network/trace/component/command/CommandDeserializer.java @@ -29,6 +29,8 @@ public static BaseCommand deserialize(final Command command) { return ConfigurationDiscoveryCommand.DESERIALIZER.deserialize(command); } else if (AsyncProfilerTaskCommand.NAME.equals(commandName)) { return AsyncProfilerTaskCommand.DESERIALIZER.deserialize(command); + } else if (PprofTaskCommand.NAME.equals(commandName)) { + return PprofTaskCommand.DESERIALIZER.deserialize(command); } throw new UnsupportedCommandException(command); } diff --git a/apm-protocol/apm-network/src/main/java/org/apache/skywalking/oap/server/network/trace/component/command/PprofTaskCommand.java b/apm-protocol/apm-network/src/main/java/org/apache/skywalking/oap/server/network/trace/component/command/PprofTaskCommand.java new file mode 100644 index 000000000000..3cbf9b5f4750 --- /dev/null +++ b/apm-protocol/apm-network/src/main/java/org/apache/skywalking/oap/server/network/trace/component/command/PprofTaskCommand.java @@ -0,0 +1,82 @@ +package org.apache.skywalking.oap.server.network.trace.component.command; + +import org.apache.skywalking.apm.network.common.v3.Command; +import org.apache.skywalking.apm.network.common.v3.KeyStringValuePair; +import java.util.List; +import lombok.Getter; + +@Getter +public class PprofTaskCommand extends BaseCommand implements Serializable, Deserializable { + public static final Deserializable DESERIALIZER = new PprofTaskCommand("", "", "", 0, 0, 0); + public static final String NAME = "PprofTaskQuery"; + /** + * pprof taskId + */ + private String taskId; + // Type of profiling (CPU/Heap/Block/Mutex/Goroutine/Threadcreate/Allocs) + private String events; + // unit is minute + private long duration; + // Unix timestamp in milliseconds when the task was created + private long createTime; + // + private int dumpPeriod; + + public PprofTaskCommand(String serialNumber, String taskId, String events, + long duration, long createTime, int dumpPeriod) { + super(NAME, serialNumber); + this.taskId = taskId; + this.duration = duration; + this.createTime = createTime; + this.dumpPeriod = dumpPeriod; + this.events = events; + } + + // public PprofTaskCommand(String serialNumber, String taskId, + // long duration, long startTime, long createTime, int dumpPeriod) { + // super(NAME, serialNumber); + // this.taskId = taskId; + // this.duration = duration; + // this.startTime = startTime; + // this.createTime = createTime; + // this.dumpPeriod = dumpPeriod; + // } + + @Override + public PprofTaskCommand deserialize(Command command) { + final List argsList = command.getArgsList(); + String taskId = null; + String events = null; + long duration = 0; + long createTime = 0; + int dumpPeriod = 0; + String serialNumber = null; + for (final KeyStringValuePair pair : argsList) { + if ("SerialNumber".equals(pair.getKey())) { + serialNumber = pair.getValue(); + } else if ("TaskId".equals(pair.getKey())) { + taskId = pair.getValue(); + } else if ("Events".equals(pair.getKey())) { + events = pair.getValue(); + } else if ("Duration".equals(pair.getKey())) { + duration = Long.parseLong(pair.getValue()); + } else if ("CreateTime".equals(pair.getKey())) { + createTime = Long.parseLong(pair.getValue()); + } else if ("DumpPeriod".equals(pair.getKey())) { + dumpPeriod = Integer.parseInt(pair.getValue()); + } + } + return new PprofTaskCommand(serialNumber, taskId, events, duration, createTime, dumpPeriod); + } + + @Override + public Command.Builder serialize() { + final Command.Builder builder = commandBuilder(); + builder.addArgs(KeyStringValuePair.newBuilder().setKey("TaskId").setValue(taskId)) + .addArgs(KeyStringValuePair.newBuilder().setKey("Events").setValue(events)) + .addArgs(KeyStringValuePair.newBuilder().setKey("Duration").setValue(String.valueOf(duration))) + .addArgs(KeyStringValuePair.newBuilder().setKey("CreateTime").setValue(String.valueOf(createTime))) + .addArgs(KeyStringValuePair.newBuilder().setKey("DumpPeriod").setValue(String.valueOf(dumpPeriod))); + return builder; + } +} \ No newline at end of file diff --git a/oap-server/server-core/pom.xml b/oap-server/server-core/pom.xml index 784da5435c03..f6d233587ad2 100644 --- a/oap-server/server-core/pom.xml +++ b/oap-server/server-core/pom.xml @@ -44,6 +44,11 @@ library-async-profiler-jfr-parser ${project.version} + + org.apache.skywalking + library-pprof-parser + ${project.version} + org.apache.skywalking telemetry-api diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModule.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModule.java index 91509759eab3..b6d1734d2f43 100755 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModule.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModule.java @@ -22,6 +22,7 @@ import java.util.List; import org.apache.skywalking.oap.server.core.analysis.meter.MeterSystem; import org.apache.skywalking.oap.server.core.cache.AsyncProfilerTaskCache; +import org.apache.skywalking.oap.server.core.cache.PprofTaskCache; import org.apache.skywalking.oap.server.core.cache.NetworkAddressAliasCache; import org.apache.skywalking.oap.server.core.cache.ProfileTaskCache; import org.apache.skywalking.oap.server.core.command.CommandService; @@ -41,6 +42,8 @@ import org.apache.skywalking.oap.server.core.profiling.continuous.ContinuousProfilingQueryService; import org.apache.skywalking.oap.server.core.profiling.ebpf.EBPFProfilingMutationService; import org.apache.skywalking.oap.server.core.profiling.ebpf.EBPFProfilingQueryService; +import org.apache.skywalking.oap.server.core.profiling.pprof.PprofMutationService; +import org.apache.skywalking.oap.server.core.profiling.pprof.PprofQueryService; import org.apache.skywalking.oap.server.core.profiling.trace.ProfileTaskMutationService; import org.apache.skywalking.oap.server.core.profiling.trace.ProfileTaskQueryService; import org.apache.skywalking.oap.server.core.query.AggregationQueryService; @@ -106,6 +109,7 @@ public Class[] services() { addManagementService(classes); addEBPFProfilingService(classes); addAsyncProfilerService(classes); + addPprofService(classes); classes.add(CommandService.class); classes.add(HierarchyService.class); @@ -137,6 +141,12 @@ private void addAsyncProfilerService(List classes) { classes.add(AsyncProfilerTaskCache.class); } + private void addPprofService(List classes) { + classes.add(PprofMutationService.class); + classes.add(PprofQueryService.class); + classes.add(PprofTaskCache.class); + } + private void addOALService(List classes) { classes.add(OALEngineLoaderService.class); } diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java index a9963f6545a2..c8d88d8fdf47 100755 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java @@ -35,6 +35,7 @@ import org.apache.skywalking.oap.server.core.cache.AsyncProfilerTaskCache; import org.apache.skywalking.oap.server.core.cache.CacheUpdateTimer; import org.apache.skywalking.oap.server.core.cache.NetworkAddressAliasCache; +import org.apache.skywalking.oap.server.core.cache.PprofTaskCache; import org.apache.skywalking.oap.server.core.cache.ProfileTaskCache; import org.apache.skywalking.oap.server.core.cluster.ClusterCoordinator; import org.apache.skywalking.oap.server.core.cluster.ClusterModule; @@ -65,6 +66,8 @@ import org.apache.skywalking.oap.server.core.profiling.continuous.ContinuousProfilingQueryService; import org.apache.skywalking.oap.server.core.profiling.ebpf.EBPFProfilingMutationService; import org.apache.skywalking.oap.server.core.profiling.ebpf.EBPFProfilingQueryService; +import org.apache.skywalking.oap.server.core.profiling.pprof.PprofMutationService; +import org.apache.skywalking.oap.server.core.profiling.pprof.PprofQueryService; import org.apache.skywalking.oap.server.core.profiling.trace.ProfileTaskMutationService; import org.apache.skywalking.oap.server.core.profiling.trace.ProfileTaskQueryService; import org.apache.skywalking.oap.server.core.query.AggregationQueryService; @@ -331,6 +334,12 @@ TTLStatusQuery.class, new TTLStatusQuery( AsyncProfilerQueryService.class, new AsyncProfilerQueryService(getManager())); this.registerServiceImplementation( AsyncProfilerTaskCache.class, new AsyncProfilerTaskCache(getManager(), moduleConfig)); + this.registerServiceImplementation( + PprofMutationService.class, new PprofMutationService(getManager())); + this.registerServiceImplementation( + PprofQueryService.class, new PprofQueryService(getManager())); + this.registerServiceImplementation( + PprofTaskCache.class, new PprofTaskCache(getManager(), moduleConfig)); this.registerServiceImplementation( EBPFProfilingMutationService.class, new EBPFProfilingMutationService(getManager())); this.registerServiceImplementation( diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/cache/CacheUpdateTimer.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/cache/CacheUpdateTimer.java index c38f90daccd0..17f7cee530f7 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/cache/CacheUpdateTimer.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/cache/CacheUpdateTimer.java @@ -27,12 +27,15 @@ import org.apache.skywalking.oap.server.core.profiling.asyncprofiler.storage.AsyncProfilerTaskRecord; import org.apache.skywalking.oap.server.core.query.type.AsyncProfilerTask; import org.apache.skywalking.oap.server.core.storage.profiling.asyncprofiler.IAsyncProfilerTaskQueryDAO; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskQueryDAO; +import org.apache.skywalking.oap.server.core.query.type.PprofTask; import org.apache.skywalking.oap.server.library.util.CollectionUtils; import org.apache.skywalking.oap.server.library.util.RunnableWithExceptionProtection; import org.apache.skywalking.oap.server.core.CoreModule; import org.apache.skywalking.oap.server.core.analysis.DisableRegister; import org.apache.skywalking.oap.server.core.analysis.TimeBucket; import org.apache.skywalking.oap.server.core.analysis.manual.networkalias.NetworkAddressAlias; +import org.apache.skywalking.oap.server.core.profiling.pprof.storage.PprofTaskRecord; import org.apache.skywalking.oap.server.core.profiling.trace.ProfileTaskRecord; import org.apache.skywalking.oap.server.core.query.type.ProfileTask; import org.apache.skywalking.oap.server.core.storage.StorageModule; @@ -46,6 +49,8 @@ public enum CacheUpdateTimer { private AsyncProfilerTaskCache asyncProfilerTaskCache; private IAsyncProfilerTaskQueryDAO asyncProfilerTaskQueryDAO; + private PprofTaskCache pprofTaskCache; + private IPprofTaskQueryDAO pprofTaskQueryDAO; private int ttl = 10; @@ -72,6 +77,10 @@ private void update(ModuleDefineHolder moduleDefineHolder) { if (!DisableRegister.INSTANCE.include(AsyncProfilerTaskRecord.INDEX_NAME)) { updateAsyncProfilerTask(moduleDefineHolder); } + + if (!DisableRegister.INSTANCE.include(PprofTaskRecord.INDEX_NAME)) { + updatePprofTask(moduleDefineHolder); + } } /** @@ -163,4 +172,43 @@ private void updateAsyncProfilerTask(ModuleDefineHolder moduleDefineHolder) { return; } + + private PprofTaskCache getPprofTaskCache(ModuleDefineHolder moduleDefineHolder) { + if (pprofTaskCache == null) { + pprofTaskCache = moduleDefineHolder.find(CoreModule.NAME) + .provider() + .getService(PprofTaskCache.class); + } + return pprofTaskCache; + } + + private IPprofTaskQueryDAO getPprofTaskQueryDAO(ModuleDefineHolder moduleDefineHolder) { + if (pprofTaskQueryDAO == null) { + pprofTaskQueryDAO = moduleDefineHolder.find(StorageModule.NAME) + .provider() + .getService(IPprofTaskQueryDAO.class); + } + return pprofTaskQueryDAO; + } + + private void updatePprofTask(ModuleDefineHolder moduleDefineHolder) { + PprofTaskCache taskCache = getPprofTaskCache(moduleDefineHolder); + IPprofTaskQueryDAO taskQueryDAO = getPprofTaskQueryDAO(moduleDefineHolder); + + try { + List taskList = taskQueryDAO.getTaskList( + null, taskCache.getCacheStartTimeBucket(), taskCache.getCacheEndTimeBucket(), null + ); + if (CollectionUtils.isEmpty(taskList)) { + return; + } + + for (PprofTask task : taskList) { + taskCache.saveTask(task.getServiceId(), task); + } + + } catch (IOException e) { + log.warn("Unable to update pprof task cache", e); + } + } } \ No newline at end of file diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/cache/PprofTaskCache.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/cache/PprofTaskCache.java new file mode 100644 index 000000000000..9ce7e60defc2 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/cache/PprofTaskCache.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.cache; + +import org.apache.skywalking.oap.server.library.module.Service; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import org.apache.skywalking.oap.server.core.CoreModuleConfig; +import org.apache.skywalking.oap.server.core.analysis.TimeBucket; +import org.apache.skywalking.oap.server.core.query.type.PprofTask; +import org.apache.skywalking.oap.server.core.storage.StorageModule; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskQueryDAO; +import org.apache.skywalking.oap.server.library.module.ModuleManager; +import java.time.Duration; +import java.util.Objects; +import java.util.concurrent.TimeUnit; + +public class PprofTaskCache implements Service { + private static final Logger LOGGER = LoggerFactory.getLogger(PprofTaskCache.class); + + private final Cache serviceId2taskCache; + private final ModuleManager moduleManager; + + private IPprofTaskQueryDAO taskQueryDAO; + + public PprofTaskCache(ModuleManager moduleManager, CoreModuleConfig moduleConfig) { + this.moduleManager = moduleManager; + long initialSize = moduleConfig.getMaxSizeOfProfileTask() / 10L; + int initialCapacitySize = (int) (initialSize > Integer.MAX_VALUE ? Integer.MAX_VALUE : initialSize); + + serviceId2taskCache = CacheBuilder.newBuilder() + .initialCapacity(initialCapacitySize) + .maximumSize(moduleConfig.getMaxSizeOfProfileTask()) + // remove old profile task data - extend to 10 minutes to ensure data availability + .expireAfterWrite(Duration.ofMinutes(10)) + .build(); + } + + private IPprofTaskQueryDAO getTaskQueryDAO() { + if (Objects.isNull(taskQueryDAO)) { + taskQueryDAO = moduleManager.find(StorageModule.NAME) + .provider() + .getService(IPprofTaskQueryDAO.class); + } + return taskQueryDAO; + } + + public PprofTask getPprofTask(String serviceId) { + PprofTask task = serviceId2taskCache.getIfPresent(serviceId); + return task; + } + + public void saveTask(String serviceId, PprofTask task) { + if (task == null) { + return ; + } + + serviceId2taskCache.put(serviceId, task); + } + + /** + * use for every db query, -5min start time + */ + public long getCacheStartTimeBucket() { + return TimeBucket.getRecordTimeBucket(System.currentTimeMillis() - TimeUnit.MINUTES.toMillis(5)); + } + + /** + * use for every db query, +5min end time(because search through task's create time) + */ + public long getCacheEndTimeBucket() { + return TimeBucket.getRecordTimeBucket(System.currentTimeMillis() + TimeUnit.MINUTES.toMillis(5)); + } + +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/command/CommandService.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/command/CommandService.java index ac7ef401769a..40a8de67e06c 100755 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/command/CommandService.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/command/CommandService.java @@ -33,6 +33,7 @@ import org.apache.skywalking.oap.server.core.query.type.AsyncProfilerEventType; import org.apache.skywalking.oap.server.core.query.type.AsyncProfilerTask; import org.apache.skywalking.oap.server.core.query.type.EBPFProfilingTaskExtension; +import org.apache.skywalking.oap.server.core.query.type.PprofTask; import org.apache.skywalking.oap.server.library.util.CollectionUtils; import org.apache.skywalking.oap.server.library.util.StringUtil; import org.apache.skywalking.oap.server.network.trace.component.command.AsyncProfilerTaskCommand; @@ -40,6 +41,7 @@ import org.apache.skywalking.oap.server.network.trace.component.command.ContinuousProfilingPolicyCommand; import org.apache.skywalking.oap.server.network.trace.component.command.EBPFProfilingTaskCommand; import org.apache.skywalking.oap.server.network.trace.component.command.EBPFProfilingTaskExtensionConfig; +import org.apache.skywalking.oap.server.network.trace.component.command.PprofTaskCommand; import org.apache.skywalking.oap.server.network.trace.component.command.ProfileTaskCommand; import org.apache.skywalking.oap.server.core.query.type.ProfileTask; import org.apache.skywalking.oap.server.library.module.ModuleManager; @@ -72,6 +74,19 @@ public AsyncProfilerTaskCommand newAsyncProfileTaskCommand(AsyncProfilerTask tas eventNames, task.getExecArgs(), task.getCreateTime()); } + /** + * Create a new pprof task command for Go agents + */ + public PprofTaskCommand newPprofTaskCommand(PprofTask task) { + final String serialNumber = UUID.randomUUID().toString(); + String events = ""; + if (task.getEvents() != null) { + events = task.getEvents().getName(); + } + return new PprofTaskCommand(serialNumber, task.getId(), events, + task.getDuration(), task.getCreateTime(), task.getDumpPeriod()); + } + /** * Used to notify the eBPF Profiling task to the eBPF agent side */ diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/PprofMutationService.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/PprofMutationService.java new file mode 100644 index 000000000000..c909790df570 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/PprofMutationService.java @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.profiling.pprof; + +import lombok.RequiredArgsConstructor; +import org.apache.skywalking.oap.server.library.module.Service; +import org.apache.skywalking.oap.server.library.module.ModuleManager; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskQueryDAO; +import org.apache.skywalking.oap.server.core.storage.StorageModule; +import org.apache.skywalking.oap.server.core.query.type.PprofEventType; +import java.io.IOException; +import java.util.List; +import org.apache.skywalking.oap.server.core.analysis.worker.NoneStreamProcessor; +import java.util.concurrent.TimeUnit; +import org.apache.skywalking.oap.server.core.analysis.TimeBucket; +import org.apache.skywalking.oap.server.core.Const; +import org.apache.skywalking.oap.server.core.query.type.PprofTaskCreationResult; +import org.apache.skywalking.oap.server.core.query.type.PprofTaskCreationType; +import org.apache.skywalking.oap.server.core.query.type.PprofTask; +import org.apache.skywalking.oap.server.core.profiling.pprof.storage.PprofTaskRecord; + +import org.apache.skywalking.oap.server.library.util.CollectionUtils; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@RequiredArgsConstructor +public class PprofMutationService implements Service { + private final ModuleManager moduleManager; + + private IPprofTaskQueryDAO taskQueryDAO; + + private IPprofTaskQueryDAO getPprofTaskDAO() { + if (taskQueryDAO == null) { + this.taskQueryDAO = moduleManager.find(StorageModule.NAME) + .provider() + .getService(IPprofTaskQueryDAO.class); + } + return taskQueryDAO; + } + + public PprofTaskCreationResult createTask(String serviceId, + List serviceInstanceIds, + int duration, + PprofEventType events, + int dumpPeriod) throws IOException { + long createTime = System.currentTimeMillis(); + // check data + PprofTaskCreationResult checkResult = checkDataSuccess( + serviceId, serviceInstanceIds, duration, createTime, events + ); + if (checkResult != null) { + return checkResult; + } + + // create task + PprofTaskRecord task = new PprofTaskRecord(); + String taskId = createTime + Const.ID_CONNECTOR + serviceId; + task.setTaskId(taskId); + task.setServiceId(serviceId); + task.setServiceInstanceIdsFromList(serviceInstanceIds); + task.setDuration(duration); + task.setEvents(events.toString()); + task.setDumpPeriod(dumpPeriod); + task.setCreateTime(createTime); + task.setTimeBucket(TimeBucket.getRecordTimeBucket(createTime)); + NoneStreamProcessor.getInstance().in(task); + return PprofTaskCreationResult.builder() + .id(task.id().build()) + .code(PprofTaskCreationType.SUCCESS) + .build(); + } + + private PprofTaskCreationResult checkDataSuccess(String serviceId, + List serviceInstanceIds, + int duration, + long createTime, + PprofEventType events) throws IOException { + String checkArgumentMessage = checkArgumentError(serviceId, serviceInstanceIds, duration, events); + if (checkArgumentMessage != null) { + return PprofTaskCreationResult.builder() + .code(PprofTaskCreationType.ARGUMENT_ERROR) + .errorReason(checkArgumentMessage) + .build(); + } + String checkTaskProfilingMessage = checkTaskProfiling(serviceId, createTime); + if (checkTaskProfilingMessage != null) { + return PprofTaskCreationResult.builder() + .code(PprofTaskCreationType.ALREADY_PROFILING_ERROR) + .errorReason(checkTaskProfilingMessage) + .build(); + } + return null; + } + + private String checkArgumentError(String serviceId, + List serviceInstanceIds, + int duration, + PprofEventType events) { + if (serviceId == null) { + return "service cannot be null"; + } + if (duration <= 0) { + return "duration cannot be negative"; + } + if (events == null) { + return "events cannot be empty"; + } + if (CollectionUtils.isEmpty(serviceInstanceIds)) { + return "serviceInstanceIds cannot be empty"; + } + return null; + } + + private String checkTaskProfiling(String serviceId, + long createTime) throws IOException { + // Each service can only enable one task at a time + long endTimeBucket = TimeBucket.getMinuteTimeBucket(createTime); + final List alreadyHaveTaskList = getPprofTaskDAO().getTaskList( + serviceId, null, endTimeBucket, 1 + ); + if (CollectionUtils.isNotEmpty(alreadyHaveTaskList)) { + for (PprofTask task : alreadyHaveTaskList) { + if (task.getCreateTime() + TimeUnit.SECONDS.toMillis(task.getDuration()) >= createTime) { + // if the endTime is greater or equal than the createTime of the newly created task, i.e. there is overlap between two tasks, it is an invalid case, it will return an error + return "current service already has monitor pprof task execute at this time"; + } + } + } + return null; + } +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/PprofQueryService.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/PprofQueryService.java new file mode 100644 index 000000000000..2d456996ac02 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/PprofQueryService.java @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.profiling.pprof; + +import org.apache.skywalking.oap.server.core.analysis.IDManager; +import org.apache.skywalking.oap.server.library.module.Service; +import org.apache.skywalking.oap.server.library.module.ModuleManager; +import org.apache.skywalking.oap.server.core.storage.StorageModule; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskQueryDAO; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofDataQueryDAO; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskLogQueryDAO; +import org.apache.skywalking.oap.server.core.query.PprofTaskLog; +import org.apache.skywalking.oap.server.core.query.input.Duration; +import org.apache.skywalking.oap.server.core.query.type.PprofTask; +import java.util.Objects; +import org.apache.skywalking.oap.server.core.profiling.pprof.storage.PprofProfilingDataRecord; +import java.io.IOException; +import org.apache.skywalking.oap.server.library.pprof.type.FrameTree; +import org.apache.skywalking.oap.server.core.query.type.PprofStackTree; +import org.apache.skywalking.oap.server.library.pprof.parser.PprofMergeBuilder; +import java.util.List; +import com.google.gson.Gson; +import java.util.stream.Collectors; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@RequiredArgsConstructor +public class PprofQueryService implements Service { + private static final Gson GSON = new Gson(); + + private final ModuleManager moduleManager; + + private IPprofTaskQueryDAO taskQueryDAO; + private IPprofDataQueryDAO dataQueryDAO; + private IPprofTaskLogQueryDAO logQueryDAO; + + private IPprofTaskQueryDAO getTaskQueryDAO() { + if (taskQueryDAO == null) { + this.taskQueryDAO = moduleManager.find(StorageModule.NAME) + .provider() + .getService(IPprofTaskQueryDAO.class); + } + return taskQueryDAO; + } + + private IPprofDataQueryDAO getPprofDataQueryDAO() { + if (dataQueryDAO == null) { + this.dataQueryDAO = moduleManager.find(StorageModule.NAME) + .provider() + .getService(IPprofDataQueryDAO.class); + } + return dataQueryDAO; + } + + private IPprofTaskLogQueryDAO getTaskLogQueryDAO() { + if (logQueryDAO == null) { + this.logQueryDAO = moduleManager.find(StorageModule.NAME) + .provider() + .getService(IPprofTaskLogQueryDAO.class); + } + return logQueryDAO; + } + + public List queryTask(String serviceId, Duration duration, Integer limit) throws IOException { + Long startTimeBucket = null; + Long endTimeBucket = null; + if (Objects.nonNull(duration)) { + startTimeBucket = duration.getStartTimeBucketInSec(); + endTimeBucket = duration.getEndTimeBucketInSec(); + } + List tasks = getTaskQueryDAO().getTaskList(serviceId, startTimeBucket, endTimeBucket, limit); + return tasks; + } + + public PprofStackTree queryPprofData(String taskId, List instanceIds) throws IOException { + List pprofDataList = getPprofDataQueryDAO().getByTaskIdAndInstances(taskId, instanceIds); + List trees = pprofDataList.stream() + .map(data -> GSON.fromJson(new String(data.getDataBinary()), FrameTree.class)) + .collect(Collectors.toList()); + FrameTree resultTree = new PprofMergeBuilder() + .merge(trees) + .build(); + return new PprofStackTree(resultTree); + } + + public List queryPprofTaskLogs(String taskId) throws IOException { + List taskLogList = getTaskLogQueryDAO().getTaskLogList(); + return findMatchedLogs(taskId, taskLogList); + } + + private List findMatchedLogs(final String taskID, final List allLogs) { + return allLogs.stream() + .filter(l -> Objects.equals(l.getId(), taskID)) + .map(this::extendTaskLog) + .collect(Collectors.toList()); + } + + private PprofTaskLog extendTaskLog(PprofTaskLog log) { + final IDManager.ServiceInstanceID.InstanceIDDefinition instanceIDDefinition = IDManager.ServiceInstanceID + .analysisId(log.getInstanceId()); + log.setInstanceName(instanceIDDefinition.getName()); + return log; + } + +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofProfilingDataDispatcher.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofProfilingDataDispatcher.java new file mode 100644 index 000000000000..5cdf1dd2b1ed --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofProfilingDataDispatcher.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + + package org.apache.skywalking.oap.server.core.profiling.pprof.storage; + +import com.google.gson.Gson; +import org.apache.skywalking.oap.server.core.analysis.SourceDispatcher; +import org.apache.skywalking.oap.server.core.source.PprofProfilingData; +import org.apache.skywalking.oap.server.core.analysis.TimeBucket; +import org.apache.skywalking.oap.server.core.analysis.worker.RecordStreamProcessor; + +public class PprofProfilingDataDispatcher implements SourceDispatcher { + private static final Gson GSON = new Gson(); + + @Override + public void dispatch(PprofProfilingData source) { + PprofProfilingDataRecord record = new PprofProfilingDataRecord(); + record.setTaskId(source.getTaskId()); + record.setInstanceId(source.getInstanceId()); + record.setEventType(source.getEventType().toString()); + record.setDataBinary(GSON.toJson(source.getFrameTree()).getBytes()); + record.setUploadTime(source.getUploadTime()); + record.setTimeBucket(TimeBucket.getRecordTimeBucket(source.getUploadTime())); + RecordStreamProcessor.getInstance().in(record); + } +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofProfilingDataRecord.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofProfilingDataRecord.java new file mode 100644 index 000000000000..17164411a257 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofProfilingDataRecord.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.profiling.pprof.storage; + +import lombok.Data; +import org.apache.skywalking.oap.server.core.analysis.record.Record; +import org.apache.skywalking.oap.server.core.analysis.worker.RecordStreamProcessor; +import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDB; +import org.apache.skywalking.oap.server.core.storage.annotation.Column; +import org.apache.skywalking.oap.server.core.analysis.Stream; +import static org.apache.skywalking.oap.server.core.source.DefaultScopeDefine.PPROF_PROFILING_DATA; +import org.apache.skywalking.oap.server.core.storage.StorageID; +import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity; +import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage; +import org.apache.skywalking.oap.server.core.storage.type.StorageBuilder; +import com.google.common.hash.Hashing; +import java.nio.charset.StandardCharsets; + +@Data +@Stream(name = PprofProfilingDataRecord.INDEX_NAME, scopeId = PPROF_PROFILING_DATA, + builder = PprofProfilingDataRecord.Builder.class, processor = RecordStreamProcessor.class) +@BanyanDB.TimestampColumn(PprofProfilingDataRecord.UPLOAD_TIME) +public class PprofProfilingDataRecord extends Record { + public static final String INDEX_NAME = "pprof_profiling_data"; + + public static final String TASK_ID = "task_id"; + public static final String EVENT_TYPE = "event_type"; + public static final String INSTANCE_ID = "instance_id"; + public static final String DATA_BINARY = "data_binary"; + public static final String UPLOAD_TIME = "upload_time"; + + @Column(name = TASK_ID) + private String taskId; + + @Column(name = INSTANCE_ID) + @BanyanDB.SeriesID(index = 0) + private String instanceId; + + @Column(name = EVENT_TYPE) + private String eventType; + + @Column(name = UPLOAD_TIME) + private long uploadTime; + + @Column(name = DATA_BINARY, storageOnly = true) + private byte[] dataBinary; + + @Override + public StorageID id() { + return new StorageID().appendMutant( + new String[]{ + TASK_ID, + INSTANCE_ID, + EVENT_TYPE, + UPLOAD_TIME + }, + Hashing.sha256().newHasher() + .putString(taskId, StandardCharsets.UTF_8) + .putString(instanceId, StandardCharsets.UTF_8) + .putString(eventType, StandardCharsets.UTF_8) + .putLong(uploadTime) + .hash().toString() + ); + } + + public static class Builder implements StorageBuilder { + @Override + public PprofProfilingDataRecord storage2Entity(final Convert2Entity converter) { + final PprofProfilingDataRecord dataTraffic = new PprofProfilingDataRecord(); + dataTraffic.setTimeBucket(((Number) converter.get(TIME_BUCKET)).longValue()); + dataTraffic.setTaskId((String) converter.get(TASK_ID)); + dataTraffic.setInstanceId((String) converter.get(INSTANCE_ID)); + dataTraffic.setUploadTime(((Number) converter.get(UPLOAD_TIME)).longValue()); + dataTraffic.setEventType((String) converter.get(EVENT_TYPE)); + dataTraffic.setDataBinary(converter.getBytes(DATA_BINARY)); + return dataTraffic; + } + + @Override + public void entity2Storage(final PprofProfilingDataRecord storageData, final Convert2Storage converter) { + converter.accept(TIME_BUCKET, storageData.getTimeBucket()); + converter.accept(TASK_ID, storageData.getTaskId()); + converter.accept(INSTANCE_ID, storageData.getInstanceId()); + converter.accept(UPLOAD_TIME, storageData.getUploadTime()); + converter.accept(EVENT_TYPE, storageData.getEventType()); + converter.accept(DATA_BINARY, storageData.getDataBinary()); + } + } +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofTaskLogRecord.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofTaskLogRecord.java new file mode 100644 index 000000000000..f3ee5ec7338e --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofTaskLogRecord.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.profiling.pprof.storage; + +import org.apache.skywalking.oap.server.core.analysis.Stream; +import org.apache.skywalking.oap.server.core.analysis.record.Record; +import org.apache.skywalking.oap.server.core.analysis.worker.RecordStreamProcessor; +import org.apache.skywalking.oap.server.core.source.ScopeDeclaration; +import org.apache.skywalking.oap.server.core.storage.StorageID; +import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDB; +import org.apache.skywalking.oap.server.core.storage.annotation.Column; +import org.apache.skywalking.oap.server.core.storage.annotation.ElasticSearch; +import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity; +import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage; +import org.apache.skywalking.oap.server.core.storage.type.StorageBuilder; + +import static org.apache.skywalking.oap.server.core.source.DefaultScopeDefine.PPROF_TASK_LOG; + +import lombok.Getter; +import lombok.Setter; + +@Getter +@Setter +@ScopeDeclaration(id = PPROF_TASK_LOG, name = "PprofTaskLog") +@Stream(name = PprofTaskLogRecord.INDEX_NAME, scopeId = PPROF_TASK_LOG, builder = PprofTaskLogRecord.Builder.class, processor = RecordStreamProcessor.class) +@BanyanDB.TimestampColumn(PprofTaskLogRecord.TIMESTAMP) +public class PprofTaskLogRecord extends Record { + public static final String INDEX_NAME = "pprof_task_log"; + public static final String TASK_ID = "task_id"; + public static final String INSTANCE_ID = "instance_id"; + public static final String OPERATION_TYPE = "operation_type"; + public static final String OPERATION_TIME = "operation_time"; + public static final String TIMESTAMP = "timestamp"; + + @Column(name = TASK_ID) + private String taskId; + @Column(name = INSTANCE_ID) + @BanyanDB.SeriesID(index = 0) + private String instanceId; + @Column(name = OPERATION_TYPE, storageOnly = true) + private int operationType; + @ElasticSearch.EnableDocValues + @Column(name = OPERATION_TIME) + private long operationTime; + @Getter + @Setter + @ElasticSearch.EnableDocValues + @Column(name = TIMESTAMP) + private long timestamp; + + @Override + public StorageID id() { + return new StorageID() + .append(TASK_ID, getTaskId()) + .append(INSTANCE_ID, getInstanceId()) + .append(OPERATION_TYPE, getOperationType()) + .append(OPERATION_TIME, getOperationTime()); + } + + public static class Builder implements StorageBuilder { + @Override + public PprofTaskLogRecord storage2Entity(final Convert2Entity converter) { + final PprofTaskLogRecord log = new PprofTaskLogRecord(); + log.setTaskId((String) converter.get(TASK_ID)); + log.setInstanceId((String) converter.get(INSTANCE_ID)); + log.setOperationType(((Number) converter.get(OPERATION_TYPE)).intValue()); + log.setOperationTime(((Number) converter.get(OPERATION_TIME)).longValue()); + log.setTimestamp(((Number) converter.get(TIMESTAMP)).longValue()); + log.setTimeBucket(((Number) converter.get(TIME_BUCKET)).longValue()); + return log; + } + + @Override + public void entity2Storage(final PprofTaskLogRecord storageData, final Convert2Storage converter) { + converter.accept(TASK_ID, storageData.getTaskId()); + converter.accept(INSTANCE_ID, storageData.getInstanceId()); + converter.accept(OPERATION_TYPE, storageData.getOperationType()); + converter.accept(OPERATION_TIME, storageData.getOperationTime()); + converter.accept(TIME_BUCKET, storageData.getTimeBucket()); + converter.accept(TIMESTAMP, storageData.getTimestamp()); + } + } +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofTaskRecord.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofTaskRecord.java new file mode 100644 index 000000000000..e00090d7fa05 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofTaskRecord.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.profiling.pprof.storage; + +import lombok.Getter; +import lombok.Setter; +import org.apache.skywalking.oap.server.core.analysis.Stream; +import org.apache.skywalking.oap.server.core.analysis.config.NoneStream; +import org.apache.skywalking.oap.server.core.analysis.worker.NoneStreamProcessor; +import org.apache.skywalking.oap.server.core.source.ScopeDeclaration; +import org.apache.skywalking.oap.server.core.storage.StorageID; +import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDB; +import org.apache.skywalking.oap.server.core.storage.annotation.Column; +import org.apache.skywalking.oap.server.core.storage.annotation.ElasticSearch; +import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity; +import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage; +import org.apache.skywalking.oap.server.core.storage.type.StorageBuilder; +import java.util.List; + +import com.google.gson.Gson; + +import static org.apache.skywalking.oap.server.core.source.DefaultScopeDefine.PPROF_TASK; + +/** + * Pprof task database bean, use none stream + */ +@Getter +@Setter +@ScopeDeclaration(id = PPROF_TASK, name = "PprofTask") +@Stream(name = PprofTaskRecord.INDEX_NAME, scopeId = PPROF_TASK, builder = PprofTaskRecord.Builder.class, processor = NoneStreamProcessor.class) +@BanyanDB.TimestampColumn(PprofTaskRecord.CREATE_TIME) +public class PprofTaskRecord extends NoneStream { + private static final Gson GSON = new Gson(); + + public static final String INDEX_NAME = "pprof_task"; + public static final String TASK_ID = "task_id"; + public static final String SERVICE_ID = "service_id"; + public static final String SERVICE_INSTANCE_IDS = "service_instance_ids"; + public static final String CREATE_TIME = "create_time"; + public static final String EVENT_TYPES = "events"; + public static final String DURATION = "duration"; + public static final String DUMP_PERIOD = "dump_period"; + + @Override + public StorageID id() { + return new StorageID().append(TASK_ID, taskId); + } + + @Column(name = SERVICE_ID) + @BanyanDB.SeriesID(index = 0) + private String serviceId; + @Column(name = SERVICE_INSTANCE_IDS) + private String serviceInstanceIds; + @Column(name = TASK_ID) + private String taskId; + @ElasticSearch.EnableDocValues + @Column(name = CREATE_TIME) + private long createTime; + @ElasticSearch.EnableDocValues + @Column(name = EVENT_TYPES) + private String events; + @Column(name = DURATION) + private int duration; + @Column(name = DUMP_PERIOD) + private int dumpPeriod; + + public static class Builder implements StorageBuilder { + + @Override + public PprofTaskRecord storage2Entity(final Convert2Entity converter) { + PprofTaskRecord record = new PprofTaskRecord(); + record.setServiceId((String) converter.get(SERVICE_ID)); + record.setServiceInstanceIds((String) converter.get(SERVICE_INSTANCE_IDS)); + record.setTaskId((String) converter.get(TASK_ID)); + record.setCreateTime(((Number) converter.get(CREATE_TIME)).longValue()); + record.setEvents((String) converter.get(EVENT_TYPES)); + record.setDuration(((Number) converter.get(DURATION)).intValue()); + record.setDumpPeriod(((Number) converter.get(DUMP_PERIOD)).intValue()); + record.setTimeBucket(((Number) converter.get(TIME_BUCKET)).longValue()); + return record; + } + + @Override + public void entity2Storage(final PprofTaskRecord storageData, final Convert2Storage converter) { + converter.accept(SERVICE_ID, storageData.getServiceId()); + converter.accept(SERVICE_INSTANCE_IDS, storageData.getServiceInstanceIds()); + converter.accept(TASK_ID, storageData.getTaskId()); + converter.accept(CREATE_TIME, storageData.getCreateTime()); + converter.accept(EVENT_TYPES, storageData.getEvents()); + converter.accept(DURATION, storageData.getDuration()); + converter.accept(DUMP_PERIOD, storageData.getDumpPeriod()); + converter.accept(TIME_BUCKET, storageData.getTimeBucket()); + } + } + + public void setServiceInstanceIdsFromList(List serviceInstanceIds) { + this.serviceInstanceIds = GSON.toJson(serviceInstanceIds); + } + +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/PprofTaskLog.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/PprofTaskLog.java new file mode 100644 index 000000000000..471bde9cdfaa --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/PprofTaskLog.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.query; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import org.apache.skywalking.oap.server.core.query.type.PprofTaskLogOperationType; + +@Setter +@Getter +@NoArgsConstructor +@AllArgsConstructor +@Builder +public class PprofTaskLog { + // task id + private String id; + + // instance + private String instanceId; + private String instanceName; + + // operation + private PprofTaskLogOperationType operationType; + private long operationTime; +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/PprofAnalyzationRequest.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/PprofAnalyzationRequest.java new file mode 100644 index 000000000000..cb62845d2bee --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/PprofAnalyzationRequest.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.query.input; + +import lombok.Getter; +import lombok.Setter; + +import java.util.List; + +@Getter +@Setter +public class PprofAnalyzationRequest { + private String taskId; + private List instanceIds; +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/PprofTaskCreationRequest.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/PprofTaskCreationRequest.java new file mode 100644 index 000000000000..af19a9466d1e --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/PprofTaskCreationRequest.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.query.input; + +import lombok.Getter; +import lombok.Setter; +import org.apache.skywalking.oap.server.core.query.type.PprofEventType; + +import java.util.List; + +@Getter +@Setter +public class PprofTaskCreationRequest { + private String serviceId; + private List serviceInstanceIds; + private int duration; + private PprofEventType events; + private int dumpPeriod; +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/PprofTaskListRequest.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/PprofTaskListRequest.java new file mode 100644 index 000000000000..db789a56228a --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/PprofTaskListRequest.java @@ -0,0 +1,12 @@ +package org.apache.skywalking.oap.server.core.query.input; + +import lombok.Getter; +import lombok.Setter; + +@Getter +@Setter +public class PprofTaskListRequest { + private String serviceId; + private Duration queryDuration; + private Integer limit; +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofAnalyzation.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofAnalyzation.java new file mode 100644 index 000000000000..ef56f268acad --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofAnalyzation.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.query.type; + +import lombok.AllArgsConstructor; +import lombok.Data; + +@Data +@AllArgsConstructor +public class PprofAnalyzation { + private PprofStackTree tree; +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofEventType.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofEventType.java new file mode 100644 index 000000000000..333b0b061dd9 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofEventType.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.query.type; + +import lombok.AllArgsConstructor; +import lombok.Getter; + +@Getter +@AllArgsConstructor +public enum PprofEventType { + CPU(0, "cpu"), + HEAP(1, "heap"), + BLOCK(2, "block"), + MUTEX(3, "mutex"), + GOROUTINE(4, "goroutine"), + THREADCREATE(5, "threadcreate"), + ALLOCS(6, "allocs"); + + + private final int code; + private final String name; + + public static PprofEventType valueOfString(String event) { + return PprofEventType.valueOf(event); + } +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofStackElement.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofStackElement.java new file mode 100644 index 000000000000..b82f3e1ec670 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofStackElement.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.query.type; + +import lombok.Getter; +import lombok.Setter; + +@Getter +@Setter + +public class PprofStackElement { + // work for tree building, id matches multiple parentId + private int id; + private int parentId; + + // stack code signature + private String codeSignature; + + private long total; + private long self; + +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofStackTree.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofStackTree.java new file mode 100644 index 000000000000..2600133dd98b --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofStackTree.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.query.type; + +import com.google.common.collect.Lists; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import org.apache.skywalking.oap.server.library.pprof.type.FrameTree; +import java.util.List; +import java.util.Objects; + +@Setter +@Getter +@NoArgsConstructor +public class PprofStackTree { + private List elements; + + private int idGen = 0; + + public PprofStackTree(FrameTree tree) { + this.elements = convertTree(-1, tree); + } + + private List convertTree(int parentId, FrameTree tree) { + PprofStackElement pprofStackElement = new PprofStackElement(); + pprofStackElement.setId(idGen++); + pprofStackElement.setParentId(parentId); + pprofStackElement.setCodeSignature(tree.getSignature()); + pprofStackElement.setTotal(tree.getTotal()); + pprofStackElement.setSelf(tree.getSelf()); + + List children = tree.getChildren(); + List result = Lists.newArrayList(pprofStackElement); + if (Objects.isNull(children) || children.isEmpty()) { + return result; + } + + for (FrameTree child : children) { + List childElements = convertTree(pprofStackElement.getId(), child); + result.addAll(childElements); + } + + return result; + } +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTask.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTask.java new file mode 100644 index 000000000000..3d8e4e3ba5bc --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTask.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.query.type; + +import java.util.List; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +@Setter +@Getter +@NoArgsConstructor +@AllArgsConstructor +@Builder +public class PprofTask { + + private String id; + private String serviceId; + private List serviceInstanceIds; + private PprofEventType events; + private long createTime; + private int duration; + private int dumpPeriod; + +} \ No newline at end of file diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskCreationResult.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskCreationResult.java new file mode 100644 index 000000000000..1e9ca4d8596c --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskCreationResult.java @@ -0,0 +1,24 @@ +package org.apache.skywalking.oap.server.core.query.type; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +/** + * create pprof task result + */ +@Setter +@Getter +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class PprofTaskCreationResult { + // ErrorReason gives detailed reason for the exception, if the code returned represents a kind of failure. + private String errorReason; + // Code defines the status of the response, i.e. success or failure. + private PprofTaskCreationType code; + // Task id, if code is SUCCESS. + private String id; +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskCreationType.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskCreationType.java new file mode 100644 index 000000000000..ddf62db10a05 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskCreationType.java @@ -0,0 +1,8 @@ +package org.apache.skywalking.oap.server.core.query.type; + +public enum PprofTaskCreationType { + SUCCESS, + ARGUMENT_ERROR, + ALREADY_PROFILING_ERROR, + ; +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskListResult.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskListResult.java new file mode 100644 index 000000000000..e3ee7a75a99d --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskListResult.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.skywalking.oap.server.core.query.type; + +import lombok.AllArgsConstructor; +import lombok.Data; + +import java.util.List; + +@Data +@AllArgsConstructor +public class PprofTaskListResult { + private String errorReason; + private List tasks; +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskLogOperationType.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskLogOperationType.java new file mode 100644 index 000000000000..e220ceb23d37 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskLogOperationType.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.query.type; + +import java.util.HashMap; +import java.util.Map; + +public enum PprofTaskLogOperationType { + NOTIFIED(1), // when sniffer has execution finished to report + EXECUTION_FINISHED(2), // when sniffer has execution finished to report + PPROF_UPLOAD_FILE_TOO_LARGE_ERROR(3), // when sniffer finished task but jfr file is to large that oap server can not receive + EXECUTION_TASK_ERROR(4) // when sniffer fails to execute its task + ; + + private final int code; + private static final Map CACHE = new HashMap(); + + static { + for (PprofTaskLogOperationType val : PprofTaskLogOperationType.values()) { + CACHE.put(val.getCode(), val); + } + } + + /** + * Parse operation type by code + */ + public static PprofTaskLogOperationType parse(int code) { + return CACHE.get(code); + } + + PprofTaskLogOperationType(int code) { + this.code = code; + } + + public int getCode() { + return this.code; + } +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskProgress.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskProgress.java new file mode 100644 index 000000000000..991e6e4f6d95 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskProgress.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.query.type; + +import lombok.Data; +import org.apache.skywalking.oap.server.core.query.PprofTaskLog; + +import java.util.List; + +@Data +public class PprofTaskProgress { + private List logs; + private List errorInstanceIds; + private List successInstanceIds; +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/DefaultScopeDefine.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/DefaultScopeDefine.java index 1d51cc193e18..f2018b4a0984 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/DefaultScopeDefine.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/DefaultScopeDefine.java @@ -151,6 +151,9 @@ public class DefaultScopeDefine { public static final int BROWSER_APP_RESOURCE_PERF = 88; public static final int BROWSER_APP_WEB_INTERACTION_PAGE_PERF = 89; public static final int SW_SPAN_ATTACHED_EVENT = 90; + public static final int PPROF_TASK = 91; + public static final int PPROF_PROFILING_DATA = 92; + public static final int PPROF_TASK_LOG = 93; /** * Catalog of scope, the metrics processor could use this to group all generated metrics by oal rt. diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/PprofProfilingData.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/PprofProfilingData.java new file mode 100644 index 000000000000..d9c23835f86c --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/PprofProfilingData.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.source; + +import lombok.Data; +import org.apache.skywalking.oap.server.core.query.type.PprofEventType; + +import static org.apache.skywalking.oap.server.core.source.DefaultScopeDefine.PPROF_PROFILING_DATA; + +@Data +@ScopeDeclaration(id = PPROF_PROFILING_DATA, name = "PprofProfilingData") +@ScopeDefaultColumn.VirtualColumnDefinition(fieldName = "entityId", columnName = "entity_id", isID = true, type = String.class) +public class PprofProfilingData extends Source { + private volatile String entityId; + + @Override + public int scope() { + return PPROF_PROFILING_DATA; + } + + @Override + public String getEntityId() { + if (entityId == null) { + return taskId + instanceId + eventType.name() + uploadTime; + } + return entityId; + } + + private String taskId; + private String instanceId; + private long uploadTime; + private PprofEventType eventType; + private Object frameTree; +} \ No newline at end of file diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/StorageModule.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/StorageModule.java index 89ac015cd0b4..cd711dedeaa7 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/StorageModule.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/StorageModule.java @@ -24,6 +24,9 @@ import org.apache.skywalking.oap.server.core.storage.profiling.asyncprofiler.IAsyncProfilerTaskLogQueryDAO; import org.apache.skywalking.oap.server.core.storage.profiling.asyncprofiler.IAsyncProfilerTaskQueryDAO; import org.apache.skywalking.oap.server.core.storage.profiling.asyncprofiler.IJFRDataQueryDAO; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskLogQueryDAO; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskQueryDAO; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofDataQueryDAO; import org.apache.skywalking.oap.server.core.storage.profiling.continuous.IContinuousProfilingPolicyDAO; import org.apache.skywalking.oap.server.core.storage.profiling.ebpf.IServiceLabelDAO; import org.apache.skywalking.oap.server.core.storage.profiling.trace.IProfileTaskLogQueryDAO; @@ -96,6 +99,9 @@ public Class[] services() { IAsyncProfilerTaskQueryDAO.class, IAsyncProfilerTaskLogQueryDAO.class, IJFRDataQueryDAO.class, + IPprofTaskQueryDAO.class, + IPprofTaskLogQueryDAO.class, + IPprofDataQueryDAO.class, StorageTTLStatusQuery.class }; } diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/profiling/pprof/IPprofDataQueryDAO.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/profiling/pprof/IPprofDataQueryDAO.java new file mode 100644 index 000000000000..523fa73d419b --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/profiling/pprof/IPprofDataQueryDAO.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.skywalking.oap.server.core.storage.profiling.pprof; + +import org.apache.skywalking.oap.server.library.module.Service; +import org.apache.skywalking.oap.server.core.profiling.pprof.storage.PprofProfilingDataRecord; +import java.io.IOException; +import java.util.List; + +public interface IPprofDataQueryDAO extends Service { + /** + * get pprof data record + * + * @param taskId taskId + * @param instanceIds instances of successfully uploaded file and parsed + * @return record list + */ + List getByTaskIdAndInstances(final String taskId, List instanceIds) throws IOException; +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/profiling/pprof/IPprofTaskLogQueryDAO.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/profiling/pprof/IPprofTaskLogQueryDAO.java new file mode 100644 index 000000000000..78571479204a --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/profiling/pprof/IPprofTaskLogQueryDAO.java @@ -0,0 +1,14 @@ +package org.apache.skywalking.oap.server.core.storage.profiling.pprof; + +import org.apache.skywalking.oap.server.core.query.PprofTaskLog; +import org.apache.skywalking.oap.server.core.storage.DAO; + +import java.io.IOException; +import java.util.List; + +public interface IPprofTaskLogQueryDAO extends DAO { + /** + * search all task log list in appoint task id + */ + List getTaskLogList() throws IOException; +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/profiling/pprof/IPprofTaskQueryDAO.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/profiling/pprof/IPprofTaskQueryDAO.java new file mode 100644 index 000000000000..4c4945654512 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/profiling/pprof/IPprofTaskQueryDAO.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.storage.profiling.pprof; + +import org.apache.skywalking.oap.server.core.storage.DAO; +import org.apache.skywalking.oap.server.core.query.type.PprofTask; +import java.io.IOException; +import java.util.List; + +public interface IPprofTaskQueryDAO extends DAO { + + /** + * search task list in appoint time bucket + * + * @param serviceId monitor service id, maybe null + * @param startTimeBucket time bucket bigger than or equals, nullable + * @param endTimeBucket time bucket smaller than or equals, nullable + * @param limit limit count, if null means query all + */ + List getTaskList(final String serviceId, final Long startTimeBucket, + final Long endTimeBucket, final Integer limit) throws IOException; + + /** + * query profile task by id + * + * @param id taskId + * @return task data + */ + PprofTask getById(final String id) throws IOException; + +} diff --git a/oap-server/server-library/library-pprof-parser/pom.xml b/oap-server/server-library/library-pprof-parser/pom.xml new file mode 100755 index 000000000000..196e38a441bd --- /dev/null +++ b/oap-server/server-library/library-pprof-parser/pom.xml @@ -0,0 +1,86 @@ + + + 4.0.0 + + + org.apache.skywalking + server-library + ${revision} + + + library-pprof-parser + + + 17 + 17 + UTF-8 + true + + + + + com.google.protobuf + protobuf-java + 3.25.5 + + + com.google.code.gson + gson + 2.10.1 + + + org.projectlombok + lombok + 1.18.30 + provided + + + + + + + org.xolstice.maven.plugins + protobuf-maven-plugin + 0.6.1 + + com.google.protobuf:protoc:3.25.5:exe:${os.detected.classifier} + + + + + compile + + + + + + org.codehaus.mojo + build-helper-maven-plugin + 3.3.0 + + + add-generated-sources + generate-sources + + add-source + + + + target/generated-sources/protobuf/java + + + + + + + + + kr.motd.maven + os-maven-plugin + 1.7.0 + + + + \ No newline at end of file diff --git a/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/parser/PprofMergeBuilder.java b/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/parser/PprofMergeBuilder.java new file mode 100644 index 000000000000..20b2cf57570b --- /dev/null +++ b/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/parser/PprofMergeBuilder.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.library.pprof.parser; + +import org.apache.skywalking.oap.server.library.pprof.type.FrameTree; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class PprofMergeBuilder { + private final Node root = new Node("root"); + + public PprofMergeBuilder merge(List trees) { + if (trees == null || trees.isEmpty()) { + return this; + } + for (FrameTree tree : trees) { + merge0(root, tree); + } + return this; + } + + public PprofMergeBuilder merge(FrameTree tree) { + merge0(root, tree); + return this; + } + + private void merge0(Node node, FrameTree tree) { + if (tree == null) { + return; + } + if (tree.getChildren() != null) { + for (FrameTree childTree : tree.getChildren()) { + Node child = getOrAddChild(node, childTree.getSignature()); + merge0(child, childTree); + } + } + node.total += tree.getTotal(); + node.self += tree.getSelf(); + } + + private Node getOrAddChild(Node parent, String signature) { + return parent.children.computeIfAbsent(signature, Node::new); + } + + public FrameTree build() { + return toFrameTree(root); + } + + private FrameTree toFrameTree(Node node) { + FrameTree tree = new FrameTree(node.signature, node.total, node.self); + for (Node child : node.children.values()) { + tree.getChildren().add(toFrameTree(child)); + } + return tree; + } + + private static class Node { + final String signature; + long total; + long self; + final Map children = new HashMap<>(); + + Node(String signature) { + this.signature = signature; + } + } +} diff --git a/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/parser/PprofParser.java b/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/parser/PprofParser.java new file mode 100644 index 000000000000..eacbf2d55f18 --- /dev/null +++ b/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/parser/PprofParser.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.library.pprof.parser; + +import com.google.perftools.profiles.ProfileProto; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import org.apache.skywalking.oap.server.library.pprof.type.FrameTree; +import org.apache.skywalking.oap.server.library.pprof.type.FrameTreeBuilder; + +/** + * Parses pprof protobuf format files and converts them to frame trees. + */ +public class PprofParser { + + public static FrameTree dumpTree(ByteBuffer buf) throws IOException { + ProfileProto.Profile profile = ProfileProto.Profile.parseFrom(buf); + FrameTree tree = new FrameTreeBuilder(profile).build(); + return tree; + } + + public static FrameTree dumpTree(String filePath) throws IOException { + File file = new File(filePath); + if (!file.exists()) { + throw new IOException("Pprof file not found: " + filePath); + } + + InputStream fileStream = new FileInputStream(file); + ProfileProto.Profile profile = ProfileProto.Profile.parseFrom(fileStream); + FrameTree tree = new FrameTreeBuilder(profile).build(); + return tree; + + } +} \ No newline at end of file diff --git a/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/FrameTree.java b/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/FrameTree.java new file mode 100755 index 000000000000..2b1587a8ec53 --- /dev/null +++ b/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/FrameTree.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.library.pprof.type; + +import com.google.gson.annotations.SerializedName; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import java.util.ArrayList; +import java.util.List; + +@Data +@NoArgsConstructor +@AllArgsConstructor +public class FrameTree { + @SerializedName("name") + private String signature; + @SerializedName("value") + private long total; + private long self; + private final List children = new ArrayList<>(); +} diff --git a/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/FrameTreeBuilder.java b/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/FrameTreeBuilder.java new file mode 100644 index 000000000000..3fe4a3b04089 --- /dev/null +++ b/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/FrameTreeBuilder.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.library.pprof.type; + +import com.google.perftools.profiles.ProfileProto; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +@Data +@NoArgsConstructor +@AllArgsConstructor +class RawFrameTree { + private long locationId; + private long total; + private long self; + private final Map children = new HashMap<>(); +} + +public class FrameTreeBuilder { + private final ProfileProto.Profile profile; + private final RawFrameTree root; + + public FrameTreeBuilder(ProfileProto.Profile profile) { + this.profile = profile; + this.root = new RawFrameTree(0, 0, 0); + } + + private FrameTree parseTree(RawFrameTree rawTree) { + FrameTree tree = new FrameTree(getSignature(rawTree.getLocationId()), rawTree.getTotal(), rawTree.getSelf()); + for (RawFrameTree rawChild : rawTree.getChildren().values()) { + FrameTree child = parseTree(rawChild); + tree.getChildren().add(child); + } + return tree; + } + + private String getSignature(long locationId) { + if (locationId == 0) { + return "root"; + } + ProfileProto.Location location = profile.getLocation((int) locationId - 1); + return location.getLineList().stream().map((line) -> { + ProfileProto.Function function = profile.getFunction((int) line.getFunctionId() - 1); + String functionName = profile.getStringTable((int) function.getName()); + return functionName + ":" + line.getLine(); + }).collect(Collectors.joining(";")); + } + + public FrameTree build() { + for (ProfileProto.Sample sample : profile.getSampleList()) { + mergeSample(sample); + } + return parseTree(this.root); + } + + private void mergeSample(ProfileProto.Sample sample) { + // merge sample data + Map children = root.getChildren(); + List locationIdList = new ArrayList<>(sample.getLocationIdList()); + // from root to leaf + Collections.reverse(locationIdList); + int size = locationIdList.size(); + for (int i = 0; i < size; i++) { + boolean isEnd = i == size - 1; + long locationId = locationIdList.get(i); + if (children.containsKey(locationId)) { + RawFrameTree child = children.get(locationId); + child.setTotal(child.getTotal() + 1); + child.setSelf(child.getSelf() + (isEnd ? 1 : 0)); + children = child.getChildren(); + } else { + RawFrameTree child = new RawFrameTree(locationId, 1, (isEnd ? 1 : 0)); + children.put(locationId, child); + children = child.getChildren(); + } + } + root.setTotal(root.getTotal() + 1); + } +} \ No newline at end of file diff --git a/oap-server/server-library/library-pprof-parser/src/main/proto/profile.proto b/oap-server/server-library/library-pprof-parser/src/main/proto/profile.proto new file mode 100755 index 000000000000..60216a3cfc8f --- /dev/null +++ b/oap-server/server-library/library-pprof-parser/src/main/proto/profile.proto @@ -0,0 +1,233 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Profile is a common stacktrace profile format. +// +// Measurements represented with this format should follow the +// following conventions: +// +// - Consumers should treat unset optional fields as if they had been +// set with their default value. +// +// - When possible, measurements should be stored in "unsampled" form +// that is most useful to humans. There should be enough +// information present to determine the original sampled values. +// +// - On-disk, the serialized proto must be gzip-compressed. +// +// - The profile is represented as a set of samples, where each sample +// references a sequence of locations, and where each location belongs +// to a mapping. +// - There is a N->1 relationship from sample.location_id entries to +// locations. For every sample.location_id entry there must be a +// unique Location with that id. +// - There is an optional N->1 relationship from locations to +// mappings. For every nonzero Location.mapping_id there must be a +// unique Mapping with that id. + +syntax = "proto3"; + +package perftools.profiles; + +option java_package = "com.google.perftools.profiles"; +option java_outer_classname = "ProfileProto"; + +message Profile { + // A description of the samples associated with each Sample.value. + // For a cpu profile this might be: + // [["cpu","nanoseconds"]] or [["wall","seconds"]] or [["syscall","count"]] + // For a heap profile, this might be: + // [["allocations","count"], ["space","bytes"]], + // If one of the values represents the number of events represented + // by the sample, by convention it should be at index 0 and use + // sample_type.unit == "count". + repeated ValueType sample_type = 1; + // The set of samples recorded in this profile. + repeated Sample sample = 2; + // Mapping from address ranges to the image/binary/library mapped + // into that address range. mapping[0] will be the main binary. + repeated Mapping mapping = 3; + // Locations referenced by samples. + repeated Location location = 4; + // Functions referenced by locations. + repeated Function function = 5; + // A common table for strings referenced by various messages. + // string_table[0] must always be "". + repeated string string_table = 6; + // frames with Function.function_name fully matching the following + // regexp will be dropped from the samples, along with their successors. + int64 drop_frames = 7; // Index into string table. + // frames with Function.function_name fully matching the following + // regexp will be kept, even if it matches drop_frames. + int64 keep_frames = 8; // Index into string table. + + // The following fields are informational, do not affect + // interpretation of results. + + // Time of collection (UTC) represented as nanoseconds past the epoch. + int64 time_nanos = 9; + // Duration of the profile, if a duration makes sense. + int64 duration_nanos = 10; + // The kind of events between sampled occurrences. + // e.g [ "cpu","cycles" ] or [ "heap","bytes" ] + ValueType period_type = 11; + // The number of events between sampled occurrences. + int64 period = 12; + // Free-form text associated with the profile. The text is displayed as is + // to the user by the tools that read profiles (e.g. by pprof). This field + // should not be used to store any machine-readable information, it is only + // for human-friendly content. The profile must stay functional if this field + // is cleaned. + repeated int64 comment = 13; // Indices into string table. + // Index into the string table of the type of the preferred sample + // value. If unset, clients should default to the last sample value. + int64 default_sample_type = 14; + // Documentation link for this profile. The URL must be absolute, + // e.g., http://pprof.example.com/cpu-profile.html + // + // The URL may be missing if the profile was generated by older code or code + // that did not bother to supply a link. + int64 doc_url = 15; // Index into string table. +} + +// ValueType describes the semantics and measurement units of a value. +message ValueType { + int64 type = 1; // Index into string table. + int64 unit = 2; // Index into string table. +} + +// Each Sample records values encountered in some program +// context. The program context is typically a stack trace, perhaps +// augmented with auxiliary information like the thread-id, some +// indicator of a higher level request being handled etc. +message Sample { + // The ids recorded here correspond to a Profile.location.id. + // The leaf is at location_id[0]. + repeated uint64 location_id = 1; + // The type and unit of each value is defined by the corresponding + // entry in Profile.sample_type. All samples must have the same + // number of values, the same as the length of Profile.sample_type. + // When aggregating multiple samples into a single sample, the + // result has a list of values that is the element-wise sum of the + // lists of the originals. + repeated int64 value = 2; + // label includes additional context for this sample. It can include + // things like a thread id, allocation size, etc. + // + // NOTE: While possible, having multiple values for the same label key is + // strongly discouraged and should never be used. Most tools (e.g. pprof) do + // not have good (or any) support for multi-value labels. And an even more + // discouraged case is having a string label and a numeric label of the same + // name on a sample. Again, possible to express, but should not be used. + repeated Label label = 3; +} + +message Label { + // Index into string table. An annotation for a sample (e.g. + // "allocation_size") with an associated value. + // Keys with "pprof::" prefix are reserved for internal use by pprof. + int64 key = 1; + + // At most one of the following must be present + int64 str = 2; // Index into string table + int64 num = 3; + + // Should only be present when num is present. + // Specifies the units of num. + // Use arbitrary string (for example, "requests") as a custom count unit. + // If no unit is specified, consumer may apply heuristic to deduce the unit. + // Consumers may also interpret units like "bytes" and "kilobytes" as memory + // units and units like "seconds" and "nanoseconds" as time units, + // and apply appropriate unit conversions to these. + int64 num_unit = 4; // Index into string table +} + +message Mapping { + // Unique nonzero id for the mapping. + uint64 id = 1; + // Address at which the binary (or DLL) is loaded into memory. + uint64 memory_start = 2; + // The limit of the address range occupied by this mapping. + uint64 memory_limit = 3; + // Offset in the binary that corresponds to the first mapped address. + uint64 file_offset = 4; + // The object this entry is loaded from. This can be a filename on + // disk for the main binary and shared libraries, or virtual + // abstractions like "[vdso]". + int64 filename = 5; // Index into string table + // A string that uniquely identifies a particular program version + // with high probability. E.g., for binaries generated by GNU tools, + // it could be the contents of the .note.gnu.build-id field. + int64 build_id = 6; // Index into string table + + // The following fields indicate the resolution of symbolic info. + bool has_functions = 7; + bool has_filenames = 8; + bool has_line_numbers = 9; + bool has_inline_frames = 10; +} + +// Describes function and line table debug information. +message Location { + // Unique nonzero id for the location. A profile could use + // instruction addresses or any integer sequence as ids. + uint64 id = 1; + // The id of the corresponding profile.Mapping for this location. + // It can be unset if the mapping is unknown or not applicable for + // this profile type. + uint64 mapping_id = 2; + // The instruction address for this location, if available. It + // should be within [Mapping.memory_start...Mapping.memory_limit] + // for the corresponding mapping. A non-leaf address may be in the + // middle of a call instruction. It is up to display tools to find + // the beginning of the instruction if necessary. + uint64 address = 3; + // Multiple line indicates this location has inlined functions, + // where the last entry represents the caller into which the + // preceding entries were inlined. + // + // E.g., if memcpy() is inlined into printf: + // line[0].function_name == "memcpy" + // line[1].function_name == "printf" + repeated Line line = 4; + // Provides an indication that multiple symbols map to this location's + // address, for example due to identical code folding by the linker. In that + // case the line information above represents one of the multiple + // symbols. This field must be recomputed when the symbolization state of the + // profile changes. + bool is_folded = 5; +} + +message Line { + // The id of the corresponding profile.Function for this line. + uint64 function_id = 1; + // Line number in source code. + int64 line = 2; + // Column number in source code. + int64 column = 3; +} + +message Function { + // Unique nonzero id for the function. + uint64 id = 1; + // Name of the function, in human-readable form if available. + int64 name = 2; // Index into string table + // Name of the function, as identified by the system. + // For instance, it can be a C++ mangled name. + int64 system_name = 3; // Index into string table + // Source file containing the function. + int64 filename = 4; // Index into string table + // Line number in source file. + int64 start_line = 5; +} \ No newline at end of file diff --git a/oap-server/server-library/library-util/src/main/java/org/apache/skywalking/oap/server/library/util/CollectionUtils.java b/oap-server/server-library/library-util/src/main/java/org/apache/skywalking/oap/server/library/util/CollectionUtils.java index d41bce7f2b49..110a4f341e8b 100644 --- a/oap-server/server-library/library-util/src/main/java/org/apache/skywalking/oap/server/library/util/CollectionUtils.java +++ b/oap-server/server-library/library-util/src/main/java/org/apache/skywalking/oap/server/library/util/CollectionUtils.java @@ -32,6 +32,10 @@ public static boolean isEmpty(List list) { return list == null || list.size() == 0; } + public static boolean isEmpty(String str) { + return str == null || str.length() == 0; + } + public static boolean isEmpty(Set set) { return set == null || set.size() == 0; } diff --git a/oap-server/server-library/pom.xml b/oap-server/server-library/pom.xml index 3e92ba599bae..93481db0742a 100644 --- a/oap-server/server-library/pom.xml +++ b/oap-server/server-library/pom.xml @@ -36,6 +36,7 @@ library-datacarrier-queue library-kubernetes-support library-async-profiler-jfr-parser + library-pprof-parser library-integration-test diff --git a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/GraphQLQueryProvider.java b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/GraphQLQueryProvider.java index 9d77c7e0e60f..4232710c0856 100644 --- a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/GraphQLQueryProvider.java +++ b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/GraphQLQueryProvider.java @@ -47,6 +47,8 @@ import org.apache.skywalking.oap.query.graphql.resolver.OndemandLogQuery; import org.apache.skywalking.oap.query.graphql.resolver.ProfileMutation; import org.apache.skywalking.oap.query.graphql.resolver.ProfileQuery; +import org.apache.skywalking.oap.query.graphql.resolver.PprofMutation; +import org.apache.skywalking.oap.query.graphql.resolver.PprofQuery; import org.apache.skywalking.oap.query.graphql.resolver.Query; import org.apache.skywalking.oap.query.graphql.resolver.RecordsQuery; import org.apache.skywalking.oap.query.graphql.resolver.TopNRecordsQuery; @@ -150,7 +152,9 @@ public void prepare() throws ServiceNotProvidedException { .resolvers(new RecordsQuery(getManager())) .file("query-protocol/hierarchy.graphqls").resolvers(new HierarchyQuery(getManager())) .file("query-protocol/async-profiler.graphqls") - .resolvers(new AsyncProfilerQuery(getManager()), new AsyncProfilerMutation(getManager())); + .resolvers(new AsyncProfilerQuery(getManager()), new AsyncProfilerMutation(getManager())) + .file("query-protocol/pprof.graphqls") + .resolvers(new PprofQuery(getManager()), new PprofMutation(getManager())); if (config.isEnableOnDemandPodLog()) { schemaBuilder diff --git a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/PprofMutation.java b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/PprofMutation.java new file mode 100644 index 000000000000..eb98f8d7825a --- /dev/null +++ b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/PprofMutation.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.query.graphql.resolver; + +import graphql.kickstart.tools.GraphQLMutationResolver; +import lombok.extern.slf4j.Slf4j; +import org.apache.skywalking.oap.server.library.module.ModuleManager; +import org.apache.skywalking.oap.server.core.CoreModule; +import org.apache.skywalking.oap.server.core.profiling.pprof.PprofMutationService; +import org.apache.skywalking.oap.server.core.query.type.PprofTaskCreationResult; +import org.apache.skywalking.oap.server.core.query.input.PprofTaskCreationRequest; + +import java.io.IOException; + +@Slf4j +public class PprofMutation implements GraphQLMutationResolver { + private final ModuleManager moduleManager; + + private PprofMutationService mutationService; + + public PprofMutation(ModuleManager moduleManager) { + this.moduleManager = moduleManager; + } + + private PprofMutationService getPprofMutationService() { + if (mutationService == null) { + this.mutationService = moduleManager.find(CoreModule.NAME) + .provider() + .getService(PprofMutationService.class); + } + return mutationService; + } + + public PprofTaskCreationResult createPprofTask(PprofTaskCreationRequest request) throws IOException { + PprofMutationService pprofMutationService = getPprofMutationService(); + return pprofMutationService.createTask(request.getServiceId(), request.getServiceInstanceIds(), + request.getDuration(), request.getEvents(), request.getDumpPeriod()); + } +} \ No newline at end of file diff --git a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/PprofQuery.java b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/PprofQuery.java new file mode 100644 index 000000000000..9b1096cfbdec --- /dev/null +++ b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/PprofQuery.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.query.graphql.resolver; + +import org.apache.skywalking.oap.server.core.CoreModule; +import groovy.util.logging.Slf4j; +import org.apache.skywalking.oap.server.library.module.ModuleManager; +import graphql.kickstart.tools.GraphQLQueryResolver; +import org.apache.skywalking.oap.server.core.profiling.pprof.PprofQueryService; +import org.apache.skywalking.oap.server.core.query.type.PprofTaskListResult; +import org.apache.skywalking.oap.server.core.query.input.PprofTaskListRequest; +import org.apache.skywalking.oap.server.core.query.type.PprofTask; +import org.apache.skywalking.oap.server.core.query.type.PprofTaskProgress; +import org.apache.skywalking.oap.server.core.query.PprofTaskLog; +import org.apache.skywalking.oap.server.core.query.type.PprofTaskLogOperationType; +import org.apache.skywalking.oap.server.core.query.type.PprofAnalyzation; +import org.apache.skywalking.oap.server.core.query.type.PprofStackTree; +import org.apache.skywalking.oap.server.core.query.input.PprofAnalyzationRequest; +import java.util.ArrayList; +import java.io.IOException; +import java.util.List; + +@Slf4j +public class PprofQuery implements GraphQLQueryResolver { + private final ModuleManager moduleManager; + + private PprofQueryService queryService; + + public PprofQuery(ModuleManager moduleManager) { + this.moduleManager = moduleManager; + } + + private PprofQueryService getPprofQueryService() { + if (queryService == null) { + this.queryService = moduleManager.find(CoreModule.NAME).provider().getService(PprofQueryService.class); + } + return queryService; + } + + public PprofTaskListResult queryPprofTaskList(PprofTaskListRequest request) throws IOException { + List tasks = getPprofQueryService().queryTask( + request.getServiceId(), request.getQueryDuration(), request.getLimit() + ); + return new PprofTaskListResult(null, tasks); + } + + public PprofAnalyzation queryPprofAnalyze(PprofAnalyzationRequest request) throws IOException { + PprofStackTree eventFrameTrees = getPprofQueryService().queryPprofData( + request.getTaskId(), request.getInstanceIds() + ); + return new PprofAnalyzation(eventFrameTrees); + } + + public PprofTaskProgress queryPprofTaskProgress(String taskId) throws IOException { + PprofTaskProgress pprofTaskProgress = new PprofTaskProgress(); + List logs = getPprofQueryService().queryPprofTaskLogs(taskId); + pprofTaskProgress.setLogs(logs); + List errorInstances = new ArrayList<>(); + List successInstances = new ArrayList<>(); + logs.forEach(log -> { + if (PprofTaskLogOperationType.EXECUTION_FINISHED.equals(log.getOperationType())) { + successInstances.add(log.getInstanceId()); + } else if (PprofTaskLogOperationType.EXECUTION_TASK_ERROR.equals(log.getOperationType()) + || PprofTaskLogOperationType.PPROF_UPLOAD_FILE_TOO_LARGE_ERROR.equals(log.getOperationType())) { + errorInstances.add(log.getInstanceId()); + } + }); + pprofTaskProgress.setErrorInstanceIds(errorInstances); + pprofTaskProgress.setSuccessInstanceIds(successInstances); + return pprofTaskProgress; + } +} diff --git a/oap-server/server-receiver-plugin/pom.xml b/oap-server/server-receiver-plugin/pom.xml index 17892a71a48e..068a86167c62 100644 --- a/oap-server/server-receiver-plugin/pom.xml +++ b/oap-server/server-receiver-plugin/pom.xml @@ -49,6 +49,7 @@ skywalking-telegraf-receiver-plugin aws-firehose-receiver skywalking-async-profiler-receiver-plugin + skywalking-pprof-receiver-plugin diff --git a/oap-server/server-receiver-plugin/receiver-proto/pom.xml b/oap-server/server-receiver-plugin/receiver-proto/pom.xml index 0c057f133cef..aaff44d8c39a 100644 --- a/oap-server/server-receiver-plugin/receiver-proto/pom.xml +++ b/oap-server/server-receiver-plugin/receiver-proto/pom.xml @@ -41,6 +41,14 @@ flatbuffers-java provided + + com.google.protobuf + protobuf-java + + + com.google.protobuf + protobuf-java-util + diff --git a/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/pom.xml b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/pom.xml new file mode 100644 index 000000000000..985a5b30702f --- /dev/null +++ b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/pom.xml @@ -0,0 +1,53 @@ + + + + + + server-receiver-plugin + org.apache.skywalking + ${revision} + + 4.0.0 + + skywalking-pprof-receiver-plugin + + + + org.apache.skywalking + library-module + ${project.version} + + + org.apache.skywalking + skywalking-sharing-server-plugin + ${project.version} + + + org.apache.skywalking + apm-network + ${project.version} + + + org.apache.skywalking + library-pprof-parser + ${project.version} + + + diff --git a/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/module/PprofModule.java b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/module/PprofModule.java new file mode 100644 index 000000000000..f768f1a7ec91 --- /dev/null +++ b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/module/PprofModule.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.receiver.pprof.module; + +import org.apache.skywalking.oap.server.library.module.ModuleDefine; + +public class PprofModule extends ModuleDefine { + public static final String NAME = "receiver-pprof"; + + public PprofModule() { + super(NAME); + } + + @Override + public Class[] services() { + return new Class[] {}; + } +} \ No newline at end of file diff --git a/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/module/PprofModuleConfig.java b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/module/PprofModuleConfig.java new file mode 100644 index 000000000000..99f5b70fc078 --- /dev/null +++ b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/module/PprofModuleConfig.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.receiver.pprof.module; + +import lombok.Getter; +import lombok.Setter; +import org.apache.skywalking.oap.server.library.module.ModuleConfig; + +@Getter +@Setter +public class PprofModuleConfig extends ModuleConfig { + /** + * Used to manage the maximum size of the pprof file that can be received, the unit is Byte + * default is 30M + */ + private int pprofMaxSize = 4 * 1024; + /** + * default is true + *

+ * If memoryParserEnabled is true, then PprofByteBufCollectionObserver will be enabled + * will use memory to receive pprof files without writing files (this is currently used). + * This can prevent the oap server from crashing due to no volume mounting. + *

+ * If memoryParserEnabled is false, then PprofFileCollectionObserver will be enabled + * which uses createTemp to write files and then reads the files for parsing. + * The advantage of this is that it reduces memory and prevents the oap server from crashing due to + * insufficient memory, but it may report an error due to no volume mounting. + */ + private boolean memoryParserEnabled = true; +} diff --git a/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/PprofModuleProvider.java b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/PprofModuleProvider.java new file mode 100644 index 000000000000..0ff1aacad79e --- /dev/null +++ b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/PprofModuleProvider.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.receiver.pprof.provider; + +import org.apache.skywalking.oap.server.core.CoreModule; +import org.apache.skywalking.oap.server.core.server.GRPCHandlerRegister; +import org.apache.skywalking.oap.server.library.module.ModuleConfig; +import org.apache.skywalking.oap.server.library.module.ModuleDefine; +import org.apache.skywalking.oap.server.library.module.ModuleProvider; +import org.apache.skywalking.oap.server.library.module.ModuleStartException; +import org.apache.skywalking.oap.server.library.module.ServiceNotProvidedException; +import org.apache.skywalking.oap.server.receiver.pprof.module.PprofModule; +import org.apache.skywalking.oap.server.receiver.pprof.module.PprofModuleConfig; +import org.apache.skywalking.oap.server.receiver.pprof.provider.handler.PprofServiceHandler; +import org.apache.skywalking.oap.server.receiver.sharing.server.SharingServerModule; + +public class PprofModuleProvider extends ModuleProvider { + private PprofModuleConfig config; + + @Override + public String name() { + return "default"; + } + + @Override + public Class module() { + return PprofModule.class; + } + + @Override + public ConfigCreator newConfigCreator() { + return new ConfigCreator() { + @Override + public Class type() { + return PprofModuleConfig.class; + } + + @Override + public void onInitialized(final PprofModuleConfig initialized) { + config = initialized; + } + }; + } + + @Override + public void prepare() throws ServiceNotProvidedException, ModuleStartException { + + } + + @Override + public void start() throws ServiceNotProvidedException, ModuleStartException { + GRPCHandlerRegister grpcHandlerRegister = getManager().find(SharingServerModule.NAME) + .provider() + .getService(GRPCHandlerRegister.class); + PprofServiceHandler pprofServiceHandler = new PprofServiceHandler(getManager(), + config.getPprofMaxSize(), config.isMemoryParserEnabled()); + grpcHandlerRegister.addHandler(pprofServiceHandler); + } + + @Override + public void notifyAfterCompleted() throws ServiceNotProvidedException, ModuleStartException { + } + + @Override + public String[] requiredModules() { + return new String[]{ + CoreModule.NAME, + SharingServerModule.NAME + }; + } + +} diff --git a/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/PprofServiceHandler.java b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/PprofServiceHandler.java new file mode 100644 index 000000000000..6fdf30538cd0 --- /dev/null +++ b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/PprofServiceHandler.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.receiver.pprof.provider.handler; + +import io.grpc.stub.StreamObserver; +import lombok.extern.slf4j.Slf4j; +import java.io.IOException; +import org.apache.skywalking.oap.server.core.query.type.PprofTaskLogOperationType; +import org.apache.skywalking.apm.network.pprof.v10.PprofData; +import org.apache.skywalking.apm.network.pprof.v10.PprofTaskGrpc; +import org.apache.skywalking.apm.network.pprof.v10.PprofCollectionResponse; +import org.apache.skywalking.apm.network.pprof.v10.PprofMetaData; + +import org.apache.skywalking.apm.network.common.v3.Commands; +import org.apache.skywalking.apm.network.pprof.v10.PprofTaskCommandQuery; + +import org.apache.skywalking.oap.server.core.CoreModule; +import org.apache.skywalking.oap.server.core.analysis.IDManager; +import org.apache.skywalking.oap.server.core.command.CommandService; +import java.util.Objects; +import org.apache.skywalking.oap.server.library.util.CollectionUtils; +import org.apache.skywalking.oap.server.core.query.type.PprofTask; +import org.apache.skywalking.oap.server.core.source.SourceReceiver; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskQueryDAO; +import org.apache.skywalking.oap.server.core.storage.StorageModule; +import org.apache.skywalking.oap.server.core.cache.PprofTaskCache; +import org.apache.skywalking.oap.server.core.profiling.pprof.storage.PprofTaskLogRecord; +import org.apache.skywalking.oap.server.core.analysis.worker.RecordStreamProcessor; +import org.apache.skywalking.oap.server.core.analysis.TimeBucket; +import java.util.concurrent.TimeUnit; + +//import org.apache.skywalking.oap.server.core.storage.StorageModule; +//import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskQueryDAO; +import org.apache.skywalking.oap.server.library.module.ModuleManager; +import org.apache.skywalking.oap.server.library.server.grpc.GRPCHandler; +import org.apache.skywalking.oap.server.network.trace.component.command.PprofTaskCommand; +import org.apache.skywalking.oap.server.receiver.pprof.provider.handler.stream.PprofByteBufCollectionObserver; +import org.apache.skywalking.oap.server.receiver.pprof.provider.handler.stream.PprofCollectionMetaData; +import org.apache.skywalking.oap.server.receiver.pprof.provider.handler.stream.PprofFileCollectionObserver; + +@Slf4j +public class PprofServiceHandler extends PprofTaskGrpc.PprofTaskImplBase implements GRPCHandler { + + private final IPprofTaskQueryDAO taskDAO; + private final SourceReceiver sourceReceiver; + private final CommandService commandService; + private final PprofTaskCache taskCache; + private final int pprofMaxSize; + private final boolean memoryParserEnabled; + + public PprofServiceHandler(ModuleManager moduleManager, int pprofMaxSize, boolean memoryParserEnabled) { + this.taskDAO = moduleManager.find(StorageModule.NAME).provider().getService(IPprofTaskQueryDAO.class); + this.commandService = moduleManager.find(CoreModule.NAME).provider().getService(CommandService.class); + this.sourceReceiver = moduleManager.find(CoreModule.NAME).provider().getService(SourceReceiver.class); + this.taskCache = moduleManager.find(CoreModule.NAME).provider().getService(PprofTaskCache.class); + this.pprofMaxSize = pprofMaxSize; + this.memoryParserEnabled = memoryParserEnabled; + } + + @Override + public StreamObserver collect(StreamObserver responseObserver) { + return memoryParserEnabled ? + new PprofByteBufCollectionObserver(taskDAO, responseObserver, sourceReceiver, pprofMaxSize) + : new PprofFileCollectionObserver(taskDAO, responseObserver, sourceReceiver, pprofMaxSize); + } + + @Override + public void getPprofTaskCommands(PprofTaskCommandQuery request, StreamObserver responseObserver) { + String serviceId = IDManager.ServiceID.buildId(request.getService(), true); + String serviceInstanceId = IDManager.ServiceInstanceID.buildId(serviceId, request.getServiceInstance()); + PprofTask task = taskCache.getPprofTask(serviceId); + // if task is null or createTime is less than lastCommandTime, return empty commands + if (Objects.isNull(task) || task.getCreateTime() <= request.getLastCommandTime() || + (!CollectionUtils.isEmpty(task.getServiceInstanceIds()) && !task.getServiceInstanceIds().contains(serviceInstanceId))) { + responseObserver.onNext(Commands.newBuilder().build()); + responseObserver.onCompleted(); + return; + } + + PprofTaskCommand pprofTaskCommand = commandService.newPprofTaskCommand(task); + Commands commands = Commands.newBuilder().addCommands(pprofTaskCommand.serialize()).build(); + responseObserver.onNext(commands); + responseObserver.onCompleted(); + recordPprofTaskLog(task, serviceInstanceId, PprofTaskLogOperationType.NOTIFIED); + } + + public static void recordPprofTaskLog(PprofTask task, String instanceId, PprofTaskLogOperationType operationType) { + PprofTaskLogRecord logRecord = new PprofTaskLogRecord(); + logRecord.setTaskId(task.getId()); + logRecord.setInstanceId(instanceId); + logRecord.setOperationType(operationType.getCode()); + logRecord.setOperationTime(System.currentTimeMillis()); + long timestamp = task.getCreateTime() + TimeUnit.SECONDS.toMillis(task.getDuration()); + logRecord.setTimestamp(timestamp); + logRecord.setTimeBucket(TimeBucket.getRecordTimeBucket(timestamp)); + RecordStreamProcessor.getInstance().in(logRecord); + } + + public static PprofCollectionMetaData parseMetaData(PprofMetaData metaData, IPprofTaskQueryDAO taskDAO) throws IOException { + String taskId = metaData.getTaskId(); + PprofTask task = taskDAO.getById(taskId); + String serviceId = IDManager.ServiceID.buildId(metaData.getService(), true); + String serviceInstanceId = IDManager.ServiceInstanceID.buildId(serviceId, metaData.getServiceInstance()); + + return PprofCollectionMetaData.builder() + .task(task) + .serviceId(serviceId) + .instanceId(serviceInstanceId) + .type(metaData.getType()) + .contentSize(metaData.getContentSize()) + .uploadTime(System.currentTimeMillis()) + .build(); + } +} \ No newline at end of file diff --git a/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/stream/PprofByteBufCollectionObserver.java b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/stream/PprofByteBufCollectionObserver.java new file mode 100644 index 000000000000..5b9f10abba0e --- /dev/null +++ b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/stream/PprofByteBufCollectionObserver.java @@ -0,0 +1,158 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.receiver.pprof.provider.handler.stream; + +import io.grpc.Status; +import io.grpc.stub.StreamObserver; +import lombok.extern.slf4j.Slf4j; +import org.apache.skywalking.oap.server.library.pprof.parser.PprofParser; +import org.apache.skywalking.oap.server.library.pprof.type.FrameTree; +import org.apache.skywalking.apm.network.pprof.v10.PprofData; +import org.apache.skywalking.oap.server.core.query.type.PprofTask; +import org.apache.skywalking.apm.network.pprof.v10.PprofCollectionResponse; +import org.apache.skywalking.apm.network.pprof.v10.PprofProfilingStatus; +import org.apache.skywalking.oap.server.core.source.PprofProfilingData; +import org.apache.skywalking.oap.server.core.source.SourceReceiver; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskQueryDAO; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Objects; +import org.apache.skywalking.oap.server.core.query.type.PprofEventType; +import org.apache.skywalking.oap.server.core.query.type.PprofTaskLogOperationType; +import static org.apache.skywalking.oap.server.receiver.pprof.provider.handler.PprofServiceHandler.parseMetaData; +import static org.apache.skywalking.oap.server.receiver.pprof.provider.handler.PprofServiceHandler.recordPprofTaskLog; + +@Slf4j +public class PprofByteBufCollectionObserver implements StreamObserver { + private final IPprofTaskQueryDAO taskDAO; + private final StreamObserver responseObserver; + private final SourceReceiver sourceReceiver; + private final int pprofMaxSize; + private PprofCollectionMetaData taskMetaData; + private ByteBuffer buf; + + public PprofByteBufCollectionObserver(IPprofTaskQueryDAO taskDAO, + StreamObserver responseObserver, + SourceReceiver sourceReceiver, int pprofMaxSize) { + this.taskDAO = taskDAO; + this.responseObserver = responseObserver; + this.sourceReceiver = sourceReceiver; + this.pprofMaxSize = pprofMaxSize; + } + + @Override + public void onNext(PprofData pprofData) { + try { + if (Objects.isNull(taskMetaData) && pprofData.hasMetadata()) { + taskMetaData = parseMetaData(pprofData.getMetadata(), taskDAO); + if (PprofProfilingStatus.PPROF_PROFILING_SUCCESS.equals(taskMetaData.getType())) { + int size = taskMetaData.getContentSize(); + log.info("pprofMaxSize: {}, Pprof data size: {}", pprofMaxSize, size); + if (pprofMaxSize >= size) { + buf = ByteBuffer.allocate(size); + // Send success response to allow client to continue uploading + responseObserver.onNext(PprofCollectionResponse.newBuilder() + .setStatus(PprofProfilingStatus.PPROF_PROFILING_SUCCESS) + .build()); + + log.info("Started collecting pprof data in memory - service: {}, serviceInstance: {}, size: {} bytes", + pprofData.getMetadata().getService(), pprofData.getMetadata().getServiceInstance(), size); + } else { + responseObserver.onNext(PprofCollectionResponse.newBuilder() + .setStatus(PprofProfilingStatus.PPROF_TERMINATED_BY_OVERSIZE) + .build()); + recordPprofTaskLog(taskMetaData.getTask(), taskMetaData.getInstanceId(), PprofTaskLogOperationType.PPROF_UPLOAD_FILE_TOO_LARGE_ERROR); + log.warn("Pprof file size {} exceeds maximum allowed size {} for service: {}, serviceInstance: {}", + size, pprofMaxSize, pprofData.getMetadata().getService(), pprofData.getMetadata().getServiceInstance()); + } + } else { + responseObserver.onNext(PprofCollectionResponse.newBuilder() + .setStatus(PprofProfilingStatus.PPROF_EXECUTION_TASK_ERROR) + .build()); + recordPprofTaskLog(taskMetaData.getTask(), taskMetaData.getInstanceId(), PprofTaskLogOperationType.EXECUTION_TASK_ERROR); + log.error("Received execution error from agent - service: {}, serviceInstance: {}, status: {}", + pprofData.getMetadata().getService(), pprofData.getMetadata().getServiceInstance(), taskMetaData.getType()); + } + } else if (pprofData.hasContent()) { + if (buf != null) { + pprofData.getContent().copyTo(buf); + log.info("Received {} bytes of pprof data", pprofData.getContent().size()); + } + } + } catch (IOException e) { + log.error("Error processing pprof data", e); + responseObserver.onError(Status.INTERNAL.withDescription("Error processing pprof data: " + e.getMessage()).asRuntimeException()); + } + } + + @Override + public void onError(Throwable throwable) { + Status status = Status.fromThrowable(throwable); + if (Status.CANCELLED.getCode() == status.getCode()) { + if (log.isDebugEnabled()) { + log.debug(throwable.getMessage(), throwable); + } + return; + } + log.error("Error in receiving pprof profiling data", throwable); + + } + + @Override + public void onCompleted() { + responseObserver.onCompleted(); + if (Objects.nonNull(buf)) { + buf.flip(); + try { + parseAndStorageData(taskMetaData, buf); + } catch (IOException e) { + log.error("Failed to parse and store pprof data", e); + } + } + } + + private void parseAndStorageData(PprofCollectionMetaData taskMetaData, ByteBuffer buf) throws IOException { + PprofTask task = taskMetaData.getTask(); + if (task == null) { + log.error("Pprof instanceId:{} has not been assigned a task but still uploaded data", taskMetaData.getInstanceId()); + return; + } + recordPprofTaskLog(task, taskMetaData.getInstanceId(), PprofTaskLogOperationType.EXECUTION_FINISHED); + parsePprofAndStorage(taskMetaData, buf); + } + + public void parsePprofAndStorage(PprofCollectionMetaData taskMetaData, ByteBuffer buf) throws IOException { + log.info("Parsing pprof file for service: {}, instance: {}", + taskMetaData.getServiceId(), taskMetaData.getInstanceId()); + PprofTask task = taskMetaData.getTask(); + FrameTree tree = PprofParser.dumpTree(buf); + PprofProfilingData data = new PprofProfilingData(); + data.setEventType(PprofEventType.valueOfString(task.getEvents().name())); + data.setFrameTree(tree); + data.setTaskId(task.getId()); + data.setInstanceId(taskMetaData.getInstanceId()); + data.setUploadTime(taskMetaData.getUploadTime()); + log.info("data eventType: {}", data.getEventType()); + log.info("data frameTree: {}", tree); + log.info("data taskId: {}", task.getId()); + log.info("data instanceId: {}", taskMetaData.getInstanceId()); + log.info("data uploadTime: {}", taskMetaData.getUploadTime()); + sourceReceiver.receive(data); + } +} diff --git a/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/stream/PprofCollectionMetaData.java b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/stream/PprofCollectionMetaData.java new file mode 100644 index 000000000000..b0986361d97c --- /dev/null +++ b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/stream/PprofCollectionMetaData.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.receiver.pprof.provider.handler.stream; + +import org.apache.skywalking.oap.server.core.query.type.PprofTask; + +import org.apache.skywalking.apm.network.pprof.v10.PprofProfilingStatus; +import lombok.Builder; +import lombok.Data; + +@Data +@Builder +public class PprofCollectionMetaData { + private PprofTask task; + private String serviceId; + private String instanceId; + private int contentSize; + private PprofProfilingStatus type; + private long uploadTime; +} diff --git a/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/stream/PprofFileCollectionObserver.java b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/stream/PprofFileCollectionObserver.java new file mode 100644 index 000000000000..a5cd831e9685 --- /dev/null +++ b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/stream/PprofFileCollectionObserver.java @@ -0,0 +1,170 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.receiver.pprof.provider.handler.stream; + +import io.grpc.Status; +import io.grpc.stub.StreamObserver; +import lombok.extern.slf4j.Slf4j; +import lombok.SneakyThrows; +import org.apache.skywalking.apm.network.pprof.v10.PprofData; +import org.apache.skywalking.apm.network.pprof.v10.PprofCollectionResponse; +import org.apache.skywalking.apm.network.pprof.v10.PprofProfilingStatus; +import org.apache.skywalking.oap.server.core.source.SourceReceiver; +import org.apache.skywalking.oap.server.core.query.type.PprofTask; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskQueryDAO; +import org.apache.skywalking.oap.server.core.source.PprofProfilingData; +import org.apache.skywalking.oap.server.library.pprof.parser.PprofParser; +import org.apache.skywalking.oap.server.library.pprof.type.FrameTree; +import org.apache.skywalking.oap.server.core.query.type.PprofEventType; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Objects; +import org.apache.skywalking.oap.server.core.query.type.PprofTaskLogOperationType; +import static org.apache.skywalking.oap.server.receiver.pprof.provider.handler.PprofServiceHandler.recordPprofTaskLog; +import static org.apache.skywalking.oap.server.receiver.pprof.provider.handler.PprofServiceHandler.parseMetaData; + +@Slf4j +public class PprofFileCollectionObserver implements StreamObserver { + private final IPprofTaskQueryDAO taskDAO; + private final StreamObserver responseObserver; + private final SourceReceiver sourceReceiver; + private final int pprofMaxSize; + private PprofCollectionMetaData taskMetaData; + private Path tempFile; + private FileOutputStream fileOutputStream; + + public PprofFileCollectionObserver(IPprofTaskQueryDAO taskDAO, + StreamObserver responseObserver, + SourceReceiver sourceReceiver, int pprofMaxSize) { + this.taskDAO = taskDAO; + this.responseObserver = responseObserver; + this.sourceReceiver = sourceReceiver; + this.pprofMaxSize = pprofMaxSize; + } + + @SneakyThrows + @Override + public void onNext(PprofData pprofData) { + if (Objects.isNull(taskMetaData) && pprofData.hasMetadata()) { + taskMetaData = parseMetaData(pprofData.getMetadata(), taskDAO); + + if (PprofProfilingStatus.PPROF_PROFILING_SUCCESS.equals(taskMetaData.getType())) { + int size = taskMetaData.getContentSize(); + if (pprofMaxSize >= size) { + // Create temporary file for pprof data + tempFile = Files.createTempFile(taskMetaData.getTask().getId() + taskMetaData.getInstanceId() + System.currentTimeMillis(), ".pprof"); + fileOutputStream = new FileOutputStream(tempFile.toFile()); + + // Send success response to allow client to continue uploading + responseObserver.onNext(PprofCollectionResponse.newBuilder() + .setStatus(PprofProfilingStatus.PPROF_PROFILING_SUCCESS) + .build()); + } else { + responseObserver.onNext(PprofCollectionResponse.newBuilder() + .setStatus(PprofProfilingStatus.PPROF_TERMINATED_BY_OVERSIZE) + .build()); + recordPprofTaskLog(taskMetaData.getTask(), taskMetaData.getInstanceId(), PprofTaskLogOperationType.PPROF_UPLOAD_FILE_TOO_LARGE_ERROR); + } + } else { + responseObserver.onNext(PprofCollectionResponse.newBuilder() + .setStatus(PprofProfilingStatus.PPROF_EXECUTION_TASK_ERROR) + .build()); + recordPprofTaskLog(taskMetaData.getTask(), taskMetaData.getInstanceId(), PprofTaskLogOperationType.EXECUTION_TASK_ERROR); + } + } else if (pprofData.hasContent()) { + if (fileOutputStream != null) { + fileOutputStream.write(pprofData.getContent().toByteArray()); + + if (log.isDebugEnabled()) { + log.debug("Received {} bytes of pprof data", pprofData.getContent().size()); + } + } + } + } + + @Override + public void onError(Throwable throwable) { + Status status = Status.fromThrowable(throwable); + if (Status.CANCELLED.getCode() == status.getCode()) { + if (log.isDebugEnabled()) { + log.debug("Pprof data collection cancelled: {}", throwable.getMessage()); + } + } else { + log.error("Error in receiving pprof profiling data", throwable); + } + + // Clean up resources + closeFileStream(); + } + + @Override + @SneakyThrows + public void onCompleted() { + responseObserver.onCompleted(); + + if (Objects.nonNull(tempFile)) { + closeFileStream(); + parseAndStorageData(taskMetaData, tempFile.toAbsolutePath().toString()); + } + } + + private void closeFileStream() { + if (fileOutputStream != null) { + try { + fileOutputStream.close(); + fileOutputStream = null; + } catch (IOException e) { + log.error("Failed to close file output stream", e); + } + } + } + + @SneakyThrows + private void parseAndStorageData(PprofCollectionMetaData taskMetaData, String fileName) { + PprofTask task = taskMetaData.getTask(); + if (task == null) { + log.error("Pprof instanceId:{} has not been assigned a task but still uploaded data", taskMetaData.getInstanceId()); + return; + } + recordPprofTaskLog(task, taskMetaData.getInstanceId(), PprofTaskLogOperationType.EXECUTION_FINISHED); + parsePprofAndStorage(taskMetaData, fileName); + } + + public void parsePprofAndStorage(PprofCollectionMetaData taskMetaData, + String fileName) throws IOException { + log.info("Parsing pprof file for service: {}, instance: {}", + taskMetaData.getServiceId(), taskMetaData.getInstanceId()); + PprofTask task = taskMetaData.getTask(); + FrameTree tree = PprofParser.dumpTree(fileName); + PprofProfilingData data = new PprofProfilingData(); + data.setEventType(PprofEventType.valueOfString(task.getEvents().name())); + data.setFrameTree(tree); + data.setTaskId(task.getId()); + data.setInstanceId(taskMetaData.getInstanceId()); + data.setUploadTime(taskMetaData.getUploadTime()); + log.info("data eventType: {}", data.getEventType()); + log.info("data frameTree: {}", tree); + log.info("data taskId: {}", task.getId()); + log.info("data instanceId: {}", taskMetaData.getInstanceId()); + log.info("data uploadTime: {}", taskMetaData.getUploadTime()); + sourceReceiver.receive(data); + } +} diff --git a/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleDefine b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleDefine new file mode 100644 index 000000000000..3ef9c8a560ef --- /dev/null +++ b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleDefine @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# + +org.apache.skywalking.oap.server.receiver.pprof.module.PprofModule diff --git a/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleProvider b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleProvider new file mode 100644 index 000000000000..e4e90929ba62 --- /dev/null +++ b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleProvider @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# + +org.apache.skywalking.oap.server.receiver.pprof.provider.PprofModuleProvider diff --git a/oap-server/server-receiver-plugin/skywalking-sharing-server-plugin/pom.xml b/oap-server/server-receiver-plugin/skywalking-sharing-server-plugin/pom.xml index 5cf914ba25c2..19b45143d0ea 100644 --- a/oap-server/server-receiver-plugin/skywalking-sharing-server-plugin/pom.xml +++ b/oap-server/server-receiver-plugin/skywalking-sharing-server-plugin/pom.xml @@ -27,4 +27,12 @@ skywalking-sharing-server-plugin jar + + + + org.apache.skywalking + library-server + ${project.version} + + \ No newline at end of file diff --git a/oap-server/server-starter/pom.xml b/oap-server/server-starter/pom.xml index d1971572aa90..7d44453471ae 100644 --- a/oap-server/server-starter/pom.xml +++ b/oap-server/server-starter/pom.xml @@ -176,6 +176,11 @@ skywalking-async-profiler-receiver-plugin ${project.version} + + org.apache.skywalking + skywalking-pprof-receiver-plugin + ${project.version} + org.apache.skywalking skywalking-telegraf-receiver-plugin diff --git a/oap-server/server-starter/src/main/resources/application.yml b/oap-server/server-starter/src/main/resources/application.yml index 30c7f4d2fa58..50b66c9c8de9 100644 --- a/oap-server/server-starter/src/main/resources/application.yml +++ b/oap-server/server-starter/src/main/resources/application.yml @@ -151,7 +151,7 @@ storage: # Since 10.2.0, the banyandb configuration is separated to an independent configuration file: `bydb.yaml`. elasticsearch: namespace: ${SW_NAMESPACE:""} - clusterNodes: ${SW_STORAGE_ES_CLUSTER_NODES:localhost:9200} + clusterNodes: 192.168.50.242:9200 protocol: ${SW_STORAGE_ES_HTTP_PROTOCOL:"http"} connectTimeout: ${SW_STORAGE_ES_CONNECT_TIMEOUT:3000} socketTimeout: ${SW_STORAGE_ES_SOCKET_TIMEOUT:30000} @@ -298,6 +298,20 @@ receiver-async-profiler: # It is recommended to use physical file mode when volume mounting is used or the tmp directory has sufficient storage. memoryParserEnabled: ${SW_RECEIVER_ASYNC_PROFILER_MEMORY_PARSER_ENABLED:true} +receiver-pprof: + selector: ${SW_RECEIVER_PPROF:default} + default: + # Used to manage the maximum size of the pprof file that can be received, the unit is Byte, default is 30M + pprofMaxSize: ${SW_RECEIVER_PPROF_MAX_SIZE:31457280} + # Used to determine whether to receive pprof in memory file or physical file mode + # + # The memory file mode have fewer local file system limitations, so they are by default. But it costs more memory. + # + # The physical file mode will use less memory when parsing and is more friendly to parsing large files. + # However, if the storage of the tmp directory in the container is insufficient, the oap server instance may crash. + # It is recommended to use physical file mode when volume mounting is used or the tmp directory has sufficient storage. + memoryParserEnabled: ${SW_RECEIVER_PPROF_MEMORY_PARSER_ENABLED:true} + receiver-zabbix: selector: ${SW_RECEIVER_ZABBIX:-} default: diff --git a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBStorageConfig.java b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBStorageConfig.java index e32891c3e472..0f533990ea02 100644 --- a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBStorageConfig.java +++ b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBStorageConfig.java @@ -97,6 +97,8 @@ public static class Global { */ private int asyncProfilerTaskQueryMaxSize; + private int pprofTaskQueryMaxSize; + private int resultWindowMaxSize = 10000; private int metadataQueryMaxSize = 5000; private int segmentQueryMaxSize = 200; diff --git a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBStorageProvider.java b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBStorageProvider.java index 62f6dea5ba15..d1f58d35bff1 100644 --- a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBStorageProvider.java +++ b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBStorageProvider.java @@ -40,6 +40,9 @@ import org.apache.skywalking.oap.server.core.storage.profiling.asyncprofiler.IAsyncProfilerTaskLogQueryDAO; import org.apache.skywalking.oap.server.core.storage.profiling.asyncprofiler.IAsyncProfilerTaskQueryDAO; import org.apache.skywalking.oap.server.core.storage.profiling.asyncprofiler.IJFRDataQueryDAO; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofDataQueryDAO; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskLogQueryDAO; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskQueryDAO; import org.apache.skywalking.oap.server.core.storage.profiling.continuous.IContinuousProfilingPolicyDAO; import org.apache.skywalking.oap.server.core.storage.profiling.ebpf.IEBPFProfilingDataDAO; import org.apache.skywalking.oap.server.core.storage.profiling.ebpf.IEBPFProfilingScheduleDAO; @@ -80,6 +83,9 @@ import org.apache.skywalking.oap.server.storage.plugin.banyandb.stream.BanyanDBAlarmQueryDAO; import org.apache.skywalking.oap.server.storage.plugin.banyandb.stream.BanyanDBAsyncProfilerTaskLogQueryDAO; import org.apache.skywalking.oap.server.storage.plugin.banyandb.stream.BanyanDBAsyncProfilerTaskQueryDAO; +import org.apache.skywalking.oap.server.storage.plugin.banyandb.stream.BanyanDBPprofDataQueryDAO; +import org.apache.skywalking.oap.server.storage.plugin.banyandb.stream.BanyanDBPprofTaskLogQueryDAO; +import org.apache.skywalking.oap.server.storage.plugin.banyandb.stream.BanyanDBPprofTaskQueryDAO; import org.apache.skywalking.oap.server.storage.plugin.banyandb.stream.BanyanDBBrowserLogQueryDAO; import org.apache.skywalking.oap.server.storage.plugin.banyandb.stream.BanyanDBContinuousProfilingPolicyDAO; import org.apache.skywalking.oap.server.storage.plugin.banyandb.stream.BanyanDBEBPFProfilingDataDAO; @@ -199,6 +205,17 @@ IAsyncProfilerTaskLogQueryDAO.class, new BanyanDBAsyncProfilerTaskLogQueryDAO(cl this.config.getGlobal().getAsyncProfilerTaskQueryMaxSize() )); this.registerServiceImplementation(IJFRDataQueryDAO.class, new BanyanDBJFRDataQueryDAO(client)); + this.registerServiceImplementation( + IPprofTaskQueryDAO.class, new BanyanDBPprofTaskQueryDAO(client, + this.config.getGlobal().getPprofTaskQueryMaxSize() + )); + this.registerServiceImplementation( + IPprofTaskLogQueryDAO.class, new BanyanDBPprofTaskLogQueryDAO(client, + this.config.getGlobal().getPprofTaskQueryMaxSize() + )); + this.registerServiceImplementation( + IPprofDataQueryDAO.class, new BanyanDBPprofDataQueryDAO(client) + ); this.registerServiceImplementation( StorageTTLStatusQuery.class, new BanyanDBTTLStatusQuery(config) diff --git a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBPprofDataQueryDAO.java b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBPprofDataQueryDAO.java new file mode 100644 index 000000000000..7d03db2f5aac --- /dev/null +++ b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBPprofDataQueryDAO.java @@ -0,0 +1,59 @@ +package org.apache.skywalking.oap.server.storage.plugin.banyandb.stream; + +import com.google.common.collect.ImmutableSet; +import org.apache.skywalking.oap.server.core.profiling.pprof.storage.PprofProfilingDataRecord; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofDataQueryDAO; +import java.util.Set; +import java.util.List; +import java.io.IOException; +import java.util.ArrayList; +import org.apache.skywalking.oap.server.library.util.StringUtil; +import org.apache.skywalking.oap.server.storage.plugin.banyandb.BanyanDBStorageClient; +import org.apache.skywalking.banyandb.v1.client.StreamQuery; +import org.apache.skywalking.banyandb.v1.client.StreamQueryResponse; +import org.apache.skywalking.oap.server.library.util.CollectionUtils; +import org.apache.skywalking.banyandb.v1.client.RowEntity; +import org.apache.skywalking.oap.server.storage.plugin.banyandb.BanyanDBConverter; + +public class BanyanDBPprofDataQueryDAO extends AbstractBanyanDBDAO implements IPprofDataQueryDAO { + private static final Set TAGS = ImmutableSet.of( + PprofProfilingDataRecord.TASK_ID, + PprofProfilingDataRecord.INSTANCE_ID, + PprofProfilingDataRecord.EVENT_TYPE, + PprofProfilingDataRecord.UPLOAD_TIME, + PprofProfilingDataRecord.DATA_BINARY + ); + + public BanyanDBPprofDataQueryDAO(BanyanDBStorageClient client) { + super(client); + } + + @Override + public List getByTaskIdAndInstances(String taskId, List instanceIds) throws IOException { + if (StringUtil.isBlank(taskId)) { + return new ArrayList<>(); + } + StreamQueryResponse resp = query(false, PprofProfilingDataRecord.INDEX_NAME, TAGS, + new QueryBuilder() { + @Override + protected void apply(StreamQuery query) { + query.and(eq(PprofProfilingDataRecord.TASK_ID, taskId)); + if (CollectionUtils.isNotEmpty(instanceIds)) { + query.and(in(PprofProfilingDataRecord.INSTANCE_ID, instanceIds)); + } + } + }); + List records = new ArrayList<>(resp.size()); + for (final RowEntity entity : resp.getElements()) { + records.add(buildProfilingDataRecord(entity)); + } + + return records; + } + + private PprofProfilingDataRecord buildProfilingDataRecord(RowEntity entity) { + final PprofProfilingDataRecord.Builder builder = new PprofProfilingDataRecord.Builder(); + BanyanDBConverter.StorageToStream storageToStream = new BanyanDBConverter.StorageToStream(PprofProfilingDataRecord.INDEX_NAME, entity); + return builder.storage2Entity(storageToStream); + } +} diff --git a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBPprofTaskLogQueryDAO.java b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBPprofTaskLogQueryDAO.java new file mode 100644 index 000000000000..cfd9d4d42daf --- /dev/null +++ b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBPprofTaskLogQueryDAO.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.banyandb.stream; + +import com.google.common.collect.ImmutableSet; +import org.apache.skywalking.banyandb.v1.client.Element; +import org.apache.skywalking.banyandb.v1.client.StreamQuery; +import org.apache.skywalking.banyandb.v1.client.StreamQueryResponse; +import org.apache.skywalking.oap.server.core.profiling.pprof.storage.PprofTaskLogRecord; +import org.apache.skywalking.oap.server.core.query.PprofTaskLog; +import org.apache.skywalking.oap.server.core.query.type.PprofTaskLogOperationType; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskLogQueryDAO; +import org.apache.skywalking.oap.server.storage.plugin.banyandb.BanyanDBStorageClient; + +import java.io.IOException; +import java.util.LinkedList; +import java.util.List; +import java.util.Set; + +/** + * {@link PprofTaskLogRecord} is a stream + */ +public class BanyanDBPprofTaskLogQueryDAO extends AbstractBanyanDBDAO implements IPprofTaskLogQueryDAO { + private static final Set TAGS = ImmutableSet.of( + PprofTaskLogRecord.OPERATION_TIME, + PprofTaskLogRecord.INSTANCE_ID, + PprofTaskLogRecord.TASK_ID, + PprofTaskLogRecord.OPERATION_TYPE + ); + + private final int queryMaxSize; + + public BanyanDBPprofTaskLogQueryDAO(BanyanDBStorageClient client, int taskQueryMaxSize) { + super(client); + // query log size use pprof task query max size * per log count + this.queryMaxSize = taskQueryMaxSize * 50; + } + + @Override + public List getTaskLogList() throws IOException { + StreamQueryResponse resp = query(false, PprofTaskLogRecord.INDEX_NAME, TAGS, + new QueryBuilder() { + @Override + public void apply(StreamQuery query) { + query.setLimit(BanyanDBPprofTaskLogQueryDAO.this.queryMaxSize); + } + }); + + final LinkedList tasks = new LinkedList<>(); + for (final Element element : resp.getElements()) { + tasks.add(buildPprofTaskLog(element)); + } + return tasks; + } + + private PprofTaskLog buildPprofTaskLog(Element data) { + int operationTypeInt = ((Number) data.getTagValue(PprofTaskLogRecord.OPERATION_TYPE)).intValue(); + PprofTaskLogOperationType operationType = PprofTaskLogOperationType.parse(operationTypeInt); + return PprofTaskLog.builder() + .id(data.getTagValue(PprofTaskLogRecord.TASK_ID)) + .instanceId(data.getTagValue(PprofTaskLogRecord.INSTANCE_ID)) + .operationType(operationType) + .operationTime(((Number) data.getTagValue(PprofTaskLogRecord.OPERATION_TIME)).longValue()) + .build(); + } +} diff --git a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBPprofTaskQueryDAO.java b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBPprofTaskQueryDAO.java new file mode 100644 index 000000000000..58488382f6f4 --- /dev/null +++ b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBPprofTaskQueryDAO.java @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.banyandb.stream; + +import com.google.common.collect.ImmutableSet; +import com.google.common.reflect.TypeToken; +import com.google.gson.Gson; +import org.apache.skywalking.banyandb.v1.client.AbstractQuery; +import org.apache.skywalking.banyandb.v1.client.RowEntity; +import org.apache.skywalking.banyandb.v1.client.StreamQuery; +import org.apache.skywalking.banyandb.v1.client.StreamQueryResponse; +import org.apache.skywalking.banyandb.v1.client.TimestampRange; +import org.apache.skywalking.oap.server.core.analysis.TimeBucket; +import org.apache.skywalking.oap.server.core.profiling.pprof.storage.PprofTaskRecord; +import org.apache.skywalking.oap.server.core.query.type.PprofTask; +import org.apache.skywalking.oap.server.core.query.type.PprofEventType; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskQueryDAO; +import org.apache.skywalking.oap.server.library.util.StringUtil; +import org.apache.skywalking.oap.server.storage.plugin.banyandb.BanyanDBStorageClient; +import lombok.extern.slf4j.Slf4j; +import java.io.IOException; +import java.lang.reflect.Type; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +@Slf4j +public class BanyanDBPprofTaskQueryDAO extends AbstractBanyanDBDAO implements IPprofTaskQueryDAO { + private static final Gson GSON = new Gson(); + private static final Set TAGS = ImmutableSet.of( + PprofTaskRecord.SERVICE_ID, + PprofTaskRecord.SERVICE_INSTANCE_IDS, + PprofTaskRecord.TASK_ID, + PprofTaskRecord.CREATE_TIME, + PprofTaskRecord.EVENT_TYPES, + PprofTaskRecord.DURATION, + PprofTaskRecord.DUMP_PERIOD + ); + + private final int queryMaxSize; + + public BanyanDBPprofTaskQueryDAO(BanyanDBStorageClient client, int queryMaxSize) { + super(client); + this.queryMaxSize = queryMaxSize; + } + + @Override + public List getTaskList(String serviceId, Long startTimeBucket, Long endTimeBucket, Integer limit) throws IOException { + long startTS = LOWER_BOUND_TIME; + long endTS = UPPER_BOUND_TIME; + if (startTimeBucket != null) { + startTS = TimeBucket.getTimestamp(startTimeBucket); + } + if (endTimeBucket != null) { + endTS = TimeBucket.getTimestamp(endTimeBucket); + } + StreamQueryResponse resp = query(false, PprofTaskRecord.INDEX_NAME, TAGS, new TimestampRange(startTS, endTS), + new QueryBuilder() { + @Override + protected void apply(StreamQuery query) { + if (StringUtil.isNotEmpty(serviceId)) { + query.and(eq(PprofTaskRecord.SERVICE_ID, serviceId)); + } + + if (limit != null) { + query.setLimit(limit); + } else { + query.setLimit(BanyanDBPprofTaskQueryDAO.this.queryMaxSize); + } + query.setOrderBy(new AbstractQuery.OrderBy(AbstractQuery.Sort.DESC)); + } + }); + + List tasks = new ArrayList<>(resp.size()); + for (final RowEntity entity : resp.getElements()) { + tasks.add(buildPprofTask(entity)); + } + return tasks; + } + + @Override + public PprofTask getById(String id) throws IOException { + StreamQueryResponse resp = query(false, PprofTaskRecord.INDEX_NAME, TAGS, + new QueryBuilder() { + @Override + protected void apply(StreamQuery query) { + if (StringUtil.isNotEmpty(id)) { + query.and(eq(PprofTaskRecord.TASK_ID, id)); + } + query.setLimit(1); + } + }); + + if (resp.size() == 0) { + return null; + } + + return buildPprofTask(resp.getElements().get(0)); + } + + private PprofTask buildPprofTask(RowEntity data) { + Type listType = new TypeToken>() { + }.getType(); + + String serviceInstanceIds = data.getTagValue(PprofTaskRecord.SERVICE_INSTANCE_IDS); + List serviceInstanceIdList = GSON.fromJson(serviceInstanceIds, listType); + + // Convert string events to PprofEventType enum + String eventsStr = data.getTagValue(PprofTaskRecord.EVENT_TYPES); + PprofEventType eventType = null; + if (StringUtil.isNotEmpty(eventsStr)) { + try { + eventType = PprofEventType.valueOfString(eventsStr); + } catch (Exception e) { + // Default to CPU if conversion fails + eventType = PprofEventType.CPU; + log.warn("Failed to parse pprof event type: {}, using CPU as default", eventsStr, e); + } + } + + return PprofTask.builder() + .id(data.getTagValue(PprofTaskRecord.TASK_ID)) + .serviceId(data.getTagValue(PprofTaskRecord.SERVICE_ID)) + .serviceInstanceIds(serviceInstanceIdList) + .createTime(((Number) data.getTagValue(PprofTaskRecord.CREATE_TIME)).longValue()) + .events(eventType) + .duration(((Number) data.getTagValue(PprofTaskRecord.DURATION)).intValue()) + .dumpPeriod(((Number) data.getTagValue(PprofTaskRecord.DUMP_PERIOD)).intValue()) + .build(); + } +} diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchProvider.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchProvider.java index cd255d99b6cf..cfbd84c4cb13 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchProvider.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchProvider.java @@ -39,6 +39,8 @@ import org.apache.skywalking.oap.server.core.storage.profiling.asyncprofiler.IAsyncProfilerTaskLogQueryDAO; import org.apache.skywalking.oap.server.core.storage.profiling.asyncprofiler.IAsyncProfilerTaskQueryDAO; import org.apache.skywalking.oap.server.core.storage.profiling.asyncprofiler.IJFRDataQueryDAO; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskLogQueryDAO; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskQueryDAO; import org.apache.skywalking.oap.server.core.storage.profiling.continuous.IContinuousProfilingPolicyDAO; import org.apache.skywalking.oap.server.core.storage.profiling.ebpf.IEBPFProfilingDataDAO; import org.apache.skywalking.oap.server.core.storage.profiling.ebpf.IEBPFProfilingScheduleDAO; @@ -81,6 +83,7 @@ import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.AlarmQueryEsDAO; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.AsyncProfilerTaskLogQueryEsDAO; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.AsyncProfilerTaskQueryEsDAO; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.PprofTaskLogQueryEsDAO; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.BrowserLogQueryEsDAO; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.ContinuousProfilingPolicyEsDAO; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.EBPFProfilingDataEsDAO; @@ -95,6 +98,7 @@ import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.ProfileTaskLogEsDAO; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.ProfileTaskQueryEsDAO; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.ProfileThreadSnapshotQueryEsDAO; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.PprofTaskQueryEsDAO; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.RecordsQueryEsDAO; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.ServiceLabelEsDAO; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.SpanAttachedEventEsDAO; @@ -271,10 +275,18 @@ IProfileThreadSnapshotQueryDAO.class, new ProfileThreadSnapshotQueryEsDAO(elasti IAsyncProfilerTaskLogQueryDAO.class, new AsyncProfilerTaskLogQueryEsDAO(elasticSearchClient, config.getAsyncProfilerTaskQueryMaxSize()) ); + this.registerServiceImplementation( + IPprofTaskLogQueryDAO.class, + new PprofTaskLogQueryEsDAO(elasticSearchClient, config.getAsyncProfilerTaskQueryMaxSize()) + ); this.registerServiceImplementation( IJFRDataQueryDAO.class, new JFRDataQueryEsDAO(elasticSearchClient) ); + this.registerServiceImplementation( + IPprofTaskQueryDAO.class, + new PprofTaskQueryEsDAO(elasticSearchClient, config.getAsyncProfilerTaskQueryMaxSize()) + ); this.registerServiceImplementation( StorageTTLStatusQuery.class, new DefaultStorageTTLStatusQuery() diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofTaskLogQueryEsDAO.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofTaskLogQueryEsDAO.java new file mode 100644 index 000000000000..4d9ce9a5e51d --- /dev/null +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofTaskLogQueryEsDAO.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query; + +import java.io.IOException; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +import org.apache.skywalking.library.elasticsearch.requests.search.Query; +import org.apache.skywalking.library.elasticsearch.requests.search.Search; +import org.apache.skywalking.library.elasticsearch.requests.search.SearchBuilder; +import org.apache.skywalking.library.elasticsearch.requests.search.Sort; +import org.apache.skywalking.library.elasticsearch.response.search.SearchHit; +import org.apache.skywalking.library.elasticsearch.response.search.SearchResponse; +import org.apache.skywalking.oap.server.core.profiling.pprof.storage.PprofTaskLogRecord; +import org.apache.skywalking.oap.server.core.query.PprofTaskLog; +import org.apache.skywalking.oap.server.core.query.type.PprofTaskLogOperationType; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskLogQueryDAO; +import org.apache.skywalking.oap.server.library.client.elasticsearch.ElasticSearchClient; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.EsDAO; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.IndexController; + +public class PprofTaskLogQueryEsDAO extends EsDAO implements IPprofTaskLogQueryDAO { + + private final int queryMaxSize; + + public PprofTaskLogQueryEsDAO(ElasticSearchClient client, int profileTaskQueryMaxSize) { + super(client); + // query log size use pprof task query max size * per log count + this.queryMaxSize = profileTaskQueryMaxSize * 50; + } + + @Override + public List getTaskLogList() throws IOException { + final String index = IndexController.LogicIndicesRegister.getPhysicalTableName(PprofTaskLogRecord.INDEX_NAME); + + final SearchBuilder search = Search.builder().query(Query.bool()); + if (IndexController.LogicIndicesRegister.isMergedTable(PprofTaskLogRecord.INDEX_NAME)) { + search.query(Query.bool().must(Query.term(IndexController.LogicIndicesRegister.RECORD_TABLE_NAME, PprofTaskLogRecord.INDEX_NAME))); + } + + search.size(queryMaxSize); + search.sort(PprofTaskLogRecord.OPERATION_TIME, Sort.Order.DESC); + + final SearchResponse response = getClient().search(index, search.build()); + + List tasks = new LinkedList<>(); + for (SearchHit hit : response.getHits().getHits()) { + tasks.add(parseTaskLog(hit)); + } + return tasks; + } + + private PprofTaskLog parseTaskLog(SearchHit data) { + Map source = data.getSource(); + + int operationTypeInt = ((Number) source.get(PprofTaskLogRecord.OPERATION_TYPE)).intValue(); + PprofTaskLogOperationType operationType = PprofTaskLogOperationType.parse(operationTypeInt); + + return PprofTaskLog.builder() + .id((String) source.get(PprofTaskLogRecord.TASK_ID)) + .instanceId((String) source.get(PprofTaskLogRecord.INSTANCE_ID)) + .operationType(operationType) + .operationTime(((Number) source.get(PprofTaskLogRecord.OPERATION_TIME)).longValue()) + .build(); + } +} diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofTaskQueryEsDAO.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofTaskQueryEsDAO.java new file mode 100644 index 000000000000..e93afbfbdef1 --- /dev/null +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofTaskQueryEsDAO.java @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query; + +import java.io.IOException; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; +import java.lang.reflect.Type; + +import org.apache.skywalking.oap.server.library.util.StringUtil; +import org.apache.skywalking.library.elasticsearch.requests.search.BoolQueryBuilder; +import org.apache.skywalking.library.elasticsearch.requests.search.Query; +import org.apache.skywalking.library.elasticsearch.requests.search.Search; +import org.apache.skywalking.library.elasticsearch.requests.search.SearchBuilder; +import org.apache.skywalking.library.elasticsearch.requests.search.Sort; +import org.apache.skywalking.library.elasticsearch.response.search.SearchHit; +import org.apache.skywalking.library.elasticsearch.response.search.SearchResponse; +import org.apache.skywalking.oap.server.core.profiling.pprof.storage.PprofTaskRecord; +import org.apache.skywalking.oap.server.core.query.type.PprofTask; +import org.apache.skywalking.oap.server.core.query.type.PprofEventType; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskQueryDAO; +import org.apache.skywalking.oap.server.library.client.elasticsearch.ElasticSearchClient; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.EsDAO; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.IndexController; + +public class PprofTaskQueryEsDAO extends EsDAO implements IPprofTaskQueryDAO { + private static final Gson GSON = new Gson(); + + private final int queryMaxSize; + + public PprofTaskQueryEsDAO(ElasticSearchClient client, int queryMaxSize) { + super(client); + this.queryMaxSize = queryMaxSize; + } + + @Override + public List getTaskList(String serviceId, Long startTimeBucket, Long endTimeBucket, Integer limit) throws IOException { + final String index = IndexController.LogicIndicesRegister.getPhysicalTableName(PprofTaskRecord.INDEX_NAME); + final BoolQueryBuilder query = Query.bool(); + if (IndexController.LogicIndicesRegister.isMergedTable(PprofTaskRecord.INDEX_NAME)) { + query.must(Query.term(IndexController.LogicIndicesRegister.RECORD_TABLE_NAME, PprofTaskRecord.INDEX_NAME)); + } + + if (StringUtil.isNotEmpty(serviceId)) { + query.must(Query.term(PprofTaskRecord.SERVICE_ID, serviceId)); + } + + if (startTimeBucket != null) { + query.must(Query.range(PprofTaskRecord.TIME_BUCKET).gte(startTimeBucket)); + } + + if (endTimeBucket != null) { + query.must(Query.range(PprofTaskRecord.TIME_BUCKET).lte(endTimeBucket)); + } + + final SearchBuilder search = Search.builder().query(query); + + if (limit != null) { + search.size(limit); + } else { + search.size(queryMaxSize); + } + + search.sort(PprofTaskRecord.CREATE_TIME, Sort.Order.DESC); + + final SearchResponse response = getClient().search(index, search.build()); + + List tasks = new LinkedList<>(); + for (SearchHit hit : response.getHits().getHits()) { + tasks.add(parseTask(hit)); + } + return tasks; + } + + @Override + public PprofTask getById(String id) throws IOException { + if (StringUtil.isEmpty(id)) { + return null; + } + final String index = IndexController.LogicIndicesRegister.getPhysicalTableName(PprofTaskRecord.INDEX_NAME); + final BoolQueryBuilder query = Query.bool(); + if (IndexController.LogicIndicesRegister.isMergedTable(PprofTaskRecord.INDEX_NAME)) { + query.must(Query.term(IndexController.LogicIndicesRegister.RECORD_TABLE_NAME, PprofTaskRecord.INDEX_NAME)); + } + query.must(Query.term(PprofTaskRecord.TASK_ID, id)); + + final SearchBuilder search = Search.builder().query(query).size(1); + final SearchResponse response = getClient().search(index, search.build()); + + if (!response.getHits().getHits().isEmpty()) { + return parseTask(response.getHits().getHits().iterator().next()); + } + return null; + } + + private PprofTask parseTask(SearchHit data) { + Map source = data.getSource(); + Type listType = new TypeToken>() { + }.getType(); + + String serviceInstanceIds = (String) source.get(PprofTaskRecord.SERVICE_INSTANCE_IDS); + + List instanceIdList = GSON.fromJson(serviceInstanceIds, listType); + + // Convert string events to PprofEventType enum + String eventsStr = (String) source.get(PprofTaskRecord.EVENT_TYPES); + PprofEventType eventType = null; + if (StringUtil.isNotEmpty(eventsStr)) { + try { + eventType = PprofEventType.valueOfString(eventsStr); + } catch (Exception e) { + // Default to CPU if conversion fails + eventType = PprofEventType.CPU; + } + } + + return PprofTask.builder() + .id((String) source.get(PprofTaskRecord.TASK_ID)) + .serviceId((String) source.get(PprofTaskRecord.SERVICE_ID)) + .serviceInstanceIds(instanceIdList) + .createTime(((Number) source.get(PprofTaskRecord.CREATE_TIME)).longValue()) + .startTime(((Number) source.get(PprofTaskRecord.START_TIME)).longValue()) + .events(eventType) + .duration(((Number) source.get(PprofTaskRecord.DURATION)).intValue()) + .dumpPeriod(((Number) source.get(PprofTaskRecord.DUMP_PERIOD)).intValue()) + .build(); + } +} From 6b4640f81cf97b5e0c34b9630d95b6a53c0ebc55 Mon Sep 17 00:00:00 2001 From: JophieQu Date: Thu, 18 Sep 2025 15:08:22 +0800 Subject: [PATCH 2/7] fix --- .../component/command/PprofTaskCommand.java | 45 +++++++++++++------ .../pprof/module/PprofModuleConfig.java | 2 +- .../src/main/resources/bydb.yml | 3 +- .../banyandb/BanyanDBStorageConfig.java | 5 ++- 4 files changed, 39 insertions(+), 16 deletions(-) diff --git a/apm-protocol/apm-network/src/main/java/org/apache/skywalking/oap/server/network/trace/component/command/PprofTaskCommand.java b/apm-protocol/apm-network/src/main/java/org/apache/skywalking/oap/server/network/trace/component/command/PprofTaskCommand.java index 3cbf9b5f4750..87210f8685da 100644 --- a/apm-protocol/apm-network/src/main/java/org/apache/skywalking/oap/server/network/trace/component/command/PprofTaskCommand.java +++ b/apm-protocol/apm-network/src/main/java/org/apache/skywalking/oap/server/network/trace/component/command/PprofTaskCommand.java @@ -1,3 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + package org.apache.skywalking.oap.server.network.trace.component.command; import org.apache.skywalking.apm.network.common.v3.Command; @@ -15,11 +33,22 @@ public class PprofTaskCommand extends BaseCommand implements Serializable, Deser private String taskId; // Type of profiling (CPU/Heap/Block/Mutex/Goroutine/Threadcreate/Allocs) private String events; - // unit is minute + /** + * run profiling for duration (minute) + */ private long duration; - // Unix timestamp in milliseconds when the task was created + /** + * task create time + */ private long createTime; - // + /** + * pprof dump period parameters. There are different dumpperiod configurations for different events. + * Here is a table of parameters. + * + *

For Block - sample an average of one blocking event per rate nanoseconds spent blocked. (default: 0)

+ *

For Mutex - sample an average of 1/rate events are reported. (default: 0)

+ * details @see pprof argument + */ private int dumpPeriod; public PprofTaskCommand(String serialNumber, String taskId, String events, @@ -32,16 +61,6 @@ public PprofTaskCommand(String serialNumber, String taskId, String events, this.events = events; } - // public PprofTaskCommand(String serialNumber, String taskId, - // long duration, long startTime, long createTime, int dumpPeriod) { - // super(NAME, serialNumber); - // this.taskId = taskId; - // this.duration = duration; - // this.startTime = startTime; - // this.createTime = createTime; - // this.dumpPeriod = dumpPeriod; - // } - @Override public PprofTaskCommand deserialize(Command command) { final List argsList = command.getArgsList(); diff --git a/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/module/PprofModuleConfig.java b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/module/PprofModuleConfig.java index 99f5b70fc078..acb92227d020 100644 --- a/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/module/PprofModuleConfig.java +++ b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/module/PprofModuleConfig.java @@ -29,7 +29,7 @@ public class PprofModuleConfig extends ModuleConfig { * Used to manage the maximum size of the pprof file that can be received, the unit is Byte * default is 30M */ - private int pprofMaxSize = 4 * 1024; + private int pprofMaxSize = 30 * 1024 * 1024; /** * default is true *

diff --git a/oap-server/server-starter/src/main/resources/bydb.yml b/oap-server/server-starter/src/main/resources/bydb.yml index 75899fee8aca..48bc7d3e1b98 100644 --- a/oap-server/server-starter/src/main/resources/bydb.yml +++ b/oap-server/server-starter/src/main/resources/bydb.yml @@ -18,7 +18,7 @@ global: # Each target is a BanyanDB server in the format of `host:port`. # If BanyanDB is deployed as a standalone server, the target should be the IP address or domain name and port of the BanyanDB server. # If BanyanDB is deployed in a cluster, the targets should be the IP address or domain name and port of the `liaison` nodes, separated by commas. - targets: ${SW_STORAGE_BANYANDB_TARGETS:127.0.0.1:17912} + targets: ${SW_STORAGE_BANYANDB_TARGETS:192.168.50.128:17912} # The maximum number of records in a bulk write request. # A larger value can improve write performance but also increases OAP and BanyanDB Server memory usage. maxBulkSize: ${SW_STORAGE_BANYANDB_MAX_BULK_SIZE:10000} @@ -43,6 +43,7 @@ global: # The batch size for querying profile data. profileDataQueryBatchSize: ${SW_STORAGE_BANYANDB_QUERY_PROFILE_DATA_BATCH_SIZE:100} asyncProfilerTaskQueryMaxSize: ${SW_STORAGE_BANYANDB_ASYNC_PROFILER_TASK_QUERY_MAX_SIZE:200} + pprofTaskQueryMaxSize: ${SW_STORAGE_BANYANDB_PPROF_TASK_QUERY_MAX_SIZE:200} # If the BanyanDB server is configured with TLS, configure the TLS cert file path and enable TLS connection. sslTrustCAPath: ${SW_STORAGE_BANYANDB_SSL_TRUST_CA_PATH:""} # Cleanup TopN rules in BanyanDB server that are not configured in the bydb-topn.yml config. diff --git a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBStorageConfig.java b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBStorageConfig.java index 0f533990ea02..e21039359dae 100644 --- a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBStorageConfig.java +++ b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBStorageConfig.java @@ -96,7 +96,10 @@ public static class Global { * single request. */ private int asyncProfilerTaskQueryMaxSize; - + /** + * Max size of {@link org.apache.skywalking.oap.server.core.query.type.PprofTask} to be fetched in a + * single request. + */ private int pprofTaskQueryMaxSize; private int resultWindowMaxSize = 10000; From a54d8dc8b1bf7a48a242fbe23aae80a51bb50b20 Mon Sep 17 00:00:00 2001 From: JophieQu Date: Wed, 24 Sep 2025 20:50:37 +0800 Subject: [PATCH 3/7] fix --- .../profiling/pprof/PprofMutationService.java | 23 +++++++--- .../pprof/parser/PprofMergeBuilder.java | 36 ++++++--------- .../library/pprof/parser/PprofParser.java | 30 +++++++++++-- .../oap/server/library/pprof/type/Frame.java | 45 +++++++++++++++++++ .../oap/server/library/pprof/type/Index.java | 34 ++++++++++++++ .../StorageModuleElasticsearchConfig.java | 5 +++ .../StorageModuleElasticsearchProvider.java | 4 +- .../query/PprofTaskQueryEsDAO.java | 16 +------ 8 files changed, 143 insertions(+), 50 deletions(-) create mode 100644 oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/Frame.java create mode 100644 oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/Index.java diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/PprofMutationService.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/PprofMutationService.java index c909790df570..ddfb45d5667a 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/PprofMutationService.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/PprofMutationService.java @@ -62,7 +62,7 @@ public PprofTaskCreationResult createTask(String serviceId, long createTime = System.currentTimeMillis(); // check data PprofTaskCreationResult checkResult = checkDataSuccess( - serviceId, serviceInstanceIds, duration, createTime, events + serviceId, serviceInstanceIds, duration, createTime, events, dumpPeriod ); if (checkResult != null) { return checkResult; @@ -90,8 +90,9 @@ private PprofTaskCreationResult checkDataSuccess(String serviceId, List serviceInstanceIds, int duration, long createTime, - PprofEventType events) throws IOException { - String checkArgumentMessage = checkArgumentError(serviceId, serviceInstanceIds, duration, events); + PprofEventType events, + int dumpPeriod) throws IOException { + String checkArgumentMessage = checkArgumentError(serviceId, serviceInstanceIds, duration, events, dumpPeriod); if (checkArgumentMessage != null) { return PprofTaskCreationResult.builder() .code(PprofTaskCreationType.ARGUMENT_ERROR) @@ -111,16 +112,24 @@ private PprofTaskCreationResult checkDataSuccess(String serviceId, private String checkArgumentError(String serviceId, List serviceInstanceIds, int duration, - PprofEventType events) { + PprofEventType events, + int dumpPeriod) { if (serviceId == null) { return "service cannot be null"; } - if (duration <= 0) { - return "duration cannot be negative"; - } if (events == null) { return "events cannot be empty"; } + if (events == PprofEventType.CPU || events == PprofEventType.BLOCK || events == PprofEventType.MUTEX) { + if (duration <= 0) { + return "duration cannot be negative"; + } + } + if (events == PprofEventType.BLOCK || events == PprofEventType.MUTEX) { + if (dumpPeriod <= 0) { + return "dumpPeriod cannot be negative"; + } + } if (CollectionUtils.isEmpty(serviceInstanceIds)) { return "serviceInstanceIds cannot be empty"; } diff --git a/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/parser/PprofMergeBuilder.java b/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/parser/PprofMergeBuilder.java index 20b2cf57570b..6937c3416eb9 100644 --- a/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/parser/PprofMergeBuilder.java +++ b/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/parser/PprofMergeBuilder.java @@ -19,12 +19,13 @@ package org.apache.skywalking.oap.server.library.pprof.parser; import org.apache.skywalking.oap.server.library.pprof.type.FrameTree; -import java.util.HashMap; +import org.apache.skywalking.oap.server.library.pprof.type.Frame; +import org.apache.skywalking.oap.server.library.pprof.type.Index; import java.util.List; -import java.util.Map; public class PprofMergeBuilder { - private final Node root = new Node("root"); + private final Index cpool = new Index<>(String.class, ""); + private final Frame root = new Frame("root"); public PprofMergeBuilder merge(List trees) { if (trees == null || trees.isEmpty()) { @@ -41,44 +42,35 @@ public PprofMergeBuilder merge(FrameTree tree) { return this; } - private void merge0(Node node, FrameTree tree) { + private void merge0(Frame frame, FrameTree tree) { if (tree == null) { return; } if (tree.getChildren() != null) { for (FrameTree childTree : tree.getChildren()) { - Node child = getOrAddChild(node, childTree.getSignature()); + Frame child = addChild(frame, childTree.getSignature()); merge0(child, childTree); } } - node.total += tree.getTotal(); - node.self += tree.getSelf(); + frame.setTotal(frame.getTotal() + tree.getTotal()); + frame.setSelf(frame.getSelf() + tree.getSelf()); } - private Node getOrAddChild(Node parent, String signature) { - return parent.children.computeIfAbsent(signature, Node::new); + private Frame addChild(Frame parent, String signature) { + int titleIndex = cpool.index(signature); + return parent.getChild(titleIndex, signature); } public FrameTree build() { return toFrameTree(root); } - private FrameTree toFrameTree(Node node) { - FrameTree tree = new FrameTree(node.signature, node.total, node.self); - for (Node child : node.children.values()) { + private FrameTree toFrameTree(Frame node) { + FrameTree tree = new FrameTree(node.getSignature(), node.getTotal(), node.getSelf()); + for (Frame child : node.values()) { tree.getChildren().add(toFrameTree(child)); } return tree; } - private static class Node { - final String signature; - long total; - long self; - final Map children = new HashMap<>(); - - Node(String signature) { - this.signature = signature; - } - } } diff --git a/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/parser/PprofParser.java b/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/parser/PprofParser.java index eacbf2d55f18..8e8acbaa2391 100644 --- a/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/parser/PprofParser.java +++ b/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/parser/PprofParser.java @@ -19,11 +19,13 @@ package org.apache.skywalking.oap.server.library.pprof.parser; import com.google.perftools.profiles.ProfileProto; +import java.io.ByteArrayInputStream; import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.nio.ByteBuffer; +import java.util.zip.GZIPInputStream; import org.apache.skywalking.oap.server.library.pprof.type.FrameTree; import org.apache.skywalking.oap.server.library.pprof.type.FrameTreeBuilder; @@ -33,9 +35,14 @@ public class PprofParser { public static FrameTree dumpTree(ByteBuffer buf) throws IOException { - ProfileProto.Profile profile = ProfileProto.Profile.parseFrom(buf); + byte[] bytes = new byte[buf.remaining()]; + buf.get(bytes); + InputStream stream = new java.io.ByteArrayInputStream(bytes); + InputStream inputStream = isGzippedBytes(bytes) ? new GZIPInputStream(stream) : stream; + ProfileProto.Profile profile = ProfileProto.Profile.parseFrom(inputStream); FrameTree tree = new FrameTreeBuilder(profile).build(); return tree; + } public static FrameTree dumpTree(String filePath) throws IOException { @@ -43,11 +50,26 @@ public static FrameTree dumpTree(String filePath) throws IOException { if (!file.exists()) { throw new IOException("Pprof file not found: " + filePath); } - InputStream fileStream = new FileInputStream(file); + InputStream stream = filePath.endsWith(".gz") || isGzipped(file) ? + new GZIPInputStream(fileStream) : fileStream; ProfileProto.Profile profile = ProfileProto.Profile.parseFrom(fileStream); FrameTree tree = new FrameTreeBuilder(profile).build(); return tree; - } -} \ No newline at end of file + + private static boolean isGzipped(File file) throws IOException { + try (FileInputStream fis = new FileInputStream(file)) { + byte[] magic = new byte[2]; + if (fis.read(magic) == 2) { + return (magic[0] == (byte) 0x1f) && (magic[1] == (byte) 0x8b); + } + } + return false; + } + + private static boolean isGzippedBytes(byte[] bytes) { + return bytes.length >= 2 && + (bytes[0] == (byte) 0x1f) && (bytes[1] == (byte) 0x8b); + } +} diff --git a/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/Frame.java b/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/Frame.java new file mode 100644 index 000000000000..65993079526d --- /dev/null +++ b/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/Frame.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.library.pprof.type; + +import java.util.HashMap; +import lombok.Getter; +import lombok.Setter; + +@Getter +@Setter +public class Frame extends HashMap { + final int key; + final String signature; + long total; + long self; + + private Frame(int key, String signature) { + this.key = key; + this.signature = signature; + } + + public Frame(String signature) { + this(signature.hashCode(), signature); + } + + public Frame getChild(int titleIndex, String signature) { + return super.computeIfAbsent(titleIndex, k -> new Frame(k, signature)); + } +} diff --git a/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/Index.java b/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/Index.java new file mode 100644 index 000000000000..30582ff5022c --- /dev/null +++ b/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/Index.java @@ -0,0 +1,34 @@ +package org.apache.skywalking.oap.server.library.pprof.type; + +import java.lang.reflect.Array; +import java.util.HashMap; + +public class Index extends HashMap { + private final Class cls; + + public Index(Class cls, T empty) { + this.cls = cls; + super.put(empty, 0); + } + + public int index(T key) { + Integer index = super.get(key); + if (index != null) { + return index; + } else { + int newIndex = super.size(); + super.put(key, newIndex); + return newIndex; + } + } + + @SuppressWarnings("unchecked") + public T[] keys() { + T[] result = (T[]) Array.newInstance(cls, size()); + for (Entry entry : entrySet()) { + result[entry.getValue()] = entry.getKey(); + } + return result; + } +} + diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchConfig.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchConfig.java index b275acd65014..d8dc848be5c3 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchConfig.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchConfig.java @@ -167,4 +167,9 @@ public class StorageModuleElasticsearchConfig extends ModuleConfig { * in a single request. */ private int asyncProfilerTaskQueryMaxSize; + /** + * Max size of {@link org.apache.skywalking.oap.server.core.query.type.PprofTask} to be fetched + * in a single request. + */ + private int pprofTaskQueryMaxSize; } diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchProvider.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchProvider.java index cfbd84c4cb13..343b4d354f09 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchProvider.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchProvider.java @@ -277,7 +277,7 @@ IProfileThreadSnapshotQueryDAO.class, new ProfileThreadSnapshotQueryEsDAO(elasti ); this.registerServiceImplementation( IPprofTaskLogQueryDAO.class, - new PprofTaskLogQueryEsDAO(elasticSearchClient, config.getAsyncProfilerTaskQueryMaxSize()) + new PprofTaskLogQueryEsDAO(elasticSearchClient, config.getPprofTaskQueryMaxSize()) ); this.registerServiceImplementation( IJFRDataQueryDAO.class, @@ -285,7 +285,7 @@ IProfileThreadSnapshotQueryDAO.class, new ProfileThreadSnapshotQueryEsDAO(elasti ); this.registerServiceImplementation( IPprofTaskQueryDAO.class, - new PprofTaskQueryEsDAO(elasticSearchClient, config.getAsyncProfilerTaskQueryMaxSize()) + new PprofTaskQueryEsDAO(elasticSearchClient, config.getPprofTaskQueryMaxSize()) ); this.registerServiceImplementation( StorageTTLStatusQuery.class, diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofTaskQueryEsDAO.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofTaskQueryEsDAO.java index e93afbfbdef1..eb41c21fb7a9 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofTaskQueryEsDAO.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofTaskQueryEsDAO.java @@ -120,26 +120,12 @@ private PprofTask parseTask(SearchHit data) { String serviceInstanceIds = (String) source.get(PprofTaskRecord.SERVICE_INSTANCE_IDS); List instanceIdList = GSON.fromJson(serviceInstanceIds, listType); - - // Convert string events to PprofEventType enum - String eventsStr = (String) source.get(PprofTaskRecord.EVENT_TYPES); - PprofEventType eventType = null; - if (StringUtil.isNotEmpty(eventsStr)) { - try { - eventType = PprofEventType.valueOfString(eventsStr); - } catch (Exception e) { - // Default to CPU if conversion fails - eventType = PprofEventType.CPU; - } - } - return PprofTask.builder() .id((String) source.get(PprofTaskRecord.TASK_ID)) .serviceId((String) source.get(PprofTaskRecord.SERVICE_ID)) .serviceInstanceIds(instanceIdList) .createTime(((Number) source.get(PprofTaskRecord.CREATE_TIME)).longValue()) - .startTime(((Number) source.get(PprofTaskRecord.START_TIME)).longValue()) - .events(eventType) + .events((PprofEventType) source.get(PprofTaskRecord.EVENT_TYPES)) .duration(((Number) source.get(PprofTaskRecord.DURATION)).intValue()) .dumpPeriod(((Number) source.get(PprofTaskRecord.DUMP_PERIOD)).intValue()) .build(); From d069e68c2bd7458d04c4909fc5c582b5067469f5 Mon Sep 17 00:00:00 2001 From: JophieQu Date: Wed, 24 Sep 2025 20:53:31 +0800 Subject: [PATCH 4/7] fix: rollback yaml --- oap-server/server-starter/src/main/resources/application.yml | 2 +- oap-server/server-starter/src/main/resources/bydb.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/oap-server/server-starter/src/main/resources/application.yml b/oap-server/server-starter/src/main/resources/application.yml index 50b66c9c8de9..a92a4604325b 100644 --- a/oap-server/server-starter/src/main/resources/application.yml +++ b/oap-server/server-starter/src/main/resources/application.yml @@ -151,7 +151,7 @@ storage: # Since 10.2.0, the banyandb configuration is separated to an independent configuration file: `bydb.yaml`. elasticsearch: namespace: ${SW_NAMESPACE:""} - clusterNodes: 192.168.50.242:9200 + clusterNodes: ${SW_STORAGE_ES_CLUSTER_NODES:localhost:9200} protocol: ${SW_STORAGE_ES_HTTP_PROTOCOL:"http"} connectTimeout: ${SW_STORAGE_ES_CONNECT_TIMEOUT:3000} socketTimeout: ${SW_STORAGE_ES_SOCKET_TIMEOUT:30000} diff --git a/oap-server/server-starter/src/main/resources/bydb.yml b/oap-server/server-starter/src/main/resources/bydb.yml index 48bc7d3e1b98..2f83d0a26147 100644 --- a/oap-server/server-starter/src/main/resources/bydb.yml +++ b/oap-server/server-starter/src/main/resources/bydb.yml @@ -18,7 +18,7 @@ global: # Each target is a BanyanDB server in the format of `host:port`. # If BanyanDB is deployed as a standalone server, the target should be the IP address or domain name and port of the BanyanDB server. # If BanyanDB is deployed in a cluster, the targets should be the IP address or domain name and port of the `liaison` nodes, separated by commas. - targets: ${SW_STORAGE_BANYANDB_TARGETS:192.168.50.128:17912} + targets: ${SW_STORAGE_BANYANDB_TARGETS:127.0.0.1:17912} # The maximum number of records in a bulk write request. # A larger value can improve write performance but also increases OAP and BanyanDB Server memory usage. maxBulkSize: ${SW_STORAGE_BANYANDB_MAX_BULK_SIZE:10000} From 1afb571ad46776e6d58685d60363079d224c868c Mon Sep 17 00:00:00 2001 From: JophieQu Date: Mon, 29 Sep 2025 17:43:44 +0800 Subject: [PATCH 5/7] fix --- .../oap/server/core/CoreModuleProvider.java | 2 +- .../oap/server/core/cache/PprofTaskCache.java | 28 ++---------- .../profiling/pprof/PprofMutationService.java | 1 - .../storage/PprofProfilingDataDispatcher.java | 1 - .../storage/PprofProfilingDataRecord.java | 9 ---- .../pprof/parser/PprofMergeBuilder.java | 15 +------ .../library/pprof/parser/PprofParser.java | 22 ++-------- .../oap/server/library/pprof/type/Frame.java | 13 ++---- .../server/library/pprof/type/FrameTree.java | 43 ++++++++++++++----- .../library/pprof/type/FrameTreeBuilder.java | 3 +- .../oap/server/library/pprof/type/Index.java | 34 --------------- .../PprofByteBufCollectionObserver.java | 8 ---- .../stream/BanyanDBPprofDataQueryDAO.java | 19 +++++++- .../query/PprofTaskLogQueryEsDAO.java | 23 ++++++++++ 14 files changed, 88 insertions(+), 133 deletions(-) delete mode 100644 oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/Index.java diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java index c8d88d8fdf47..9d8f15ad56a2 100755 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java @@ -339,7 +339,7 @@ TTLStatusQuery.class, new TTLStatusQuery( this.registerServiceImplementation( PprofQueryService.class, new PprofQueryService(getManager())); this.registerServiceImplementation( - PprofTaskCache.class, new PprofTaskCache(getManager(), moduleConfig)); + PprofTaskCache.class, new PprofTaskCache(moduleConfig)); this.registerServiceImplementation( EBPFProfilingMutationService.class, new EBPFProfilingMutationService(getManager())); this.registerServiceImplementation( diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/cache/PprofTaskCache.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/cache/PprofTaskCache.java index 9ce7e60defc2..304826f5efd3 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/cache/PprofTaskCache.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/cache/PprofTaskCache.java @@ -19,50 +19,28 @@ package org.apache.skywalking.oap.server.core.cache; import org.apache.skywalking.oap.server.library.module.Service; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; import org.apache.skywalking.oap.server.core.CoreModuleConfig; import org.apache.skywalking.oap.server.core.analysis.TimeBucket; import org.apache.skywalking.oap.server.core.query.type.PprofTask; -import org.apache.skywalking.oap.server.core.storage.StorageModule; -import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskQueryDAO; -import org.apache.skywalking.oap.server.library.module.ModuleManager; import java.time.Duration; -import java.util.Objects; import java.util.concurrent.TimeUnit; public class PprofTaskCache implements Service { - private static final Logger LOGGER = LoggerFactory.getLogger(PprofTaskCache.class); - private final Cache serviceId2taskCache; - private final ModuleManager moduleManager; - - private IPprofTaskQueryDAO taskQueryDAO; - - public PprofTaskCache(ModuleManager moduleManager, CoreModuleConfig moduleConfig) { - this.moduleManager = moduleManager; + public PprofTaskCache(CoreModuleConfig moduleConfig) { long initialSize = moduleConfig.getMaxSizeOfProfileTask() / 10L; int initialCapacitySize = (int) (initialSize > Integer.MAX_VALUE ? Integer.MAX_VALUE : initialSize); serviceId2taskCache = CacheBuilder.newBuilder() .initialCapacity(initialCapacitySize) .maximumSize(moduleConfig.getMaxSizeOfProfileTask()) - // remove old profile task data - extend to 10 minutes to ensure data availability - .expireAfterWrite(Duration.ofMinutes(10)) + // remove old profile task data + .expireAfterWrite(Duration.ofMinutes(1)) .build(); } - private IPprofTaskQueryDAO getTaskQueryDAO() { - if (Objects.isNull(taskQueryDAO)) { - taskQueryDAO = moduleManager.find(StorageModule.NAME) - .provider() - .getService(IPprofTaskQueryDAO.class); - } - return taskQueryDAO; - } - public PprofTask getPprofTask(String serviceId) { PprofTask task = serviceId2taskCache.getIfPresent(serviceId); return task; diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/PprofMutationService.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/PprofMutationService.java index ddfb45d5667a..3d8264e90644 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/PprofMutationService.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/PprofMutationService.java @@ -67,7 +67,6 @@ public PprofTaskCreationResult createTask(String serviceId, if (checkResult != null) { return checkResult; } - // create task PprofTaskRecord task = new PprofTaskRecord(); String taskId = createTime + Const.ID_CONNECTOR + serviceId; diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofProfilingDataDispatcher.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofProfilingDataDispatcher.java index 5cdf1dd2b1ed..21df992f1d54 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofProfilingDataDispatcher.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofProfilingDataDispatcher.java @@ -32,7 +32,6 @@ public void dispatch(PprofProfilingData source) { PprofProfilingDataRecord record = new PprofProfilingDataRecord(); record.setTaskId(source.getTaskId()); record.setInstanceId(source.getInstanceId()); - record.setEventType(source.getEventType().toString()); record.setDataBinary(GSON.toJson(source.getFrameTree()).getBytes()); record.setUploadTime(source.getUploadTime()); record.setTimeBucket(TimeBucket.getRecordTimeBucket(source.getUploadTime())); diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofProfilingDataRecord.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofProfilingDataRecord.java index 17164411a257..c88704b75491 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofProfilingDataRecord.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/storage/PprofProfilingDataRecord.java @@ -38,9 +38,7 @@ @BanyanDB.TimestampColumn(PprofProfilingDataRecord.UPLOAD_TIME) public class PprofProfilingDataRecord extends Record { public static final String INDEX_NAME = "pprof_profiling_data"; - public static final String TASK_ID = "task_id"; - public static final String EVENT_TYPE = "event_type"; public static final String INSTANCE_ID = "instance_id"; public static final String DATA_BINARY = "data_binary"; public static final String UPLOAD_TIME = "upload_time"; @@ -51,9 +49,6 @@ public class PprofProfilingDataRecord extends Record { @Column(name = INSTANCE_ID) @BanyanDB.SeriesID(index = 0) private String instanceId; - - @Column(name = EVENT_TYPE) - private String eventType; @Column(name = UPLOAD_TIME) private long uploadTime; @@ -67,13 +62,11 @@ public StorageID id() { new String[]{ TASK_ID, INSTANCE_ID, - EVENT_TYPE, UPLOAD_TIME }, Hashing.sha256().newHasher() .putString(taskId, StandardCharsets.UTF_8) .putString(instanceId, StandardCharsets.UTF_8) - .putString(eventType, StandardCharsets.UTF_8) .putLong(uploadTime) .hash().toString() ); @@ -87,7 +80,6 @@ public PprofProfilingDataRecord storage2Entity(final Convert2Entity converter) { dataTraffic.setTaskId((String) converter.get(TASK_ID)); dataTraffic.setInstanceId((String) converter.get(INSTANCE_ID)); dataTraffic.setUploadTime(((Number) converter.get(UPLOAD_TIME)).longValue()); - dataTraffic.setEventType((String) converter.get(EVENT_TYPE)); dataTraffic.setDataBinary(converter.getBytes(DATA_BINARY)); return dataTraffic; } @@ -98,7 +90,6 @@ public void entity2Storage(final PprofProfilingDataRecord storageData, final Con converter.accept(TASK_ID, storageData.getTaskId()); converter.accept(INSTANCE_ID, storageData.getInstanceId()); converter.accept(UPLOAD_TIME, storageData.getUploadTime()); - converter.accept(EVENT_TYPE, storageData.getEventType()); converter.accept(DATA_BINARY, storageData.getDataBinary()); } } diff --git a/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/parser/PprofMergeBuilder.java b/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/parser/PprofMergeBuilder.java index 6937c3416eb9..3e4e0f9f0082 100644 --- a/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/parser/PprofMergeBuilder.java +++ b/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/parser/PprofMergeBuilder.java @@ -20,11 +20,9 @@ import org.apache.skywalking.oap.server.library.pprof.type.FrameTree; import org.apache.skywalking.oap.server.library.pprof.type.Frame; -import org.apache.skywalking.oap.server.library.pprof.type.Index; import java.util.List; public class PprofMergeBuilder { - private final Index cpool = new Index<>(String.class, ""); private final Frame root = new Frame("root"); public PprofMergeBuilder merge(List trees) { @@ -57,20 +55,11 @@ private void merge0(Frame frame, FrameTree tree) { } private Frame addChild(Frame parent, String signature) { - int titleIndex = cpool.index(signature); - return parent.getChild(titleIndex, signature); + return parent.getChild(signature); } public FrameTree build() { - return toFrameTree(root); - } - - private FrameTree toFrameTree(Frame node) { - FrameTree tree = new FrameTree(node.getSignature(), node.getTotal(), node.getSelf()); - for (Frame child : node.values()) { - tree.getChildren().add(toFrameTree(child)); - } - return tree; + return FrameTree.buildTree(root); } } diff --git a/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/parser/PprofParser.java b/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/parser/PprofParser.java index 8e8acbaa2391..9fe8ab776b3f 100644 --- a/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/parser/PprofParser.java +++ b/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/parser/PprofParser.java @@ -38,7 +38,7 @@ public static FrameTree dumpTree(ByteBuffer buf) throws IOException { byte[] bytes = new byte[buf.remaining()]; buf.get(bytes); InputStream stream = new java.io.ByteArrayInputStream(bytes); - InputStream inputStream = isGzippedBytes(bytes) ? new GZIPInputStream(stream) : stream; + InputStream inputStream = new GZIPInputStream(stream); ProfileProto.Profile profile = ProfileProto.Profile.parseFrom(inputStream); FrameTree tree = new FrameTreeBuilder(profile).build(); return tree; @@ -51,25 +51,9 @@ public static FrameTree dumpTree(String filePath) throws IOException { throw new IOException("Pprof file not found: " + filePath); } InputStream fileStream = new FileInputStream(file); - InputStream stream = filePath.endsWith(".gz") || isGzipped(file) ? - new GZIPInputStream(fileStream) : fileStream; - ProfileProto.Profile profile = ProfileProto.Profile.parseFrom(fileStream); + InputStream stream = new GZIPInputStream(fileStream); + ProfileProto.Profile profile = ProfileProto.Profile.parseFrom(stream); FrameTree tree = new FrameTreeBuilder(profile).build(); return tree; } - - private static boolean isGzipped(File file) throws IOException { - try (FileInputStream fis = new FileInputStream(file)) { - byte[] magic = new byte[2]; - if (fis.read(magic) == 2) { - return (magic[0] == (byte) 0x1f) && (magic[1] == (byte) 0x8b); - } - } - return false; - } - - private static boolean isGzippedBytes(byte[] bytes) { - return bytes.length >= 2 && - (bytes[0] == (byte) 0x1f) && (bytes[1] == (byte) 0x8b); - } } diff --git a/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/Frame.java b/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/Frame.java index 65993079526d..194829d9262b 100644 --- a/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/Frame.java +++ b/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/Frame.java @@ -25,21 +25,16 @@ @Getter @Setter public class Frame extends HashMap { - final int key; final String signature; long total; long self; - private Frame(int key, String signature) { - this.key = key; - this.signature = signature; - } public Frame(String signature) { - this(signature.hashCode(), signature); + this.signature = signature; } - - public Frame getChild(int titleIndex, String signature) { - return super.computeIfAbsent(titleIndex, k -> new Frame(k, signature)); + + public Frame getChild(String signature) { + return super.computeIfAbsent(signature.hashCode(), k -> new Frame(signature)); } } diff --git a/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/FrameTree.java b/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/FrameTree.java index 2b1587a8ec53..9a2f240ccdd3 100755 --- a/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/FrameTree.java +++ b/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/FrameTree.java @@ -18,21 +18,42 @@ package org.apache.skywalking.oap.server.library.pprof.type; -import com.google.gson.annotations.SerializedName; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; import java.util.ArrayList; import java.util.List; - -@Data -@NoArgsConstructor -@AllArgsConstructor +import lombok.Getter; +@Getter public class FrameTree { - @SerializedName("name") private String signature; - @SerializedName("value") private long total; private long self; - private final List children = new ArrayList<>(); + private List children; + + public FrameTree(Frame frame) { + this.signature = frame.getSignature(); + this.total = frame.getTotal(); + this.self = frame.getSelf(); + this.children = new ArrayList<>(frame.size()); + } + public FrameTree(String signature, long total, long self) { + this.signature = signature; + this.total = total; + this.self = self; + this.children = new ArrayList<>(); + } + + public static FrameTree buildTree(Frame frame) { + if (frame == null) return null; + + FrameTree frameTree = new FrameTree(frame); + // has children? + if (!frame.isEmpty()) { + frameTree.children = new ArrayList<>(frame.size()); + // build tree + for (Frame childFrame : frame.values()) { + FrameTree childFrameTree = buildTree(childFrame); + frameTree.children.add(childFrameTree); + } + } + return frameTree; + } } diff --git a/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/FrameTreeBuilder.java b/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/FrameTreeBuilder.java index 3fe4a3b04089..c42f504ad8a2 100644 --- a/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/FrameTreeBuilder.java +++ b/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/FrameTreeBuilder.java @@ -56,7 +56,6 @@ private FrameTree parseTree(RawFrameTree rawTree) { } return tree; } - private String getSignature(long locationId) { if (locationId == 0) { return "root"; @@ -87,11 +86,13 @@ private void mergeSample(ProfileProto.Sample sample) { boolean isEnd = i == size - 1; long locationId = locationIdList.get(i); if (children.containsKey(locationId)) { + // if the child exists, merge the sample data RawFrameTree child = children.get(locationId); child.setTotal(child.getTotal() + 1); child.setSelf(child.getSelf() + (isEnd ? 1 : 0)); children = child.getChildren(); } else { + // if the child does not exist, create a new child RawFrameTree child = new RawFrameTree(locationId, 1, (isEnd ? 1 : 0)); children.put(locationId, child); children = child.getChildren(); diff --git a/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/Index.java b/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/Index.java deleted file mode 100644 index 30582ff5022c..000000000000 --- a/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/Index.java +++ /dev/null @@ -1,34 +0,0 @@ -package org.apache.skywalking.oap.server.library.pprof.type; - -import java.lang.reflect.Array; -import java.util.HashMap; - -public class Index extends HashMap { - private final Class cls; - - public Index(Class cls, T empty) { - this.cls = cls; - super.put(empty, 0); - } - - public int index(T key) { - Integer index = super.get(key); - if (index != null) { - return index; - } else { - int newIndex = super.size(); - super.put(key, newIndex); - return newIndex; - } - } - - @SuppressWarnings("unchecked") - public T[] keys() { - T[] result = (T[]) Array.newInstance(cls, size()); - for (Entry entry : entrySet()) { - result[entry.getValue()] = entry.getKey(); - } - return result; - } -} - diff --git a/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/stream/PprofByteBufCollectionObserver.java b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/stream/PprofByteBufCollectionObserver.java index 5b9f10abba0e..833305e8f0b9 100644 --- a/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/stream/PprofByteBufCollectionObserver.java +++ b/oap-server/server-receiver-plugin/skywalking-pprof-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/pprof/provider/handler/stream/PprofByteBufCollectionObserver.java @@ -63,31 +63,23 @@ public void onNext(PprofData pprofData) { taskMetaData = parseMetaData(pprofData.getMetadata(), taskDAO); if (PprofProfilingStatus.PPROF_PROFILING_SUCCESS.equals(taskMetaData.getType())) { int size = taskMetaData.getContentSize(); - log.info("pprofMaxSize: {}, Pprof data size: {}", pprofMaxSize, size); if (pprofMaxSize >= size) { buf = ByteBuffer.allocate(size); // Send success response to allow client to continue uploading responseObserver.onNext(PprofCollectionResponse.newBuilder() .setStatus(PprofProfilingStatus.PPROF_PROFILING_SUCCESS) .build()); - - log.info("Started collecting pprof data in memory - service: {}, serviceInstance: {}, size: {} bytes", - pprofData.getMetadata().getService(), pprofData.getMetadata().getServiceInstance(), size); } else { responseObserver.onNext(PprofCollectionResponse.newBuilder() .setStatus(PprofProfilingStatus.PPROF_TERMINATED_BY_OVERSIZE) .build()); recordPprofTaskLog(taskMetaData.getTask(), taskMetaData.getInstanceId(), PprofTaskLogOperationType.PPROF_UPLOAD_FILE_TOO_LARGE_ERROR); - log.warn("Pprof file size {} exceeds maximum allowed size {} for service: {}, serviceInstance: {}", - size, pprofMaxSize, pprofData.getMetadata().getService(), pprofData.getMetadata().getServiceInstance()); } } else { responseObserver.onNext(PprofCollectionResponse.newBuilder() .setStatus(PprofProfilingStatus.PPROF_EXECUTION_TASK_ERROR) .build()); recordPprofTaskLog(taskMetaData.getTask(), taskMetaData.getInstanceId(), PprofTaskLogOperationType.EXECUTION_TASK_ERROR); - log.error("Received execution error from agent - service: {}, serviceInstance: {}, status: {}", - pprofData.getMetadata().getService(), pprofData.getMetadata().getServiceInstance(), taskMetaData.getType()); } } else if (pprofData.hasContent()) { if (buf != null) { diff --git a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBPprofDataQueryDAO.java b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBPprofDataQueryDAO.java index 7d03db2f5aac..cdb573a9a8ef 100644 --- a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBPprofDataQueryDAO.java +++ b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBPprofDataQueryDAO.java @@ -1,3 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + package org.apache.skywalking.oap.server.storage.plugin.banyandb.stream; import com.google.common.collect.ImmutableSet; @@ -19,7 +37,6 @@ public class BanyanDBPprofDataQueryDAO extends AbstractBanyanDBDAO implements IP private static final Set TAGS = ImmutableSet.of( PprofProfilingDataRecord.TASK_ID, PprofProfilingDataRecord.INSTANCE_ID, - PprofProfilingDataRecord.EVENT_TYPE, PprofProfilingDataRecord.UPLOAD_TIME, PprofProfilingDataRecord.DATA_BINARY ); diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofTaskLogQueryEsDAO.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofTaskLogQueryEsDAO.java index 4d9ce9a5e51d..6f9f16606e9e 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofTaskLogQueryEsDAO.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofTaskLogQueryEsDAO.java @@ -82,3 +82,26 @@ private PprofTaskLog parseTaskLog(SearchHit data) { .build(); } } + + + + + + + + + + + + + + + + + + + + + + + From 2578bd168b53ac8f6fa8303741046b957f11e727 Mon Sep 17 00:00:00 2001 From: JophieQu Date: Mon, 29 Sep 2025 22:33:39 +0800 Subject: [PATCH 6/7] fix --- apm-protocol/apm-network/src/main/proto | 2 +- docs/en/changes/changes.md | 1 + .../query/input/PprofTaskListRequest.java | 18 ++++++++++++++++++ .../query/type/PprofTaskCreationResult.java | 18 ++++++++++++++++++ .../query/type/PprofTaskCreationType.java | 18 ++++++++++++++++++ .../pprof/IPprofTaskLogQueryDAO.java | 19 ++++++++++++++++++- .../src/main/resources/query-protocol | 2 +- 7 files changed, 75 insertions(+), 3 deletions(-) diff --git a/apm-protocol/apm-network/src/main/proto b/apm-protocol/apm-network/src/main/proto index 157baf3e7a97..d5882f167677 160000 --- a/apm-protocol/apm-network/src/main/proto +++ b/apm-protocol/apm-network/src/main/proto @@ -1 +1 @@ -Subproject commit 157baf3e7a9710d3041e9caec39eecab7e0c5a82 +Subproject commit d5882f167677626479d7e1df3933f1024eecd3bc diff --git a/docs/en/changes/changes.md b/docs/en/changes/changes.md index bc3886219b8b..603b34bec16d 100644 --- a/docs/en/changes/changes.md +++ b/docs/en/changes/changes.md @@ -46,6 +46,7 @@ * Add UI dashboard for Ruby runtime metrics. * Tracing Query Execution HTTP APIs: make the argument `service layer` optional. * GraphQL API: metadata, topology, log and trace support query by name. +* Support pprof profiling feature #### UI diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/PprofTaskListRequest.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/PprofTaskListRequest.java index db789a56228a..a2304a04ba95 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/PprofTaskListRequest.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/PprofTaskListRequest.java @@ -1,3 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + package org.apache.skywalking.oap.server.core.query.input; import lombok.Getter; diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskCreationResult.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskCreationResult.java index 1e9ca4d8596c..10f7653e9aa6 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskCreationResult.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskCreationResult.java @@ -1,3 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + package org.apache.skywalking.oap.server.core.query.type; import lombok.AllArgsConstructor; diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskCreationType.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskCreationType.java index ddf62db10a05..14519f62d3c0 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskCreationType.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/PprofTaskCreationType.java @@ -1,3 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + package org.apache.skywalking.oap.server.core.query.type; public enum PprofTaskCreationType { diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/profiling/pprof/IPprofTaskLogQueryDAO.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/profiling/pprof/IPprofTaskLogQueryDAO.java index 78571479204a..ba6b652dc396 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/profiling/pprof/IPprofTaskLogQueryDAO.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/profiling/pprof/IPprofTaskLogQueryDAO.java @@ -1,8 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + package org.apache.skywalking.oap.server.core.storage.profiling.pprof; import org.apache.skywalking.oap.server.core.query.PprofTaskLog; import org.apache.skywalking.oap.server.core.storage.DAO; - import java.io.IOException; import java.util.List; diff --git a/oap-server/server-query-plugin/query-graphql-plugin/src/main/resources/query-protocol b/oap-server/server-query-plugin/query-graphql-plugin/src/main/resources/query-protocol index abf4c4d1588d..ed68593ec32e 160000 --- a/oap-server/server-query-plugin/query-graphql-plugin/src/main/resources/query-protocol +++ b/oap-server/server-query-plugin/query-graphql-plugin/src/main/resources/query-protocol @@ -1 +1 @@ -Subproject commit abf4c4d1588d16facae4a696032d5f8b68a4ccaf +Subproject commit ed68593ec32ecd2cb56326bede0cd7b9f85b7bec From 4c03a81fb9380f26e7020d030e59688cbc2e738c Mon Sep 17 00:00:00 2001 From: JophieQu Date: Fri, 10 Oct 2025 17:55:57 +0800 Subject: [PATCH 7/7] add jdbc & es storage --- .../oap/server/core/cache/PprofTaskCache.java | 2 +- .../src/main/resources/application.yml | 1 + .../StorageModuleElasticsearchProvider.java | 14 +- .../query/PprofDataQueryEsDAO.java | 72 ++++++++ .../query/PprofTaskLogQueryEsDAO.java | 21 ++- .../query/PprofTaskQueryEsDAO.java | 19 +-- .../storage-jdbc-hikaricp-plugin/pom.xml | 5 + .../jdbc/common/JDBCStorageProvider.java | 18 ++ .../common/dao/JDBCPprofDataQueryDAO.java | 92 +++++++++++ .../common/dao/JDBCPprofTaskLogQueryDAO.java | 83 ++++++++++ .../common/dao/JDBCPprofTaskQueryDAO.java | 155 ++++++++++++++++++ 11 files changed, 454 insertions(+), 28 deletions(-) create mode 100644 oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofDataQueryEsDAO.java create mode 100644 oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCPprofDataQueryDAO.java create mode 100644 oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCPprofTaskLogQueryDAO.java create mode 100644 oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCPprofTaskQueryDAO.java diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/cache/PprofTaskCache.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/cache/PprofTaskCache.java index 304826f5efd3..73ac6dcfa347 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/cache/PprofTaskCache.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/cache/PprofTaskCache.java @@ -36,7 +36,7 @@ public PprofTaskCache(CoreModuleConfig moduleConfig) { serviceId2taskCache = CacheBuilder.newBuilder() .initialCapacity(initialCapacitySize) .maximumSize(moduleConfig.getMaxSizeOfProfileTask()) - // remove old profile task data + // remove old pprof task data .expireAfterWrite(Duration.ofMinutes(1)) .build(); } diff --git a/oap-server/server-starter/src/main/resources/application.yml b/oap-server/server-starter/src/main/resources/application.yml index a92a4604325b..e6b084cc3891 100644 --- a/oap-server/server-starter/src/main/resources/application.yml +++ b/oap-server/server-starter/src/main/resources/application.yml @@ -184,6 +184,7 @@ storage: segmentQueryMaxSize: ${SW_STORAGE_ES_QUERY_SEGMENT_SIZE:200} profileTaskQueryMaxSize: ${SW_STORAGE_ES_QUERY_PROFILE_TASK_SIZE:200} asyncProfilerTaskQueryMaxSize: ${SW_STORAGE_ES_QUERY_ASYNC_PROFILER_TASK_SIZE:200} + pprofTaskQueryMaxSize: ${SW_STORAGE_ES_QUERY_PPROF_TASK_SIZE:200} profileDataQueryBatchSize: ${SW_STORAGE_ES_QUERY_PROFILE_DATA_BATCH_SIZE:100} oapAnalyzer: ${SW_STORAGE_ES_OAP_ANALYZER:"{\"analyzer\":{\"oap_analyzer\":{\"type\":\"stop\"}}}"} # the oap analyzer. oapLogAnalyzer: ${SW_STORAGE_ES_OAP_LOG_ANALYZER:"{\"analyzer\":{\"oap_log_analyzer\":{\"type\":\"standard\"}}}"} # the oap log analyzer. It could be customized by the ES analyzer configuration to support more language log formats, such as Chinese log, Japanese log and etc. diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchProvider.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchProvider.java index 343b4d354f09..3cea61495b1d 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchProvider.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchProvider.java @@ -41,6 +41,7 @@ import org.apache.skywalking.oap.server.core.storage.profiling.asyncprofiler.IJFRDataQueryDAO; import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskLogQueryDAO; import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskQueryDAO; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofDataQueryDAO; import org.apache.skywalking.oap.server.core.storage.profiling.continuous.IContinuousProfilingPolicyDAO; import org.apache.skywalking.oap.server.core.storage.profiling.ebpf.IEBPFProfilingDataDAO; import org.apache.skywalking.oap.server.core.storage.profiling.ebpf.IEBPFProfilingScheduleDAO; @@ -92,6 +93,7 @@ import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.ESEventQueryDAO; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.HierarchyQueryEsDAO; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.JFRDataQueryEsDAO; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.PprofDataQueryEsDAO; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.LogQueryEsDAO; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.MetadataQueryEsDAO; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.MetricsQueryEsDAO; @@ -275,10 +277,6 @@ IProfileThreadSnapshotQueryDAO.class, new ProfileThreadSnapshotQueryEsDAO(elasti IAsyncProfilerTaskLogQueryDAO.class, new AsyncProfilerTaskLogQueryEsDAO(elasticSearchClient, config.getAsyncProfilerTaskQueryMaxSize()) ); - this.registerServiceImplementation( - IPprofTaskLogQueryDAO.class, - new PprofTaskLogQueryEsDAO(elasticSearchClient, config.getPprofTaskQueryMaxSize()) - ); this.registerServiceImplementation( IJFRDataQueryDAO.class, new JFRDataQueryEsDAO(elasticSearchClient) @@ -287,6 +285,14 @@ IProfileThreadSnapshotQueryDAO.class, new ProfileThreadSnapshotQueryEsDAO(elasti IPprofTaskQueryDAO.class, new PprofTaskQueryEsDAO(elasticSearchClient, config.getPprofTaskQueryMaxSize()) ); + this.registerServiceImplementation( + IPprofTaskLogQueryDAO.class, + new PprofTaskLogQueryEsDAO(elasticSearchClient, config.getPprofTaskQueryMaxSize()) + ); + this.registerServiceImplementation( + IPprofDataQueryDAO.class, + new PprofDataQueryEsDAO(elasticSearchClient) + ); this.registerServiceImplementation( StorageTTLStatusQuery.class, new DefaultStorageTTLStatusQuery() diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofDataQueryEsDAO.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofDataQueryEsDAO.java new file mode 100644 index 000000000000..2dc2b03d7be1 --- /dev/null +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofDataQueryEsDAO.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + package org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query; + +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofDataQueryDAO; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.EsDAO; +import org.apache.skywalking.oap.server.library.client.elasticsearch.ElasticSearchClient; +import org.apache.skywalking.oap.server.core.profiling.pprof.storage.PprofProfilingDataRecord; +import org.apache.skywalking.oap.server.library.util.StringUtil; +import org.apache.skywalking.oap.server.library.util.CollectionUtils; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.IndexController; +import org.apache.skywalking.library.elasticsearch.requests.search.BoolQueryBuilder; +import org.apache.skywalking.library.elasticsearch.requests.search.Query; +import org.apache.skywalking.library.elasticsearch.requests.search.SearchBuilder; +import org.apache.skywalking.library.elasticsearch.requests.search.Search; +import org.apache.skywalking.library.elasticsearch.response.search.SearchHit; +import org.apache.skywalking.library.elasticsearch.response.search.SearchResponse; +import com.google.common.collect.Lists; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.ElasticSearchConverter; +import java.util.Map; +import java.util.ArrayList; +import java.util.List; + +public class PprofDataQueryEsDAO extends EsDAO implements IPprofDataQueryDAO { + public PprofDataQueryEsDAO(ElasticSearchClient client) { + super(client); + } + + @Override + public List getByTaskIdAndInstances(String taskId, List instanceIds) { + if (StringUtil.isBlank(taskId) || CollectionUtils.isEmpty(instanceIds)) { + return new ArrayList<>(); + } + final String index = IndexController.LogicIndicesRegister.getPhysicalTableName(PprofProfilingDataRecord.INDEX_NAME); + final BoolQueryBuilder query = Query.bool(); + if (IndexController.LogicIndicesRegister.isMergedTable(PprofProfilingDataRecord.INDEX_NAME)) { + query.must(Query.term(IndexController.LogicIndicesRegister.RECORD_TABLE_NAME, PprofProfilingDataRecord.INDEX_NAME)); + } + query.must(Query.term(PprofProfilingDataRecord.TASK_ID, taskId)); + query.must(Query.terms(PprofProfilingDataRecord.INSTANCE_ID, instanceIds)); + final SearchBuilder search = Search.builder().query(query); + final SearchResponse response = getClient().search(index, search.build()); + List dataRecords = Lists.newArrayList(); + for (SearchHit searchHit : response.getHits().getHits()) { + dataRecords.add(parseData(searchHit)); + } + return dataRecords; + } + + private PprofProfilingDataRecord parseData(SearchHit data) { + final Map sourceAsMap = data.getSource(); + final PprofProfilingDataRecord.Builder builder = new PprofProfilingDataRecord.Builder(); + return builder.storage2Entity(new ElasticSearchConverter.ToEntity(PprofProfilingDataRecord.INDEX_NAME, sourceAsMap)); + } +} diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofTaskLogQueryEsDAO.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofTaskLogQueryEsDAO.java index 6f9f16606e9e..87f20d4498bf 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofTaskLogQueryEsDAO.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofTaskLogQueryEsDAO.java @@ -23,6 +23,7 @@ import java.util.List; import java.util.Map; +import org.apache.skywalking.library.elasticsearch.requests.search.BoolQueryBuilder; import org.apache.skywalking.library.elasticsearch.requests.search.Query; import org.apache.skywalking.library.elasticsearch.requests.search.Search; import org.apache.skywalking.library.elasticsearch.requests.search.SearchBuilder; @@ -50,30 +51,28 @@ public PprofTaskLogQueryEsDAO(ElasticSearchClient client, int profileTaskQueryMa @Override public List getTaskLogList() throws IOException { final String index = IndexController.LogicIndicesRegister.getPhysicalTableName(PprofTaskLogRecord.INDEX_NAME); - - final SearchBuilder search = Search.builder().query(Query.bool()); + final BoolQueryBuilder query = Query.bool(); if (IndexController.LogicIndicesRegister.isMergedTable(PprofTaskLogRecord.INDEX_NAME)) { - search.query(Query.bool().must(Query.term(IndexController.LogicIndicesRegister.RECORD_TABLE_NAME, PprofTaskLogRecord.INDEX_NAME))); + query.must(Query.term(IndexController.LogicIndicesRegister.RECORD_TABLE_NAME, PprofTaskLogRecord.INDEX_NAME)); } - search.size(queryMaxSize); - search.sort(PprofTaskLogRecord.OPERATION_TIME, Sort.Order.DESC); - + final SearchBuilder search = + Search.builder().query(query) + .sort(PprofTaskLogRecord.OPERATION_TIME, Sort.Order.DESC) + .size(queryMaxSize); final SearchResponse response = getClient().search(index, search.build()); List tasks = new LinkedList<>(); - for (SearchHit hit : response.getHits().getHits()) { - tasks.add(parseTaskLog(hit)); + for (SearchHit searchHit : response.getHits().getHits()) { + tasks.add(buildPprofTaskLog(searchHit)); } return tasks; } - private PprofTaskLog parseTaskLog(SearchHit data) { + private PprofTaskLog buildPprofTaskLog(SearchHit data) { Map source = data.getSource(); - int operationTypeInt = ((Number) source.get(PprofTaskLogRecord.OPERATION_TYPE)).intValue(); PprofTaskLogOperationType operationType = PprofTaskLogOperationType.parse(operationTypeInt); - return PprofTaskLog.builder() .id((String) source.get(PprofTaskLogRecord.TASK_ID)) .instanceId((String) source.get(PprofTaskLogRecord.INSTANCE_ID)) diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofTaskQueryEsDAO.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofTaskQueryEsDAO.java index eb41c21fb7a9..555e39cabe4e 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofTaskQueryEsDAO.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofTaskQueryEsDAO.java @@ -23,6 +23,7 @@ import java.util.List; import java.util.Map; import com.google.gson.Gson; +import java.util.Objects; import com.google.gson.reflect.TypeToken; import java.lang.reflect.Type; @@ -54,8 +55,8 @@ public PprofTaskQueryEsDAO(ElasticSearchClient client, int queryMaxSize) { @Override public List getTaskList(String serviceId, Long startTimeBucket, Long endTimeBucket, Integer limit) throws IOException { - final String index = IndexController.LogicIndicesRegister.getPhysicalTableName(PprofTaskRecord.INDEX_NAME); - final BoolQueryBuilder query = Query.bool(); + String index = IndexController.LogicIndicesRegister.getPhysicalTableName(PprofTaskRecord.INDEX_NAME); + BoolQueryBuilder query = Query.bool(); if (IndexController.LogicIndicesRegister.isMergedTable(PprofTaskRecord.INDEX_NAME)) { query.must(Query.term(IndexController.LogicIndicesRegister.RECORD_TABLE_NAME, PprofTaskRecord.INDEX_NAME)); } @@ -71,14 +72,8 @@ public List getTaskList(String serviceId, Long startTimeBucket, Long if (endTimeBucket != null) { query.must(Query.range(PprofTaskRecord.TIME_BUCKET).lte(endTimeBucket)); } - - final SearchBuilder search = Search.builder().query(query); - - if (limit != null) { - search.size(limit); - } else { - search.size(queryMaxSize); - } + SearchBuilder search = Search.builder().query(query); + search.size(Objects.requireNonNullElse(limit, queryMaxSize)); search.sort(PprofTaskRecord.CREATE_TIME, Sort.Order.DESC); @@ -92,7 +87,7 @@ public List getTaskList(String serviceId, Long startTimeBucket, Long } @Override - public PprofTask getById(String id) throws IOException { + public PprofTask getById(String id) { if (StringUtil.isEmpty(id)) { return null; } @@ -125,7 +120,7 @@ private PprofTask parseTask(SearchHit data) { .serviceId((String) source.get(PprofTaskRecord.SERVICE_ID)) .serviceInstanceIds(instanceIdList) .createTime(((Number) source.get(PprofTaskRecord.CREATE_TIME)).longValue()) - .events((PprofEventType) source.get(PprofTaskRecord.EVENT_TYPES)) + .events(PprofEventType.valueOfString((String) source.get(PprofTaskRecord.EVENT_TYPES))) .duration(((Number) source.get(PprofTaskRecord.DURATION)).intValue()) .dumpPeriod(((Number) source.get(PprofTaskRecord.DUMP_PERIOD)).intValue()) .build(); diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/pom.xml b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/pom.xml index 3285d110a4e1..c56a24e86a19 100644 --- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/pom.xml +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/pom.xml @@ -48,6 +48,11 @@ org.postgresql postgresql + + mysql + mysql-connector-java + 8.0.13 + org.testcontainers diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/JDBCStorageProvider.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/JDBCStorageProvider.java index 2fd5cbcb4927..7891c1c7b122 100644 --- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/JDBCStorageProvider.java +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/JDBCStorageProvider.java @@ -33,6 +33,9 @@ import org.apache.skywalking.oap.server.core.storage.profiling.asyncprofiler.IAsyncProfilerTaskLogQueryDAO; import org.apache.skywalking.oap.server.core.storage.profiling.asyncprofiler.IAsyncProfilerTaskQueryDAO; import org.apache.skywalking.oap.server.core.storage.profiling.asyncprofiler.IJFRDataQueryDAO; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskLogQueryDAO; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskQueryDAO; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofDataQueryDAO; import org.apache.skywalking.oap.server.core.storage.profiling.continuous.IContinuousProfilingPolicyDAO; import org.apache.skywalking.oap.server.core.storage.profiling.ebpf.IEBPFProfilingDataDAO; import org.apache.skywalking.oap.server.core.storage.profiling.ebpf.IEBPFProfilingScheduleDAO; @@ -66,6 +69,9 @@ import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao.JDBCAlarmQueryDAO; import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao.JDBCAsyncProfilerTaskLogQueryDAO; import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao.JDBCAsyncProfilerTaskQueryDAO; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao.JDBCPprofDataQueryDAO; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao.JDBCPprofTaskLogQueryDAO; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao.JDBCPprofTaskQueryDAO; import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao.JDBCBatchDAO; import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao.JDBCBrowserLogQueryDAO; import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao.JDBCContinuousProfilingPolicyDAO; @@ -255,6 +261,18 @@ public void prepare() throws ServiceNotProvidedException, ModuleStartException { IJFRDataQueryDAO.class, new JDBCJFRDataQueryDAO(jdbcClient, tableHelper) ); + this.registerServiceImplementation( + IPprofTaskQueryDAO.class, + new JDBCPprofTaskQueryDAO(jdbcClient, tableHelper) + ); + this.registerServiceImplementation( + IPprofTaskLogQueryDAO.class, + new JDBCPprofTaskLogQueryDAO(jdbcClient, tableHelper) + ); + this.registerServiceImplementation( + IPprofDataQueryDAO.class, + new JDBCPprofDataQueryDAO(jdbcClient, tableHelper) + ); this.registerServiceImplementation( StorageTTLStatusQuery.class, new DefaultStorageTTLStatusQuery() diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCPprofDataQueryDAO.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCPprofDataQueryDAO.java new file mode 100644 index 000000000000..6a0aa9055f4a --- /dev/null +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCPprofDataQueryDAO.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + package org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao; + + import lombok.RequiredArgsConstructor; + import lombok.SneakyThrows; + import org.apache.skywalking.oap.server.core.profiling.pprof.storage.PprofProfilingDataRecord; + import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofDataQueryDAO; + import org.apache.skywalking.oap.server.library.client.jdbc.hikaricp.JDBCClient; + import org.apache.skywalking.oap.server.library.util.CollectionUtils; + import org.apache.skywalking.oap.server.library.util.StringUtil; + import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.JDBCEntityConverters; + import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.JDBCTableInstaller; + import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.TableHelper; + + import java.io.IOException; + import java.sql.ResultSet; + import java.util.ArrayList; + import java.util.Base64; + import java.util.List; + + @RequiredArgsConstructor + public class JDBCPprofDataQueryDAO implements IPprofDataQueryDAO { + private final JDBCClient jdbcClient; + private final TableHelper tableHelper; + + @Override + @SneakyThrows + public List getByTaskIdAndInstances(String taskId, List instanceIds) throws IOException { + if (StringUtil.isBlank(taskId)) { + return new ArrayList<>(); + } + List tables = tableHelper.getTablesWithinTTL(PprofProfilingDataRecord.INDEX_NAME); + List results = new ArrayList<>(); + for (final var table : tables) { + List condition = new ArrayList<>(4); + StringBuilder sql = new StringBuilder() + .append("select * from ").append(table) + .append(" where ").append(JDBCTableInstaller.TABLE_COLUMN).append(" = ?"); + condition.add(PprofProfilingDataRecord.INDEX_NAME); + + if (CollectionUtils.isNotEmpty(instanceIds)) { + sql.append(" and ").append(PprofProfilingDataRecord.INSTANCE_ID).append(" in (?) "); + String joinedInstanceIds = String.join(",", instanceIds); + condition.add(joinedInstanceIds); + } + + results.addAll( + jdbcClient.executeQuery( + sql.toString(), + resultSet -> { + final var result = new ArrayList(); + while (resultSet.next()) { + result.add(parseData(resultSet)); + } + return result; + }, + condition.toArray(new Object[0])) + ); + } + return results; + } + + private PprofProfilingDataRecord parseData(ResultSet data) { + final PprofProfilingDataRecord.Builder builder = new PprofProfilingDataRecord.Builder(); + PprofProfilingDataRecord PprofProfilingDataRecord = builder.storage2Entity(JDBCEntityConverters.toEntity(data)); + byte[] dataBinary = PprofProfilingDataRecord.getDataBinary(); + if (dataBinary != null) { + byte[] decodeResult = Base64.getDecoder().decode(dataBinary); + PprofProfilingDataRecord.setDataBinary(decodeResult); + } + return PprofProfilingDataRecord; + } + } + \ No newline at end of file diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCPprofTaskLogQueryDAO.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCPprofTaskLogQueryDAO.java new file mode 100644 index 000000000000..ea6da7dd81d1 --- /dev/null +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCPprofTaskLogQueryDAO.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + package org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao; + +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import org.apache.skywalking.oap.server.core.profiling.pprof.storage.PprofTaskLogRecord; +import org.apache.skywalking.oap.server.core.query.PprofTaskLog; +import org.apache.skywalking.oap.server.core.query.type.PprofTaskLogOperationType; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskLogQueryDAO; +import org.apache.skywalking.oap.server.library.client.jdbc.hikaricp.JDBCClient; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.JDBCTableInstaller; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.SQLAndParameters; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.TableHelper; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +@RequiredArgsConstructor +public class JDBCPprofTaskLogQueryDAO implements IPprofTaskLogQueryDAO { + private final JDBCClient jdbcClient; + private final TableHelper tableHelper; + + @Override + @SneakyThrows + public List getTaskLogList() { + List tables = tableHelper.getTablesWithinTTL(PprofTaskLogRecord.INDEX_NAME); + final List results = new ArrayList(); + for (String table : tables) { + SQLAndParameters sqlAndParameters = buildSQL(table); + List logs = jdbcClient.executeQuery( + sqlAndParameters.sql(), + resultSet -> { + final List tasks = new ArrayList<>(); + while (resultSet.next()) { + tasks.add(parseLog(resultSet)); + } + return tasks; + }, + sqlAndParameters.parameters()); + results.addAll(logs); + } + return results; + } + + private SQLAndParameters buildSQL(String table) { + StringBuilder sql = new StringBuilder(); + List parameters = new ArrayList<>(2); + sql.append("select * from ").append(table) + .append(" where ").append(JDBCTableInstaller.TABLE_COLUMN).append(" = ?"); + parameters.add(PprofTaskLogRecord.INDEX_NAME); + sql.append(" order by ").append(PprofTaskLogRecord.OPERATION_TIME).append(" desc"); + return new SQLAndParameters(sql.toString(), parameters); + } + + private PprofTaskLog parseLog(ResultSet data) throws SQLException { + return PprofTaskLog.builder() + .id(data.getString(PprofTaskLogRecord.TASK_ID)) + .instanceId(data.getString(PprofTaskLogRecord.INSTANCE_ID)) + .operationType(PprofTaskLogOperationType.parse(data.getInt(PprofTaskLogRecord.OPERATION_TYPE))) + .operationTime(data.getLong(PprofTaskLogRecord.OPERATION_TIME)) + .build(); + } +} diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCPprofTaskQueryDAO.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCPprofTaskQueryDAO.java new file mode 100644 index 000000000000..645accbb4d42 --- /dev/null +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCPprofTaskQueryDAO.java @@ -0,0 +1,155 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao; + +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import org.apache.skywalking.oap.server.core.profiling.pprof.storage.PprofTaskRecord; +import org.apache.skywalking.oap.server.core.query.type.PprofEventType; +import org.apache.skywalking.oap.server.core.query.type.PprofTask; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskQueryDAO; +import org.apache.skywalking.oap.server.library.client.jdbc.hikaricp.JDBCClient; +import org.apache.skywalking.oap.server.library.util.StringUtil; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.JDBCTableInstaller; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.TableHelper; + +import java.io.IOException; +import java.lang.reflect.Type; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +import static java.util.stream.Collectors.toList; + +@RequiredArgsConstructor +public class JDBCPprofTaskQueryDAO implements IPprofTaskQueryDAO { + private static final Gson GSON = new Gson(); + + private final JDBCClient jdbcClient; + private final TableHelper tableHelper; + + @Override + @SneakyThrows + public List getTaskList(String serviceId, Long startTimeBucket, Long endTimeBucket, Integer limit) throws IOException { + final var results = new ArrayList(); + final var tables = startTimeBucket == null || endTimeBucket == null ? + tableHelper.getTablesWithinTTL(PprofTaskRecord.INDEX_NAME) : + tableHelper.getTablesForRead(PprofTaskRecord.INDEX_NAME, startTimeBucket, endTimeBucket); + for (final var table : tables) { + List condition = new ArrayList<>(4); + StringBuilder sql = new StringBuilder() + .append("select * from ").append(table) + .append(" where ").append(JDBCTableInstaller.TABLE_COLUMN).append(" = ?"); + condition.add(PprofTaskRecord.INDEX_NAME); + + if (StringUtil.isNotEmpty(serviceId)) { + sql.append(" and ").append(PprofTaskRecord.SERVICE_ID).append("=? "); + condition.add(serviceId); + } + + if (startTimeBucket != null) { + sql.append(" and ").append(PprofTaskRecord.TIME_BUCKET).append(" >= ? "); + condition.add(startTimeBucket); + } + + if (endTimeBucket != null) { + sql.append(" and ").append(PprofTaskRecord.TIME_BUCKET).append(" <= ? "); + condition.add(endTimeBucket); + } + + sql.append(" ORDER BY ").append(PprofTaskRecord.CREATE_TIME).append(" DESC "); + + if (limit != null) { + sql.append(" LIMIT ").append(limit); + } + + results.addAll( + jdbcClient.executeQuery( + sql.toString(), + resultSet -> { + final var tasks = new ArrayList(); + while (resultSet.next()) { + tasks.add(buildPprofTask(resultSet)); + } + return tasks; + }, + condition.toArray(new Object[0])) + ); + } + return limit == null ? + results : + results + .stream() + .limit(limit) + .collect(toList()); + } + + @Override + @SneakyThrows + public PprofTask getById(String id) throws IOException { + final var tables = tableHelper.getTablesWithinTTL(PprofTaskRecord.INDEX_NAME); + for (String table : tables) { + final StringBuilder sql = new StringBuilder(); + final List condition = new ArrayList<>(1); + sql.append("select * from ").append(table) + .append(" where ") + .append(JDBCTableInstaller.TABLE_COLUMN).append(" = ? ") + .append(" and ") + .append(PprofTaskRecord.TASK_ID + "=? LIMIT 1"); + condition.add(PprofTaskRecord.INDEX_NAME); + condition.add(id); + + final var r = jdbcClient.executeQuery( + sql.toString(), + resultSet -> { + if (resultSet.next()) { + return buildPprofTask(resultSet); + } + return null; + }, + condition.toArray(new Object[0])); + if (r != null) { + return r; + } + } + return null; + } + + private PprofTask buildPprofTask(ResultSet data) throws SQLException { + Type listType = new TypeToken>() { + }.getType(); + String events = data.getString(PprofTaskRecord.EVENT_TYPES); + String serviceInstanceIds = data.getString(PprofTaskRecord.SERVICE_INSTANCE_IDS); + List serviceInstanceIdList = GSON.fromJson(serviceInstanceIds, listType); + return PprofTask.builder() + .id(data.getString(PprofTaskRecord.TASK_ID)) + .serviceId(data.getString(PprofTaskRecord.SERVICE_ID)) + .serviceInstanceIds(serviceInstanceIdList) + .createTime(data.getLong(PprofTaskRecord.CREATE_TIME)) + .duration(data.getInt(PprofTaskRecord.DURATION)) + .events(PprofEventType.valueOfString(events)) + .dumpPeriod(data.getInt(PprofTaskRecord.DUMP_PERIOD)) + .build(); + } + +}