diff --git a/changelog/unreleased/SOLR-17436-v2-metrics-api.yml b/changelog/unreleased/SOLR-17436-v2-metrics-api.yml new file mode 100644 index 000000000000..e6378c056480 --- /dev/null +++ b/changelog/unreleased/SOLR-17436-v2-metrics-api.yml @@ -0,0 +1,8 @@ +# See https://github.com/apache/solr/blob/main/dev-docs/changelog.adoc +title: Create a v2 equivalent for /admin/metrics +type: added # added, changed, fixed, deprecated, removed, dependency_update, security, other +authors: + - name: Isabelle Giguère +links: + - name: SOLR-17436 + url: https://issues.apache.org/jira/browse/SOLR-17436 diff --git a/solr/api/src/java/org/apache/solr/client/api/endpoint/MetricsApi.java b/solr/api/src/java/org/apache/solr/client/api/endpoint/MetricsApi.java new file mode 100644 index 000000000000..64efee83591d --- /dev/null +++ b/solr/api/src/java/org/apache/solr/client/api/endpoint/MetricsApi.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.api.endpoint; + +import static org.apache.solr.client.api.util.Constants.RAW_OUTPUT_PROPERTY; + +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.extensions.Extension; +import io.swagger.v3.oas.annotations.extensions.ExtensionProperty; +import io.swagger.v3.oas.annotations.media.Schema; +import jakarta.ws.rs.GET; +import jakarta.ws.rs.HeaderParam; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.QueryParam; +import jakarta.ws.rs.core.StreamingOutput; + +/** V2 API definitions to fetch metrics. */ +@Path("/metrics") +public interface MetricsApi { + + @GET + @Operation( + summary = "Retrieve metrics gathered by Solr.", + tags = {"metrics"}, + extensions = { + @Extension(properties = {@ExtensionProperty(name = RAW_OUTPUT_PROPERTY, value = "true")}) + }) + StreamingOutput getMetrics( + @HeaderParam("Accept") String acceptHeader, + @Parameter( + schema = + @Schema( + name = "node", + description = "Name of the node to which proxy the request.", + defaultValue = "all")) + @QueryParam(value = "node") + String node, + @Parameter(schema = @Schema(name = "name", description = "The metric name to filter on.")) + @QueryParam(value = "name") + String name, + @Parameter( + schema = @Schema(name = "category", description = "The category label to filter on.")) + @QueryParam(value = "category") + String category, + @Parameter( + schema = + @Schema( + name = "core", + description = + "TThe core name to filter on. More than one core can be specified in a comma-separated list.")) + @QueryParam(value = "core") + String core, + @Parameter( + schema = + @Schema(name = "collection", description = "The collection name to filter on. ")) + @QueryParam(value = "collection") + String collection, + @Parameter(schema = @Schema(name = "shard", description = "The shard name to filter on.")) + @QueryParam(value = "shard") + String shard, + @Parameter( + schema = + @Schema( + name = "replica_type", + description = "The replica type to filter on.", + allowableValues = {"NRT", "TLOG", "PULL"})) + @QueryParam(value = "replica_type") + String replicaType); +} diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java index c2f0fd3d500b..76c65ed62539 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java @@ -874,11 +874,16 @@ public static void checkDiskSpace( new ModifiableSolrParams() .add("key", indexSizeMetricName) .add("key", freeDiskSpaceMetricName); + // TODO: SOLR-17955 SolrResponse rsp = new MetricsRequest(params).process(cloudManager.getSolrClient()); + if (rsp == null) { + log.warn("No Solr response available from parent shard leader."); + return; + } Number size = (Number) rsp.getResponse()._get(List.of("metrics", indexSizeMetricName), null); if (size == null) { - log.warn("cannot verify information for parent shard leader"); + log.warn("missing index size information for parent shard leader"); return; } double indexSize = size.doubleValue(); diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.java index 24c7dc83b0c9..de83b5a1ab8e 100644 --- a/solr/core/src/java/org/apache/solr/core/SolrCore.java +++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java @@ -17,8 +17,6 @@ package org.apache.solr.core; import static org.apache.solr.common.params.CommonParams.PATH; -import static org.apache.solr.handler.admin.MetricsHandler.OPEN_METRICS_WT; -import static org.apache.solr.handler.admin.MetricsHandler.PROMETHEUS_METRICS_WT; import static org.apache.solr.metrics.SolrCoreMetricManager.COLLECTION_ATTR; import static org.apache.solr.metrics.SolrCoreMetricManager.CORE_ATTR; import static org.apache.solr.metrics.SolrCoreMetricManager.REPLICA_TYPE_ATTR; @@ -3103,8 +3101,8 @@ public PluginBag getResponseWriters() { m.put("csv", new CSVResponseWriter()); m.put("schema.xml", new SchemaXmlResponseWriter()); m.put("smile", new SmileResponseWriter()); - m.put(PROMETHEUS_METRICS_WT, new PrometheusResponseWriter()); - m.put(OPEN_METRICS_WT, new PrometheusResponseWriter()); + m.put(MetricUtils.PROMETHEUS_METRICS_WT, new PrometheusResponseWriter()); + m.put(MetricUtils.OPEN_METRICS_WT, new PrometheusResponseWriter()); m.put(ReplicationAPIBase.FILE_STREAM, getFileStreamWriter()); DEFAULT_RESPONSE_WRITERS = Collections.unmodifiableMap(m); } diff --git a/solr/core/src/java/org/apache/solr/handler/admin/AdminHandlersProxy.java b/solr/core/src/java/org/apache/solr/handler/admin/AdminHandlersProxy.java index 5f253c4ec4e7..a91db17d9bbb 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/AdminHandlersProxy.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/AdminHandlersProxy.java @@ -33,15 +33,18 @@ import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.request.GenericSolrRequest; +import org.apache.solr.client.solrj.request.GenericV2SolrRequest; import org.apache.solr.client.solrj.response.InputStreamResponseParser; import org.apache.solr.cloud.ZkController; import org.apache.solr.common.SolrException; +import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.params.SolrParams; import org.apache.solr.common.util.NamedList; import org.apache.solr.core.CoreContainer; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; +import org.apache.solr.util.stats.MetricUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -55,17 +58,34 @@ public class AdminHandlersProxy { private static final String PARAM_NODE = "node"; private static final long PROMETHEUS_FETCH_TIMEOUT_SECONDS = 10; - /** Proxy this request to a different remote node if 'node' or 'nodes' parameter is provided */ + /** + * Proxy this request to a different remote node's V1 API if 'node' or 'nodes' parameter is + * provided. For V2, use {@link AdminHandlersProxy#maybeProxyToNodes(String, SolrQueryRequest, + * SolrQueryResponse, CoreContainer)} + */ public static boolean maybeProxyToNodes( SolrQueryRequest req, SolrQueryResponse rsp, CoreContainer container) throws IOException, SolrServerException, InterruptedException { + return maybeProxyToNodes("V1", req, rsp, container); + } + + /** + * Proxy this request to a different remote node's selected API version if 'node' or 'nodes' + * parameter is provided + */ + public static boolean maybeProxyToNodes( + String apiVersion, SolrQueryRequest req, SolrQueryResponse rsp, CoreContainer container) + throws IOException, SolrServerException, InterruptedException { String pathStr = req.getPath(); ModifiableSolrParams params = new ModifiableSolrParams(req.getParams()); // Check if response format is Prometheus/OpenMetrics - String wt = params.get("wt"); - boolean isPrometheusFormat = "prometheus".equals(wt) || "openmetrics".equals(wt); + String wt = params.get(CommonParams.WT); + boolean isPrometheusFormat = + MetricUtils.PROMETHEUS_METRICS_WT.equals(wt) + || MetricUtils.OPEN_METRICS_WT.equals(wt) + || (wt == null && pathStr.endsWith("/metrics")); if (isPrometheusFormat) { // Prometheus format: use singular 'node' parameter for single-node proxy @@ -75,7 +95,7 @@ public static boolean maybeProxyToNodes( } params.remove(PARAM_NODE); - handlePrometheusSingleNode(nodeName, pathStr, params, container, rsp); + handlePrometheusSingleNode(apiVersion, nodeName, pathStr, params, container, rsp); } else { // Other formats (JSON/XML): use plural 'nodes' parameter for multi-node aggregation String nodeNames = req.getParams().get(PARAM_NODES); @@ -85,7 +105,7 @@ public static boolean maybeProxyToNodes( params.remove(PARAM_NODES); Set nodes = resolveNodes(nodeNames, container); - handleNamedListFormat(nodes, pathStr, params, container.getZkController(), rsp); + handleNamedListFormat(apiVersion, nodes, pathStr, params, container.getZkController(), rsp); } return true; @@ -93,6 +113,7 @@ public static boolean maybeProxyToNodes( /** Handle non-Prometheus formats using the existing NamedList approach. */ private static void handleNamedListFormat( + String apiVersion, Set nodes, String pathStr, SolrParams params, @@ -101,7 +122,7 @@ private static void handleNamedListFormat( Map>> responses = new LinkedHashMap<>(); for (String node : nodes) { - responses.put(node, callRemoteNode(node, pathStr, params, zkController)); + responses.put(node, callRemoteNode(apiVersion, node, pathStr, params, zkController)); } for (Map.Entry>> entry : responses.entrySet()) { @@ -125,8 +146,12 @@ private static void handleNamedListFormat( } /** Makes a remote request asynchronously. */ - public static CompletableFuture> callRemoteNode( - String nodeName, String uriPath, SolrParams params, ZkController zkController) { + private static CompletableFuture> callRemoteNode( + String apiVersion, + String nodeName, + String uriPath, + SolrParams params, + ZkController zkController) { // Validate that the node exists in the cluster if (!zkController.zkStateReader.getClusterState().getLiveNodes().contains(nodeName)) { @@ -137,13 +162,17 @@ public static CompletableFuture> callRemoteNode( log.debug("Proxying {} request to node {}", uriPath, nodeName); URI baseUri = URI.create(zkController.zkStateReader.getBaseUrlForNodeName(nodeName)); - SolrRequest proxyReq = new GenericSolrRequest(SolrRequest.METHOD.GET, uriPath, params); + + SolrRequest proxyReq = createRequest(apiVersion, uriPath, params); // Set response parser based on wt parameter to ensure correct format is used - String wt = params.get("wt"); - if ("prometheus".equals(wt) || "openmetrics".equals(wt)) { + String wt = params.get(CommonParams.WT); + if (MetricUtils.PROMETHEUS_METRICS_WT.equals(wt) || MetricUtils.OPEN_METRICS_WT.equals(wt)) { proxyReq.setResponseParser(new InputStreamResponseParser(wt)); } + if (wt == null && uriPath.endsWith("/metrics")) { + proxyReq.setResponseParser(new InputStreamResponseParser(MetricUtils.PROMETHEUS_METRICS_WT)); + } try { return zkController @@ -195,6 +224,7 @@ private static Set resolveNodes(String nodeNames, CoreContainer containe * @param rsp the response to populate */ private static void handlePrometheusSingleNode( + String apiVersion, String nodeName, String pathStr, ModifiableSolrParams params, @@ -205,7 +235,7 @@ private static void handlePrometheusSingleNode( // Keep wt=prometheus for the remote request so MetricsHandler accepts it // The InputStreamResponseParser will return the Prometheus text in a "stream" key Future> response = - callRemoteNode(nodeName, pathStr, params, container.getZkController()); + callRemoteNode(apiVersion, nodeName, pathStr, params, container.getZkController()); try { try { @@ -220,4 +250,12 @@ private static void handlePrometheusSingleNode( throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, t); } } + + private static SolrRequest createRequest( + String apiVersion, String uriPath, SolrParams params) { + if (apiVersion.equalsIgnoreCase("V1")) { + return new GenericSolrRequest(SolrRequest.METHOD.GET, uriPath, params); + } + return new GenericV2SolrRequest(SolrRequest.METHOD.GET, uriPath, params); + } } diff --git a/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java index 13b4d044c7e2..53405797f336 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java @@ -17,28 +17,21 @@ package org.apache.solr.handler.admin; -import io.prometheus.metrics.model.snapshots.CounterSnapshot; -import io.prometheus.metrics.model.snapshots.GaugeSnapshot; -import io.prometheus.metrics.model.snapshots.HistogramSnapshot; -import io.prometheus.metrics.model.snapshots.InfoSnapshot; import io.prometheus.metrics.model.snapshots.MetricSnapshot; import io.prometheus.metrics.model.snapshots.MetricSnapshots; import java.util.ArrayList; -import java.util.HashMap; +import java.util.Collection; import java.util.List; -import java.util.Map; import java.util.Set; import java.util.SortedMap; -import java.util.TreeMap; import java.util.function.BiConsumer; -import java.util.regex.Pattern; +import org.apache.solr.api.JerseyResource; import org.apache.solr.common.SolrException; import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.params.SolrParams; -import org.apache.solr.common.util.CommonTestInjection; -import org.apache.solr.common.util.StrUtils; import org.apache.solr.core.CoreContainer; import org.apache.solr.handler.RequestHandlerBase; +import org.apache.solr.handler.admin.api.GetMetrics; import org.apache.solr.metrics.SolrMetricManager; import org.apache.solr.metrics.otel.FilterablePrometheusMetricReader; import org.apache.solr.request.SolrQueryRequest; @@ -46,6 +39,7 @@ import org.apache.solr.response.SolrQueryResponse; import org.apache.solr.security.AuthorizationContext; import org.apache.solr.security.PermissionNameProvider; +import org.apache.solr.util.stats.MetricUtils; /** Request handler to return metrics */ public class MetricsHandler extends RequestHandlerBase implements PermissionNameProvider { @@ -61,25 +55,9 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionName public static final String EXPR_PARAM = "expr"; public static final String TYPE_PARAM = "type"; - // Prometheus filtering parameters - public static final String CATEGORY_PARAM = "category"; - public static final String CORE_PARAM = "core"; - public static final String COLLECTION_PARAM = "collection"; - public static final String SHARD_PARAM = "shard"; - public static final String REPLICA_TYPE_PARAM = "replica_type"; - public static final String METRIC_NAME_PARAM = "name"; - private static final Set labelFilterKeys = - Set.of(CATEGORY_PARAM, CORE_PARAM, COLLECTION_PARAM, SHARD_PARAM, REPLICA_TYPE_PARAM); - - public static final String PROMETHEUS_METRICS_WT = "prometheus"; - public static final String OPEN_METRICS_WT = "openmetrics"; - public static final String ALL = "all"; - private static final Pattern KEY_SPLIT_REGEX = - Pattern.compile("(? injectedSysProps = CommonTestInjection.injectAdditionalProps(); private final boolean enabled; public MetricsHandler(CoreContainer coreContainer) { @@ -115,7 +93,8 @@ public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throw if (format == null) { req.setParams(SolrParams.wrapDefaults(params, SolrParams.of("wt", "prometheus"))); - } else if (!PROMETHEUS_METRICS_WT.equals(format) && !OPEN_METRICS_WT.equals(format)) { + } else if (!MetricUtils.PROMETHEUS_METRICS_WT.equals(format) + && !MetricUtils.OPEN_METRICS_WT.equals(format)) { throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, "Only Prometheus and OpenMetrics metric formats supported. Unsupported format requested: " @@ -139,13 +118,13 @@ private void handleRequest(SolrParams params, BiConsumer consume return; } - Set metricNames = readParamsAsSet(params, METRIC_NAME_PARAM); - SortedMap> labelFilters = labelFilters(params); + Set metricNames = MetricUtils.readParamsAsSet(params, MetricUtils.METRIC_NAME_PARAM); + SortedMap> labelFilters = MetricUtils.labelFilters(params); if (metricNames.isEmpty() && labelFilters.isEmpty()) { consumer.accept( "metrics", - mergeSnapshots( + MetricUtils.mergeSnapshots( metricManager.getPrometheusMetricReaders().values().stream() .flatMap(r -> r.collect().stream()) .toList())); @@ -160,119 +139,10 @@ private void handleRequest(SolrParams params, BiConsumer consume } // Merge all filtered snapshots and return the merged result - MetricSnapshots mergedSnapshots = mergeSnapshots(allSnapshots); + MetricSnapshots mergedSnapshots = MetricUtils.mergeSnapshots(allSnapshots); consumer.accept("metrics", mergedSnapshots); } - private SortedMap> labelFilters(SolrParams params) { - SortedMap> labelFilters = new TreeMap<>(); - labelFilterKeys.forEach( - (paramName) -> { - Set filterValues = readParamsAsSet(params, paramName); - if (!filterValues.isEmpty()) { - labelFilters.put(paramName, filterValues); - } - }); - - return labelFilters; - } - - private Set readParamsAsSet(SolrParams params, String paramName) { - String[] paramValues = params.getParams(paramName); - if (paramValues == null || paramValues.length == 0) { - return Set.of(); - } - - List paramSet = new ArrayList<>(); - for (String param : paramValues) { - paramSet.addAll(StrUtils.splitSmart(param, ',')); - } - return Set.copyOf(paramSet); - } - - /** - * Merge a collection of individual {@link MetricSnapshot} instances into one {@link - * MetricSnapshots}. This is necessary because we create a {@link - * io.opentelemetry.sdk.metrics.SdkMeterProvider} per Solr core resulting in duplicate metric - * names across cores which is an illegal format if under the same prometheus grouping. - */ - private MetricSnapshots mergeSnapshots(List snapshots) { - Map counterSnapshotMap = new HashMap<>(); - Map gaugeSnapshotMap = new HashMap<>(); - Map histogramSnapshotMap = new HashMap<>(); - InfoSnapshot otelInfoSnapshots = null; - - for (MetricSnapshot snapshot : snapshots) { - String metricName = snapshot.getMetadata().getPrometheusName(); - - switch (snapshot) { - case CounterSnapshot counterSnapshot -> { - CounterSnapshot.Builder builder = - counterSnapshotMap.computeIfAbsent( - metricName, - k -> { - var base = - CounterSnapshot.builder() - .name(counterSnapshot.getMetadata().getName()) - .help(counterSnapshot.getMetadata().getHelp()); - return counterSnapshot.getMetadata().hasUnit() - ? base.unit(counterSnapshot.getMetadata().getUnit()) - : base; - }); - counterSnapshot.getDataPoints().forEach(builder::dataPoint); - } - case GaugeSnapshot gaugeSnapshot -> { - GaugeSnapshot.Builder builder = - gaugeSnapshotMap.computeIfAbsent( - metricName, - k -> { - var base = - GaugeSnapshot.builder() - .name(gaugeSnapshot.getMetadata().getName()) - .help(gaugeSnapshot.getMetadata().getHelp()); - return gaugeSnapshot.getMetadata().hasUnit() - ? base.unit(gaugeSnapshot.getMetadata().getUnit()) - : base; - }); - gaugeSnapshot.getDataPoints().forEach(builder::dataPoint); - } - case HistogramSnapshot histogramSnapshot -> { - HistogramSnapshot.Builder builder = - histogramSnapshotMap.computeIfAbsent( - metricName, - k -> { - var base = - HistogramSnapshot.builder() - .name(histogramSnapshot.getMetadata().getName()) - .help(histogramSnapshot.getMetadata().getHelp()); - return histogramSnapshot.getMetadata().hasUnit() - ? base.unit(histogramSnapshot.getMetadata().getUnit()) - : base; - }); - histogramSnapshot.getDataPoints().forEach(builder::dataPoint); - } - case InfoSnapshot infoSnapshot -> { - // InfoSnapshot is a special case in that each SdkMeterProvider will create a duplicate - // metric called target_info containing OTEL SDK metadata. Only one of these need to be - // kept - if (otelInfoSnapshots == null) - otelInfoSnapshots = - new InfoSnapshot(infoSnapshot.getMetadata(), infoSnapshot.getDataPoints()); - } - default -> { - // Handle unexpected snapshot types gracefully - } - } - } - - MetricSnapshots.Builder snapshotsBuilder = MetricSnapshots.builder(); - counterSnapshotMap.values().forEach(b -> snapshotsBuilder.metricSnapshot(b.build())); - gaugeSnapshotMap.values().forEach(b -> snapshotsBuilder.metricSnapshot(b.build())); - histogramSnapshotMap.values().forEach(b -> snapshotsBuilder.metricSnapshot(b.build())); - if (otelInfoSnapshots != null) snapshotsBuilder.metricSnapshot(otelInfoSnapshots); - return snapshotsBuilder.build(); - } - @Override public String getDescription() { return "A handler to return all the metrics gathered by Solr"; @@ -282,4 +152,14 @@ public String getDescription() { public Category getCategory() { return Category.ADMIN; } + + @Override + public Collection> getJerseyResources() { + return List.of(GetMetrics.class); + } + + @Override + public Boolean registerV2() { + return Boolean.TRUE; + } } diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/GetMetrics.java b/solr/core/src/java/org/apache/solr/handler/admin/api/GetMetrics.java new file mode 100644 index 000000000000..6de821a162ce --- /dev/null +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/GetMetrics.java @@ -0,0 +1,186 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.handler.admin.api; + +import io.prometheus.metrics.model.snapshots.MetricSnapshot; +import io.prometheus.metrics.model.snapshots.MetricSnapshots; +import jakarta.inject.Inject; +import jakarta.ws.rs.WebApplicationException; +import jakarta.ws.rs.core.StreamingOutput; +import java.io.IOException; +import java.io.OutputStream; +import java.lang.invoke.MethodHandles; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.SortedMap; +import org.apache.solr.client.api.endpoint.MetricsApi; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.params.CommonParams; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.common.params.SolrParams; +import org.apache.solr.core.CoreContainer; +import org.apache.solr.handler.admin.AdminHandlersProxy; +import org.apache.solr.jersey.PermissionName; +import org.apache.solr.metrics.SolrMetricManager; +import org.apache.solr.metrics.otel.FilterablePrometheusMetricReader; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.response.PrometheusResponseWriter; +import org.apache.solr.response.SolrQueryResponse; +import org.apache.solr.security.PermissionNameProvider; +import org.apache.solr.util.stats.MetricUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * V2 API implementation to fetch metrics gathered by Solr. + * + *

This API is analogous to the v1 /admin/metrics endpoint. + */ +public class GetMetrics extends AdminAPIBase implements MetricsApi { + + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + private final SolrMetricManager metricManager; + private final boolean enabled; + + @Inject + public GetMetrics( + CoreContainer coreContainer, + SolrQueryRequest solrQueryRequest, + SolrQueryResponse solrQueryResponse) { + super(coreContainer, solrQueryRequest, solrQueryResponse); + this.metricManager = coreContainer.getMetricManager(); + this.enabled = coreContainer.getConfig().getMetricsConfig().isEnabled(); + } + + @Override + @PermissionName(PermissionNameProvider.Name.METRICS_READ_PERM) + public StreamingOutput getMetrics( + String acceptHeader, + String node, + String name, + String category, + String core, + String collection, + String shard, + String replicaType) { + + // Convert request params into SolrParams, to reuse existing code. + ModifiableSolrParams params = + new ModifiableSolrParams( + Map.of( + MetricUtils.NODE_PARAM, new String[] {node}, + MetricUtils.METRIC_NAME_PARAM, new String[] {name}, + MetricUtils.CATEGORY_PARAM, new String[] {category}, + MetricUtils.CORE_PARAM, new String[] {core}, + MetricUtils.COLLECTION_PARAM, new String[] {collection}, + MetricUtils.SHARD_PARAM, new String[] {shard}, + MetricUtils.REPLICA_TYPE_PARAM, new String[] {replicaType})); + + solrQueryRequest.setParams(params); + + validateRequest(acceptHeader); + + // TODO: fix AdminHandlersProxy to support V2 + if (proxyToNodes()) { + return null; + } + + // Using the same logic, same methods, as in MetricsHandler.handleRequest + Set metricNames = MetricUtils.readParamsAsSet(params, MetricUtils.METRIC_NAME_PARAM); + SortedMap> labelFilters = MetricUtils.labelFilters(params); + + return doGetMetrics(metricNames, labelFilters); + } + + private void validateRequest(String acceptHeader) { + if (!enabled) { + log.info("Metrics not enabled"); + throw new SolrException( + SolrException.ErrorCode.INVALID_STATE, "Metrics collection is disabled"); + } + + if (metricManager == null) { + log.info("SolrMetricManager instance not initialized"); + throw new SolrException( + SolrException.ErrorCode.INVALID_STATE, "SolrMetricManager instance not initialized"); + } + + // Should handle 'Accept' header only, but a lot of code still expects 'wt'. + if (acceptHeader == null) { + log.info("Set default wt=prometheus"); + solrQueryRequest.setParams( + SolrParams.wrapDefaults( + solrQueryRequest.getParams(), + SolrParams.of(CommonParams.WT, MetricUtils.PROMETHEUS_METRICS_WT))); + } else if (!PrometheusResponseWriter.CONTENT_TYPE_PROMETHEUS.equals(acceptHeader) + && !PrometheusResponseWriter.CONTENT_TYPE_OPEN_METRICS.equals(acceptHeader)) { + log.info("Unsupported format requested"); + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + "Only Prometheus and OpenMetrics metric formats supported. Unsupported format requested: " + + acceptHeader); + } + } + + private boolean proxyToNodes() { + try { + if (coreContainer != null + && AdminHandlersProxy.maybeProxyToNodes( + "V2", solrQueryRequest, solrQueryResponse, coreContainer)) { + return true; // Request was proxied to other node + } + } catch (Exception e) { + log.warn("Exception proxying to other node", e); + } + return false; + } + + private StreamingOutput doGetMetrics( + Set metricNames, SortedMap> labelFilters) { + + List snapshots = new ArrayList<>(); + + if ((metricNames == null || metricNames.isEmpty()) && labelFilters.isEmpty()) { + snapshots.addAll( + metricManager.getPrometheusMetricReaders().values().stream() + .flatMap(r -> r.collect().stream()) + .toList()); + } else { + for (FilterablePrometheusMetricReader reader : + metricManager.getPrometheusMetricReaders().values()) { + MetricSnapshots filteredSnapshots = reader.collect(metricNames, labelFilters); + filteredSnapshots.forEach(snapshots::add); + } + } + + return writeMetricSnapshots(MetricUtils.mergeSnapshots(snapshots)); + } + + private StreamingOutput writeMetricSnapshots(MetricSnapshots snapshots) { + return new StreamingOutput() { + @Override + public void write(OutputStream output) throws IOException, WebApplicationException { + PrometheusResponseWriter writer = new PrometheusResponseWriter(); + writer.writeMetricSnapshots(output, solrQueryRequest, snapshots); + output.flush(); + } + }; + } +} diff --git a/solr/core/src/java/org/apache/solr/handler/api/V2ApiUtils.java b/solr/core/src/java/org/apache/solr/handler/api/V2ApiUtils.java index 9a69d4e23646..737a63cef1a1 100644 --- a/solr/core/src/java/org/apache/solr/handler/api/V2ApiUtils.java +++ b/solr/core/src/java/org/apache/solr/handler/api/V2ApiUtils.java @@ -32,6 +32,7 @@ import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.StrUtils; import org.apache.solr.common.util.Utils; +import org.apache.solr.response.PrometheusResponseWriter; import org.apache.solr.response.RawResponseWriter; import org.apache.solr.response.SolrQueryResponse; @@ -105,6 +106,10 @@ public static String getMediaTypeFromWtParam(SolrParams params, String defaultMe return JAVABIN_CONTENT_TYPE_V2; case FILE_STREAM: return RawResponseWriter.CONTENT_TYPE; + case "prometheus": + return PrometheusResponseWriter.CONTENT_TYPE_PROMETHEUS; + case "openmetrics": + return PrometheusResponseWriter.CONTENT_TYPE_OPEN_METRICS; default: return defaultMediaType; } diff --git a/solr/core/src/java/org/apache/solr/jersey/JerseyApplications.java b/solr/core/src/java/org/apache/solr/jersey/JerseyApplications.java index 3ac1ef79600a..6eed07d2af82 100644 --- a/solr/core/src/java/org/apache/solr/jersey/JerseyApplications.java +++ b/solr/core/src/java/org/apache/solr/jersey/JerseyApplications.java @@ -48,6 +48,8 @@ public CoreContainerApp() { register(MessageBodyWriters.XmlMessageBodyWriter.class, 5); register(MessageBodyWriters.CsvMessageBodyWriter.class, 5); register(MessageBodyWriters.RawMessageBodyWriter.class, 5); + register(MessageBodyWriters.PrometheusMessageBodyWriter.class, 5); + register(MessageBodyWriters.OpenmetricsMessageBodyWriter.class, 5); register(MessageBodyReaders.CachingJsonMessageBodyReader.class, 2); register(SolrJacksonMapper.class); diff --git a/solr/core/src/java/org/apache/solr/jersey/MessageBodyWriters.java b/solr/core/src/java/org/apache/solr/jersey/MessageBodyWriters.java index 4af087ead58a..4057b8eb7736 100644 --- a/solr/core/src/java/org/apache/solr/jersey/MessageBodyWriters.java +++ b/solr/core/src/java/org/apache/solr/jersey/MessageBodyWriters.java @@ -38,6 +38,7 @@ import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.CSVResponseWriter; import org.apache.solr.response.JavaBinResponseWriter; +import org.apache.solr.response.PrometheusResponseWriter; import org.apache.solr.response.QueryResponseWriter; import org.apache.solr.response.RawResponseWriter; import org.apache.solr.response.SolrQueryResponse; @@ -107,6 +108,39 @@ public String getSupportedMediaType() { } } + // Not sure if required. Ref. org.apache.solr.handler.admin.api.GetMetrics and + // org.apache.solr.response.PrometheusResponseWriter + @Produces(PrometheusResponseWriter.CONTENT_TYPE_PROMETHEUS) + public static class PrometheusMessageBodyWriter extends BaseMessageBodyWriter + implements MessageBodyWriter { + @Override + public QueryResponseWriter createResponseWriter() { + return new PrometheusResponseWriter(); + } + + @Override + public String getSupportedMediaType() { + return PrometheusResponseWriter.CONTENT_TYPE_PROMETHEUS; + } + } + + // Not sure if required. Ref. org.apache.solr.handler.admin.api.GetMetrics and + // org.apache.solr.response.PrometheusResponseWriter + @Produces(PrometheusResponseWriter.CONTENT_TYPE_OPEN_METRICS) + public static class OpenmetricsMessageBodyWriter extends BaseMessageBodyWriter + implements MessageBodyWriter { + @Override + public QueryResponseWriter createResponseWriter() { + // same writer handles both Prometheus and OpenMetrics + return new PrometheusResponseWriter(); + } + + @Override + public String getSupportedMediaType() { + return PrometheusResponseWriter.CONTENT_TYPE_OPEN_METRICS; + } + } + public abstract static class BaseMessageBodyWriter implements MessageBodyWriter { @Context protected ResourceContext resourceContext; diff --git a/solr/core/src/java/org/apache/solr/response/PrometheusResponseWriter.java b/solr/core/src/java/org/apache/solr/response/PrometheusResponseWriter.java index 01a2af194210..d56a85b9bbe5 100644 --- a/solr/core/src/java/org/apache/solr/response/PrometheusResponseWriter.java +++ b/solr/core/src/java/org/apache/solr/response/PrometheusResponseWriter.java @@ -16,8 +16,6 @@ */ package org.apache.solr.response; -import static org.apache.solr.handler.admin.MetricsHandler.OPEN_METRICS_WT; - import io.prometheus.metrics.expositionformats.OpenMetricsTextFormatWriter; import io.prometheus.metrics.expositionformats.PrometheusTextFormatWriter; import io.prometheus.metrics.model.snapshots.MetricSnapshots; @@ -27,15 +25,19 @@ import java.nio.charset.StandardCharsets; import org.apache.solr.common.params.CommonParams; import org.apache.solr.handler.admin.MetricsHandler; +import org.apache.solr.handler.admin.api.GetMetrics; import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.util.stats.MetricUtils; -/** Response writer for Prometheus metrics. This is used only by the {@link MetricsHandler} */ -@SuppressWarnings(value = "unchecked") +/** + * Response writer for Prometheus metrics. This is used only by the {@link MetricsHandler} and V2 + * API implementation {@link GetMetrics} + */ public class PrometheusResponseWriter implements QueryResponseWriter { // not TextQueryResponseWriter because Prometheus libs work with an OutputStream - private static final String CONTENT_TYPE_PROMETHEUS = "text/plain; version=0.0.4"; - private static final String CONTENT_TYPE_OPEN_METRICS = + public static final String CONTENT_TYPE_PROMETHEUS = "text/plain; version=0.0.4"; + public static final String CONTENT_TYPE_OPEN_METRICS = "application/openmetrics-text; version=1.0.0; charset=utf-8"; @Override @@ -64,6 +66,12 @@ public void write( throw new IOException("No metrics found in response"); } MetricSnapshots snapshots = (MetricSnapshots) metrics; + writeMetricSnapshots(out, request, snapshots); + } + + /** Write MetricSnapshots in Prometheus or OpenMetrics format */ + public void writeMetricSnapshots( + OutputStream out, SolrQueryRequest request, MetricSnapshots snapshots) throws IOException { if (writeOpenMetricsFormat(request)) { new OpenMetricsTextFormatWriter(false, true).write(out, snapshots); } else { @@ -78,7 +86,7 @@ public String getContentType(SolrQueryRequest request, SolrQueryResponse respons private boolean writeOpenMetricsFormat(SolrQueryRequest request) { String wt = request.getParams().get(CommonParams.WT); - if (OPEN_METRICS_WT.equals(wt)) { + if (MetricUtils.OPEN_METRICS_WT.equals(wt)) { return true; } diff --git a/solr/core/src/java/org/apache/solr/response/QueryResponseWriter.java b/solr/core/src/java/org/apache/solr/response/QueryResponseWriter.java index f15feac9f67f..83469dfece15 100644 --- a/solr/core/src/java/org/apache/solr/response/QueryResponseWriter.java +++ b/solr/core/src/java/org/apache/solr/response/QueryResponseWriter.java @@ -45,6 +45,9 @@ public interface QueryResponseWriter extends NamedListInitializedPlugin { public static String CONTENT_TYPE_XML_UTF8 = "application/xml; charset=UTF-8"; public static String CONTENT_TYPE_TEXT_UTF8 = "text/plain; charset=UTF-8"; + public static final String CONTENT_TYPE_PROMETHEUS = "text/plain; version=0.0.4"; + public static final String CONTENT_TYPE_OPEN_METRICS = + "application/openmetrics-text; version=1.0.0; charset=utf-8"; /** * Writes the response to the {@link OutputStream}. {@code contentType} is from {@link diff --git a/solr/core/src/java/org/apache/solr/util/stats/MetricUtils.java b/solr/core/src/java/org/apache/solr/util/stats/MetricUtils.java index 5c878ab64763..506799af323c 100644 --- a/solr/core/src/java/org/apache/solr/util/stats/MetricUtils.java +++ b/solr/core/src/java/org/apache/solr/util/stats/MetricUtils.java @@ -18,9 +18,25 @@ import com.codahale.metrics.Snapshot; import com.codahale.metrics.Timer; +import io.prometheus.metrics.model.snapshots.CounterSnapshot; +import io.prometheus.metrics.model.snapshots.GaugeSnapshot; +import io.prometheus.metrics.model.snapshots.HistogramSnapshot; +import io.prometheus.metrics.model.snapshots.InfoSnapshot; +import io.prometheus.metrics.model.snapshots.MetricSnapshot; +import io.prometheus.metrics.model.snapshots.MetricSnapshots; import java.lang.management.OperatingSystemMXBean; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.SortedMap; +import java.util.TreeMap; import java.util.concurrent.TimeUnit; +import org.apache.solr.common.params.SolrParams; import org.apache.solr.common.util.NamedList; +import org.apache.solr.common.util.StrUtils; /** Metrics specific utility functions. */ public class MetricUtils { @@ -49,6 +65,39 @@ public class MetricUtils { private static final String P999 = "p999"; private static final String P999_MS = P999 + MS; + // 'wt' values for V1 Metrics API + public static final String PROMETHEUS_METRICS_WT = "prometheus"; + public static final String OPEN_METRICS_WT = "openmetrics"; + + // Metrics API query params + public static final String NODE_PARAM = "node"; + public static final String CATEGORY_PARAM = "category"; + public static final String CORE_PARAM = "core"; + public static final String COLLECTION_PARAM = "collection"; + public static final String SHARD_PARAM = "shard"; + public static final String REPLICA_TYPE_PARAM = "replica_type"; + public static final String METRIC_NAME_PARAM = "name"; + + private static final Set labelFilterKeys = + Set.of( + MetricUtils.CATEGORY_PARAM, + MetricUtils.CORE_PARAM, + MetricUtils.COLLECTION_PARAM, + MetricUtils.SHARD_PARAM, + MetricUtils.REPLICA_TYPE_PARAM); + + /** + * These are well-known implementations of {@link java.lang.management.OperatingSystemMXBean}. + * Some of them provide additional useful properties beyond those declared by the interface. + */ + public static String[] OS_MXBEAN_CLASSES = + new String[] { + OperatingSystemMXBean.class.getName(), + "com.sun.management.OperatingSystemMXBean", + "com.sun.management.UnixOperatingSystemMXBean", + "com.ibm.lang.management.OperatingSystemMXBean" + }; + /** * Adds metrics from a Timer to a NamedList, using well-known back-compat names. * @@ -89,14 +138,139 @@ public static double bytesToMegabytes(long bytes) { } /** - * These are well-known implementations of {@link java.lang.management.OperatingSystemMXBean}. - * Some of them provide additional useful properties beyond those declared by the interface. + * Merge a collection of individual {@link MetricSnapshot} instances into one {@link + * MetricSnapshots}. This is necessary because we create a {@link + * io.opentelemetry.sdk.metrics.SdkMeterProvider} per Solr core resulting in duplicate metric + * names across cores which is an illegal format if under the same prometheus grouping. */ - public static String[] OS_MXBEAN_CLASSES = - new String[] { - OperatingSystemMXBean.class.getName(), - "com.sun.management.OperatingSystemMXBean", - "com.sun.management.UnixOperatingSystemMXBean", - "com.ibm.lang.management.OperatingSystemMXBean" - }; + public static MetricSnapshots mergeSnapshots(List snapshots) { + Map counterSnapshotMap = new HashMap<>(); + Map gaugeSnapshotMap = new HashMap<>(); + Map histogramSnapshotMap = new HashMap<>(); + InfoSnapshot otelInfoSnapshots = null; + + for (MetricSnapshot snapshot : snapshots) { + String metricName = snapshot.getMetadata().getPrometheusName(); + + switch (snapshot) { + case CounterSnapshot counterSnapshot -> { + CounterSnapshot.Builder builder = + counterSnapshotMap.computeIfAbsent( + metricName, + k -> { + var base = + CounterSnapshot.builder() + .name(counterSnapshot.getMetadata().getName()) + .help(counterSnapshot.getMetadata().getHelp()); + return counterSnapshot.getMetadata().hasUnit() + ? base.unit(counterSnapshot.getMetadata().getUnit()) + : base; + }); + counterSnapshot.getDataPoints().forEach(builder::dataPoint); + } + case GaugeSnapshot gaugeSnapshot -> { + GaugeSnapshot.Builder builder = + gaugeSnapshotMap.computeIfAbsent( + metricName, + k -> { + var base = + GaugeSnapshot.builder() + .name(gaugeSnapshot.getMetadata().getName()) + .help(gaugeSnapshot.getMetadata().getHelp()); + return gaugeSnapshot.getMetadata().hasUnit() + ? base.unit(gaugeSnapshot.getMetadata().getUnit()) + : base; + }); + gaugeSnapshot.getDataPoints().forEach(builder::dataPoint); + } + case HistogramSnapshot histogramSnapshot -> { + HistogramSnapshot.Builder builder = + histogramSnapshotMap.computeIfAbsent( + metricName, + k -> { + var base = + HistogramSnapshot.builder() + .name(histogramSnapshot.getMetadata().getName()) + .help(histogramSnapshot.getMetadata().getHelp()); + return histogramSnapshot.getMetadata().hasUnit() + ? base.unit(histogramSnapshot.getMetadata().getUnit()) + : base; + }); + histogramSnapshot.getDataPoints().forEach(builder::dataPoint); + } + case InfoSnapshot infoSnapshot -> { + // InfoSnapshot is a special case in that each SdkMeterProvider will create a duplicate + // metric called target_info containing OTEL SDK metadata. Only one of these need to be + // kept + if (otelInfoSnapshots == null) + otelInfoSnapshots = + new InfoSnapshot(infoSnapshot.getMetadata(), infoSnapshot.getDataPoints()); + } + default -> { + // Handle unexpected snapshot types gracefully + } + } + } + + MetricSnapshots.Builder snapshotsBuilder = MetricSnapshots.builder(); + counterSnapshotMap.values().forEach(b -> snapshotsBuilder.metricSnapshot(b.build())); + gaugeSnapshotMap.values().forEach(b -> snapshotsBuilder.metricSnapshot(b.build())); + histogramSnapshotMap.values().forEach(b -> snapshotsBuilder.metricSnapshot(b.build())); + if (otelInfoSnapshots != null) snapshotsBuilder.metricSnapshot(otelInfoSnapshots); + return snapshotsBuilder.build(); + } + + /** Gather label filters */ + public static SortedMap> labelFilters(SolrParams params) { + SortedMap> labelFilters = new TreeMap<>(); + labelFilterKeys.forEach( + (paramName) -> { + Set filterValues = readParamsAsSet(params, paramName); + if (!filterValues.isEmpty()) { + labelFilters.put(paramName, filterValues); + } + }); + + return labelFilters; + } + + /** Add label filters to the filters map */ + public static void addLabelFilters(String value, Map> filters) { + labelFilterKeys.forEach( + (paramName) -> { + Set filterValues = paramValueAsSet(value); + if (!filterValues.isEmpty()) { + filters.put(paramName, filterValues); + } + }); + } + + /** Split the coma-separated param values into a set */ + public static Set paramValueAsSet(String paramValue) { + String[] values = paramValue.split(","); + List valuesSet = new ArrayList<>(); + for (String value : values) { + valuesSet.add(value); + } + return Set.copyOf(valuesSet); + } + + /** + * Read Solr parameters as a Set. + * + *

Could probably be moved to a more generic utility class, but only used in MetricsHandler and + * GetMetrics resource. + */ + public static Set readParamsAsSet(SolrParams params, String paramName) { + String[] paramValues = params.getParams(paramName); + if (paramValues == null || paramValues.length == 0) { + return Set.of(); + } + + Set paramSet = new HashSet<>(); + for (String param : paramValues) { + if (param != null && param.length() > 0) paramSet.addAll(StrUtils.splitSmart(param, ',')); + } + return paramSet; + } } diff --git a/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java b/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java index 920b1aa2f3b2..758588d74655 100644 --- a/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java @@ -59,7 +59,6 @@ import org.apache.solr.client.solrj.response.Group; import org.apache.solr.client.solrj.response.GroupCommand; import org.apache.solr.client.solrj.response.GroupResponse; -import org.apache.solr.client.solrj.response.InputStreamResponseParser; import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.client.solrj.response.UpdateResponse; import org.apache.solr.cloud.api.collections.CollectionHandlingUtils; @@ -1283,7 +1282,6 @@ private Long getNumCommits(HttpSolrClient sourceClient) throws SolrServerExcepti .withSocketTimeout(60000, TimeUnit.MILLISECONDS) .build()) { var req = new MetricsRequest(SolrParams.of("wt", "prometheus")); - req.setResponseParser(new InputStreamResponseParser("prometheus")); NamedList resp = client.request(req); try (InputStream in = (InputStream) resp.get("stream")) { diff --git a/solr/core/src/test/org/apache/solr/cloud/TestBaseStatsCacheCloud.java b/solr/core/src/test/org/apache/solr/cloud/TestBaseStatsCacheCloud.java index 23b6721de339..a85bc5ab9cf8 100644 --- a/solr/core/src/test/org/apache/solr/cloud/TestBaseStatsCacheCloud.java +++ b/solr/core/src/test/org/apache/solr/cloud/TestBaseStatsCacheCloud.java @@ -27,7 +27,6 @@ import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.client.solrj.request.MetricsRequest; import org.apache.solr.client.solrj.request.UpdateRequest; -import org.apache.solr.client.solrj.response.InputStreamResponseParser; import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrInputDocument; @@ -132,7 +131,6 @@ public void testBasicStats() throws Exception { for (JettySolrRunner jettySolrRunner : cluster.getJettySolrRunners()) { try (SolrClient client = getHttpSolrClient(jettySolrRunner.getBaseUrl().toString())) { var req = new MetricsRequest(SolrParams.of("wt", "prometheus")); - req.setResponseParser(new InputStreamResponseParser("prometheus")); NamedList resp = client.request(req); try (InputStream in = (InputStream) resp.get("stream")) { diff --git a/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java b/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java index 7cf04d543756..7ade78cb6c6e 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java @@ -29,6 +29,7 @@ import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; import org.apache.solr.security.AuthorizationContext; +import org.apache.solr.util.stats.MetricUtils; import org.junit.BeforeClass; import org.junit.Test; @@ -53,8 +54,8 @@ public void testMetricNamesFiltering() throws Exception { CommonParams.QT, CommonParams.METRICS_PATH, CommonParams.WT, - MetricsHandler.PROMETHEUS_METRICS_WT, - MetricsHandler.METRIC_NAME_PARAM, + MetricUtils.PROMETHEUS_METRICS_WT, + MetricUtils.METRIC_NAME_PARAM, expectedRequestsMetricName), resp); var metrics = resp.getValues().get("metrics"); @@ -80,8 +81,8 @@ public void testMultipleMetricNamesFiltering() throws Exception { CommonParams.QT, CommonParams.METRICS_PATH, CommonParams.WT, - MetricsHandler.PROMETHEUS_METRICS_WT, - MetricsHandler.METRIC_NAME_PARAM, + MetricUtils.PROMETHEUS_METRICS_WT, + MetricUtils.METRIC_NAME_PARAM, expectedRequestsMetricName + "," + expectedSearcherMetricName), resp); @@ -105,8 +106,8 @@ public void testNonExistentMetricNameFiltering() throws Exception { CommonParams.QT, CommonParams.METRICS_PATH, CommonParams.WT, - MetricsHandler.PROMETHEUS_METRICS_WT, - MetricsHandler.METRIC_NAME_PARAM, + MetricUtils.PROMETHEUS_METRICS_WT, + MetricUtils.METRIC_NAME_PARAM, nonexistentMetricName), resp); var metrics = (MetricSnapshots) resp.getValues().get("metrics"); @@ -126,8 +127,8 @@ public void testLabelFiltering() throws Exception { CommonParams.QT, CommonParams.METRICS_PATH, CommonParams.WT, - MetricsHandler.PROMETHEUS_METRICS_WT, - MetricsHandler.CATEGORY_PARAM, + MetricUtils.PROMETHEUS_METRICS_WT, + MetricUtils.CATEGORY_PARAM, "QUERY"), resp); var metrics = (MetricSnapshots) resp.getValues().get("metrics"); @@ -137,7 +138,7 @@ public void testLabelFiltering() throws Exception { ms.getDataPoints() .forEach( (dp) -> { - assertEquals("QUERY", dp.getLabels().get(MetricsHandler.CATEGORY_PARAM)); + assertEquals("QUERY", dp.getLabels().get(MetricUtils.CATEGORY_PARAM)); }); }); @@ -156,8 +157,8 @@ public void testMultipleLabelFiltering() throws Exception { CommonParams.QT, CommonParams.METRICS_PATH, CommonParams.WT, - MetricsHandler.PROMETHEUS_METRICS_WT, - MetricsHandler.CATEGORY_PARAM, + MetricUtils.PROMETHEUS_METRICS_WT, + MetricUtils.CATEGORY_PARAM, "QUERY" + "," + "SEARCHER"), resp); @@ -168,10 +169,8 @@ public void testMultipleLabelFiltering() throws Exception { .forEach( (dp) -> { assertTrue( - dp.getLabels().get(MetricsHandler.CATEGORY_PARAM).equals("QUERY") - || dp.getLabels() - .get(MetricsHandler.CATEGORY_PARAM) - .equals("SEARCHER")); + dp.getLabels().get(MetricUtils.CATEGORY_PARAM).equals("QUERY") + || dp.getLabels().get(MetricUtils.CATEGORY_PARAM).equals("SEARCHER")); }); }); @@ -188,8 +187,8 @@ public void testNonExistentLabelFiltering() throws Exception { CommonParams.QT, CommonParams.METRICS_PATH, CommonParams.WT, - MetricsHandler.PROMETHEUS_METRICS_WT, - MetricsHandler.CORE_PARAM, + MetricUtils.PROMETHEUS_METRICS_WT, + MetricUtils.CORE_PARAM, "nonexistent_core_name"), resp); @@ -210,10 +209,10 @@ public void testMixedLabelFiltering() throws Exception { CommonParams.QT, CommonParams.METRICS_PATH, CommonParams.WT, - MetricsHandler.PROMETHEUS_METRICS_WT, - MetricsHandler.CORE_PARAM, + MetricUtils.PROMETHEUS_METRICS_WT, + MetricUtils.CORE_PARAM, "collection1", - MetricsHandler.CATEGORY_PARAM, + MetricUtils.CATEGORY_PARAM, "SEARCHER"), resp); @@ -224,8 +223,8 @@ public void testMixedLabelFiltering() throws Exception { .forEach( (dp) -> { assertTrue( - dp.getLabels().get(MetricsHandler.CATEGORY_PARAM).equals("SEARCHER") - && dp.getLabels().get(MetricsHandler.CORE_PARAM).equals("collection1")); + dp.getLabels().get(MetricUtils.CATEGORY_PARAM).equals("SEARCHER") + && dp.getLabels().get(MetricUtils.CORE_PARAM).equals("collection1")); }); }); @@ -245,10 +244,10 @@ public void testMetricNamesAndLabelFiltering() throws Exception { CommonParams.QT, CommonParams.METRICS_PATH, CommonParams.WT, - MetricsHandler.PROMETHEUS_METRICS_WT, - MetricsHandler.CATEGORY_PARAM, + MetricUtils.PROMETHEUS_METRICS_WT, + MetricUtils.CATEGORY_PARAM, "CORE", - MetricsHandler.METRIC_NAME_PARAM, + MetricUtils.METRIC_NAME_PARAM, expectedMetricName), resp); @@ -256,7 +255,7 @@ public void testMetricNamesAndLabelFiltering() throws Exception { assertEquals(1, metrics.size()); var actualDatapoint = metrics.get(0).getDataPoints().getFirst(); assertEquals(expectedMetricName, metrics.get(0).getMetadata().getPrometheusName()); - assertEquals("CORE", actualDatapoint.getLabels().get(MetricsHandler.CATEGORY_PARAM)); + assertEquals("CORE", actualDatapoint.getLabels().get(MetricUtils.CATEGORY_PARAM)); handler.close(); } diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/GetMetricsTest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/GetMetricsTest.java new file mode 100644 index 000000000000..571c7e6dddbc --- /dev/null +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/GetMetricsTest.java @@ -0,0 +1,293 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.handler.admin.api; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.security.KeyManagementException; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; +import java.security.UnrecoverableKeyException; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.function.Consumer; +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.cloud.MiniSolrCloudCluster; +import org.apache.solr.response.PrometheusResponseWriter; +import org.apache.solr.util.SSLTestConfig; +import org.apache.solr.util.stats.MetricUtils; +import org.eclipse.jetty.client.ContentResponse; +import org.eclipse.jetty.client.HttpClient; +import org.eclipse.jetty.http.HttpFields.Mutable; +import org.eclipse.jetty.http.HttpHeader; +import org.eclipse.jetty.http.HttpMethod; +import org.eclipse.jetty.util.ssl.SslContextFactory; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +/** + * Unit tests for {@link GetMetrics} + * + *

These tests are not using any extension of HttpSolrClientBase: + * + *

    + *
  • to avoid dependency on SolrJ, SolrRequest, SolrParams, NamedList + *
  • to be able to send the "Accept" header + *
+ * + *

Tests in this class make plain RESTful HTTP GET requests, with "Accept" header and query + * parameters, to test GetMetrics/MetricsApi. + * + *

See also: TestMetricsRequest, in SolrJ + */ +public class GetMetricsTest extends SolrTestCaseJ4 { + + // No need for the full output + private static final int MAX_OUTPUT = 1024; + + private static final int TIMEOUT = 15000; + + private static HttpClient jettyHttpClient; + private static String metricsV2Url; + private static MiniSolrCloudCluster cluster; + + @BeforeClass + public static void beforeClass() throws Exception { + Path tempDir = createTempDir(); + copyMinConf(tempDir); + Files.copy( + SolrTestCaseJ4.TEST_PATH().resolve("solr.xml"), + tempDir.resolve("solr.xml"), + StandardCopyOption.REPLACE_EXISTING); + MiniSolrCloudCluster.Builder clusterBuilder = new MiniSolrCloudCluster.Builder(2, tempDir); + cluster = clusterBuilder.withSolrXml(tempDir.resolve("solr.xml")).build(); + + metricsV2Url = cluster.getJettySolrRunner(0).getBaseURLV2().toString().concat("/metrics"); + + // useSsl = true, clientAuth = false + SSLTestConfig sslConfig = new SSLTestConfig(true, false); + // trustAll = true + SslContextFactory.Client factory = new SslContextFactory.Client(true); + try { + factory.setSslContext(sslConfig.buildClientSSLContext()); + } catch (KeyManagementException + | UnrecoverableKeyException + | NoSuchAlgorithmException + | KeyStoreException e) { + throw new IllegalStateException( + "Unable to setup https scheme for HTTPClient to test SSL.", e); + } + + jettyHttpClient = new HttpClient(); + jettyHttpClient.setConnectTimeout(TIMEOUT); + jettyHttpClient.setSslContextFactory(factory); + jettyHttpClient.setMaxConnectionsPerDestination(1); + jettyHttpClient.setMaxRequestsQueuedPerDestination(1); + } + + @AfterClass + public static void afterClass() throws Exception { + jettyHttpClient.destroy(); + cluster.shutdown(); + } + + @Before + public void beforeTest() throws Exception { + // stop and start Jetty client for each test, otherwise, it seems responses get mixed! + jettyHttpClient.start(); + } + + @After + public void afterTest() throws Exception { + jettyHttpClient.stop(); + } + + @Test + public void testGetMetricsDefault() + throws IOException, + InterruptedException, + ExecutionException, + TimeoutException, + SolrServerException { + ContentResponse response = null; + try { + response = + jettyHttpClient + .newRequest(metricsV2Url) + .timeout(TIMEOUT, TimeUnit.MILLISECONDS) + .method(HttpMethod.GET) + .send(); + } catch (InterruptedException | ExecutionException | TimeoutException e) { + Assert.fail("Should not throw exception: " + e.getClass() + ". message: " + e.getMessage()); + return; + } + Assert.assertEquals(200, response.getStatus()); + + String str = readMaxOut(response.getContent()); + Assert.assertTrue(str.contains("# HELP")); + Assert.assertTrue(str.contains("# TYPE")); + } + + @Test + public void testGetMetricsPrometheus() + throws IOException, InterruptedException, TimeoutException, ExecutionException { + ContentResponse response = null; + try { + response = + jettyHttpClient + .newRequest(metricsV2Url) + .timeout(TIMEOUT, TimeUnit.MILLISECONDS) + .method(HttpMethod.GET) + .headers( + new Consumer() { + + @Override + public void accept(Mutable arg0) { + arg0.add(HttpHeader.ACCEPT, PrometheusResponseWriter.CONTENT_TYPE_PROMETHEUS); + } + }) + .send(); + } catch (InterruptedException | ExecutionException | TimeoutException e) { + Assert.fail("Should not throw exception: " + e.getClass() + ". message: " + e.getMessage()); + return; + } + Assert.assertEquals(200, response.getStatus()); + Assert.assertEquals("text/plain", response.getMediaType()); + } + + @Test + public void testGetMetricsOpenMetrics() + throws IOException, InterruptedException, TimeoutException, ExecutionException { + ContentResponse response = null; + try { + response = + jettyHttpClient + .newRequest(metricsV2Url) + .timeout(TIMEOUT, TimeUnit.MILLISECONDS) + .method(HttpMethod.GET) + .headers( + new Consumer() { + + @Override + public void accept(Mutable arg0) { + arg0.add( + HttpHeader.ACCEPT, PrometheusResponseWriter.CONTENT_TYPE_OPEN_METRICS); + } + }) + .send(); + } catch (InterruptedException | ExecutionException | TimeoutException e) { + Assert.fail("Should not throw exception: " + e.getClass() + ". message: " + e.getMessage()); + return; + } + Assert.assertEquals(200, response.getStatus()); + Assert.assertEquals("application/openmetrics-text", response.getMediaType()); + } + + @Test + public void testGetMetricsCategoryParams() throws IOException { + String expected = """ + category="QUERY" + """; + + ContentResponse response = null; + try { + response = + jettyHttpClient + .newRequest(metricsV2Url) + .timeout(TIMEOUT, TimeUnit.MILLISECONDS) + .param(MetricUtils.CATEGORY_PARAM, "QUERY") + .method(HttpMethod.GET) + .headers( + new Consumer() { + + @Override + public void accept(Mutable arg0) { + arg0.add(HttpHeader.ACCEPT, PrometheusResponseWriter.CONTENT_TYPE_PROMETHEUS); + } + }) + .send(); + } catch (InterruptedException | ExecutionException | TimeoutException e) { + Assert.fail("Should not throw exception: " + e.getClass() + ". message: " + e.getMessage()); + return; + } + Assert.assertEquals(200, response.getStatus()); + + String str = readMaxOut(response.getContent()); + Assert.assertTrue(str.contains(expected.trim())); + Assert.assertFalse(str.contains("category=\"CORE\"")); + Assert.assertFalse(str.contains("category=\"UPDATE\"")); + } + + @Test + public void testGetMetricsProxyToNode() throws IOException { + URL otherUrl = cluster.getJettySolrRunner(1).getBaseURLV2(); + String otherNode = otherUrl.getHost() + ":" + otherUrl.getPort() + "_solr"; + + ContentResponse response = null; + try { + response = + jettyHttpClient + .newRequest(metricsV2Url) + .timeout(TIMEOUT, TimeUnit.MILLISECONDS) + .param(MetricUtils.NODE_PARAM, otherNode) + .method(HttpMethod.GET) + .send(); + } catch (InterruptedException | ExecutionException | TimeoutException e) { + Assert.fail("Should not throw exception: " + e.getClass() + ". message: " + e.getMessage()); + return; + } + // HTTP 204: no content to test + Assert.assertEquals(204, response.getStatus()); + + String unknownNode = "unknown.host:1234_solr"; + try { + response = + jettyHttpClient + .newRequest(metricsV2Url) + .timeout(TIMEOUT, TimeUnit.MILLISECONDS) + .param(MetricUtils.NODE_PARAM, unknownNode) + .method(HttpMethod.GET) + .send(); + } catch (InterruptedException | ExecutionException | TimeoutException e) { + Assert.fail("Should not throw exception: " + e.getClass() + ". message: " + e.getMessage()); + return; + } + // Unknown host is ignored, returns the default response + Assert.assertEquals(200, response.getStatus()); + } + + private static String readMaxOut(byte[] bytes) throws IOException { + int max = bytes.length > MAX_OUTPUT ? MAX_OUTPUT : bytes.length; + String str = ""; + try (ByteArrayOutputStream out = new ByteArrayOutputStream(max); ) { + out.write(bytes, 0, max); + str = out.toString(StandardCharsets.UTF_8); + } + return str; + } +} diff --git a/solr/core/src/test/org/apache/solr/response/TestPrometheusResponseWriter.java b/solr/core/src/test/org/apache/solr/response/TestPrometheusResponseWriter.java index d9d2493eb3a1..530a2ba60093 100644 --- a/solr/core/src/test/org/apache/solr/response/TestPrometheusResponseWriter.java +++ b/solr/core/src/test/org/apache/solr/response/TestPrometheusResponseWriter.java @@ -67,8 +67,7 @@ public static void beforeClass() throws Exception { public void testPrometheusStructureOutput() throws Exception { ModifiableSolrParams params = new ModifiableSolrParams(); params.set("wt", "prometheus"); - var req = new MetricsRequest(params); - req.setResponseParser(new InputStreamResponseParser("prometheus")); + var req = new MetricsRequest(params); // response parser set in MetricsRequest constructor try (SolrClient adminClient = getHttpSolrClient(solrTestRule.getBaseUrl())) { NamedList res = adminClient.request(req); diff --git a/solr/core/src/test/org/apache/solr/response/TestPrometheusResponseWriterCloud.java b/solr/core/src/test/org/apache/solr/response/TestPrometheusResponseWriterCloud.java index ffc254723e11..508bd9d04d57 100644 --- a/solr/core/src/test/org/apache/solr/response/TestPrometheusResponseWriterCloud.java +++ b/solr/core/src/test/org/apache/solr/response/TestPrometheusResponseWriterCloud.java @@ -22,7 +22,6 @@ import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.client.solrj.request.MetricsRequest; import org.apache.solr.client.solrj.request.SolrQuery; -import org.apache.solr.client.solrj.response.InputStreamResponseParser; import org.apache.solr.cloud.SolrCloudTestCase; import org.apache.solr.common.params.SolrParams; import org.apache.solr.common.util.NamedList; @@ -68,7 +67,6 @@ public void testPrometheusCloudLabels() throws Exception { solrClient.query("collection1", query); var req = new MetricsRequest(SolrParams.of("wt", "prometheus")); - req.setResponseParser(new InputStreamResponseParser("prometheus")); NamedList resp = solrClient.request(req); try (InputStream in = (InputStream) resp.get("stream")) { @@ -98,7 +96,6 @@ public void testCollectionDeletePrometheusOutput() throws Exception { solrClient.query("collection2", query); var req = new MetricsRequest(SolrParams.of("wt", "prometheus")); - req.setResponseParser(new InputStreamResponseParser("prometheus")); NamedList resp = solrClient.request(req); diff --git a/solr/modules/jwt-auth/src/test/org/apache/solr/security/jwt/JWTAuthPluginTest.java b/solr/modules/jwt-auth/src/test/org/apache/solr/security/jwt/JWTAuthPluginTest.java index 0d6cc759cb2f..fb80668b9f73 100644 --- a/solr/modules/jwt-auth/src/test/org/apache/solr/security/jwt/JWTAuthPluginTest.java +++ b/solr/modules/jwt-auth/src/test/org/apache/solr/security/jwt/JWTAuthPluginTest.java @@ -212,6 +212,8 @@ public void initFromSecurityJSONUrlJwk() throws Exception { JWTAuthPlugin.JWTAuthenticationResponse resp = plugin.authenticate(testHeader); assertEquals(JWT_VALIDATION_EXCEPTION, resp.getAuthCode()); + System.out.println( + "initFromSecurityJSONUrlJwk error message: " + resp.getJwtException().getMessage()); assertTrue(resp.getJwtException().getMessage().contains("Connection refused")); } diff --git a/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/TestDistributedTracing.java b/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/TestDistributedTracing.java index 97916c007ca9..e3eb646896c8 100644 --- a/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/TestDistributedTracing.java +++ b/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/TestDistributedTracing.java @@ -17,8 +17,6 @@ package org.apache.solr.opentelemetry; -import static org.apache.solr.handler.admin.MetricsHandler.PROMETHEUS_METRICS_WT; - import io.opentelemetry.api.GlobalOpenTelemetry; import io.opentelemetry.api.trace.TracerProvider; import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter; @@ -43,6 +41,7 @@ import org.apache.solr.cloud.SolrCloudTestCase; import org.apache.solr.common.SolrDocumentList; import org.apache.solr.common.util.NamedList; +import org.apache.solr.util.stats.MetricUtils; import org.apache.solr.util.tracing.TraceUtils; import org.junit.AfterClass; import org.junit.Before; @@ -136,7 +135,7 @@ public void testAdminApi() throws Exception { CloudSolrClient cloudClient = cluster.getSolrClient(); MetricsRequest request = new MetricsRequest(); - request.setResponseParser(new InputStreamResponseParser(PROMETHEUS_METRICS_WT)); + request.setResponseParser(new InputStreamResponseParser(MetricUtils.PROMETHEUS_METRICS_WT)); NamedList rsp = cloudClient.request(request); ((InputStream) rsp.get("stream")).close(); var finishedSpans = getAndClearSpans(); diff --git a/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/TestMetricExemplars.java b/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/TestMetricExemplars.java index 6eaf0c07e175..0c8ba50078f2 100644 --- a/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/TestMetricExemplars.java +++ b/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/TestMetricExemplars.java @@ -26,7 +26,6 @@ import org.apache.solr.client.solrj.impl.CloudSolrClient; import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.client.solrj.request.MetricsRequest; -import org.apache.solr.client.solrj.response.InputStreamResponseParser; import org.apache.solr.cloud.SolrCloudTestCase; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.util.NamedList; @@ -79,7 +78,6 @@ public void testOpenMetricExemplars() throws Exception { var expectedTrace = getRootTraceId(spans); var req = new MetricsRequest(new ModifiableSolrParams().set("wt", "openmetrics")); - req.setResponseParser(new InputStreamResponseParser("openmetrics")); NamedList resp = cloudClient.request(req); try (InputStream in = (InputStream) resp.get("stream")) { diff --git a/solr/solr-ref-guide/modules/deployment-guide/pages/metrics-reporting.adoc b/solr/solr-ref-guide/modules/deployment-guide/pages/metrics-reporting.adoc index 09590b44e722..7d0e79a2c0a5 100644 --- a/solr/solr-ref-guide/modules/deployment-guide/pages/metrics-reporting.adoc +++ b/solr/solr-ref-guide/modules/deployment-guide/pages/metrics-reporting.adoc @@ -132,7 +132,13 @@ Metrics collection for index merges can be configured in the `` section == Metrics API -The `/admin/metrics` endpoint natively provides access to all metrics in Prometheus format by default. You can also specify `wt=prometheus` as a parameter for Prometheus format or `wt=openmetrics` for OpenMetrics format. More information on the data models is provided in the sections below. +The `/metrics` endpoint natively provides access to all metrics in Prometheus format by default. You can also specify `wt=prometheus` as a parameter for Prometheus format or `wt=openmetrics` for OpenMetrics format. More information on the data models is provided in the sections below. + +[NOTE] +==== +The V2 `/metrics` endpoint is equivalent to the V1 `/admin/metrics` endpoint. +Examples on this page show only the V2 endpoint. +==== === Prometheus @@ -148,14 +154,14 @@ The `prometheus-config.yml` file needs to be configured for a Prometheus server ---- scrape_configs: - job_name: 'solr' - metrics_path: "/solr/admin/metrics" + metrics_path: "/api/metrics" static_configs: - targets: ['localhost:8983', 'localhost:7574'] ---- === OpenMetrics -OpenMetrics format is available from the `/admin/metrics` endpoint by providing the `wt=openmetrics` parameter or by passing the Accept header `application/openmetrics-text;version=1.0.0`. OpenMetrics is an extension of the Prometheus format that adds additional metadata and exemplars. +OpenMetrics format is available from the `/metrics` endpoint by providing the `wt=openmetrics` parameter or by passing the Accept header `application/openmetrics-text;version=1.0.0`. OpenMetrics is an extension of the Prometheus format that adds additional metadata and exemplars. See https://prometheus.io/docs/specs/om/open_metrics_spec/[OpenMetrics Spec] documentation for more information. @@ -173,7 +179,7 @@ A basic `prometheus-config.yml` configuration for a Prometheus server in SolrClo ---- scrape_configs: - job_name: 'solr' - metrics_path: "/solr/admin/metrics" + metrics_path: "/api/metrics" static_configs: - targets: ['localhost:8983', 'localhost:7574'] params: @@ -256,26 +262,26 @@ The replica type to filter on. Valid values are NRT, TLOG, or PULL. This attribu Request only metrics from the `foobar` collection: [source,text] -http://localhost:8983/solr/admin/metrics?collection=foobar +http://localhost:8983/api/metrics?collection=foobar Request only the metrics with a category label of QUERY or UPDATE: [source,text] -http://localhost:8983/solr/admin/metrics?category=QUERY,UPDATE +http://localhost:8983/api/metrics?category=QUERY,UPDATE Request only `solr_core_requests_total` metrics from the `foobar_shard1_replica_n1` core: [source,text] -http://localhost:8983/solr/admin/metrics?name=solr_core_requests_total&core=foobar_shard1_replica_n1 +http://localhost:8983/api/metrics?name=solr_core_requests_total&core=foobar_shard1_replica_n1 Request only the core index size `solr_core_index_size_bytes` metrics from collections labeled `foo` and `bar`: [source,text] -http://localhost:8983/solr/admin/metrics?name=solr_core_index_size_bytes&collection=foo,bar +http://localhost:8983/api/metrics?name=solr_core_index_size_bytes&collection=foo,bar == OTLP -For users who do not use or support pulling metrics in Prometheus format with the `/admin/metrics` API, Solr also supports pushing metrics natively with https://opentelemetry.io/docs/specs/otlp/[OTLP], which is a vendor-agnostic protocol for pushing metrics via gRPC or HTTP. +For users who do not use or support pulling metrics in Prometheus format with the `/metrics` API, Solr also supports pushing metrics natively with https://opentelemetry.io/docs/specs/otlp/[OTLP], which is a vendor-agnostic protocol for pushing metrics via gRPC or HTTP. OTLP is widely supported by many tools, vendors, and pipelines. See the OpenTelemetry https://opentelemetry.io/ecosystem/vendors/[vendors list] for more details on available and compatible options. @@ -339,7 +345,7 @@ Endpoint to send OTLP metrics to using the HTTP protocol. === OpenTelemetry Collector setup -The https://opentelemetry.io/docs/collector/[OpenTelemetry Collector] is a powerful process that allows users to decouple their metrics pipeline and route to their preferred backend. It natively supports metrics being pushed to it via OTLP and/or scraping the `/admin/metrics` Prometheus endpoint supported by Solr. You can push both metrics and traces to the collector via OTLP as a single pipeline. +The https://opentelemetry.io/docs/collector/[OpenTelemetry Collector] is a powerful process that allows users to decouple their metrics pipeline and route to their preferred backend. It natively supports metrics being pushed to it via OTLP and/or scraping the `/metrics` Prometheus endpoint supported by Solr. You can push both metrics and traces to the collector via OTLP as a single pipeline. A simple setup to route metrics from Solr -> OpenTelemetry Collector -> Prometheus can be configured with the following OpenTelemetry Collector configuration file: diff --git a/solr/solrj-zookeeper/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java b/solr/solrj-zookeeper/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java index ef6bb5dbd21a..f036d294f60d 100644 --- a/solr/solrj-zookeeper/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java +++ b/solr/solrj-zookeeper/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java @@ -35,7 +35,6 @@ import org.apache.solr.client.solrj.cloud.NodeStateProvider; import org.apache.solr.client.solrj.request.GenericSolrRequest; import org.apache.solr.client.solrj.request.MetricsRequest; -import org.apache.solr.client.solrj.response.InputStreamResponseParser; import org.apache.solr.client.solrj.response.JavaBinResponseParser; import org.apache.solr.client.solrj.response.SimpleSolrResponse; import org.apache.solr.common.MapWriter; @@ -214,7 +213,6 @@ static void processMetricStream( params.add("name", String.join(",", metricNames)); var req = new MetricsRequest(params); - req.setResponseParser(new InputStreamResponseParser("prometheus")); String baseUrl = ctx.zkClientClusterStateProvider.getZkStateReader().getBaseUrlForNodeName(solrNode); diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/request/MetricsRequest.java b/solr/solrj/src/java/org/apache/solr/client/solrj/request/MetricsRequest.java index 82c10659586f..6b7de8903cd1 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/request/MetricsRequest.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/request/MetricsRequest.java @@ -17,15 +17,16 @@ package org.apache.solr.client.solrj.request; import org.apache.solr.client.solrj.SolrRequest; -import org.apache.solr.client.solrj.SolrResponse; -import org.apache.solr.client.solrj.response.SolrResponseBase; +import org.apache.solr.client.solrj.response.InputStreamResponse; +import org.apache.solr.client.solrj.response.InputStreamResponseParser; +import org.apache.solr.common.SolrException; import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.params.SolrParams; import org.apache.solr.common.util.NamedList; -/** Request to "/admin/metrics" */ -public class MetricsRequest extends SolrRequest { +/** Request to V1 "/admin/metrics" or V2 "/metrics" */ +public class MetricsRequest extends SolrRequest { private static final long serialVersionUID = 1L; @@ -36,12 +37,37 @@ public MetricsRequest() { this(new ModifiableSolrParams()); } + /** + * @param path the HTTP path to use for this request. Supports V1 "/admin/metrics" (default) or V2 + * "/metrics" + */ + public MetricsRequest(String path) { + this(path, new ModifiableSolrParams()); + } + /** * @param params the Solr parameters to use for this request. */ public MetricsRequest(SolrParams params) { - super(METHOD.GET, CommonParams.METRICS_PATH, SolrRequestType.ADMIN); + this(CommonParams.METRICS_PATH, params); + } + + /** + * @param params the Solr parameters to use for this request. + */ + public MetricsRequest(String path, SolrParams params) { + super(METHOD.GET, path, SolrRequestType.ADMIN); + if (!path.endsWith("/metrics")) { + throw new SolrException( + SolrException.ErrorCode.INVALID_STATE, "Request path not supported: " + path); + } this.params = params; + // Set response parser according to "wt". + if ("openmetrics".equals(params.get(CommonParams.WT))) { + setResponseParser(new InputStreamResponseParser("openmetrics")); + } else { + setResponseParser(new InputStreamResponseParser("prometheus")); + } } @Override @@ -50,8 +76,17 @@ public SolrParams getParams() { } @Override - protected SolrResponse createResponse(NamedList namedList) { - SolrResponseBase resp = new SolrResponseBase(); - return (SolrResponse) resp; + protected InputStreamResponse createResponse(NamedList namedList) { + return new InputStreamResponse(); + } + + @Override + public ApiVersion getApiVersion() { + if (CommonParams.METRICS_PATH.equals(getPath())) { + // (/solr) /admin/metrics + return ApiVersion.V1; + } + // Ref. org.apache.solr.client.api.endpoint.MetricsApi : /metrics + return ApiVersion.V2; } } diff --git a/solr/solrj/src/test-files/solrj/solr/solr-metrics-enabled.xml b/solr/solrj/src/test-files/solrj/solr/solr-metrics-enabled.xml new file mode 100644 index 000000000000..a48c29f7e278 --- /dev/null +++ b/solr/solrj/src/test-files/solrj/solr/solr-metrics-enabled.xml @@ -0,0 +1,50 @@ + + + + + + + + + + ${shareSchema:false} + ${configSetBaseDir:configsets} + ${coreRootDirectory:.} + ${solr.tests.security.allow.urls:} + + + ${urlScheme:} + ${socketTimeout:90000} + ${connTimeout:15000} + + + + 127.0.0.1 + ${hostPort:8983} + ${solr.zookeeper.client.timeout:30000} + 0 + ${distribUpdateConnTimeout:45000} + ${distribUpdateSoTimeout:340000} + ${zkCredentialsProvider:org.apache.solr.common.cloud.DefaultZkCredentialsProvider} + ${zkACLProvider:org.apache.solr.common.cloud.DefaultZkACLProvider} + ${zkCredentialsInjector:org.apache.solr.common.cloud.DefaultZkCredentialsInjector} + + + diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/request/TestMetricsRequest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/request/TestMetricsRequest.java new file mode 100644 index 000000000000..c360ddb85d4c --- /dev/null +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/request/TestMetricsRequest.java @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.solrj.request; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeoutException; +import org.apache.commons.io.file.PathUtils; +import org.apache.solr.client.solrj.SolrClient.SolrClientFunction; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.jetty.HttpJettySolrClient; +import org.apache.solr.client.solrj.response.InputStreamResponseParser; +import org.apache.solr.cloud.MiniSolrCloudCluster; +import org.apache.solr.cloud.SolrCloudTestCase; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.common.params.SolrParams; +import org.apache.solr.util.stats.MetricUtils; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +/** Test {@link MetricsRequest}. */ +public class TestMetricsRequest extends SolrCloudTestCase { + + private static final String METRICS_V2_PATH = "/metrics"; + + private static HttpJettySolrClient httpClient; + private static MiniSolrCloudCluster cluster; + + @BeforeClass + public static void beforeClass() throws Exception { + Path tempDir = createTempDir(); + Path testFilesDir = getFile("solrj/solr/collection1").getParent(); + PathUtils.copyDirectory(testFilesDir, tempDir); + Files.copy( + testFilesDir.resolve("solr-metrics-enabled.xml"), + tempDir.resolve("solr.xml"), + StandardCopyOption.REPLACE_EXISTING); + MiniSolrCloudCluster.Builder clusterBuilder = new MiniSolrCloudCluster.Builder(2, tempDir); + cluster = clusterBuilder.withSolrXml(tempDir.resolve("solr.xml")).build(); + + HttpJettySolrClient.Builder clientBuilder = + new HttpJettySolrClient.Builder(cluster.getJettySolrRunner(0).getBaseUrl().toString()); + httpClient = clientBuilder.build(); + } + + @AfterClass + public static void afterClass() throws Exception { + httpClient.close(); + cluster.shutdown(); + } + + @Test + public void testGetMetricsV2() + throws IOException, + InterruptedException, + ExecutionException, + TimeoutException, + SolrServerException { + MetricsRequest solrRequest = new MetricsRequest(METRICS_V2_PATH); + String str = + httpClient.requestWithBaseUrl( + cluster.getJettySolrRunner(0).getBaseURLV2().toString(), + new SolrClientFunction() { + + @Override + public String apply(HttpJettySolrClient c) throws IOException, SolrServerException { + return InputStreamResponseParser.consumeResponseToString(c.request(solrRequest)); + } + }); + + Assert.assertTrue(str.contains("# HELP")); + Assert.assertTrue(str.contains("# TYPE")); + } + + @Test + public void testGetMetricsV2ParamName() + throws IOException, + InterruptedException, + ExecutionException, + TimeoutException, + SolrServerException { + String requestedName = "solr_disk_space_megabytes"; + String notRequested = "solr_client_request_duration_milliseconds_bucket"; + SolrParams params = + new ModifiableSolrParams( + Map.of(MetricUtils.METRIC_NAME_PARAM, new String[] {requestedName})); + MetricsRequest solrRequest = new MetricsRequest(METRICS_V2_PATH, params); + String str = + httpClient.requestWithBaseUrl( + cluster.getJettySolrRunner(0).getBaseURLV2().toString(), + new SolrClientFunction() { + + @Override + public String apply(HttpJettySolrClient c) throws IOException, SolrServerException { + return InputStreamResponseParser.consumeResponseToString(c.request(solrRequest)); + } + }); + + Assert.assertTrue(str.contains("# HELP")); + Assert.assertTrue(str.contains("# TYPE")); + Assert.assertTrue(str.contains(requestedName)); + Assert.assertFalse(str.contains(notRequested)); + } + + @Test + public void testGetMetricsV1() + throws IOException, + InterruptedException, + ExecutionException, + TimeoutException, + SolrServerException { + MetricsRequest solrRequest = new MetricsRequest(); + String str = + httpClient.requestWithBaseUrl( + cluster.getJettySolrRunner(0).getBaseUrl().toString(), + new SolrClientFunction() { + + @Override + public String apply(HttpJettySolrClient c) throws IOException, SolrServerException { + return InputStreamResponseParser.consumeResponseToString(c.request(solrRequest)); + } + }); + + Assert.assertTrue(str.contains("# HELP")); + Assert.assertTrue(str.contains("# TYPE")); + } +} diff --git a/solr/test-framework/src/java/org/apache/solr/util/SolrJMetricTestUtils.java b/solr/test-framework/src/java/org/apache/solr/util/SolrJMetricTestUtils.java index 1ec456202c6c..7669c3755cf0 100644 --- a/solr/test-framework/src/java/org/apache/solr/util/SolrJMetricTestUtils.java +++ b/solr/test-framework/src/java/org/apache/solr/util/SolrJMetricTestUtils.java @@ -24,7 +24,6 @@ import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.jetty.HttpJettySolrClient; import org.apache.solr.client.solrj.request.MetricsRequest; -import org.apache.solr.client.solrj.response.InputStreamResponseParser; import org.apache.solr.common.params.SolrParams; import org.apache.solr.common.util.NamedList; @@ -33,7 +32,6 @@ public final class SolrJMetricTestUtils { public static double getPrometheusMetricValue(SolrClient solrClient, String metricName) throws SolrServerException, IOException { var req = new MetricsRequest(SolrParams.of("wt", "prometheus")); - req.setResponseParser(new InputStreamResponseParser("prometheus")); NamedList resp = solrClient.request(req); try (InputStream in = (InputStream) resp.get("stream")) { @@ -52,7 +50,6 @@ public static Double getNumCoreRequests( try (var client = new HttpJettySolrClient.Builder(baseUrl).build()) { var req = new MetricsRequest(SolrParams.of("wt", "prometheus")); - req.setResponseParser(new InputStreamResponseParser("prometheus")); NamedList resp = client.request(req); try (InputStream in = (InputStream) resp.get("stream")) { @@ -80,7 +77,6 @@ public static Double getNumNodeRequestErrors(String baseUrl, String category, St try (var client = new HttpJettySolrClient.Builder(baseUrl).build()) { var req = new MetricsRequest(SolrParams.of("wt", "prometheus")); - req.setResponseParser(new InputStreamResponseParser("prometheus")); NamedList resp = client.request(req); try (InputStream in = (InputStream) resp.get("stream")) {