diff --git a/.github/workflows/e2e.ci.yaml b/.github/workflows/e2e.ci.yaml index 7cac01b..0869c4f 100644 --- a/.github/workflows/e2e.ci.yaml +++ b/.github/workflows/e2e.ci.yaml @@ -52,15 +52,6 @@ jobs: - name: Run SWCK oap-ui-agent-satellite Test config: test/e2e/swck/oap-ui-agent-satellite.yaml name: ${{ matrix.test.name }} - env: - OAP_TAG: e6bbebe107096bf50ed79e40a88f230bd1237c3d - OAP_REPO: ghcr.io/apache/skywalking/oap - UI_TAG: e6bbebe107096bf50ed79e40a88f230bd1237c3d - UI_REPO: ghcr.io/apache/skywalking/ui - SATELLITE_TAG: v35bfaff6352b4dc351a706772796a1f79b651c14 - SATELLITE_REPO: ghcr.io/apache/skywalking-satellite/skywalking-satellite - BANYANDB_TAG: 27cfc5d0eed6891df69eacdf524611cb9fe241e3 - BANYANDB_REPO: ghcr.io/apache/skywalking-banyandb steps: - uses: actions/checkout@v2 - name: Login to ghcr @@ -74,7 +65,7 @@ jobs: with: go-version: '1.24' - name: ${{ matrix.test.name }} - uses: apache/skywalking-infra-e2e@7e4b5b68716fdb7b79b21fa8908f9db497e1b115 + uses: apache/skywalking-infra-e2e@8c21e43e241a32a54bdf8eeceb9099eb27e5e9b4 with: e2e-file: $GITHUB_WORKSPACE/${{ matrix.test.config }} - if: ${{ failure() }} diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..764f0f5 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,115 @@ +# CLAUDE.md + +## Project Overview + +Apache SkyWalking Helm Charts — Helm 3 charts for deploying SkyWalking and related components on Kubernetes. + +## Repository Structure + +``` +chart/ + skywalking/ # Main SkyWalking chart (OAP, UI, Satellite) + Chart.yaml # Chart metadata and dependencies + values.yaml # Default values + values-my-es.yaml # Example values for external Elasticsearch + templates/ + _helpers.tpl # Shared template helpers (naming, env vars, init containers) + oap-*.yaml # OAP server templates + ui-*.yaml # UI templates + satellite-*.yaml # Satellite templates + NOTES.txt # Post-install notes + adapter/ # SWCK Adapter chart + operator/ # SWCK Operator chart +test/e2e/ # E2E test configs (skywalking-infra-e2e format) + e2e-elasticsearch.yaml + e2e-banyandb-*.yaml + values.yaml # Test-specific value overrides + swck/ # SWCK-specific e2e tests +.github/workflows/ + e2e.ci.yaml # CI pipeline running all e2e tests +``` + +## Chart Dependencies + +Defined in `chart/skywalking/Chart.yaml`: +- **eck-operator** (3.3.1) — ECK operator, condition: `eckOperator.enabled` +- **eck-elasticsearch** (0.18.1, alias: `elasticsearch`) — ECK-managed ES, condition: `elasticsearch.enabled` +- **postgresql** (12.1.2) — Bitnami PostgreSQL, condition: `postgresql.enabled` +- **skywalking-banyandb-helm** (alias: `banyandb`) — BanyanDB, condition: `banyandb.enabled` + +## Key Conventions + +### Template Helpers (`_helpers.tpl`) +- `skywalking.fullname` — base name for all resources +- `skywalking.oap.fullname` / `skywalking.ui.fullname` / `skywalking.satellite.fullname` — component names +- `skywalking.elasticsearch.fullname` — ECK Elasticsearch resource name (service is `{name}-es-http`) +- `skywalking.containers.wait-for-storage` — init container that waits for the configured storage backend +- `skywalking.oap.envs.storage` — storage-specific environment variables for OAP + +### Storage Pattern +Each storage backend (elasticsearch, postgresql, banyandb) follows the same pattern: +- `*.enabled` — deploy the backend as a subchart +- `*.config.*` — connection settings for external instances (when `enabled: false`) +- `_helpers.tpl` handles both embedded and external modes in `wait-for-storage` and `oap.envs.storage` + +### ECK Elasticsearch +- ECK auto-generates an auth secret: `{fullname}-es-elastic-user` with key `elastic` +- HTTP TLS is disabled by default (`http.tls.selfSignedCertificate.disabled: true`) for OAP connectivity +- Node topology is configured via `elasticsearch.nodeSets[]` (count, config, podTemplate, volumeClaimTemplates) + +## Common Commands + +```shell +# Update chart dependencies +helm dep up chart/skywalking + +# Template rendering (for validation) +helm template test chart/skywalking \ + --set oap.image.tag=10.3.0 \ + --set oap.storageType=elasticsearch \ + --set ui.image.tag=10.3.0 + +# Template with external ES (no ECK) +helm template test chart/skywalking \ + --set oap.image.tag=10.3.0 \ + --set oap.storageType=elasticsearch \ + --set ui.image.tag=10.3.0 \ + --set elasticsearch.enabled=false \ + --set eckOperator.enabled=false + +# Package chart +make package + +# Clean build artifacts +make clean +``` + +## Required Values + +These must be set explicitly for any deployment: +- `oap.image.tag` +- `oap.storageType` (`elasticsearch`, `postgresql`, or `banyandb`) +- `ui.image.tag` + +## E2E Tests + +Tests use [skywalking-infra-e2e](https://github.com/apache/skywalking-infra-e2e). Each `.yaml` file under `test/e2e/` defines setup steps, triggers, and verification queries. The CI workflow is in `.github/workflows/e2e.ci.yaml`. + +## Docs to Keep in Sync + +When modifying chart configuration, update all of: +1. `chart/skywalking/values.yaml` — default values +2. `chart/skywalking/README.md` — parameter tables +3. `README.md` — install examples and user-facing docs +4. `chart/skywalking/values-my-es.yaml` — external ES example (if ES-related) +5. `test/e2e/values.yaml` — test overrides (if defaults change) + +## Git Workflow + +- **Do not push directly to master.** Always create a feature branch and open a PR. +- Branch naming example: `upgrade-elasticsearch-eck-8.18.8` +- **Do not add Claude as co-author** in commit messages. + +## License + +All files must include the Apache 2.0 license header. diff --git a/README.md b/README.md index 8240d1f..d1519dc 100644 --- a/README.md +++ b/README.md @@ -57,6 +57,7 @@ helm install "${SKYWALKING_RELEASE_NAME}" \ --set oap.image.tag=10.3.0 \ --set oap.storageType=banyandb \ --set ui.image.tag=10.3.0 \ + --set eckOperator.enabled=false \ --set elasticsearch.enabled=false \ --set banyandb.enabled=true \ --set banyandb.image.tag=0.9.0 @@ -138,19 +139,22 @@ helm install "${SKYWALKING_RELEASE_NAME}" ${REPO}/skywalking -n "${SKYWALKING_RE --set ui.image.tag=10.3.0 ``` -Because ElasticSearch recommends to use the corresponding Helm Chart version of the ElasticSearch version, -if you want to use a specific version of ElasticSearch, you have to change the ElasticSearch Helm Chart version in -`dependencies` section in `Chart.yaml` file, which requires you to install from the source codes. -Or you should deploy the desired ElasticSearch version first by yourself, and then deploy SkyWalking to use the -existing ElasticSearch by setting the following section: +Elasticsearch is now deployed via [ECK (Elastic Cloud on Kubernetes)](https://github.com/elastic/cloud-on-k8s). +By default, the chart deploys the ECK operator and an Elasticsearch 8.18.8 cluster. +If you already have the ECK operator installed, set `eckOperator.enabled=false`. + +To use an existing external Elasticsearch instead, disable the embedded deployment: ```yaml +eckOperator: + enabled: false + elasticsearch: - enabled: true - config: # For users of an existing elasticsearch cluster,takes effect when `elasticsearch.enabled` is false + enabled: false + config: + host: elasticsearch-es-http port: http: 9200 - host: elasticsearch # es service on kubernetes or host user: "xxx" # [optional] password: "xxx" # [optional] ``` @@ -183,7 +187,7 @@ helm install "${SKYWALKING_RELEASE_NAME}" ${REPO}/skywalking -n "${SKYWALKING_RE ## Install a specific version of SkyWalking with an existing database -If you want to use a specific version of ElasticSearch as storage solution, for instance, modify the connection information to the existing ElasticSearch cluster in file [`values-my-es.yaml`](chart/skywalking/values-my-es.yaml). +If you want to use an existing Elasticsearch cluster as storage solution, modify the connection information in file [`values-my-es.yaml`](chart/skywalking/values-my-es.yaml). ```shell script helm install "${SKYWALKING_RELEASE_NAME}" ${REPO}/skywalking -n "${SKYWALKING_RELEASE_NAMESPACE}" \ diff --git a/chart/skywalking/Chart.yaml b/chart/skywalking/Chart.yaml index 049cdc6..65660da 100644 --- a/chart/skywalking/Chart.yaml +++ b/chart/skywalking/Chart.yaml @@ -30,8 +30,13 @@ maintainers: email: kezhenxu94@163.com dependencies: -- name: elasticsearch - version: ~7.17.1 +- name: eck-operator + version: 3.3.1 + repository: https://helm.elastic.co/ + condition: eckOperator.enabled +- name: eck-elasticsearch + alias: elasticsearch + version: 0.18.1 repository: https://helm.elastic.co/ condition: elasticsearch.enabled - name: postgresql diff --git a/chart/skywalking/README.md b/chart/skywalking/README.md index dc48fa8..27b0053 100644 --- a/chart/skywalking/README.md +++ b/chart/skywalking/README.md @@ -97,68 +97,6 @@ The following table lists the configurable parameters of the Skywalking chart an | `oapInit.nodeSelector` | OAP init job labels for master pod assignment | `{}` | | `oapInit.tolerations` | OAP init job tolerations | `[]` | | `oapInit.extraPodLabels` | OAP init job metadata labels | `[]` | -| `elasticsearch.enabled` | Spin up a new elasticsearch cluster for SkyWalking | `true` | -| `elasticsearch.clusterName` | This will be used as the Elasticsearch [cluster.name](https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster.name.html) and should be unique per cluster in the namespace | `elasticsearch` | -| `elasticsearch.nodeGroup` | This is the name that will be used for each group of nodes in the cluster. The name will be `clusterName-nodeGroup-X` | `master` | -| `elasticsearch.masterService` | Optional. The service name used to connect to the masters. You only need to set this if your master `nodeGroup` is set to something other than `master`. See [Clustering and Node Discovery](#clustering-and-node-discovery) for more information. | `` | -| `elasticsearch.roles` | A hash map with the [specific roles](https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-node.html) for the node group | `master: true`
`data: true`
`ingest: true` | -| `elasticsearch.replicas` | Kubernetes replica count for the statefulset (i.e. how many pods) | `3` | -| `elasticsearch.minimumMasterNodes` | The value for [discovery.zen.minimum_master_nodes](https://www.elastic.co/guide/en/elasticsearch/reference/6.7/discovery-settings.html#minimum_master_nodes). Should be set to `(master_eligible_nodes / 2) + 1`. Ignored in Elasticsearch versions >= 7. | `2` | -| `elasticsearch.esMajorVersion` | Used to set major version specific configuration. If you are using a custom image and not running the default Elasticsearch version you will need to set this to the version you are running (e.g. `esMajorVersion: 6`) | `""` | -| `elasticsearch.esConfig` | Allows you to add any config files in `/usr/share/elasticsearch/config/` such as `elasticsearch.yml` and `log4j2.properties`. See [values.yaml](./values.yaml) for an example of the formatting. | `{}` | -| `elasticsearch.extraEnvs` | Extra [environment variables](https://kubernetes.io/docs/tasks/inject-data-application/define-environment-variable-container/#using-environment-variables-inside-of-your-config) which will be appended to the `env:` definition for the container | `[]` | -| `elasticsearch.extraVolumes` | Templatable string of additional volumes to be passed to the `tpl` function | `""` | -| `elasticsearch.extraVolumeMounts` | Templatable string of additional volumeMounts to be passed to the `tpl` function | `""` | -| `elasticsearch.extraInitContainers` | Templatable string of additional init containers to be passed to the `tpl` function | `""` | -| `elasticsearch.secretMounts` | Allows you easily mount a secret as a file inside the statefulset. Useful for mounting certificates and other secrets. See [values.yaml](./values.yaml) for an example | `[]` | -| `elasticsearch.image` | The Elasticsearch docker image | `docker.elastic.co/elasticsearch/elasticsearch` | -| `elasticsearch.imageTag` | The Elasticsearch docker image tag | `7.5.1` | -| `elasticsearch.imagePullPolicy` | The Kubernetes [imagePullPolicy](https://kubernetes.io/docs/concepts/containers/images/#updating-images) value | `IfNotPresent` | -| `elasticsearch.podAnnotations` | Configurable [annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/) applied to all Elasticsearch pods | `{}` | -| `elasticsearch.labels` | Configurable [label](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/) applied to all Elasticsearch pods | `{}` | -| `elasticsearch.esJavaOpts` | [Java options](https://www.elastic.co/guide/en/elasticsearch/reference/current/jvm-options.html) for Elasticsearch. This is where you should configure the [jvm heap size](https://www.elastic.co/guide/en/elasticsearch/reference/current/heap-size.html) | `-Xmx1g -Xms1g` | -| `elasticsearch.resources` | Allows you to set the [resources](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/) for the statefulset | `requests.cpu: 100m`
`requests.memory: 2Gi`
`limits.cpu: 1000m`
`limits.memory: 2Gi` | -| `elasticsearch.initResources` | Allows you to set the [resources](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/) for the initContainer in the statefulset | {} | -| `elasticsearch.sidecarResources` | Allows you to set the [resources](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/) for the sidecar containers in the statefulset | {} | -| `elasticsearch.networkHost` | Value for the [network.host Elasticsearch setting](https://www.elastic.co/guide/en/elasticsearch/reference/current/network.host.html) | `0.0.0.0` | -| `elasticsearch.volumeClaimTemplate` | Configuration for the [volumeClaimTemplate for statefulsets](https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#stable-storage). You will want to adjust the storage (default `30Gi`) and the `storageClassName` if you are using a different storage class | `accessModes: [ "ReadWriteOnce" ]`
`resources.requests.storage: 30Gi` | -| `elasticsearch.persistence.annotations` | Additional persistence annotations for the `volumeClaimTemplate` | `{}` | -| `elasticsearch.persistence.enabled` | Enables a persistent volume for Elasticsearch data. Can be disabled for nodes that only have [roles](https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-node.html) which don't require persistent data. | `true` | -| `elasticsearch.priorityClassName` | The [name of the PriorityClass](https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#priorityclass). No default is supplied as the PriorityClass must be created first. | `""` | -| `elasticsearch.antiAffinityTopologyKey` | The [anti-affinity topology key](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity). By default this will prevent multiple Elasticsearch nodes from running on the same Kubernetes node | `kubernetes.io/hostname` | -| `elasticsearch.antiAffinity` | Setting this to hard enforces the [anti-affinity rules](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity). If it is set to soft it will be done "best effort". Other values will be ignored. | `hard` | -| `elasticsearch.nodeAffinity` | Value for the [node affinity settings](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#node-affinity-beta-feature) | `{}` | -| `elasticsearch.podManagementPolicy` | By default Kubernetes [deploys statefulsets serially](https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#pod-management-policies). This deploys them in parallel so that they can discover eachother | `Parallel` | -| `elasticsearch.protocol` | The protocol that will be used for the readinessProbe. Change this to `https` if you have `xpack.security.http.ssl.enabled` set | `http` | -| `elasticsearch.httpPort` | The http port that Kubernetes will use for the healthchecks and the service. If you change this you will also need to set [http.port](https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-http.html#_settings) in `extraEnvs` | `9200` | -| `elasticsearch.transportPort` | The transport port that Kubernetes will use for the service. If you change this you will also need to set [transport port configuration](https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-transport.html#_transport_settings) in `extraEnvs` | `9300` | -| `elasticsearch.service.labels` | Labels to be added to non-headless service | `{}` | -| `elasticsearch.service.labelsHeadless` | Labels to be added to headless service | `{}` | -| `elasticsearch.service.type` | Type of elasticsearch service. [Service Types](https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types) | `ClusterIP` | -| `elasticsearch.service.nodePort` | Custom [nodePort](https://kubernetes.io/docs/concepts/services-networking/service/#nodeport) port that can be set if you are using `service.type: nodePort`. | `` | -| `elasticsearch.service.annotations` | Annotations that Kubernetes will use for the service. This will configure load balancer if `service.type` is `LoadBalancer` [Annotations](https://kubernetes.io/docs/concepts/services-networking/service/#ssl-support-on-aws) | `{}` | -| `elasticsearch.service.httpPortName` | The name of the http port within the service | `http` | -| `elasticsearch.service.transportPortName` | The name of the transport port within the service | `transport` | -| `elasticsearch.updateStrategy` | The [updateStrategy](https://kubernetes.io/docs/tutorials/stateful-application/basic-stateful-set/#updating-statefulsets) for the statefulset. By default Kubernetes will wait for the cluster to be green after upgrading each pod. Setting this to `OnDelete` will allow you to manually delete each pod during upgrades | `RollingUpdate` | -| `elasticsearch.maxUnavailable` | The [maxUnavailable](https://kubernetes.io/docs/tasks/run-application/configure-pdb/#specifying-a-poddisruptionbudget) value for the pod disruption budget. By default this will prevent Kubernetes from having more than 1 unhealthy pod in the node group | `1` | -| `elasticsearch.fsGroup (DEPRECATED)` | The Group ID (GID) for [securityContext.fsGroup](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/) so that the Elasticsearch user can read from the persistent volume | `` | -| `elasticsearch.podSecurityContext` | Allows you to set the [securityContext](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod) for the pod | `fsGroup: 1000`
`runAsUser: 1000` | -| `elasticsearch.securityContext` | Allows you to set the [securityContext](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container) for the container | `capabilities.drop:[ALL]`
`runAsNonRoot: true`
`runAsUser: 1000` | -| `elasticsearch.terminationGracePeriod` | The [terminationGracePeriod](https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods) in seconds used when trying to stop the pod | `120` | -| `elasticsearch.sysctlInitContainer.enabled` | Allows you to disable the sysctlInitContainer if you are setting vm.max_map_count with another method | `true` | -| `elasticsearch.sysctlVmMaxMapCount` | Sets the [sysctl vm.max_map_count](https://www.elastic.co/guide/en/elasticsearch/reference/current/vm-max-map-count.html#vm-max-map-count) needed for Elasticsearch | `262144` | -| `elasticsearch.readinessProbe` | Configuration fields for the [readinessProbe](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/) | `failureThreshold: 3`
`initialDelaySeconds: 10`
`periodSeconds: 10`
`successThreshold: 3`
`timeoutSeconds: 5` | -| `elasticsearch.clusterHealthCheckParams` | The [Elasticsearch cluster health status params](https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster-health.html#request-params) that will be used by readinessProbe command | `wait_for_status=green&timeout=1s` | -| `elasticsearch.imagePullSecrets` | Configuration for [imagePullSecrets](https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/#create-a-pod-that-uses-your-secret) so that you can use a private registry for your image | `[]` | -| `elasticsearch.nodeSelector` | Configurable [nodeSelector](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector) so that you can target specific nodes for your Elasticsearch cluster | `{}` | -| `elasticsearch.tolerations` | Configurable [tolerations](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/) | `[]` | -| `elasticsearch.ingress` | Configurable [ingress](https://kubernetes.io/docs/concepts/services-networking/ingress/) to expose the Elasticsearch service. See [`values.yaml`](./values.yaml) for an example | `enabled: false` | -| `elasticsearch.schedulerName` | Name of the [alternate scheduler](https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/#specify-schedulers-for-pods) | `nil` | -| `elasticsearch.masterTerminationFix` | A workaround needed for Elasticsearch < 7.2 to prevent master status being lost during restarts [#63](https://github.com/elastic/helm-charts/issues/63) | `false` | -| `elasticsearch.lifecycle` | Allows you to add lifecycle configuration. See [values.yaml](./values.yaml) for an example of the formatting. | `{}` | -| `elasticsearch.keystore` | Allows you map Kubernetes secrets into the keystore. See the [config example](/elasticsearch/examples/config/values.yaml) and [how to use the keystore](#how-to-use-the-keystore) | `[]` | -| `elasticsearch.rbac` | Configuration for creating a role, role binding and service account as part of this helm chart with `create: true`. Also can be used to reference an external service account with `serviceAccountName: "externalServiceAccountName"`. | `create: false`
`serviceAccountName: ""` | -| `elasticsearch.podSecurityPolicy` | Configuration for create a pod security policy with minimal permissions to run this Helm chart with `create: true`. Also can be used to reference an external pod security policy with `name: "externalPodSecurityPolicy"` | `create: false`
`name: ""` | | `satellite.name` | Satellite deployment name | `satellite` | | `satellite.replicas` | Satellite k8s deployment replicas | `1` | | `satellite.enabled` | Is enable Satellite | `false` | @@ -177,6 +115,66 @@ The following table lists the configurable parameters of the Skywalking chart an | `satellite.env` | Satellite environment variables | `[]` | | `satellite.securityContext` | Allows you to set the [securityContext](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod) for the pod | `fsGroup: 1000`
`runAsUser: 1000` | +### Elasticsearch (ECK) + +Elasticsearch is deployed via [ECK (Elastic Cloud on Kubernetes)](https://github.com/elastic/cloud-on-k8s). +The chart includes the ECK operator and an `eck-elasticsearch` subchart. Set `eckOperator.enabled=false` if the ECK operator is already installed in your cluster. + +#### Top-level parameters + +| Parameter | Description | Default | +|---|---|---| +| `eckOperator.enabled` | Deploy the ECK operator | `true` | +| `elasticsearch.enabled` | Deploy an ECK-managed Elasticsearch cluster | `true` | +| `elasticsearch.version` | Elasticsearch version to deploy | `8.18.8` | +| `elasticsearch.fullnameOverride` | Override the Elasticsearch resource name. The ECK service will be `{name}-es-http` | `""` | +| `elasticsearch.labels` | Labels applied to the Elasticsearch resource | `{}` | +| `elasticsearch.annotations` | Annotations applied to the Elasticsearch resource | `{}` | +| `elasticsearch.http` | HTTP layer settings. TLS is disabled by default for OAP connectivity | `tls.selfSignedCertificate.disabled: true` | +| `elasticsearch.secureSettings` | [Secure settings](https://www.elastic.co/guide/en/cloud-on-k8s/current/k8s-es-secure-settings.html) to inject from Kubernetes secrets | `[]` | +| `elasticsearch.updateStrategy` | [Update strategy](https://www.elastic.co/guide/en/cloud-on-k8s/current/k8s-update-strategy.html) controlling simultaneous changes | `{}` | +| `elasticsearch.volumeClaimDeletePolicy` | Policy for PVC deletion on scale-down or cluster deletion | `""` | +| `elasticsearch.ingress.enabled` | Enable ingress to expose Elasticsearch externally | `false` | + +#### External Elasticsearch (when `elasticsearch.enabled` is `false`) + +| Parameter | Description | Default | +|---|---|---| +| `elasticsearch.config.host` | Elasticsearch host | `elasticsearch` | +| `elasticsearch.config.port.http` | Elasticsearch HTTP port | `9200` | +| `elasticsearch.config.user` | Elasticsearch user (optional) | `""` | +| `elasticsearch.config.password` | Elasticsearch password (optional) | `""` | + +#### Node sets (`elasticsearch.nodeSets[]`) + +ECK [node sets](https://www.elastic.co/guide/en/cloud-on-k8s/current/k8s-node-configuration.html) define the topology of the Elasticsearch cluster. Each entry in the list creates a group of Elasticsearch nodes. + +| Parameter | Description | Default | +|---|---|---| +| `nodeSets[].name` | Name of the node set | `default` | +| `nodeSets[].count` | Number of Elasticsearch nodes in this set | `3` | +| `nodeSets[].config` | Elasticsearch configuration (e.g. `node.store.allow_mmap`, `node.roles`) | `node.store.allow_mmap: false` | +| `nodeSets[].volumeClaimTemplates` | Persistent storage for Elasticsearch data | `[]` (ECK default: EmptyDir) | + +#### Pod template (`elasticsearch.nodeSets[].podTemplate`) + +The pod template follows standard Kubernetes Pod spec nested under `podTemplate.spec`. This controls scheduling, resources, init containers, etc. + +| Parameter | Description | Default | +|---|---|---| +| `podTemplate.metadata.annotations` | Pod annotations (e.g. `iam.amazonaws.com/role`) | `{}` | +| `podTemplate.metadata.labels` | Extra pod labels | `{}` | +| `podTemplate.spec.affinity` | Pod [affinity](https://www.elastic.co/guide/en/cloud-on-k8s/current/k8s-advanced-node-scheduling.html) rules | not set | +| `podTemplate.spec.nodeSelector` | Node selector for pod assignment | not set | +| `podTemplate.spec.tolerations` | Pod tolerations | not set | +| `podTemplate.spec.imagePullSecrets` | Image pull secrets | not set | +| `podTemplate.spec.priorityClassName` | Priority class name | not set | +| `podTemplate.spec.terminationGracePeriodSeconds` | Grace period for pod termination | not set | +| `podTemplate.spec.initContainers` | Init containers (e.g. sysctl `vm.max_map_count`) | not set | +| `podTemplate.spec.containers[].resources` | Container resource requests & limits | `requests: 100m cpu, 2Gi mem`
`limits: 2Gi mem` | +| `podTemplate.spec.containers[].env` | Environment variables (e.g. `ES_JAVA_OPTS`) | not set | +| `podTemplate.spec.containers[].securityContext` | Container-level security context | not set (ECK managed) | + Specify each parameter using the `--set key=value[,key=value]` argument to `helm install`. For example, ```console diff --git a/chart/skywalking/templates/NOTES.txt b/chart/skywalking/templates/NOTES.txt index 0f0370d..1da5c4d 100644 --- a/chart/skywalking/templates/NOTES.txt +++ b/chart/skywalking/templates/NOTES.txt @@ -46,12 +46,3 @@ Get the UI URL by running these commands: kubectl port-forward svc/{{ include "skywalking.ui.fullname" . }} 8080:{{ .Values.ui.service.externalPort }} --namespace {{ .Release.Namespace }} {{- end }} -{{- if .Values.elasticsearch.enabled }} -{{- if .Values.elasticsearch.persistence.enabled }} -{{- else }} -################################################################################# -###### WARNING: Persistence is disabled!!! You will lose your data when ##### -###### the SkyWalking's storage ES pod is terminated. ##### -################################################################################# -{{- end }} -{{- end }} diff --git a/chart/skywalking/templates/_helpers.tpl b/chart/skywalking/templates/_helpers.tpl index d23a7cf..1e71ccb 100644 --- a/chart/skywalking/templates/_helpers.tpl +++ b/chart/skywalking/templates/_helpers.tpl @@ -89,13 +89,25 @@ Create the name of the service account to use for the satellite cluster {{ default (include "skywalking.satellite.fullname" .) .Values.serviceAccounts.satellite }} {{- end -}} +{{/* +Expand the fullname of the ECK-managed Elasticsearch resource. +The ECK operator creates a service named {fullname}-es-http. +*/}} +{{- define "skywalking.elasticsearch.fullname" -}} +{{- if .Values.elasticsearch.fullnameOverride -}} +{{- .Values.elasticsearch.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-elasticsearch" .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} + {{- define "skywalking.containers.wait-for-storage" -}} {{- if eq .Values.oap.storageType "elasticsearch" }} - name: wait-for-elasticsearch image: {{ .Values.initContainer.image }}:{{ .Values.initContainer.tag }} imagePullPolicy: IfNotPresent {{- if .Values.elasticsearch.enabled }} - command: ['sh', '-c', 'for i in $(seq 1 60); do nc -z -w3 {{ .Values.elasticsearch.clusterName }}-{{ .Values.elasticsearch.nodeGroup }} {{ .Values.elasticsearch.httpPort }} && exit 0 || sleep 5; done; exit 1'] + command: ['sh', '-c', 'for i in $(seq 1 60); do nc -z -w3 {{ include "skywalking.elasticsearch.fullname" . }}-es-http 9200 && exit 0 || sleep 5; done; exit 1'] {{- else }} command: ['sh', '-c', 'for i in $(seq 1 60); do nc -z -w3 {{ .Values.elasticsearch.config.host }} {{ .Values.elasticsearch.config.port.http }} && exit 0 || sleep 5; done; exit 1'] {{- end }} @@ -134,11 +146,19 @@ Create the name of the service account to use for the satellite cluster {{- if eq .Values.oap.storageType "elasticsearch" }} - name: SW_STORAGE_ES_CLUSTER_NODES {{- if .Values.elasticsearch.enabled }} - value: "{{ .Values.elasticsearch.clusterName }}-{{ .Values.elasticsearch.nodeGroup }}:{{ .Values.elasticsearch.httpPort }}" + value: "{{ include "skywalking.elasticsearch.fullname" . }}-es-http:9200" {{- else }} value: "{{ .Values.elasticsearch.config.host }}:{{ .Values.elasticsearch.config.port.http }}" {{- end }} - {{- if not .Values.elasticsearch.enabled }} + {{- if .Values.elasticsearch.enabled }} +- name: SW_ES_USER + value: "elastic" +- name: SW_ES_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "skywalking.elasticsearch.fullname" . }}-es-elastic-user + key: elastic + {{- else }} {{- if .Values.elasticsearch.config.user }} - name: SW_ES_USER value: "{{ .Values.elasticsearch.config.user }}" diff --git a/chart/skywalking/values-my-es.yaml b/chart/skywalking/values-my-es.yaml index c2a785f..3bd18a6 100644 --- a/chart/skywalking/values-my-es.yaml +++ b/chart/skywalking/values-my-es.yaml @@ -26,9 +26,12 @@ ui: image: tag: 10.0.0 +eckOperator: + enabled: false + elasticsearch: enabled: false - config: # For users of an existing elasticsearch cluster,takes effect when `elasticsearch.enabled` is false + config: # For users of an existing elasticsearch cluster, takes effect when `elasticsearch.enabled` is false host: elasticsearch-es-http port: http: 9200 diff --git a/chart/skywalking/values.yaml b/chart/skywalking/values.yaml index e9f14ce..497b34f 100644 --- a/chart/skywalking/values.yaml +++ b/chart/skywalking/values.yaml @@ -194,251 +194,156 @@ oapInit: extraPodLabels: {} # sidecar.istio.io/inject: false +# ECK Operator settings +# Set eckOperator.enabled to false if the ECK operator is already installed in your cluster +eckOperator: + enabled: true + +# Elasticsearch managed by ECK (eck-elasticsearch chart) +# ref: https://github.com/elastic/cloud-on-k8s elasticsearch: enabled: true + version: 8.18.8 + fullnameOverride: "" + + # For users of an existing elasticsearch cluster, takes effect when `elasticsearch.enabled` is false config: - # For users of an existing elasticsearch cluster,takes effect when `elasticsearch.enabled` is false + host: elasticsearch # es service on kubernetes or host port: http: 9200 - host: elasticsearch # es service on kubernetes or host - user: "xxx" # [optional] - password: "xxx" # [optional] - clusterName: "elasticsearch" - nodeGroup: "master" - - # The service that non master groups will try to connect to when joining the cluster - # This should be set to clusterName + "-" + nodeGroup for your master group - masterService: "" - - # Elasticsearch roles that will be applied to this nodeGroup - # These will be set as environment variables. E.g. node.master=true - roles: - master: "true" - ingest: "true" - data: "true" - - replicas: 3 - minimumMasterNodes: 2 - - esMajorVersion: "" - - # Allows you to add any config files in /usr/share/elasticsearch/config/ - # such as elasticsearch.yml and log4j2.properties - esConfig: {} - # elasticsearch.yml: | - # key: - # nestedkey: value - # log4j2.properties: | - # key = value - - # Extra environment variables to append to this nodeGroup - # This will be appended to the current 'env:' key. You can use any of the kubernetes env - # syntax here - extraEnvs: [] - # - name: MY_ENVIRONMENT_VAR - # value: the_value_goes_here - - # A list of secrets and their paths to mount inside the pod - # This is useful for mounting certificates for security and for mounting - # the X-Pack license - secretMounts: [] - # - name: elastic-certificates - # secretName: elastic-certificates - # path: /usr/share/elasticsearch/config/certs - - podAnnotations: {} - # iam.amazonaws.com/role: es-cluster + user: "" # [optional] + password: "" # [optional] - # additionals labels + # Labels that will be applied to the Elasticsearch resource labels: {} - esJavaOpts: "-Xmx1g -Xms1g" - - resources: - requests: - cpu: "100m" - memory: "2Gi" - limits: - cpu: "1000m" - memory: "2Gi" - - initResources: {} - # limits: - # cpu: "25m" - # # memory: "128Mi" - # requests: - # cpu: "25m" - # memory: "128Mi" - - sidecarResources: {} - # limits: - # cpu: "25m" - # # memory: "128Mi" - # requests: - # cpu: "25m" - # memory: "128Mi" - - networkHost: "0.0.0.0" - - volumeClaimTemplate: - accessModes: [ "ReadWriteOnce" ] - resources: - requests: - storage: 30Gi - - rbac: - create: false - serviceAccountName: "" - - podSecurityPolicy: - create: false - name: "" - spec: - privileged: true - fsGroup: - rule: RunAsAny - runAsUser: - rule: RunAsAny - seLinux: - rule: RunAsAny - supplementalGroups: - rule: RunAsAny - volumes: - - secret - - configMap - - persistentVolumeClaim - - persistence: - enabled: false - annotations: {} - - extraVolumes: "" - # - name: extras - # emptyDir: {} - - extraVolumeMounts: "" - # - name: extras - # mountPath: /usr/share/extras - # readOnly: true - - extraInitContainers: "" - # - name: do-something - # image: busybox - # command: ['do', 'something'] - - # This is the PriorityClass settings as defined in - # https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#priorityclass - priorityClassName: "" - - # By default this will make sure two pods don't end up on the same node - # Changing this to a region would allow you to spread pods across regions - antiAffinityTopologyKey: "kubernetes.io/hostname" - - # Hard means that by default pods will only be scheduled if there are enough nodes for them - # and that they will never end up on the same node. Setting this to soft will do this "best effort" - antiAffinity: "hard" - - # This is the node affinity settings as defined in - # https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#node-affinity-beta-feature - nodeAffinity: {} - - # The default is to deploy all pods serially. By setting this to parallel all pods are started at - # the same time when bootstrapping the cluster - podManagementPolicy: "Parallel" - - protocol: http - httpPort: 9200 - transportPort: 9300 - - service: - labels: {} - labelsHeadless: {} - type: ClusterIP - nodePort: "" - annotations: {} - httpPortName: http - transportPortName: transport - - updateStrategy: RollingUpdate - - # This is the max unavailable setting for the pod disruption budget - # The default value of 1 will make sure that kubernetes won't allow more than 1 - # of your pods to be unavailable during maintenance - maxUnavailable: 1 - - podSecurityContext: - fsGroup: 1000 - runAsUser: 1000 - - # The following value is deprecated, - # please use the above podSecurityContext.fsGroup instead - fsGroup: "" - - securityContext: - capabilities: - drop: - - ALL - # readOnlyRootFilesystem: true - runAsNonRoot: true - runAsUser: 1000 - - # How long to wait for elasticsearch to stop gracefully - terminationGracePeriod: 120 - - sysctlVmMaxMapCount: 262144 - - readinessProbe: - failureThreshold: 3 - initialDelaySeconds: 10 - periodSeconds: 10 - successThreshold: 3 - timeoutSeconds: 5 - - # https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster-health.html#request-params wait_for_status - clusterHealthCheckParams: "wait_for_status=green&timeout=1s" - - ## Use an alternate scheduler. - ## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/ - ## - schedulerName: "" - - imagePullSecrets: [] - nodeSelector: {} - tolerations: [] - - # Enabling this will publically expose your Elasticsearch instance. - # Only enable this if you have security enabled on your cluster + # Annotations that will be applied to the Elasticsearch resource + annotations: {} + + # Settings to control how Elasticsearch will be accessed + # ref: https://www.elastic.co/guide/en/cloud-on-k8s/current/k8s-accessing-elastic-services.html + http: + # Disable TLS on HTTP layer so OAP can connect without certificate trust configuration. + # To re-enable TLS, remove the tls section or set disabled to false, + # and configure OAP to trust the self-signed certificate. + tls: + selfSignedCertificate: + disabled: true + # service: + # spec: + # type: LoadBalancer + + # Control Elasticsearch Secure Settings + # ref: https://www.elastic.co/guide/en/cloud-on-k8s/current/k8s-es-secure-settings.html + secureSettings: [] + # - secretName: one-secure-settings-secret + + # Settings for limiting the number of simultaneous changes to an Elasticsearch resource + # ref: https://www.elastic.co/guide/en/cloud-on-k8s/current/k8s-update-strategy.html + updateStrategy: {} + # changeBudget: + # maxSurge: 3 + # maxUnavailable: 1 + + # VolumeClaimDeletePolicy sets the policy for handling deletion of PersistentVolumeClaims for all NodeSets. + # Possible values are DeleteOnScaledownOnly and DeleteOnScaledownAndClusterDeletion. + volumeClaimDeletePolicy: "" + + # Node configuration settings + # ref: https://www.elastic.co/guide/en/cloud-on-k8s/current/k8s-node-configuration.html + nodeSets: + - name: default + count: 3 + config: + # For production workloads, it is strongly recommended to increase the kernel setting + # vm.max_map_count to 262144 via an initContainer and leave node.store.allow_mmap unset. + # ref: https://www.elastic.co/guide/en/cloud-on-k8s/current/k8s-virtual-memory.html + node.store.allow_mmap: false + # Persistent storage for Elasticsearch data + # volumeClaimTemplates: + # - metadata: + # name: elasticsearch-data + # spec: + # accessModes: + # - ReadWriteOnce + # resources: + # requests: + # storage: 30Gi + # # storageClassName: standard + podTemplate: + metadata: {} + # annotations: + # iam.amazonaws.com/role: es-cluster + # labels: + # my-label: my-value + spec: + # Pod scheduling constraints + # ref: https://www.elastic.co/guide/en/cloud-on-k8s/current/k8s-advanced-node-scheduling.html + + # affinity: + # nodeAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # nodeSelectorTerms: + # - matchExpressions: + # - key: topology.kubernetes.io/zone + # operator: In + # values: + # - us-east-1a + + # nodeSelector: {} + + # tolerations: [] + + # imagePullSecrets: [] + + # priorityClassName: "" + + # terminationGracePeriodSeconds: 120 + + # Init containers (e.g. to set vm.max_map_count or install plugins) + # initContainers: + # - command: + # - sh + # - "-c" + # - sysctl -w vm.max_map_count=262144 + # name: sysctl + # securityContext: + # privileged: true + # runAsUser: 0 + + containers: + - name: elasticsearch + resources: + requests: + cpu: 100m + memory: 2Gi + limits: + memory: 2Gi + # Environment variables (replaces the old extraEnvs and esJavaOpts) + # env: + # - name: ES_JAVA_OPTS + # value: "-Xms1g -Xmx1g" + + # Container-level security context (typically set by ECK automatically) + # securityContext: + # capabilities: + # drop: + # - ALL + # runAsNonRoot: true + # runAsUser: 1000 + + # Ingress configuration for exposing Elasticsearch externally ingress: enabled: false annotations: {} # kubernetes.io/ingress.class: nginx # kubernetes.io/tls-acme: "true" - path: / hosts: - - chart-example.local - tls: [] - # - secretName: chart-example-tls - # hosts: - # - chart-example.local - - nameOverride: "" - fullnameOverride: "" - - # https://github.com/elastic/helm-charts/issues/63 - masterTerminationFix: false - - lifecycle: {} - # preStop: - # exec: - # command: ["/bin/sh", "-c", "echo Hello from the postStart handler > /usr/share/message"] - # postStart: - # exec: - # command: ["/bin/sh", "-c", "echo Hello from the postStart handler > /usr/share/message"] - - sysctlInitContainer: - enabled: true - - keystore: [] + - host: chart-example.local + path: / + tls: + enabled: false + # secretName: chart-example-tls postgresql: enabled: false # Whether to start a demo postgresql deployment, don't use this for production. diff --git a/test/e2e/e2e-elasticsearch.yaml b/test/e2e/e2e-elasticsearch.yaml index 9e735fb..26e8cfe 100644 --- a/test/e2e/e2e-elasticsearch.yaml +++ b/test/e2e/e2e-elasticsearch.yaml @@ -40,13 +40,17 @@ setup: kubectl label namespace default istio-injection=enabled - name: Install helm command: bash test/e2e/setup-e2e-shell/install.sh helm - - name: Install SkyWalking + - name: Install ECK operator command: | helm dep up chart/skywalking + helm -n istio-system install eck-operator chart/skywalking/charts/eck-operator-3.3.1.tgz \ + --create-namespace + kubectl -n istio-system rollout status --watch --timeout=120s statefulset/elastic-operator + - name: Install SkyWalking + command: | helm -n istio-system install skywalking chart/skywalking \ --set fullnameOverride=skywalking \ - --set elasticsearch.replicas=1 \ - --set elasticsearch.minimumMasterNodes=1 \ + --set eckOperator.enabled=false \ --set oap.env.SW_ENVOY_METRIC_ALS_HTTP_ANALYSIS=k8s-mesh \ --set oap.env.SW_ENVOY_METRIC_ALS_TCP_ANALYSIS=k8s-mesh \ --set oap.env.K8S_SERVICE_NAME_RULE='e2e::${service.metadata.name}' \ diff --git a/test/e2e/env b/test/e2e/env index 373a447..2d3ff73 100644 --- a/test/e2e/env +++ b/test/e2e/env @@ -13,13 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -SW_AGENT_JAVA_COMMIT=f0245864e4388a388fe7445b56b6ce7cedc94aaf -SW_AGENT_NGINX_LUA_COMMIT=c3cee4841798a147d83b96a10914d4ac0e11d0aa -SW_AGENT_NODEJS_COMMIT=4f9a91dad3dfd8cfe5ba8f7bd06b39e11eb5e65e -SW_AGENT_GO_COMMIT=154de50628e82e590941585411299459e352317d -SW_AGENT_PYTHON_COMMIT=c76a6ec51a478ac91abb20ec8f22a99b8d4d6a58 -SW_AGENT_CLIENT_JS_COMMIT=af0565a67d382b683c1dbd94c379b7080db61449 -SW_AGENT_CLIENT_JS_TEST_COMMIT=4f1eb1dcdbde3ec4a38534bf01dded4ab5d2f016 -SW_KUBERNETES_COMMIT_SHA=6fe5e6f0d3b7686c6be0457733e825ee68cb9b35 - -SW_CTL_COMMIT=7bbac156e74e8fdc0c5bd6f377be9f0b30420176 +OAP_TAG=a0cec0ca237792497d2da0b65757d11f58c3f342 +OAP_REPO=ghcr.io/apache/skywalking/oap +UI_TAG=a0cec0ca237792497d2da0b65757d11f58c3f342 +UI_REPO=ghcr.io/apache/skywalking/ui +SATELLITE_TAG=v35bfaff6352b4dc351a706772796a1f79b651c14 +SATELLITE_REPO=ghcr.io/apache/skywalking-satellite/skywalking-satellite +BANYANDB_TAG=4b9ac7545a7084514bdaea7f0b8b5b7b13b745a5 +BANYANDB_REPO=ghcr.io/apache/skywalking-banyandb +SW_CTL_COMMIT=9a1beab08413ce415a00a8547a238a14691c5655 diff --git a/test/e2e/expected/metrics-has-value.yml b/test/e2e/expected/metrics-has-value.yml index 37b19de..cae6688 100644 --- a/test/e2e/expected/metrics-has-value.yml +++ b/test/e2e/expected/metrics-has-value.yml @@ -22,7 +22,7 @@ results: values: {{- contains .values }} - id: {{ notEmpty .id }} - value: {{ .value }} + value: {{ notEmpty .value }} owner: null traceid: null - id: {{ notEmpty .id }} diff --git a/test/e2e/env.swck b/test/e2e/swck/deploy-elasticsearch.yaml similarity index 54% rename from test/e2e/env.swck rename to test/e2e/swck/deploy-elasticsearch.yaml index fd128cc..bc3180c 100644 --- a/test/e2e/env.swck +++ b/test/e2e/swck/deploy-elasticsearch.yaml @@ -13,14 +13,41 @@ # See the License for the specific language governing permissions and # limitations under the License. -SW_AGENT_JAVA_COMMIT=fbdfc42b4d825ba33205d646ddaaaad20c005cb8 -SW_AGENT_SATELLITE_COMMIT=1f3c08a5af19f8522f2a40d9339c45fa816bfe07 -SW_AGENT_NGINX_LUA_COMMIT=c3cee4841798a147d83b96a10914d4ac0e11d0aa -SW_AGENT_NODEJS_COMMIT=e755659c7f308d3b5589619778c8360308cb14f8 -SW_AGENT_GO_COMMIT=4af380c2db6243106b0fc650b6003ce3b3eb82a0 -SW_AGENT_PYTHON_COMMIT=50388c55428d742d73d9733278f04173585de80d -SW_AGENT_CLIENT_JS_COMMIT=af0565a67d382b683c1dbd94c379b7080db61449 -SW_AGENT_CLIENT_JS_TEST_COMMIT=4f1eb1dcdbde3ec4a38534bf01dded4ab5d2f016 -SW_KUBERNETES_COMMIT_SHA=0f3ec68e5a7e1608cec8688716b848ed15e971e5 - -SW_CTL_COMMIT=9d2d0edccda2afe5cf24f1e632142f40e80e8fa4 +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: es +spec: + serviceName: es + replicas: 1 + selector: + matchLabels: + app: es + template: + metadata: + labels: + app: es + spec: + containers: + - name: es + image: docker.elastic.co/elasticsearch/elasticsearch:8.18.8 + env: + - name: discovery.type + value: single-node + - name: xpack.security.enabled + value: "false" + ports: + - containerPort: 9200 +--- +apiVersion: v1 +kind: Service +metadata: + name: es-out +spec: + type: ClusterIP + selector: + app: es + ports: + - port: 9200 + targetPort: 9200 + protocol: TCP diff --git a/test/e2e/swck/oap-agent-adapter-hpa.yaml b/test/e2e/swck/oap-agent-adapter-hpa.yaml index b3c30cf..3bea0e0 100644 --- a/test/e2e/swck/oap-agent-adapter-hpa.yaml +++ b/test/e2e/swck/oap-agent-adapter-hpa.yaml @@ -16,7 +16,7 @@ setup: env: kind file: ../kind.yaml - init-system-environment: ../env.swck + init-system-environment: ../env kind: expose-ports: - namespace: skywalking-system @@ -63,10 +63,26 @@ setup: - namespace: skywalking-swck-system resource: deployment/skywalking-swck-operator for: condition=Available - - name: Setup oapserver and ui + - name: Setup BanyanDB + command: | + export BANYANDB_VERSION=$(yq e '.dependencies[] | select(.name == "skywalking-banyandb-helm") | .version' chart/skywalking/Chart.yaml) + helm install banyandb oci://registry-1.docker.io/apache/skywalking-banyandb-helm \ + --version ${BANYANDB_VERSION} \ + --set fullnameOverride=banyandb \ + --set image.repository=$BANYANDB_REPO \ + --set image.tag=$BANYANDB_TAG \ + --set standalone.enabled=true \ + --set cluster.enabled=false \ + --set etcd.enabled=false + wait: + - namespace: default + resource: pod + for: condition=Ready + - name: Setup oapserver command: | kubectl create namespace skywalking-system - curl https://raw.githubusercontent.com/apache/skywalking-swck/v0.9.0/test/e2e/skywalking-components.yaml | grep -B20 "\-\-\-" | kubectl apply -f - + sed "s#OAP_IMAGE_PLACEHOLDER#${OAP_REPO}:${OAP_TAG}#g;s#UI_IMAGE_PLACEHOLDER#${UI_REPO}:${UI_TAG}#g;s#PLACEHOLDER#latest#g" \ + test/e2e/swck/skywalking-components.yaml | grep -B30 "\-\-\-" | kubectl apply -f - wait: - namespace: skywalking-system resource: OAPServer/skywalking-system diff --git a/test/e2e/swck/oap-agent-banyandb.yaml b/test/e2e/swck/oap-agent-banyandb.yaml index aa3ccd5..45038e0 100644 --- a/test/e2e/swck/oap-agent-banyandb.yaml +++ b/test/e2e/swck/oap-agent-banyandb.yaml @@ -16,18 +16,15 @@ setup: env: kind file: ../kind.yaml - init-system-environment: ../env.swck + init-system-environment: ../env kind: expose-ports: - namespace: skywalking-system resource: service/demo port: 8085 - - namespace: default - resource: service/default-oap + - namespace: skywalking-system + resource: service/skywalking-system-oap port: 12800 - - namespace: default - resource: service/banyandb-test-banyandb-grpc - port: 17912 steps: - name: Install yq command: bash test/e2e/setup-e2e-shell/install.sh yq @@ -54,22 +51,29 @@ setup: - namespace: skywalking-swck-system resource: deployment/skywalking-swck-operator for: condition=Available - - name: label control plane for banyandb affinity test - command: | - kubectl label nodes kind-control-plane storage=banyandb - - name: remove taint on control plane and enable schedule - command: | - kubectl taint nodes kind-control-plane node-role.kubernetes.io/master- - - name: Install banyandb + - name: Setup BanyanDB command: | - curl https://raw.githubusercontent.com/apache/skywalking-swck/v0.9.0/test/e2e/deploy-banyandb.yaml | kubectl apply -f - + export BANYANDB_VERSION=$(yq e '.dependencies[] | select(.name == "skywalking-banyandb-helm") | .version' chart/skywalking/Chart.yaml) + helm install banyandb oci://registry-1.docker.io/apache/skywalking-banyandb-helm \ + --version ${BANYANDB_VERSION} \ + --set fullnameOverride=banyandb \ + --set image.repository=$BANYANDB_REPO \ + --set image.tag=$BANYANDB_TAG \ + --set standalone.enabled=true \ + --set cluster.enabled=false \ + --set etcd.enabled=false + wait: + - namespace: default + resource: pod + for: condition=Ready - name: setup oapserver and ui command: | kubectl create namespace skywalking-system - curl https://raw.githubusercontent.com/apache/skywalking-swck/v0.9.0/test/e2e/skywalking-components-with-banyandb.yaml | kubectl apply -f - + sed "s#OAP_IMAGE_PLACEHOLDER#${OAP_REPO}:${OAP_TAG}#g;s#UI_IMAGE_PLACEHOLDER#${UI_REPO}:${UI_TAG}#g;s#PLACEHOLDER#latest#g" \ + test/e2e/swck/skywalking-components.yaml | kubectl apply -f - wait: - - namespace: default - resource: OAPServer/default + - namespace: skywalking-system + resource: OAPServer/skywalking-system for: condition=Available - namespace: skywalking-system resource: UI/skywalking-system @@ -77,7 +81,7 @@ setup: - name: Setup java agent demo command: | kubectl label namespace skywalking-system swck-injection=enabled - curl https://raw.githubusercontent.com/apache/skywalking-swck/v0.9.0/test/e2e/demo.yaml | sed 's/oap-service/default-oap.default/' | kubectl apply -f - + curl https://raw.githubusercontent.com/apache/skywalking-swck/v0.9.0/test/e2e/demo.yaml | sed 's/oap-service/skywalking-system-oap.skywalking-system/' | kubectl apply -f - wait: - namespace: skywalking-system resource: deployment/demo @@ -91,7 +95,7 @@ cleanup: trigger: action: http interval: 10s - times: 5 + times: 0 url: http://${service_demo_host}:${service_demo_8085}/hello method: GET @@ -103,11 +107,11 @@ verify: # the interval between two attempts, e.g. 10s, 1m. interval: 10s cases: - - query: swctl --display yaml --base-url=http://${service_default_oap_host}:${service_default_oap_12800}/graphql service ls + - query: swctl --display yaml --base-url=http://${service_skywalking_system_oap_host}:${service_skywalking_system_oap_12800}/graphql service ls expected: ../expected/swck-demo-service.yaml - - query: swctl --display yaml --base-url=http://${service_default_oap_host}:${service_default_oap_12800}/graphql metrics linear --name service_cpm --service-name Your_ApplicationName | yq e 'to_entries' - + - query: swctl --display yaml --base-url=http://${service_skywalking_system_oap_host}:${service_skywalking_system_oap_12800}/graphql metrics linear --name service_cpm --service-name Your_ApplicationName | yq e 'to_entries' - expected: ../expected/metrics.yaml - - query: swctl --display yaml --base-url=http://${service_default_oap_host}:${service_default_oap_12800}/graphql endpoint list --keyword=hello --service-name Your_ApplicationName + - query: swctl --display yaml --base-url=http://${service_skywalking_system_oap_host}:${service_skywalking_system_oap_12800}/graphql endpoint list --keyword=hello --service-name Your_ApplicationName expected: ../expected/endpoint.yaml - - query: swctl --display yaml --base-url=http://${service_default_oap_host}:${service_default_oap_12800}/graphql metrics linear --name endpoint_cpm --endpoint-name GET:/hello --service-name Your_ApplicationName | yq e 'to_entries' - + - query: swctl --display yaml --base-url=http://${service_skywalking_system_oap_host}:${service_skywalking_system_oap_12800}/graphql metrics linear --name endpoint_cpm --endpoint-name GET:/hello --service-name Your_ApplicationName | yq e 'to_entries' - expected: ../expected/metrics.yaml diff --git a/test/e2e/swck/oap-ui-agent-elasticsearch.yaml b/test/e2e/swck/oap-ui-agent-elasticsearch.yaml index 85e4e6a..81f4b60 100644 --- a/test/e2e/swck/oap-ui-agent-elasticsearch.yaml +++ b/test/e2e/swck/oap-ui-agent-elasticsearch.yaml @@ -16,18 +16,15 @@ setup: env: kind file: ../kind.yaml - init-system-environment: ../env.swck + init-system-environment: ../env kind: expose-ports: - namespace: skywalking-system resource: service/demo port: 8085 - - namespace: default - resource: service/default-oap + - namespace: skywalking-system + resource: service/skywalking-system-oap port: 12800 - - namespace: default - resource: service/es-out - port: 9200 steps: - name: Install yq command: bash test/e2e/setup-e2e-shell/install.sh yq @@ -56,18 +53,16 @@ setup: for: condition=Available - name: setup elasticsearch command: | - curl https://raw.githubusercontent.com/apache/skywalking-swck/v0.9.0/test/e2e/deploy-elasticsearch.yaml | kubectl apply -f - + kubectl apply -f test/e2e/swck/deploy-elasticsearch.yaml kubectl rollout status --watch --timeout=120s statefulset/es - - name: setup storage(use the external type) - command: | - curl https://raw.githubusercontent.com/apache/skywalking-swck/v0.9.0/test/e2e/external-storage.yaml | kubectl apply -f - - name: setup oapserver and ui command: | kubectl create namespace skywalking-system - curl https://raw.githubusercontent.com/apache/skywalking-swck/v0.9.0/test/e2e/skywalking-components-with-storage.yaml | kubectl apply -f - + sed "s#OAP_IMAGE_PLACEHOLDER#${OAP_REPO}:${OAP_TAG}#g;s#UI_IMAGE_PLACEHOLDER#${UI_REPO}:${UI_TAG}#g;s#PLACEHOLDER#latest#g" \ + test/e2e/swck/skywalking-components-elasticsearch.yaml | kubectl apply -f - wait: - - namespace: default - resource: OAPServer/default + - namespace: skywalking-system + resource: OAPServer/skywalking-system for: condition=Available - namespace: skywalking-system resource: UI/skywalking-system @@ -75,7 +70,7 @@ setup: - name: Setup java agent demo command: | kubectl label namespace skywalking-system swck-injection=enabled - curl https://raw.githubusercontent.com/apache/skywalking-swck/v0.9.0/test/e2e/demo.yaml | sed 's/oap-service/default-oap.default/' | kubectl apply -f - + curl https://raw.githubusercontent.com/apache/skywalking-swck/v0.9.0/test/e2e/demo.yaml | sed 's/oap-service/skywalking-system-oap.skywalking-system/' | kubectl apply -f - wait: - namespace: skywalking-system resource: deployment/demo @@ -89,7 +84,7 @@ cleanup: trigger: action: http interval: 10s - times: 5 + times: 0 url: http://${service_demo_host}:${service_demo_8085}/hello method: GET @@ -101,11 +96,11 @@ verify: # the interval between two attempts, e.g. 10s, 1m. interval: 10s cases: - - query: swctl --display yaml --base-url=http://${service_default_oap_host}:${service_default_oap_12800}/graphql service ls + - query: swctl --display yaml --base-url=http://${service_skywalking_system_oap_host}:${service_skywalking_system_oap_12800}/graphql service ls expected: ../expected/swck-demo-service.yaml - - query: swctl --display yaml --base-url=http://${service_default_oap_host}:${service_default_oap_12800}/graphql metrics linear --name service_cpm --service-name Your_ApplicationName | yq e 'to_entries' - + - query: swctl --display yaml --base-url=http://${service_skywalking_system_oap_host}:${service_skywalking_system_oap_12800}/graphql metrics linear --name service_cpm --service-name Your_ApplicationName | yq e 'to_entries' - expected: ../expected/metrics.yaml - - query: swctl --display yaml --base-url=http://${service_default_oap_host}:${service_default_oap_12800}/graphql endpoint list --keyword=hello --service-name Your_ApplicationName + - query: swctl --display yaml --base-url=http://${service_skywalking_system_oap_host}:${service_skywalking_system_oap_12800}/graphql endpoint list --keyword=hello --service-name Your_ApplicationName expected: ../expected/endpoint.yaml - - query: swctl --display yaml --base-url=http://${service_default_oap_host}:${service_default_oap_12800}/graphql metrics linear --name endpoint_cpm --endpoint-name GET:/hello --service-name Your_ApplicationName | yq e 'to_entries' - + - query: swctl --display yaml --base-url=http://${service_skywalking_system_oap_host}:${service_skywalking_system_oap_12800}/graphql metrics linear --name endpoint_cpm --endpoint-name GET:/hello --service-name Your_ApplicationName | yq e 'to_entries' - expected: ../expected/metrics.yaml diff --git a/test/e2e/swck/oap-ui-agent-oapserverconfig.yaml b/test/e2e/swck/oap-ui-agent-oapserverconfig.yaml index 88db1b3..ae837da 100644 --- a/test/e2e/swck/oap-ui-agent-oapserverconfig.yaml +++ b/test/e2e/swck/oap-ui-agent-oapserverconfig.yaml @@ -16,7 +16,7 @@ setup: env: kind file: ../kind.yaml - init-system-environment: ../env.swck + init-system-environment: ../env kind: expose-ports: - namespace: skywalking-system @@ -54,10 +54,26 @@ setup: - namespace: skywalking-swck-system resource: deployment/skywalking-swck-operator for: condition=Available + - name: Setup BanyanDB + command: | + export BANYANDB_VERSION=$(yq e '.dependencies[] | select(.name == "skywalking-banyandb-helm") | .version' chart/skywalking/Chart.yaml) + helm install banyandb oci://registry-1.docker.io/apache/skywalking-banyandb-helm \ + --version ${BANYANDB_VERSION} \ + --set fullnameOverride=banyandb \ + --set image.repository=$BANYANDB_REPO \ + --set image.tag=$BANYANDB_TAG \ + --set standalone.enabled=true \ + --set cluster.enabled=false \ + --set etcd.enabled=false + wait: + - namespace: default + resource: pod + for: condition=Ready - name: setup oapserver and ui - command: | + command: | kubectl create namespace skywalking-system - curl https://raw.githubusercontent.com/apache/skywalking-swck/v0.9.0/test/e2e/skywalking-components.yaml | kubectl apply -f - + sed "s#OAP_IMAGE_PLACEHOLDER#${OAP_REPO}:${OAP_TAG}#g;s#UI_IMAGE_PLACEHOLDER#${UI_REPO}:${UI_TAG}#g;s#PLACEHOLDER#latest#g" \ + test/e2e/swck/skywalking-components.yaml | kubectl apply -f - wait: - namespace: skywalking-system resource: OAPServer/skywalking-system @@ -73,13 +89,13 @@ setup: - namespace: skywalking-system resource: deployment/demo for: condition=Available - - name: setup oapserverconfig(static configuration) - command: | - export oap_podname=$(kubectl get pod -lapp=oap -n skywalking-system -o jsonpath='{.items[*].metadata.name}') - curl https://raw.githubusercontent.com/apache/skywalking-swck/v0.9.0/test/e2e/oapserverconfig-demo.yaml | kubectl apply -f - - - name: wait the old pod teminated + - name: setup oapserverconfig(static configuration) and wait for pod restart command: | - while kubectl get pod $oap_podname -n skywalking-system > /dev/null; \ + oap_podname=$(kubectl get pod -lapp=oap -n skywalking-system -o jsonpath='{.items[*].metadata.name}') + curl https://raw.githubusercontent.com/apache/skywalking-swck/v0.9.0/test/e2e/oapserverconfig-demo.yaml | \ + sed '/env:/a\ - name: SW_STORAGE\n value: banyandb\n - name: SW_STORAGE_BANYANDB_TARGETS\n value: banyandb-grpc.default:17912' | \ + kubectl apply -f - + while kubectl get pod $oap_podname -n skywalking-system > /dev/null 2>&1; \ do \ sleep 0.1; \ done @@ -89,7 +105,7 @@ setup: wait: - namespace: skywalking-system resource: deployment/songs-deployment - for: condition=Available + for: condition=Available - name: setup oapserverdynamicconfig(dynamic configuration) command: | curl https://raw.githubusercontent.com/apache/skywalking-swck/v0.9.0/test/e2e/oapserverdynamicconfig-demo.yaml | sed 's/default: 0.1/default: 500/' | kubectl apply -f - @@ -102,7 +118,7 @@ cleanup: trigger: action: http interval: 5s - times: 5 + times: 0 url: http://${service_demo_host}:${service_demo_8085}/hello method: GET diff --git a/test/e2e/swck/oap-ui-agent-satellite.yaml b/test/e2e/swck/oap-ui-agent-satellite.yaml index 0526dec..e5294c1 100644 --- a/test/e2e/swck/oap-ui-agent-satellite.yaml +++ b/test/e2e/swck/oap-ui-agent-satellite.yaml @@ -16,7 +16,7 @@ setup: env: kind file: ../kind.yaml - init-system-environment: ../env.swck + init-system-environment: ../env kind: expose-ports: - namespace: skywalking-system @@ -51,10 +51,26 @@ setup: - namespace: skywalking-swck-system resource: deployment/skywalking-swck-operator for: condition=Available + - name: Setup BanyanDB + command: | + export BANYANDB_VERSION=$(yq e '.dependencies[] | select(.name == "skywalking-banyandb-helm") | .version' chart/skywalking/Chart.yaml) + helm install banyandb oci://registry-1.docker.io/apache/skywalking-banyandb-helm \ + --version ${BANYANDB_VERSION} \ + --set fullnameOverride=banyandb \ + --set image.repository=$BANYANDB_REPO \ + --set image.tag=$BANYANDB_TAG \ + --set standalone.enabled=true \ + --set cluster.enabled=false \ + --set etcd.enabled=false + wait: + - namespace: default + resource: pod + for: condition=Ready - name: setup oapserver and ui command: | kubectl create namespace skywalking-system - curl https://raw.githubusercontent.com/apache/skywalking-swck/v0.9.0/test/e2e/skywalking-components-with-satellite.yaml | kubectl apply -f - + sed "s#OAP_IMAGE_PLACEHOLDER#${OAP_REPO}:${OAP_TAG}#g;s#UI_IMAGE_PLACEHOLDER#${UI_REPO}:${UI_TAG}#g;s#SATELLITE_IMAGE_PLACEHOLDER#${SATELLITE_REPO}:${SATELLITE_TAG}#g;s#PLACEHOLDER#latest#g" \ + test/e2e/swck/skywalking-components-satellite.yaml | kubectl apply -f - wait: - namespace: skywalking-system resource: OAPServer/skywalking-system diff --git a/test/e2e/swck/skywalking-components-elasticsearch.yaml b/test/e2e/swck/skywalking-components-elasticsearch.yaml new file mode 100644 index 0000000..9df8ef3 --- /dev/null +++ b/test/e2e/swck/skywalking-components-elasticsearch.yaml @@ -0,0 +1,49 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# OAPServer with Elasticsearch storage +apiVersion: operator.skywalking.apache.org/v1alpha1 +kind: OAPServer +metadata: + name: skywalking-system + namespace: skywalking-system +spec: + version: PLACEHOLDER + instances: 1 + image: OAP_IMAGE_PLACEHOLDER + service: + template: + type: ClusterIP + config: + - name: SW_STORAGE + value: elasticsearch + - name: SW_STORAGE_ES_CLUSTER_NODES + value: es-out.default:9200 +--- +apiVersion: operator.skywalking.apache.org/v1alpha1 +kind: UI +metadata: + name: skywalking-system + namespace: skywalking-system +spec: + version: PLACEHOLDER + instances: 1 + image: UI_IMAGE_PLACEHOLDER + OAPServerAddress: http://skywalking-system-oap.skywalking-system:12800 + service: + template: + type: ClusterIP + ingress: + host: demo.ui.skywalking diff --git a/test/e2e/swck/skywalking-components-satellite.yaml b/test/e2e/swck/skywalking-components-satellite.yaml new file mode 100644 index 0000000..7b8f4c6 --- /dev/null +++ b/test/e2e/swck/skywalking-components-satellite.yaml @@ -0,0 +1,67 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# OAPServer with BanyanDB storage and satellite meter config +apiVersion: operator.skywalking.apache.org/v1alpha1 +kind: OAPServer +metadata: + name: skywalking-system + namespace: skywalking-system +spec: + version: PLACEHOLDER + instances: 1 + image: OAP_IMAGE_PLACEHOLDER + config: + - name: SW_STORAGE + value: banyandb + - name: SW_STORAGE_BANYANDB_TARGETS + value: banyandb-grpc.default:17912 + - name: SW_METER_ANALYZER_ACTIVE_FILES + value: satellite + service: + template: + type: ClusterIP +--- +apiVersion: operator.skywalking.apache.org/v1alpha1 +kind: Satellite +metadata: + name: skywalking-system + namespace: skywalking-system +spec: + version: PLACEHOLDER + instances: 1 + image: SATELLITE_IMAGE_PLACEHOLDER + config: + - name: SATELLITE_TELEMETRY_EXPORT_TYPE + value: metrics_service + service: + template: + type: ClusterIP +--- +apiVersion: operator.skywalking.apache.org/v1alpha1 +kind: UI +metadata: + name: skywalking-system + namespace: skywalking-system +spec: + version: PLACEHOLDER + instances: 1 + image: UI_IMAGE_PLACEHOLDER + OAPServerAddress: http://skywalking-system-oap.skywalking-system:12800 + service: + template: + type: ClusterIP + ingress: + host: demo.ui.skywalking diff --git a/test/e2e/swck/skywalking-components.yaml b/test/e2e/swck/skywalking-components.yaml new file mode 100644 index 0000000..63ad62f --- /dev/null +++ b/test/e2e/swck/skywalking-components.yaml @@ -0,0 +1,49 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# OAPServer with BanyanDB storage +apiVersion: operator.skywalking.apache.org/v1alpha1 +kind: OAPServer +metadata: + name: skywalking-system + namespace: skywalking-system +spec: + version: PLACEHOLDER + instances: 1 + image: OAP_IMAGE_PLACEHOLDER + service: + template: + type: ClusterIP + config: + - name: SW_STORAGE + value: banyandb + - name: SW_STORAGE_BANYANDB_TARGETS + value: banyandb-grpc.default:17912 +--- +apiVersion: operator.skywalking.apache.org/v1alpha1 +kind: UI +metadata: + name: skywalking-system + namespace: skywalking-system +spec: + version: PLACEHOLDER + instances: 1 + image: UI_IMAGE_PLACEHOLDER + OAPServerAddress: http://skywalking-system-oap.skywalking-system:12800 + service: + template: + type: ClusterIP + ingress: + host: demo.ui.skywalking diff --git a/test/e2e/values.yaml b/test/e2e/values.yaml index 19b9bae..ebbd206 100644 --- a/test/e2e/values.yaml +++ b/test/e2e/values.yaml @@ -19,8 +19,21 @@ oap: serviceName: e2e::${LABELS."service.istio.io/canonical-name"} serviceInstanceName: ${NAME} elasticsearch: - esConfig: - elasticsearch.yml: | + nodeSets: + - name: default + count: 1 + config: + node.store.allow_mmap: false cluster.routing.allocation.disk.watermark.low: 90% cluster.routing.allocation.disk.watermark.high: 99% cluster.routing.allocation.disk.watermark.flood_stage: 99% + podTemplate: + spec: + containers: + - name: elasticsearch + resources: + limits: + memory: 2Gi + requests: + cpu: 100m + memory: 2Gi