diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 009500b..e877eb9 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -90,6 +90,8 @@ jobs: sed -E "s|image: +cloudstack-csi-driver|image: ${REGISTRY_NAME}/cloudstack-csi-driver:${VERSION}|" deploy/k8s/controller-deployment.yaml >> manifest.yaml echo "---" >> manifest.yaml sed -E "s|image: +cloudstack-csi-driver|image: ${REGISTRY_NAME}/cloudstack-csi-driver:${VERSION}|" deploy/k8s/node-daemonset.yaml >> manifest.yaml + echo "---" >> manifest.yaml + cat deploy/k8s/volume-snapshot-class.yaml >> manifest.yaml - name: Create Release id: create_release @@ -102,6 +104,16 @@ jobs: draft: false prerelease: false + - name: Upload Snapshot CRDs Asset + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} + asset_path: deploy/k8s/00-snapshot-crds.yaml + asset_name: snapshot-crds.yaml + asset_content_type: application/x-yaml + - name: Upload Release Asset uses: actions/upload-release-asset@v1 env: diff --git a/README.md b/README.md index fa7bc62..b8d3c53 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,7 @@ +**Fork Notice:** + +This repo is a fork of the [Leaseweb's] (https://github.com/leaseweb/cloudstack-csi-driver) maitained cloudstack-csi-driver, which is in-turn a fork of [Apalia's](https://github.com/apalia/cloudstack-csi-driver) cloudstack-csi-driver + # CloudStack CSI Driver [![Go Reference](https://pkg.go.dev/badge/github.com/shapeblue/cloudstack-csi-driver.svg)](https://pkg.go.dev/github.com/shapeblue/cloudstack-csi-driver) @@ -76,6 +80,11 @@ The storage class must also have a parameter named `csi.cloudstack.apache.org/disk-offering-id` whose value is the CloudStack disk offering ID. +**Reclaim Policy**: Storage classes can have a `reclaimPolicy` of either `Delete` or `Retain`. If no `reclaimPolicy` is specified, it defaults to `Delete`. + +- `Delete`: When a PVC is deleted or a CKS cluster (Managed Kubernetes Cluster in CloudStack) is deleted, the associated persistent volumes and their underlying CloudStack disk volumes will be automatically removed. +- `Retain`: Persistent volumes and their underlying CloudStack disk volumes will be preserved even after PVC deletion or cluster deletion, allowing for manual recovery or data preservation. + #### Using cloudstack-csi-sc-syncer The tool `cloudstack-csi-sc-syncer` may also be used to synchronize CloudStack @@ -83,6 +92,16 @@ disk offerings to Kubernetes storage classes. [More info...](./cmd/cloudstack-csi-sc-syncer/README.md) +> **Note:** The VolumeSnapshot CRDs (CustomResourceDefinitions) of version 8.3.0 are installed in this deployment. If you use a different version, please ensure compatibility with your Kubernetes cluster and CSI sidecars. + + +``` +kubectl apply -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/v8.3.0/client/config/crd/snapshot.storage.k8s.io_volumesnapshotclasses.yaml +kubectl apply -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/v8.3.0/client/config/crd/snapshot.storage.k8s.io_volumesnapshots.yaml +kubectl apply -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/v8.3.0/client/config/crd/snapshot.storage.k8s.io_volumesnapshotcontents.yaml + +``` + ### Usage Example: @@ -106,6 +125,115 @@ To build the container images: make container ``` + +## Volume Snapshots + +**NOTE:** To create volume snapshots in KVM, make sure to set the `kvm.snapshot.enabled` global setting to true and restart the Management Server + +### Volume snapshot creation +For Volume snapshots to be created, the following configurations need to be applied: + +``` +kubectl apply -f deploy/k8s/00-snapshot-crds.yaml # Installs the VolumeSnapshotClass, VolumeSnapshotContent and VolumeSnapshtot CRDs +kubectl apply -f deploy/k8s/volume-snapshot-class.yaml # Defines VolumeSnapshotClass for CloudStack CSI driver +``` + +Once the CRDs are installed, the snapshot can be taken by applying: +``` +kubectl apply ./examples/k8s/snapshot/snapshot.yaml +``` + +In order to take the snapshot of a volume, `persistentVolumeClaimName` should be set to the right PVC name that is bound to the volume whose snapshot is to be taken. + +You can check CloudStack volume snapshots if the snapshot was successfully created. If for any reason there was an issue, it can be investgated by checking the logs of the cloudstack-csi-controller pods: cloudstack-csi-controller, csi-snapshotter and snapshot-controller containers + +``` +kubectl logs -f -n kube-system # defaults to tailing logs of cloudstack-csi-controller +kubectl logs -f -n kube-system -c csi-snapshotter +kubectl logs -f -n kube-system -c snapshot-controller +``` + +### Restoring a Volume snapshot + +To restore a volume snapshot: +1. Restore a snapshot and Use it in a pod +* Create a PVC from the snapshot - for example ./examples/k8s/snapshot/pvc-from-snapshot.yaml +* Apply the configuration: +``` +kubectl apply -f ./examples/k8s/snapshot/pvc-from-snapshot.yaml +``` +* Create a pod that uses the restored PVC; example pod config ./examples/k8s/snapshot/restore-pod.yaml +``` +kubectl apply -f ./examples/k8s/snapshot/restore-pod.yaml +``` +2. To restore a snapshot when using a deployment +Update the deployment to point to the restored PVC + +``` +spec: + volumes: + - name: app-volume + persistentVolumeClaim: + claimName: pvc-from-snapshot +``` + + +### Deletion of a volume snapshot + +To delete a volume snapshot +One can simlpy delete the volume snapshot created in kubernetes using + +``` +kubectl delete volumesnapshot snapshot-1 # here, snapshot-1 is the name of the snapshot created +``` + +#### Troubleshooting issues with volume snapshot deletion +If for whatever reason, snapshot deletion gets stuck, one can troubleshoot the issue doing the following: + +* Inspect the snapshot + +``` +kubectl get volumesnapshot [-n ] -o yaml +``` + +Look for the following section: +``` +metadata: + finalizers: + - snapshot.storage.kubernetes.io/volumesnapshot-as-source +``` + +If finalizers are present, Kubernetes will not delete the resource until they are removed or resolved. + +* Patch to Remove Finalizers + +``` +kubectl patch volumesnapshot [-n ] --type=merge -p '{"metadata":{"finalizers":[]}}' +``` + +**Caution:** This bypasses cleanup logic. Use only if you're certain the snapshot is no longer needed at the CSI/backend level + +### What happens when you restore a volume from a snapshot +* The CSI external-provisioner (a container in the cloudstack-csi-controller pod) sees the new PVC and notices it references a snapshot +* The CSI driver's `CreateVolume` method is called with a `VolumeContentSource` that contains the snapshot ID +* The CSI driver creates a new volume from the snapshot (using the CloudStack's createVolume API) +* The new volume is now available as a PV (persistent volume) and is bound to the new PVC +* The volume is NOT attached to any node just by restoring from a snapshot, the volume is only attached to a node when a Pod that uses the new PVC is scheduled on a node +* The CSI driver's `ControllerPublishVolume` and `NodePublishVolume` methods are called to attach and mount the volume to the node where the Pod is running + +Hence to debug any issues during restoring a snapshot, check the logs of the cloudstack-csi-controller, external-provisioner containers + +``` +kubectl logs -f -n kube-system # defaults to tailing logs of cloudstack-csi-controller +kubectl logs -f -n kube-system -c external-provisioner +``` + +## Additional General Notes: + +**Node Scheduling Best Practices**: When deploying applications that require specific node placement, use `nodeSelector` or `nodeAffinity` instead of `nodeName`. The `nodeName` field bypasses the Kubernetes scheduler, which can cause issues with storage provisioning. When a StorageClass has `volumeBindingMode: WaitForFirstConsumer`, the CSI controller relies on scheduler decisions to properly bind PVCs. Using `nodeName` prevents this scheduling integration, potentially causing PVC binding failures. + +**Network CIDR Considerations**: When deploying CKS (CloudStack Kubernetes Service) clusters on pre-existing networks, avoid using the `10.0.0.0/16` CIDR range as it conflicts with Calico's default pod network configuration. This overlap can prevent proper CSI driver initialization and may cause networking issues within the cluster. + ## See also - [CloudStack Kubernetes Provider](https://github.com/apache/cloudstack-kubernetes-provider) - Kubernetes Cloud Controller Manager for Apache CloudStack diff --git a/cmd/cloudstack-csi-driver/Dockerfile b/cmd/cloudstack-csi-driver/Dockerfile index 4c260e8..95d1710 100644 --- a/cmd/cloudstack-csi-driver/Dockerfile +++ b/cmd/cloudstack-csi-driver/Dockerfile @@ -14,7 +14,9 @@ RUN apk add --no-cache \ # blkid, mount and umount are required by k8s.io/mount-utils \ blkid \ mount \ - umount + umount \ + # Provides udevadm for device path detection \ + udev COPY ./bin/cloudstack-csi-driver /cloudstack-csi-driver -ENTRYPOINT ["/cloudstack-csi-driver"] \ No newline at end of file +ENTRYPOINT ["/cloudstack-csi-driver"] diff --git a/deploy/k8s/00-snapshot-crds.yaml b/deploy/k8s/00-snapshot-crds.yaml new file mode 100644 index 0000000..b6a402b --- /dev/null +++ b/deploy/k8s/00-snapshot-crds.yaml @@ -0,0 +1,954 @@ +## CRD for VolumeSnapshotClass from https://github.com/kubernetes-csi/external-snapshotter/blob/v8.3.0/client/config/crd/groupsnapshot.storage.k8s.io_volumegroupsnapshotclasses.yaml +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/814" + controller-gen.kubebuilder.io/version: v0.15.0 + name: volumesnapshotclasses.snapshot.storage.k8s.io +spec: + group: snapshot.storage.k8s.io + names: + kind: VolumeSnapshotClass + listKind: VolumeSnapshotClassList + plural: volumesnapshotclasses + shortNames: + - vsclass + - vsclasses + singular: volumesnapshotclass + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .driver + name: Driver + type: string + - description: Determines whether a VolumeSnapshotContent created through the + VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted. + jsonPath: .deletionPolicy + name: DeletionPolicy + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + description: |- + VolumeSnapshotClass specifies parameters that a underlying storage system uses when + creating a volume snapshot. A specific VolumeSnapshotClass is used by specifying its + name in a VolumeSnapshot object. + VolumeSnapshotClasses are non-namespaced + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + deletionPolicy: + description: |- + deletionPolicy determines whether a VolumeSnapshotContent created through + the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted. + Supported values are "Retain" and "Delete". + "Retain" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are kept. + "Delete" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are deleted. + Required. + enum: + - Delete + - Retain + type: string + driver: + description: |- + driver is the name of the storage driver that handles this VolumeSnapshotClass. + Required. + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + parameters: + additionalProperties: + type: string + description: |- + parameters is a key-value map with storage driver specific parameters for creating snapshots. + These values are opaque to Kubernetes. + type: object + required: + - deletionPolicy + - driver + type: object + served: true + storage: true + subresources: {} + - additionalPrinterColumns: + - jsonPath: .driver + name: Driver + type: string + - description: Determines whether a VolumeSnapshotContent created through the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted. + jsonPath: .deletionPolicy + name: DeletionPolicy + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + # This indicates the v1beta1 version of the custom resource is deprecated. + # API requests to this version receive a warning in the server response. + deprecated: true + # This overrides the default warning returned to clients making v1beta1 API requests. + deprecationWarning: "snapshot.storage.k8s.io/v1beta1 VolumeSnapshotClass is deprecated; use snapshot.storage.k8s.io/v1 VolumeSnapshotClass" + schema: + openAPIV3Schema: + description: VolumeSnapshotClass specifies parameters that a underlying storage system uses when creating a volume snapshot. A specific VolumeSnapshotClass is used by specifying its name in a VolumeSnapshot object. VolumeSnapshotClasses are non-namespaced + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + deletionPolicy: + description: deletionPolicy determines whether a VolumeSnapshotContent created through the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted. Supported values are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are kept. "Delete" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are deleted. Required. + enum: + - Delete + - Retain + type: string + driver: + description: driver is the name of the storage driver that handles this VolumeSnapshotClass. Required. + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + parameters: + additionalProperties: + type: string + description: parameters is a key-value map with storage driver specific parameters for creating snapshots. These values are opaque to Kubernetes. + type: object + required: + - deletionPolicy + - driver + type: object + served: false + storage: false + subresources: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +## VolumeSnapshotContent CRD - https://github.com/kubernetes-csi/external-snapshotter/blob/v8.3.0/client/config/crd/snapshot.storage.k8s.io_volumesnapshotcontents.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/955" + name: volumesnapshotcontents.snapshot.storage.k8s.io +spec: + group: snapshot.storage.k8s.io + names: + kind: VolumeSnapshotContent + listKind: VolumeSnapshotContentList + plural: volumesnapshotcontents + shortNames: + - vsc + - vscs + singular: volumesnapshotcontent + scope: Cluster + versions: + - additionalPrinterColumns: + - description: Indicates if the snapshot is ready to be used to restore a volume. + jsonPath: .status.readyToUse + name: ReadyToUse + type: boolean + - description: Represents the complete size of the snapshot in bytes + jsonPath: .status.restoreSize + name: RestoreSize + type: integer + - description: Determines whether this VolumeSnapshotContent and its physical + snapshot on the underlying storage system should be deleted when its bound + VolumeSnapshot is deleted. + jsonPath: .spec.deletionPolicy + name: DeletionPolicy + type: string + - description: Name of the CSI driver used to create the physical snapshot on + the underlying storage system. + jsonPath: .spec.driver + name: Driver + type: string + - description: Name of the VolumeSnapshotClass to which this snapshot belongs. + jsonPath: .spec.volumeSnapshotClassName + name: VolumeSnapshotClass + type: string + - description: Name of the VolumeSnapshot object to which this VolumeSnapshotContent + object is bound. + jsonPath: .spec.volumeSnapshotRef.name + name: VolumeSnapshot + type: string + - description: Namespace of the VolumeSnapshot object to which this VolumeSnapshotContent + object is bound. + jsonPath: .spec.volumeSnapshotRef.namespace + name: VolumeSnapshotNamespace + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + description: |- + VolumeSnapshotContent represents the actual "on-disk" snapshot object in the + underlying storage system + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: |- + spec defines properties of a VolumeSnapshotContent created by the underlying storage system. + Required. + properties: + deletionPolicy: + description: |- + deletionPolicy determines whether this VolumeSnapshotContent and its physical snapshot on + the underlying storage system should be deleted when its bound VolumeSnapshot is deleted. + Supported values are "Retain" and "Delete". + "Retain" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are kept. + "Delete" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are deleted. + For dynamically provisioned snapshots, this field will automatically be filled in by the + CSI snapshotter sidecar with the "DeletionPolicy" field defined in the corresponding + VolumeSnapshotClass. + For pre-existing snapshots, users MUST specify this field when creating the + VolumeSnapshotContent object. + Required. + enum: + - Delete + - Retain + type: string + driver: + description: |- + driver is the name of the CSI driver used to create the physical snapshot on + the underlying storage system. + This MUST be the same as the name returned by the CSI GetPluginName() call for + that driver. + Required. + type: string + source: + description: |- + source specifies whether the snapshot is (or should be) dynamically provisioned + or already exists, and just requires a Kubernetes object representation. + This field is immutable after creation. + Required. + properties: + snapshotHandle: + description: |- + snapshotHandle specifies the CSI "snapshot_id" of a pre-existing snapshot on + the underlying storage system for which a Kubernetes object representation + was (or should be) created. + This field is immutable. + type: string + x-kubernetes-validations: + - message: snapshotHandle is immutable + rule: self == oldSelf + volumeHandle: + description: |- + volumeHandle specifies the CSI "volume_id" of the volume from which a snapshot + should be dynamically taken from. + This field is immutable. + type: string + x-kubernetes-validations: + - message: volumeHandle is immutable + rule: self == oldSelf + type: object + x-kubernetes-validations: + - message: volumeHandle is required once set + rule: '!has(oldSelf.volumeHandle) || has(self.volumeHandle)' + - message: snapshotHandle is required once set + rule: '!has(oldSelf.snapshotHandle) || has(self.snapshotHandle)' + - message: exactly one of volumeHandle and snapshotHandle must be + set + rule: (has(self.volumeHandle) && !has(self.snapshotHandle)) || (!has(self.volumeHandle) + && has(self.snapshotHandle)) + sourceVolumeMode: + description: |- + SourceVolumeMode is the mode of the volume whose snapshot is taken. + Can be either “Filesystem” or “Block”. + If not specified, it indicates the source volume's mode is unknown. + This field is immutable. + This field is an alpha field. + type: string + x-kubernetes-validations: + - message: sourceVolumeMode is immutable + rule: self == oldSelf + volumeSnapshotClassName: + description: |- + name of the VolumeSnapshotClass from which this snapshot was (or will be) + created. + Note that after provisioning, the VolumeSnapshotClass may be deleted or + recreated with different set of values, and as such, should not be referenced + post-snapshot creation. + type: string + volumeSnapshotRef: + description: |- + volumeSnapshotRef specifies the VolumeSnapshot object to which this + VolumeSnapshotContent object is bound. + VolumeSnapshot.Spec.VolumeSnapshotContentName field must reference to + this VolumeSnapshotContent's name for the bidirectional binding to be valid. + For a pre-existing VolumeSnapshotContent object, name and namespace of the + VolumeSnapshot object MUST be provided for binding to happen. + This field is immutable after creation. + Required. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: |- + If referring to a piece of an object instead of an entire object, this string + should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to have some well-defined way of + referencing a part of an object. + TODO: this design is not final and this field is subject to change in the future. + type: string + kind: + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + namespace: + description: |- + Namespace of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ + type: string + resourceVersion: + description: |- + Specific resourceVersion to which this reference is made, if any. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency + type: string + uid: + description: |- + UID of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids + type: string + type: object + x-kubernetes-map-type: atomic + x-kubernetes-validations: + - message: both spec.volumeSnapshotRef.name and spec.volumeSnapshotRef.namespace + must be set + rule: has(self.name) && has(self.__namespace__) + required: + - deletionPolicy + - driver + - source + - volumeSnapshotRef + type: object + x-kubernetes-validations: + - message: sourceVolumeMode is required once set + rule: '!has(oldSelf.sourceVolumeMode) || has(self.sourceVolumeMode)' + status: + description: status represents the current information of a snapshot. + properties: + creationTime: + description: |- + creationTime is the timestamp when the point-in-time snapshot is taken + by the underlying storage system. + In dynamic snapshot creation case, this field will be filled in by the + CSI snapshotter sidecar with the "creation_time" value returned from CSI + "CreateSnapshot" gRPC call. + For a pre-existing snapshot, this field will be filled with the "creation_time" + value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. + If not specified, it indicates the creation time is unknown. + The format of this field is a Unix nanoseconds time encoded as an int64. + On Unix, the command `date +%s%N` returns the current time in nanoseconds + since 1970-01-01 00:00:00 UTC. + format: int64 + type: integer + error: + description: |- + error is the last observed error during snapshot creation, if any. + Upon success after retry, this error field will be cleared. + properties: + message: + description: |- + message is a string detailing the encountered error during snapshot + creation if specified. + NOTE: message may be logged, and it should not contain sensitive + information. + type: string + time: + description: time is the timestamp when the error was encountered. + format: date-time + type: string + type: object + readyToUse: + description: |- + readyToUse indicates if a snapshot is ready to be used to restore a volume. + In dynamic snapshot creation case, this field will be filled in by the + CSI snapshotter sidecar with the "ready_to_use" value returned from CSI + "CreateSnapshot" gRPC call. + For a pre-existing snapshot, this field will be filled with the "ready_to_use" + value returned from the CSI "ListSnapshots" gRPC call if the driver supports it, + otherwise, this field will be set to "True". + If not specified, it means the readiness of a snapshot is unknown. + type: boolean + restoreSize: + description: |- + restoreSize represents the complete size of the snapshot in bytes. + In dynamic snapshot creation case, this field will be filled in by the + CSI snapshotter sidecar with the "size_bytes" value returned from CSI + "CreateSnapshot" gRPC call. + For a pre-existing snapshot, this field will be filled with the "size_bytes" + value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. + When restoring a volume from this snapshot, the size of the volume MUST NOT + be smaller than the restoreSize if it is specified, otherwise the restoration will fail. + If not specified, it indicates that the size is unknown. + format: int64 + minimum: 0 + type: integer + snapshotHandle: + description: |- + snapshotHandle is the CSI "snapshot_id" of a snapshot on the underlying storage system. + If not specified, it indicates that dynamic snapshot creation has either failed + or it is still in progress. + type: string + volumeGroupSnapshotHandle: + description: |- + VolumeGroupSnapshotHandle is the CSI "group_snapshot_id" of a group snapshot + on the underlying storage system. + type: string + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} + - additionalPrinterColumns: + - description: Indicates if the snapshot is ready to be used to restore a volume. + jsonPath: .status.readyToUse + name: ReadyToUse + type: boolean + - description: Represents the complete size of the snapshot in bytes + jsonPath: .status.restoreSize + name: RestoreSize + type: integer + - description: Determines whether this VolumeSnapshotContent and its physical snapshot on the underlying storage system should be deleted when its bound VolumeSnapshot is deleted. + jsonPath: .spec.deletionPolicy + name: DeletionPolicy + type: string + - description: Name of the CSI driver used to create the physical snapshot on the underlying storage system. + jsonPath: .spec.driver + name: Driver + type: string + - description: Name of the VolumeSnapshotClass to which this snapshot belongs. + jsonPath: .spec.volumeSnapshotClassName + name: VolumeSnapshotClass + type: string + - description: Name of the VolumeSnapshot object to which this VolumeSnapshotContent object is bound. + jsonPath: .spec.volumeSnapshotRef.name + name: VolumeSnapshot + type: string + - description: Namespace of the VolumeSnapshot object to which this VolumeSnapshotContent object is bound. + jsonPath: .spec.volumeSnapshotRef.namespace + name: VolumeSnapshotNamespace + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + # This indicates the v1beta1 version of the custom resource is deprecated. + # API requests to this version receive a warning in the server response. + deprecated: true + # This overrides the default warning returned to clients making v1beta1 API requests. + deprecationWarning: "snapshot.storage.k8s.io/v1beta1 VolumeSnapshotContent is deprecated; use snapshot.storage.k8s.io/v1 VolumeSnapshotContent" + schema: + openAPIV3Schema: + description: VolumeSnapshotContent represents the actual "on-disk" snapshot object in the underlying storage system + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + spec: + description: spec defines properties of a VolumeSnapshotContent created by the underlying storage system. Required. + properties: + deletionPolicy: + description: deletionPolicy determines whether this VolumeSnapshotContent and its physical snapshot on the underlying storage system should be deleted when its bound VolumeSnapshot is deleted. Supported values are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are kept. "Delete" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are deleted. For dynamically provisioned snapshots, this field will automatically be filled in by the CSI snapshotter sidecar with the "DeletionPolicy" field defined in the corresponding VolumeSnapshotClass. For pre-existing snapshots, users MUST specify this field when creating the VolumeSnapshotContent object. Required. + enum: + - Delete + - Retain + type: string + driver: + description: driver is the name of the CSI driver used to create the physical snapshot on the underlying storage system. This MUST be the same as the name returned by the CSI GetPluginName() call for that driver. Required. + type: string + source: + description: source specifies whether the snapshot is (or should be) dynamically provisioned or already exists, and just requires a Kubernetes object representation. This field is immutable after creation. Required. + properties: + snapshotHandle: + description: snapshotHandle specifies the CSI "snapshot_id" of a pre-existing snapshot on the underlying storage system for which a Kubernetes object representation was (or should be) created. This field is immutable. + type: string + volumeHandle: + description: volumeHandle specifies the CSI "volume_id" of the volume from which a snapshot should be dynamically taken from. This field is immutable. + type: string + type: object + volumeSnapshotClassName: + description: name of the VolumeSnapshotClass from which this snapshot was (or will be) created. Note that after provisioning, the VolumeSnapshotClass may be deleted or recreated with different set of values, and as such, should not be referenced post-snapshot creation. + type: string + volumeSnapshotRef: + description: volumeSnapshotRef specifies the VolumeSnapshot object to which this VolumeSnapshotContent object is bound. VolumeSnapshot.Spec.VolumeSnapshotContentName field must reference to this VolumeSnapshotContent's name for the bidirectional binding to be valid. For a pre-existing VolumeSnapshotContent object, name and namespace of the VolumeSnapshot object MUST be provided for binding to happen. This field is immutable after creation. Required. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead of an entire object, this string should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. For example, if the object reference is to a container within a pod, this would take on a value like: "spec.containers{name}" (where "name" refers to the name of the container that triggered the event) or if no container name is specified "spec.containers[2]" (container with index 2 in this pod). This syntax is chosen only to have some well-defined way of referencing a part of an object. TODO: this design is not final and this field is subject to change in the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + required: + - deletionPolicy + - driver + - source + - volumeSnapshotRef + type: object + status: + description: status represents the current information of a snapshot. + properties: + creationTime: + description: creationTime is the timestamp when the point-in-time snapshot is taken by the underlying storage system. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "creation_time" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "creation_time" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. If not specified, it indicates the creation time is unknown. The format of this field is a Unix nanoseconds time encoded as an int64. On Unix, the command `date +%s%N` returns the current time in nanoseconds since 1970-01-01 00:00:00 UTC. + format: int64 + type: integer + error: + description: error is the last observed error during snapshot creation, if any. Upon success after retry, this error field will be cleared. + properties: + message: + description: 'message is a string detailing the encountered error during snapshot creation if specified. NOTE: message may be logged, and it should not contain sensitive information.' + type: string + time: + description: time is the timestamp when the error was encountered. + format: date-time + type: string + type: object + readyToUse: + description: readyToUse indicates if a snapshot is ready to be used to restore a volume. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "ready_to_use" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "ready_to_use" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it, otherwise, this field will be set to "True". If not specified, it means the readiness of a snapshot is unknown. + type: boolean + restoreSize: + description: restoreSize represents the complete size of the snapshot in bytes. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "size_bytes" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "size_bytes" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. When restoring a volume from this snapshot, the size of the volume MUST NOT be smaller than the restoreSize if it is specified, otherwise the restoration will fail. If not specified, it indicates that the size is unknown. + format: int64 + minimum: 0 + type: integer + snapshotHandle: + description: snapshotHandle is the CSI "snapshot_id" of a snapshot on the underlying storage system. If not specified, it indicates that dynamic snapshot creation has either failed or it is still in progress. + type: string + type: object + required: + - spec + type: object + served: false + storage: false + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +## VolumeSnapshot CRD - https://github.com/kubernetes-csi/external-snapshotter/blob/v8.3.0/client/config/crd/snapshot.storage.k8s.io_volumesnapshots.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/814" + name: volumesnapshots.snapshot.storage.k8s.io +spec: + group: snapshot.storage.k8s.io + names: + kind: VolumeSnapshot + listKind: VolumeSnapshotList + plural: volumesnapshots + shortNames: + - vs + singular: volumesnapshot + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Indicates if the snapshot is ready to be used to restore a volume. + jsonPath: .status.readyToUse + name: ReadyToUse + type: boolean + - description: If a new snapshot needs to be created, this contains the name of + the source PVC from which this snapshot was (or will be) created. + jsonPath: .spec.source.persistentVolumeClaimName + name: SourcePVC + type: string + - description: If a snapshot already exists, this contains the name of the existing + VolumeSnapshotContent object representing the existing snapshot. + jsonPath: .spec.source.volumeSnapshotContentName + name: SourceSnapshotContent + type: string + - description: Represents the minimum size of volume required to rehydrate from + this snapshot. + jsonPath: .status.restoreSize + name: RestoreSize + type: string + - description: The name of the VolumeSnapshotClass requested by the VolumeSnapshot. + jsonPath: .spec.volumeSnapshotClassName + name: SnapshotClass + type: string + - description: Name of the VolumeSnapshotContent object to which the VolumeSnapshot + object intends to bind to. Please note that verification of binding actually + requires checking both VolumeSnapshot and VolumeSnapshotContent to ensure + both are pointing at each other. Binding MUST be verified prior to usage of + this object. + jsonPath: .status.boundVolumeSnapshotContentName + name: SnapshotContent + type: string + - description: Timestamp when the point-in-time snapshot was taken by the underlying + storage system. + jsonPath: .status.creationTime + name: CreationTime + type: date + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + description: |- + VolumeSnapshot is a user's request for either creating a point-in-time + snapshot of a persistent volume, or binding to a pre-existing snapshot. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: |- + spec defines the desired characteristics of a snapshot requested by a user. + More info: https://kubernetes.io/docs/concepts/storage/volume-snapshots#volumesnapshots + Required. + properties: + source: + description: |- + source specifies where a snapshot will be created from. + This field is immutable after creation. + Required. + properties: + persistentVolumeClaimName: + description: |- + persistentVolumeClaimName specifies the name of the PersistentVolumeClaim + object representing the volume from which a snapshot should be created. + This PVC is assumed to be in the same namespace as the VolumeSnapshot + object. + This field should be set if the snapshot does not exists, and needs to be + created. + This field is immutable. + type: string + x-kubernetes-validations: + - message: persistentVolumeClaimName is immutable + rule: self == oldSelf + volumeSnapshotContentName: + description: |- + volumeSnapshotContentName specifies the name of a pre-existing VolumeSnapshotContent + object representing an existing volume snapshot. + This field should be set if the snapshot already exists and only needs a representation in Kubernetes. + This field is immutable. + type: string + x-kubernetes-validations: + - message: volumeSnapshotContentName is immutable + rule: self == oldSelf + type: object + x-kubernetes-validations: + - message: persistentVolumeClaimName is required once set + rule: '!has(oldSelf.persistentVolumeClaimName) || has(self.persistentVolumeClaimName)' + - message: volumeSnapshotContentName is required once set + rule: '!has(oldSelf.volumeSnapshotContentName) || has(self.volumeSnapshotContentName)' + - message: exactly one of volumeSnapshotContentName and persistentVolumeClaimName + must be set + rule: (has(self.volumeSnapshotContentName) && !has(self.persistentVolumeClaimName)) + || (!has(self.volumeSnapshotContentName) && has(self.persistentVolumeClaimName)) + volumeSnapshotClassName: + description: |- + VolumeSnapshotClassName is the name of the VolumeSnapshotClass + requested by the VolumeSnapshot. + VolumeSnapshotClassName may be left nil to indicate that the default + SnapshotClass should be used. + A given cluster may have multiple default Volume SnapshotClasses: one + default per CSI Driver. If a VolumeSnapshot does not specify a SnapshotClass, + VolumeSnapshotSource will be checked to figure out what the associated + CSI Driver is, and the default VolumeSnapshotClass associated with that + CSI Driver will be used. If more than one VolumeSnapshotClass exist for + a given CSI Driver and more than one have been marked as default, + CreateSnapshot will fail and generate an event. + Empty string is not allowed for this field. + type: string + x-kubernetes-validations: + - message: volumeSnapshotClassName must not be the empty string when + set + rule: size(self) > 0 + required: + - source + type: object + status: + description: |- + status represents the current information of a snapshot. + Consumers must verify binding between VolumeSnapshot and + VolumeSnapshotContent objects is successful (by validating that both + VolumeSnapshot and VolumeSnapshotContent point at each other) before + using this object. + properties: + boundVolumeSnapshotContentName: + description: |- + boundVolumeSnapshotContentName is the name of the VolumeSnapshotContent + object to which this VolumeSnapshot object intends to bind to. + If not specified, it indicates that the VolumeSnapshot object has not been + successfully bound to a VolumeSnapshotContent object yet. + NOTE: To avoid possible security issues, consumers must verify binding between + VolumeSnapshot and VolumeSnapshotContent objects is successful (by validating that + both VolumeSnapshot and VolumeSnapshotContent point at each other) before using + this object. + type: string + creationTime: + description: |- + creationTime is the timestamp when the point-in-time snapshot is taken + by the underlying storage system. + In dynamic snapshot creation case, this field will be filled in by the + snapshot controller with the "creation_time" value returned from CSI + "CreateSnapshot" gRPC call. + For a pre-existing snapshot, this field will be filled with the "creation_time" + value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. + If not specified, it may indicate that the creation time of the snapshot is unknown. + format: date-time + type: string + error: + description: |- + error is the last observed error during snapshot creation, if any. + This field could be helpful to upper level controllers(i.e., application controller) + to decide whether they should continue on waiting for the snapshot to be created + based on the type of error reported. + The snapshot controller will keep retrying when an error occurs during the + snapshot creation. Upon success, this error field will be cleared. + properties: + message: + description: |- + message is a string detailing the encountered error during snapshot + creation if specified. + NOTE: message may be logged, and it should not contain sensitive + information. + type: string + time: + description: time is the timestamp when the error was encountered. + format: date-time + type: string + type: object + readyToUse: + description: |- + readyToUse indicates if the snapshot is ready to be used to restore a volume. + In dynamic snapshot creation case, this field will be filled in by the + snapshot controller with the "ready_to_use" value returned from CSI + "CreateSnapshot" gRPC call. + For a pre-existing snapshot, this field will be filled with the "ready_to_use" + value returned from the CSI "ListSnapshots" gRPC call if the driver supports it, + otherwise, this field will be set to "True". + If not specified, it means the readiness of a snapshot is unknown. + type: boolean + restoreSize: + type: string + description: |- + restoreSize represents the minimum size of volume required to create a volume + from this snapshot. + In dynamic snapshot creation case, this field will be filled in by the + snapshot controller with the "size_bytes" value returned from CSI + "CreateSnapshot" gRPC call. + For a pre-existing snapshot, this field will be filled with the "size_bytes" + value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. + When restoring a volume from this snapshot, the size of the volume MUST NOT + be smaller than the restoreSize if it is specified, otherwise the restoration will fail. + If not specified, it indicates that the size is unknown. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + volumeGroupSnapshotName: + description: |- + VolumeGroupSnapshotName is the name of the VolumeGroupSnapshot of which this + VolumeSnapshot is a part of. + type: string + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} + - additionalPrinterColumns: + - description: Indicates if the snapshot is ready to be used to restore a volume. + jsonPath: .status.readyToUse + name: ReadyToUse + type: boolean + - description: If a new snapshot needs to be created, this contains the name of the source PVC from which this snapshot was (or will be) created. + jsonPath: .spec.source.persistentVolumeClaimName + name: SourcePVC + type: string + - description: If a snapshot already exists, this contains the name of the existing VolumeSnapshotContent object representing the existing snapshot. + jsonPath: .spec.source.volumeSnapshotContentName + name: SourceSnapshotContent + type: string + - description: Represents the minimum size of volume required to rehydrate from this snapshot. + jsonPath: .status.restoreSize + name: RestoreSize + type: string + - description: The name of the VolumeSnapshotClass requested by the VolumeSnapshot. + jsonPath: .spec.volumeSnapshotClassName + name: SnapshotClass + type: string + - description: Name of the VolumeSnapshotContent object to which the VolumeSnapshot object intends to bind to. Please note that verification of binding actually requires checking both VolumeSnapshot and VolumeSnapshotContent to ensure both are pointing at each other. Binding MUST be verified prior to usage of this object. + jsonPath: .status.boundVolumeSnapshotContentName + name: SnapshotContent + type: string + - description: Timestamp when the point-in-time snapshot was taken by the underlying storage system. + jsonPath: .status.creationTime + name: CreationTime + type: date + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + # This indicates the v1beta1 version of the custom resource is deprecated. + # API requests to this version receive a warning in the server response. + deprecated: true + # This overrides the default warning returned to clients making v1beta1 API requests. + deprecationWarning: "snapshot.storage.k8s.io/v1beta1 VolumeSnapshot is deprecated; use snapshot.storage.k8s.io/v1 VolumeSnapshot" + schema: + openAPIV3Schema: + description: VolumeSnapshot is a user's request for either creating a point-in-time snapshot of a persistent volume, or binding to a pre-existing snapshot. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + spec: + description: 'spec defines the desired characteristics of a snapshot requested by a user. More info: https://kubernetes.io/docs/concepts/storage/volume-snapshots#volumesnapshots Required.' + properties: + source: + description: source specifies where a snapshot will be created from. This field is immutable after creation. Required. + properties: + persistentVolumeClaimName: + description: persistentVolumeClaimName specifies the name of the PersistentVolumeClaim object representing the volume from which a snapshot should be created. This PVC is assumed to be in the same namespace as the VolumeSnapshot object. This field should be set if the snapshot does not exists, and needs to be created. This field is immutable. + type: string + volumeSnapshotContentName: + description: volumeSnapshotContentName specifies the name of a pre-existing VolumeSnapshotContent object representing an existing volume snapshot. This field should be set if the snapshot already exists and only needs a representation in Kubernetes. This field is immutable. + type: string + type: object + volumeSnapshotClassName: + description: 'VolumeSnapshotClassName is the name of the VolumeSnapshotClass requested by the VolumeSnapshot. VolumeSnapshotClassName may be left nil to indicate that the default SnapshotClass should be used. A given cluster may have multiple default Volume SnapshotClasses: one default per CSI Driver. If a VolumeSnapshot does not specify a SnapshotClass, VolumeSnapshotSource will be checked to figure out what the associated CSI Driver is, and the default VolumeSnapshotClass associated with that CSI Driver will be used. If more than one VolumeSnapshotClass exist for a given CSI Driver and more than one have been marked as default, CreateSnapshot will fail and generate an event. Empty string is not allowed for this field.' + type: string + required: + - source + type: object + status: + description: status represents the current information of a snapshot. Consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent objects is successful (by validating that both VolumeSnapshot and VolumeSnapshotContent point at each other) before using this object. + properties: + boundVolumeSnapshotContentName: + description: 'boundVolumeSnapshotContentName is the name of the VolumeSnapshotContent object to which this VolumeSnapshot object intends to bind to. If not specified, it indicates that the VolumeSnapshot object has not been successfully bound to a VolumeSnapshotContent object yet. NOTE: To avoid possible security issues, consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent objects is successful (by validating that both VolumeSnapshot and VolumeSnapshotContent point at each other) before using this object.' + type: string + creationTime: + description: creationTime is the timestamp when the point-in-time snapshot is taken by the underlying storage system. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "creation_time" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "creation_time" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. If not specified, it may indicate that the creation time of the snapshot is unknown. + format: date-time + type: string + error: + description: error is the last observed error during snapshot creation, if any. This field could be helpful to upper level controllers(i.e., application controller) to decide whether they should continue on waiting for the snapshot to be created based on the type of error reported. The snapshot controller will keep retrying when an error occurs during the snapshot creation. Upon success, this error field will be cleared. + properties: + message: + description: 'message is a string detailing the encountered error during snapshot creation if specified. NOTE: message may be logged, and it should not contain sensitive information.' + type: string + time: + description: time is the timestamp when the error was encountered. + format: date-time + type: string + type: object + readyToUse: + description: readyToUse indicates if the snapshot is ready to be used to restore a volume. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "ready_to_use" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "ready_to_use" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it, otherwise, this field will be set to "True". If not specified, it means the readiness of a snapshot is unknown. + type: boolean + restoreSize: + type: string + description: restoreSize represents the minimum size of volume required to create a volume from this snapshot. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "size_bytes" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "size_bytes" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. When restoring a volume from this snapshot, the size of the volume MUST NOT be smaller than the restoreSize if it is specified, otherwise the restoration will fail. If not specified, it indicates that the size is unknown. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + required: + - spec + type: object + served: false + storage: false + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/deploy/k8s/controller-deployment.yaml b/deploy/k8s/controller-deployment.yaml index 5859453..18dc81e 100644 --- a/deploy/k8s/controller-deployment.yaml +++ b/deploy/k8s/controller-deployment.yaml @@ -25,14 +25,16 @@ spec: serviceAccountName: cloudstack-csi-controller affinity: podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchExpressions: - - key: "app.kubernetes.io/name" - operator: In - values: - - cloudstack-csi-controller - topologyKey: "kubernetes.io/hostname" + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: "app.kubernetes.io/name" + operator: In + values: + - cloudstack-csi-controller + topologyKey: "kubernetes.io/hostname" nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: @@ -176,8 +178,45 @@ spec: type: RuntimeDefault readOnlyRootFilesystem: true allowPrivilegeEscalation: false - + - name: csi-snapshotter + image: registry.k8s.io/sig-storage/csi-snapshotter:v6.3.0 + args: + - "--v=5" + - "--csi-address=$(CSI_ADDRESS)" + - "--leader-election" + - "--leader-election-lease-duration=30s" + - "--leader-election-renew-deadline=20s" + - "--leader-election-retry-period=10s" + env: + - name: CSI_ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + resources: + limits: + cpu: 400m + memory: 200Mi + requests: + cpu: 10m + memory: 20Mi + - name: snapshot-controller + image: registry.k8s.io/sig-storage/snapshot-controller:v6.3.0 + args: + - "--v=5" + - "--leader-election" + - "--leader-election-lease-duration=30s" + - "--leader-election-renew-deadline=20s" + - "--leader-election-retry-period=10s" + resources: + limits: + cpu: 400m + memory: 200Mi + requests: + cpu: 10m + memory: 20Mi - name: liveness-probe + imagePullPolicy: IfNotPresent image: registry.k8s.io/sig-storage/livenessprobe:v2.12.0 args: - "--v=4" diff --git a/deploy/k8s/node-daemonset.yaml b/deploy/k8s/node-daemonset.yaml index 672e715..bcf163f 100644 --- a/deploy/k8s/node-daemonset.yaml +++ b/deploy/k8s/node-daemonset.yaml @@ -64,6 +64,8 @@ spec: mountPath: /dev - name: cloud-init-dir mountPath: /run/cloud-init/ + - name: sys-dir + mountPath: /sys # Comment the above 2 lines and uncomment the next 2 lines for Ignition support # - name: ignition-dir # mountPath: /run/metadata @@ -177,6 +179,10 @@ spec: hostPath: path: /run/cloud-init/ type: Directory + - name: sys-dir + hostPath: + path: /sys + type: Directory # Comment the above 4 lines and uncomment the next 4 lines for Ignition support # - name: ignition-dir # hostPath: diff --git a/deploy/k8s/rbac.yaml b/deploy/k8s/rbac.yaml index 664e97c..64fc918 100644 --- a/deploy/k8s/rbac.yaml +++ b/deploy/k8s/rbac.yaml @@ -36,6 +36,9 @@ rules: - apiGroups: ["storage.k8s.io"] resources: ["volumeattachments/status"] verbs: ["patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots", "volumesnapshots/status", "volumesnapshotclasses", "volumesnapshotcontents", "volumesnapshotcontents/status"] + verbs: ["get", "list", "watch", "update", "create", "patch", "delete"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding @@ -83,3 +86,35 @@ roleRef: kind: ClusterRole name: cloudstack-csi-node-role apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cloudstack-csi-snapshotter-role +rules: +- apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] +- apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete"] +- apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list", "watch", "update"] +- apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents/status"] + verbs: ["update"] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: cloudstack-csi-snapshotter-binding +subjects: +- kind: ServiceAccount + name: cloudstack-csi-controller + namespace: kube-system +roleRef: + kind: ClusterRole + name: cloudstack-csi-snapshotter-role + apiGroup: rbac.authorization.k8s.io \ No newline at end of file diff --git a/deploy/k8s/volume-snapshot-class.yaml b/deploy/k8s/volume-snapshot-class.yaml new file mode 100644 index 0000000..4d14a7d --- /dev/null +++ b/deploy/k8s/volume-snapshot-class.yaml @@ -0,0 +1,6 @@ +apiVersion: snapshot.storage.k8s.io/v1 +kind: VolumeSnapshotClass +metadata: + name: cloudstack-snapshot +driver: csi.cloudstack.apache.org +deletionPolicy: Delete # Deleting the snapshot object in Kubernetes with delete the snapshot in CloudStack; You can use policy Retain if the snapshot shouldn't be deleted from CloudStack \ No newline at end of file diff --git a/examples/k8s/snapshot/pvc-from-snapshot.yaml b/examples/k8s/snapshot/pvc-from-snapshot.yaml new file mode 100644 index 0000000..89a8182 --- /dev/null +++ b/examples/k8s/snapshot/pvc-from-snapshot.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: pvc-from-snapshot +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + dataSource: + name: snapshot-1 + kind: VolumeSnapshot + apiGroup: snapshot.storage.k8s.io + storageClassName: cloudstack-custom diff --git a/examples/k8s/snapshot/pvc.yaml b/examples/k8s/snapshot/pvc.yaml new file mode 100644 index 0000000..0b42906 --- /dev/null +++ b/examples/k8s/snapshot/pvc.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: my-pvc +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + storageClassName: cloudstack-custom diff --git a/examples/k8s/snapshot/restore-pod.yaml b/examples/k8s/snapshot/restore-pod.yaml new file mode 100644 index 0000000..f8e9c22 --- /dev/null +++ b/examples/k8s/snapshot/restore-pod.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Pod +metadata: + name: restore-pod +spec: + containers: + - name: app + image: busybox + command: [ "sleep", "3600" ] + volumeMounts: + - mountPath: /data + name: data + volumes: + - name: data + persistentVolumeClaim: + claimName: pvc-from-snapshot \ No newline at end of file diff --git a/examples/k8s/snapshot/snapshot.yaml b/examples/k8s/snapshot/snapshot.yaml new file mode 100644 index 0000000..0d71b43 --- /dev/null +++ b/examples/k8s/snapshot/snapshot.yaml @@ -0,0 +1,8 @@ +apiVersion: snapshot.storage.k8s.io/v1 +kind: VolumeSnapshot +metadata: + name: snapshot-1 +spec: + volumeSnapshotClassName: cloudstack-snapshot + source: + persistentVolumeClaimName: example-pvc diff --git a/pkg/cloud/cloud.go b/pkg/cloud/cloud.go index a8b5417..0c21e3d 100644 --- a/pkg/cloud/cloud.go +++ b/pkg/cloud/cloud.go @@ -23,6 +23,11 @@ type Interface interface { AttachVolume(ctx context.Context, volumeID, vmID string) (string, error) DetachVolume(ctx context.Context, volumeID string) error ExpandVolume(ctx context.Context, volumeID string, newSizeInGB int64) error + + CreateVolumeFromSnapshot(ctx context.Context, zoneID, name, domainID, projectID, snapshotID string, sizeInGB int64) (*Volume, error) + GetSnapshotByID(ctx context.Context, snapshotID string) (*Snapshot, error) + CreateSnapshot(ctx context.Context, volumeID string) (*Snapshot, error) + DeleteSnapshot(ctx context.Context, snapshotID string) error } // Volume represents a CloudStack volume. @@ -34,12 +39,27 @@ type Volume struct { Size int64 DiskOfferingID string + DomainID string + ProjectID string ZoneID string VirtualMachineID string DeviceID string } +type Snapshot struct { + ID string + Name string + Size int64 + + DomainID string + ProjectID string + ZoneID string + + VolumeID string + CreatedAt string +} + // VM represents a CloudStack Virtual Machine. type VM struct { ID string @@ -55,11 +75,12 @@ var ( // client is the implementation of Interface. type client struct { *cloudstack.CloudStackClient + projectID string } // New creates a new cloud connector, given its configuration. func New(config *Config) Interface { csClient := cloudstack.NewAsyncClient(config.APIURL, config.APIKey, config.SecretKey, config.VerifySSL) - return &client{csClient} + return &client{csClient, config.ProjectID} } diff --git a/pkg/cloud/config.go b/pkg/cloud/config.go index 86e8c57..0024dff 100644 --- a/pkg/cloud/config.go +++ b/pkg/cloud/config.go @@ -12,6 +12,7 @@ type Config struct { APIKey string SecretKey string VerifySSL bool + ProjectID string } // csConfig wraps the config for the CloudStack cloud provider. @@ -40,6 +41,7 @@ func ReadConfig(configFilePath string) (*Config, error) { return &Config{ APIURL: cfg.Global.APIURL, APIKey: cfg.Global.APIKey, + ProjectID: cfg.Global.ProjectID, SecretKey: cfg.Global.SecretKey, VerifySSL: !cfg.Global.SSLNoVerify, }, nil diff --git a/pkg/cloud/fake/fake.go b/pkg/cloud/fake/fake.go index 23218bc..64e02e6 100644 --- a/pkg/cloud/fake/fake.go +++ b/pkg/cloud/fake/fake.go @@ -12,9 +12,11 @@ import ( ) const zoneID = "a1887604-237c-4212-a9cd-94620b7880fa" +const snapshotID = "9d076136-657b-4c84-b279-455da3ea484c" type fakeConnector struct { node *cloud.VM + snapshot *cloud.Snapshot volumesByID map[string]cloud.Volume volumesByName map[string]cloud.Volume } @@ -36,8 +38,18 @@ func New() cloud.Interface { ZoneID: zoneID, } + snapshot := &cloud.Snapshot{ + ID: "9d076136-657b-4c84-b279-455da3ea484c", + Name: "pvc-vol-snap-1", + DomainID: "51f0fcb5-db16-4637-94f5-30131010214f", + ZoneID: zoneID, + VolumeID: "4f1f610d-6f17-4ff9-9228-e4062af93e54", + CreatedAt: "2025-07-07 16:13:06", + } + return &fakeConnector{ node: node, + snapshot: snapshot, volumesByID: map[string]cloud.Volume{volume.ID: volume}, volumesByName: map[string]cloud.Volume{volume.Name: volume}, } @@ -124,3 +136,19 @@ func (f *fakeConnector) ExpandVolume(_ context.Context, volumeID string, newSize return cloud.ErrNotFound } + +func (f *fakeConnector) CreateVolumeFromSnapshot(ctx context.Context, zoneID, name, domainID, projectID, snapshotID string, sizeInGB int64) (*cloud.Volume, error) { + return nil, nil +} + +func (f *fakeConnector) GetSnapshotByID(ctx context.Context, snapshotID ...string) (*cloud.Snapshot, error) { + return f.snapshot, nil +} + +func (f *fakeConnector) CreateSnapshot(ctx context.Context, volumeID string) (*cloud.Snapshot, error) { + return f.snapshot, nil +} + +func (f *fakeConnector) DeleteSnapshot(ctx context.Context, snapshotID string) error { + return nil +} diff --git a/pkg/cloud/snapshots.go b/pkg/cloud/snapshots.go new file mode 100644 index 0000000..74830e1 --- /dev/null +++ b/pkg/cloud/snapshots.go @@ -0,0 +1,83 @@ +package cloud + +import ( + "context" + "strings" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "k8s.io/klog/v2" +) + +func (c *client) GetSnapshotByID(ctx context.Context, snapshotID string) (*Snapshot, error) { + logger := klog.FromContext(ctx) + p := c.Snapshot.NewListSnapshotsParams() + if snapshotID != "" { + p.SetId(snapshotID) + } + if c.projectID != "" { + p.SetProjectid(c.projectID) + } + logger.V(2).Info("CloudStack API call", "command", "ListSnapshots", "params", map[string]string{ + "id": snapshotID, + "projectid": c.projectID, + }) + l, err := c.Snapshot.ListSnapshots(p) + if err != nil { + return nil, err + } + if l.Count == 0 { + return nil, ErrNotFound + } + if l.Count > 1 { + return nil, ErrTooManyResults + } + snapshot := l.Snapshots[0] + s := Snapshot{ + ID: snapshot.Id, + Name: snapshot.Name, + DomainID: snapshot.Domainid, + ProjectID: snapshot.Projectid, + ZoneID: snapshot.Zoneid, + VolumeID: snapshot.Volumeid, + } + + return &s, nil +} + +func (c *client) CreateSnapshot(ctx context.Context, volumeID string) (*Snapshot, error) { + logger := klog.FromContext(ctx) + p := c.Snapshot.NewCreateSnapshotParams(volumeID) + + logger.V(2).Info("CloudStack API call", "command", "CreateSnapshot", "params", map[string]string{ + "volumeid": volumeID, + }) + + snapshot, err := c.Snapshot.CreateSnapshot(p) + if err != nil { + return nil, status.Errorf(codes.Internal, "Error %v", err) + } + + snap := Snapshot{ + ID: snapshot.Id, + Name: snapshot.Name, + Size: snapshot.Virtualsize, + DomainID: snapshot.Domainid, + ProjectID: snapshot.Projectid, + ZoneID: snapshot.Zoneid, + VolumeID: snapshot.Volumeid, + CreatedAt: snapshot.Created, + } + return &snap, nil +} + +func (c *client) DeleteSnapshot(ctx context.Context, snapshotID string) error { + p := c.Snapshot.NewDeleteSnapshotParams(snapshotID) + _, err := c.Snapshot.DeleteSnapshot(p) + if err != nil && strings.Contains(err.Error(), "4350") { + // CloudStack error InvalidParameterValueException + return ErrNotFound + } + + return err +} diff --git a/pkg/cloud/vms.go b/pkg/cloud/vms.go index 68a0505..2e98f64 100644 --- a/pkg/cloud/vms.go +++ b/pkg/cloud/vms.go @@ -10,8 +10,12 @@ func (c *client) GetVMByID(ctx context.Context, vmID string) (*VM, error) { logger := klog.FromContext(ctx) p := c.VirtualMachine.NewListVirtualMachinesParams() p.SetId(vmID) + if c.projectID != "" { + p.SetProjectid(c.projectID) + } logger.V(2).Info("CloudStack API call", "command", "ListVirtualMachines", "params", map[string]string{ - "id": vmID, + "id": vmID, + "projectID": c.projectID, }) l, err := c.VirtualMachine.ListVirtualMachines(p) if err != nil { @@ -24,7 +28,7 @@ func (c *client) GetVMByID(ctx context.Context, vmID string) (*VM, error) { return nil, ErrTooManyResults } vm := l.VirtualMachines[0] - + logger.V(2).Info("Returning VM", "vmID", vm.Id, "zoneID", vm.Zoneid) return &VM{ ID: vm.Id, ZoneID: vm.Zoneid, diff --git a/pkg/cloud/volumes.go b/pkg/cloud/volumes.go index 831cdec..3d1b363 100644 --- a/pkg/cloud/volumes.go +++ b/pkg/cloud/volumes.go @@ -29,6 +29,8 @@ func (c *client) listVolumes(p *cloudstack.ListVolumesParams) (*Volume, error) { Name: vol.Name, Size: vol.Size, DiskOfferingID: vol.Diskofferingid, + DomainID: vol.Domainid, + ProjectID: vol.Projectid, ZoneID: vol.Zoneid, VirtualMachineID: vol.Virtualmachineid, DeviceID: strconv.FormatInt(vol.Deviceid, 10), @@ -41,8 +43,12 @@ func (c *client) GetVolumeByID(ctx context.Context, volumeID string) (*Volume, e logger := klog.FromContext(ctx) p := c.Volume.NewListVolumesParams() p.SetId(volumeID) + if c.projectID != "" { + p.SetProjectid(c.projectID) + } logger.V(2).Info("CloudStack API call", "command", "ListVolumes", "params", map[string]string{ - "id": volumeID, + "id": volumeID, + "projectid": c.projectID, }) return c.listVolumes(p) @@ -66,11 +72,15 @@ func (c *client) CreateVolume(ctx context.Context, diskOfferingID, zoneID, name p.SetZoneid(zoneID) p.SetName(name) p.SetSize(sizeInGB) + if c.projectID != "" { + p.SetProjectid(c.projectID) + } logger.V(2).Info("CloudStack API call", "command", "CreateVolume", "params", map[string]string{ "diskofferingid": diskOfferingID, "zoneid": zoneID, "name": name, "size": strconv.FormatInt(sizeInGB, 10), + "projectid": c.projectID, }) vol, err := c.Volume.CreateVolume(p) if err != nil { @@ -153,3 +163,44 @@ func (c *client) ExpandVolume(ctx context.Context, volumeID string, newSizeInGB return nil } + +func (c *client) CreateVolumeFromSnapshot(ctx context.Context, zoneID, name, domainID, projectID, snapshotID string, sizeInGB int64) (*Volume, error) { + logger := klog.FromContext(ctx) + + p := c.Volume.NewCreateVolumeParams() + p.SetZoneid(zoneID) + if projectID != "" { + p.SetProjectid(projectID) + } + p.SetName(name) + p.SetSize(sizeInGB) + p.SetSnapshotid(snapshotID) + + logger.V(2).Info("CloudStack API call", "command", "CreateVolume", "params", map[string]string{ + "name": name, + "size": strconv.FormatInt(sizeInGB, 10), + "snapshotid": snapshotID, + "projectid": projectID, + "zoneid": zoneID, + }) + // Execute the API call to create volume from snapshot + vol, err := c.Volume.CreateVolume(p) + if err != nil { + // Handle the error accordingly + return nil, fmt.Errorf("failed to create volume from snapshot '%s': %w", snapshotID, err) + } + + v := Volume{ + ID: vol.Id, + Name: vol.Name, + Size: vol.Size, + DiskOfferingID: vol.Diskofferingid, + DomainID: vol.Domainid, + ProjectID: vol.Projectid, + ZoneID: vol.Zoneid, + VirtualMachineID: vol.Virtualmachineid, + DeviceID: strconv.FormatInt(vol.Deviceid, 10), + } + + return &v, nil +} diff --git a/pkg/driver/controller.go b/pkg/driver/controller.go index 70dffa3..293ba16 100644 --- a/pkg/driver/controller.go +++ b/pkg/driver/controller.go @@ -2,14 +2,17 @@ package driver import ( "context" + "encoding/json" "errors" "fmt" "math/rand" + "time" "github.com/container-storage-interface/spec/lib/go/csi" "github.com/kubernetes-csi/csi-lib-utils/protosanitizer" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/timestamppb" "k8s.io/klog/v2" "github.com/shapeblue/cloudstack-csi-driver/pkg/cloud" @@ -108,6 +111,14 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol return resp, nil } + // Check if this is a volume from snapshot + var snapshotID string + if src := req.GetVolumeContentSource(); src != nil { + if snap := src.GetSnapshot(); snap != nil { + snapshotID = snap.GetSnapshotId() + } + } + // We have to create the volume. // Determine volume size using requested capacity range. @@ -116,6 +127,46 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol return nil, status.Error(codes.InvalidArgument, err.Error()) } + // If creating from snapshot, get the snapshot size + var snapshotSizeGiB int64 + if snapshotID != "" { + logger.Info("Creating volume from snapshot", "snapshotID", snapshotID) + // Call the cloud connector's CreateVolumeFromSnapshot if implemented + printVolumeAsJSON(req) + snapshot, err := cs.connector.GetSnapshotByID(ctx, snapshotID) + if errors.Is(err, cloud.ErrNotFound) { + return nil, status.Errorf(codes.NotFound, "Snapshot %v not found", snapshotID) + } else if err != nil { + // Error with CloudStack + return nil, status.Errorf(codes.Internal, "Error %v", err) + } + + logger.Info("PVC created with", "size", sizeInGB) + snapshotSizeGiB = util.RoundUpBytesToGB(snapshot.Size) + if snapshotSizeGiB > sizeInGB { + logger.Info("Snapshot size is greater than the request PVC, creating volume from snapshot of size", "snapshot size:", snapshotSizeGiB) + sizeInGB = snapshotSizeGiB + } + + volFromSnapshot, err := cs.connector.CreateVolumeFromSnapshot(ctx, snapshot.ZoneID, name, snapshot.DomainID, snapshot.ProjectID, snapshotID, sizeInGB) + if err != nil { + return nil, status.Errorf(codes.Internal, "Cannot create volume from snapshot %s: %v", snapshotID, err.Error()) + } + + resp := &csi.CreateVolumeResponse{ + Volume: &csi.Volume{ + VolumeId: volFromSnapshot.ID, + CapacityBytes: volFromSnapshot.Size, + VolumeContext: req.GetParameters(), + ContentSource: req.GetVolumeContentSource(), + AccessibleTopology: []*csi.Topology{ + Topology{ZoneID: volFromSnapshot.ZoneID}.ToCSI(), + }, + }, + } + return resp, nil + } + // Determine zone using topology constraints. var zoneID string topologyRequirement := req.GetAccessibilityRequirements() @@ -159,7 +210,7 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol VolumeId: volID, CapacityBytes: util.GigaBytesToBytes(sizeInGB), VolumeContext: req.GetParameters(), - // ContentSource: req.GetVolumeContentSource(), TODO: snapshot support. + ContentSource: req.GetVolumeContentSource(), AccessibleTopology: []*csi.Topology{ Topology{ZoneID: zoneID}.ToCSI(), }, @@ -169,6 +220,15 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol return resp, nil } +func printVolumeAsJSON(vol *csi.CreateVolumeRequest) { + b, err := json.MarshalIndent(vol, "", " ") + if err != nil { + klog.Errorf("Failed to marshal CreateVolumeRequest to JSON: %v", err) + return + } + klog.V(5).Infof("CreateVolumeRequest as JSON:\n%s", string(b)) +} + func checkVolumeSuitable(vol *cloud.Volume, diskOfferingID string, capRange *csi.CapacityRange, topologyRequirement *csi.TopologyRequirement, ) (bool, string) { @@ -230,7 +290,7 @@ func determineSize(req *csi.CreateVolumeRequest) (int64, error) { func (cs *controllerServer) DeleteVolume(ctx context.Context, req *csi.DeleteVolumeRequest) (*csi.DeleteVolumeResponse, error) { logger := klog.FromContext(ctx) - logger.V(6).Info("DeleteVolume: called", "args", *req) + logger.V(4).Info("DeleteVolume: called", "args", *req) if req.GetVolumeId() == "" { return nil, status.Error(codes.InvalidArgument, "Volume ID missing in request") @@ -265,6 +325,69 @@ func (cs *controllerServer) DeleteVolume(ctx context.Context, req *csi.DeleteVol return &csi.DeleteVolumeResponse{}, nil } +func (cs *controllerServer) CreateSnapshot(ctx context.Context, req *csi.CreateSnapshotRequest) (*csi.CreateSnapshotResponse, error) { + klog.V(4).Infof("CreateSnapshot") + + volumeID := req.GetSourceVolumeId() + volume, err := cs.connector.GetVolumeByID(ctx, volumeID) + if errors.Is(err, cloud.ErrNotFound) { + return nil, status.Errorf(codes.NotFound, "Volume %v not found", volumeID) + } else if err != nil { + // Error with CloudStack + return nil, status.Errorf(codes.Internal, "Error %v", err) + } + klog.V(4).Infof("CreateSnapshot of volume: %s", volume) + snapshot, err := cs.connector.CreateSnapshot(ctx, volume.ID) + if err != nil { + return nil, status.Errorf(codes.Internal, "Failed to create snapshot for volume %s: %v", volume.ID, err.Error()) + } + + t, err := time.Parse("2006-01-02T15:04:05-0700", snapshot.CreatedAt) + if err != nil { + return nil, status.Errorf(codes.Internal, "Failed to parse snapshot creation time: %v", err) + } + + // Convert to Timestamp protobuf + ts := timestamppb.New(t) + + resp := &csi.CreateSnapshotResponse{ + Snapshot: &csi.Snapshot{ + SnapshotId: snapshot.ID, + SourceVolumeId: volume.ID, + CreationTime: ts, + ReadyToUse: true, + // We leave the optional SizeBytes field unset as the size of a block storage snapshot is the size of the difference to the volume or previous snapshots, k8s however expects the size to be the size of the restored volume. + }, + } + return resp, nil + +} + +func (cs *controllerServer) DeleteSnapshot(ctx context.Context, req *csi.DeleteSnapshotRequest) (*csi.DeleteSnapshotResponse, error) { + snapshotID := req.GetSnapshotId() + + if snapshotID == "" { + return nil, status.Error(codes.InvalidArgument, "Snapshot ID missing in request") + } + + klog.V(4).Infof("DeleteSnapshot for snapshotID: %s", snapshotID) + + snapshot, err := cs.connector.GetSnapshotByID(ctx, snapshotID) + if errors.Is(err, cloud.ErrNotFound) { + return nil, status.Errorf(codes.NotFound, "Snapshot %v not found", snapshotID) + } else if err != nil { + // Error with CloudStack + return nil, status.Errorf(codes.Internal, "Error %v", err) + } + + err = cs.connector.DeleteSnapshot(ctx, snapshot.ID) + if err != nil && !errors.Is(err, cloud.ErrNotFound) { + return nil, status.Errorf(codes.Internal, "Cannot delete snapshot %s: %s", snapshotID, err.Error()) + } + + return &csi.DeleteSnapshotResponse{}, nil +} + func (cs *controllerServer) ControllerPublishVolume(ctx context.Context, req *csi.ControllerPublishVolumeRequest) (*csi.ControllerPublishVolumeResponse, error) { logger := klog.FromContext(ctx) logger.V(6).Info("ControllerPublishVolume: called", "args", *req) @@ -557,6 +680,20 @@ func (cs *controllerServer) ControllerGetCapabilities(ctx context.Context, req * }, }, }, + &csi.ControllerServiceCapability{ + Type: &csi.ControllerServiceCapability_Rpc{ + Rpc: &csi.ControllerServiceCapability_RPC{ + Type: csi.ControllerServiceCapability_RPC_CREATE_DELETE_SNAPSHOT, + }, + }, + }, + &csi.ControllerServiceCapability{ + Type: &csi.ControllerServiceCapability_Rpc{ + Rpc: &csi.ControllerServiceCapability_RPC{ + Type: csi.ControllerServiceCapability_RPC_LIST_SNAPSHOTS, + }, + }, + }, }, } diff --git a/pkg/mount/mount.go b/pkg/mount/mount.go index 6e45aaf..578c539 100644 --- a/pkg/mount/mount.go +++ b/pkg/mount/mount.go @@ -79,21 +79,22 @@ func (m *mounter) GetBlockSizeBytes(devicePath string) (int64, error) { } func (m *mounter) GetDevicePath(ctx context.Context, volumeID string) (string, error) { + logger := klog.FromContext(ctx) backoff := wait.Backoff{ - Duration: 1 * time.Second, - Factor: 1.1, - Steps: 15, + Duration: 2 * time.Second, + Factor: 1.5, + Steps: 20, } var devicePath string err := wait.ExponentialBackoffWithContext(ctx, backoff, func(context.Context) (bool, error) { - path, err := m.getDevicePathBySerialID(volumeID) + path, err := m.getDevicePathBySerialID(ctx, volumeID) if err != nil { return false, err } if path != "" { devicePath = path - + logger.V(4).Info("Device path found", "volumeID", volumeID, "devicePath", path) return true, nil } m.probeVolume(ctx) @@ -110,7 +111,27 @@ func (m *mounter) GetDevicePath(ctx context.Context, volumeID string) (string, e return devicePath, nil } -func (m *mounter) getDevicePathBySerialID(volumeID string) (string, error) { +func (m *mounter) getDevicePathBySerialID(ctx context.Context, volumeID string) (string, error) { + logger := klog.FromContext(ctx) + + // First try XenServer device paths + xenDevicePath, err := m.getDevicePathForXenServer(ctx, volumeID) + if err != nil { + logger.V(4).Info("Failed to get XenServer device path", "volumeID", volumeID, "error", err) + } + if xenDevicePath != "" { + return xenDevicePath, nil + } + + // Try VMware device paths + vmwareDevicePath, err := m.getDevicePathForVMware(ctx, volumeID) + if err != nil { + logger.V(4).Info("Failed to get VMware device path", "volumeID", volumeID, "error", err) + } + if vmwareDevicePath != "" { + return vmwareDevicePath, nil + } + // Fall back to standard device paths (for KVM) sourcePathPrefixes := []string{"virtio-", "scsi-", "scsi-0QEMU_QEMU_HARDDISK_"} serial := diskUUIDToSerial(volumeID) for _, prefix := range sourcePathPrefixes { @@ -120,6 +141,7 @@ func (m *mounter) getDevicePathBySerialID(volumeID string) (string, error) { return source, nil } if !os.IsNotExist(err) { + logger.Error(err, "Failed to stat device path", "path", source) return "", err } } @@ -127,6 +149,119 @@ func (m *mounter) getDevicePathBySerialID(volumeID string) (string, error) { return "", nil } +func (m *mounter) getDevicePathForXenServer(ctx context.Context, volumeID string) (string, error) { + logger := klog.FromContext(ctx) + + for i := 'b'; i <= 'z'; i++ { + devicePath := fmt.Sprintf("/dev/xvd%c", i) + logger.V(5).Info("Checking XenServer device path", "devicePath", devicePath, "volumeID", volumeID) + + if _, err := os.Stat(devicePath); err == nil { + isBlock, err := m.IsBlockDevice(devicePath) + if err == nil && isBlock { + if m.verifyDevice(ctx, devicePath, volumeID) { + logger.V(4).Info("Found and verified XenServer device", "devicePath", devicePath, "volumeID", volumeID) + return devicePath, nil + } + } + } + } + return "", fmt.Errorf("device not found for volume %s", volumeID) +} + +func (m *mounter) getDevicePathForVMware(ctx context.Context, volumeID string) (string, error) { + logger := klog.FromContext(ctx) + + // Loop through /dev/sdb to /dev/sdz (/dev/sda -> the root disk) + for i := 'b'; i <= 'z'; i++ { + devicePath := fmt.Sprintf("/dev/sd%c", i) + logger.V(5).Info("Checking VMware device path", "devicePath", devicePath, "volumeID", volumeID) + + if _, err := os.Stat(devicePath); err == nil { + isBlock, err := m.IsBlockDevice(devicePath) + if err == nil && isBlock { + if m.verifyDevice(ctx, devicePath, volumeID) { + logger.V(4).Info("Found and verified VMware device", "devicePath", devicePath, "volumeID", volumeID) + return devicePath, nil + } + } + } + } + return "", fmt.Errorf("device not found for volume %s", volumeID) +} + +func (m *mounter) verifyDevice(ctx context.Context, devicePath string, volumeID string) bool { + logger := klog.FromContext(ctx) + + size, err := m.GetBlockSizeBytes(devicePath) + if err != nil { + logger.V(4).Info("Failed to get device size", "devicePath", devicePath, "volumeID", volumeID, "error", err) + return false + } + logger.V(5).Info("Device size retrieved", "devicePath", devicePath, "volumeID", volumeID, "sizeBytes", size) + + mounted, err := m.isDeviceMounted(devicePath) + if err != nil { + logger.V(4).Info("Failed to check if device is mounted", "devicePath", devicePath, "volumeID", volumeID, "error", err) + return false + } + if mounted { + logger.V(4).Info("Device is already mounted", "devicePath", devicePath, "volumeID", volumeID) + return false + } + + props, err := m.getDeviceProperties(devicePath) + if err != nil { + logger.V(4).Info("Failed to get device properties", "devicePath", devicePath, "volumeID", volumeID, "error", err) + return false + } + logger.V(5).Info("Device properties retrieved", "devicePath", devicePath, "volumeID", volumeID, "properties", props) + + return true +} + +func (m *mounter) isDeviceMounted(devicePath string) (bool, error) { + output, err := m.Exec.Command("grep", devicePath, "/proc/mounts").Output() + if err != nil { + if strings.Contains(err.Error(), "exit status 1") { + return false, nil + } + return false, err + } + return len(output) > 0, nil +} + +func (m *mounter) isDeviceInUse(devicePath string) (bool, error) { + output, err := m.Exec.Command("lsof", devicePath).Output() + if err != nil { + if strings.Contains(err.Error(), "exit status 1") { + return false, nil + } + return false, err + } + return len(output) > 0, nil +} + +func (m *mounter) getDeviceProperties(devicePath string) (map[string]string, error) { + output, err := m.Exec.Command("udevadm", "info", "--query=property", devicePath).Output() + if err != nil { + return nil, err + } + + props := make(map[string]string) + for _, line := range strings.Split(string(output), "\n") { + if line == "" { + continue + } + parts := strings.Split(line, "=") + if len(parts) == 2 { + props[parts[0]] = parts[1] + } + } + + return props, nil +} + func (m *mounter) probeVolume(ctx context.Context) { logger := klog.FromContext(ctx) logger.V(2).Info("Scanning SCSI host")