feat: csi-cinder storage capacity

Available capacity of disk storage

Signed-off-by: Serge Logvinov <[email protected]>

charts/cinder-csi-plugin/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v1
 appVersion: v1.31.0
 description: Cinder CSI Chart for OpenStack
 name: openstack-cinder-csi
-version: 2.31.2
+version: 2.31.3
 home: https://github.com/kubernetes/cloud-provider-openstack
 icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png
 maintainers:

charts/cinder-csi-plugin/templates/cinder-csi-driver.yaml

@@ -5,6 +5,9 @@ metadata:
 spec:
   attachRequired: true
   podInfoOnMount: true
+  {{- if .Values.csi.provisioner.capacity }}
+  storageCapacity: true
+  {{- end }}
   volumeLifecycleModes:
   - Persistent
   - Ephemeral

charts/cinder-csi-plugin/templates/controllerplugin-deployment.yaml

@@ -73,6 +73,10 @@ spec:
             - "--default-fstype=ext4"
             - "--feature-gates=Topology={{ .Values.csi.provisioner.topology }}"
             - "--extra-create-metadata"
+            {{- if .Values.csi.provisioner.capacity }}
+            - "--enable-capacity"
+            - "--capacity-ownerref-level=2"
+            {{- end }}
             {{- if .Values.csi.provisioner.extraArgs }}
             {{- with .Values.csi.provisioner.extraArgs }}
             {{- tpl . $ | trim | nindent 12 }}

charts/cinder-csi-plugin/values.yaml

@@ -15,6 +15,7 @@ csi:
     extraEnv: []
   provisioner:
     topology: "true"
+    capacity: false
     image:
       repository: registry.k8s.io/sig-storage/csi-provisioner
       tag: v3.6.2

docs/cinder-csi-plugin/features.md

@@ -5,6 +5,7 @@
 - [Plugin Features](#plugin-features)
   - [Dynamic Provisioning](#dynamic-provisioning)
   - [Topology](#topology)
+  - [Storage capacity](#storage-capacity)
   - [Block Volume](#block-volume)
   - [Volume Expansion](#volume-expansion)
     - [Rescan on in-use volume resize](#rescan-on-in-use-volume-resize)
@@ -24,7 +25,7 @@
 
 Dynamic Provisioning uses persistence volume claim (PVC) to request the Kubernetes to create the Cinder volume on behalf of user and consumes the volume from inside container.
 
-For usage, refer [sample app](./examples.md#dynamic-volume-provisioning)  
+For usage, refer [sample app](./examples.md#dynamic-volume-provisioning)
 
 ## Topology
 
@@ -35,10 +36,22 @@ This feature enables driver to consider the topology constraints while creating
   `topology.cinder.csi.openstack.org/zone` : Availability by Zone
 * `allowedTopologies` can be specified in storage class to restrict the topology of provisioned volumes to specific zones and should be used as replacement of `availability` parameter.
 * To disable: set `--feature-gates=Topology=false` in external-provisioner (container `csi-provisioner` of `csi-cinder-controllerplugin`).
-  * If using Helm, it can be disabled by setting `Values.csi.provisioner.topology: "false"` 
+  * If using Helm, it can be disabled by setting `Values.csi.provisioner.topology: "false"`
 
 For usage, refer [sample app](./examples.md#use-topology)
 
+## Storage Capacity
+
+This feature enables driver to consider the storage capacity constraints while creating the volume. For more info, refer [Capacity Support](https://github.com/kubernetes-csi/external-provisioner/blob/master/README.md#capacity-support)
+
+The driver will expose the storage capacity limits of the openstack project.
+The capacity is calculated based on the quota, which is a sum of all the volumes and snapshots created by the csi project account.
+
+* Disabled by default
+* To enable: set `--enable-capacity` and `--capacity-ownerref-level=2` in external-provisioner (container `csi-provisioner` of `csi-cinder-controllerplugin`), you need to have `StorageCapacity: true` in CSIDriver object.
+  * If using Helm, it can be enabled by setting `Values.csi.provisioner.capacity: "true"`
+  * To change frequency of capacity check, set `--capacity-check-interval` in external-provisioner
+
 ## Block Volume
 
 Cinder volumes to be exposed inside containers as a block device instead of as a mounted file system. The corresponding CSI feature (CSIBlockVolume) is GA since Kubernetes 1.18.
@@ -51,7 +64,7 @@ For usage, refer [sample app](./examples.md#using-block-volume)
 
 ## Volume Expansion
 
-Driver supports both `Offline` and `Online` resize of cinder volumes. Cinder online resize support is available since cinder 3.42 microversion. 
+Driver supports both `Offline` and `Online` resize of cinder volumes. Cinder online resize support is available since cinder 3.42 microversion.
 The same should be supported by underlying OpenStack Cloud to avail the feature.
 
 * As of kubernetes v1.16, Volume Expansion is a beta feature and enabled by default.
@@ -81,7 +94,7 @@ Two different Kubernetes features allow volumes to follow the Pod's lifecycle: C
 
 This feature allows CSI volumes to be directly embedded in the Pod specification instead of a PersistentVolume. Volumes specified in this way are ephemeral and do not persist across Pod restarts.
 
-* As of Kubernetes v1.16 this feature is beta so enabled by default. 
+* As of Kubernetes v1.16 this feature is beta so enabled by default.
 * To enable this feature for CSI Driver, `volumeLifecycleModes` needs to be specified in [CSIDriver](../../manifests/cinder-csi-plugin/csi-cinder-driver.yaml) object. The driver can run in `Persistent` mode, `Ephemeral` or in both modes.
 * `podInfoOnMount` must be `true` to use this feature.
 * For usage, refer [sample app](./examples.md#deploy-app-using-inline-volumes)

docs/cinder-csi-plugin/using-cinder-csi-plugin.md

@@ -248,6 +248,7 @@ helm install --namespace kube-system --name cinder-csi ./charts/cinder-csi-plugi
 
 * [Dynamic Provisioning](./features.md#dynamic-provisioning)
 * [Topology](./features.md#topology)
+* [Storage capacity](./features.md#storage-capacity)
 * [Raw Block Volume](./features.md#block-volume)
 * [Volume Expansion](./features.md#volume-expansion)
 * [Volume Cloning](./features.md#volume-cloning)

manifests/cinder-csi-plugin/cinder-csi-controllerplugin.yaml

@@ -45,6 +45,8 @@ spec:
             - "--timeout=3m"
             - "--default-fstype=ext4"
             - "--feature-gates=Topology=true"
+            - "--enable-capacity"
+            - "--capacity-ownerref-level=2"
             - "--extra-create-metadata"
             - "--leader-election=true"
           env:

manifests/cinder-csi-plugin/csi-cinder-driver.yaml

@@ -5,6 +5,7 @@ metadata:
 spec:
   attachRequired: true
   podInfoOnMount: true
+  storageCapacity: true
   volumeLifecycleModes:
   - Persistent
   - Ephemeral

pkg/csi/cinder/controllerserver.go

@@ -33,6 +33,7 @@ import (
 	"google.golang.org/grpc/status"
 	"google.golang.org/protobuf/types/known/timestamppb"
 
+	corev1 "k8s.io/api/core/v1"
 	"k8s.io/cloud-provider-openstack/pkg/csi/cinder/openstack"
 	"k8s.io/cloud-provider-openstack/pkg/util"
 	cpoerrors "k8s.io/cloud-provider-openstack/pkg/util/errors"
@@ -949,7 +950,28 @@ func (cs *controllerServer) ValidateVolumeCapabilities(ctx context.Context, req
 }
 
 func (cs *controllerServer) GetCapacity(ctx context.Context, req *csi.GetCapacityRequest) (*csi.GetCapacityResponse, error) {
-	return nil, status.Error(codes.Unimplemented, "GetCapacity is not yet implemented")
+	klog.V(4).Infof("GetCapacity: called with args %+v", protosanitizer.StripSecrets(*req))
+
+	topology := req.GetAccessibleTopology()
+	if topology == nil {
+		return &csi.GetCapacityResponse{}, nil
+	}
+
+	region := topology.GetSegments()[corev1.LabelTopologyRegion]
+
+	cloud, cloudExist := cs.Clouds[region]
+	if !cloudExist {
+		return nil, status.Error(codes.InvalidArgument, "[GetCapacity] specified cloud undefined")
+	}
+
+	availableCapacity, err := cloud.GetFreeQuotaStorageSpace()
+	if err != nil {
+		return nil, status.Errorf(codes.Internal, "[GetCapacity]: failed with error %v", err)
+	}
+
+	return &csi.GetCapacityResponse{
+		AvailableCapacity: int64(availableCapacity * 1024 * 1024 * 1024),
+	}, nil
 }
 
 func (cs *controllerServer) ControllerGetVolume(ctx context.Context, req *csi.ControllerGetVolumeRequest) (*csi.ControllerGetVolumeResponse, error) {

pkg/csi/cinder/driver.go

@@ -99,6 +99,7 @@ func NewDriver(o *DriverOpts) *Driver {
 			csi.ControllerServiceCapability_RPC_PUBLISH_UNPUBLISH_VOLUME,
 			csi.ControllerServiceCapability_RPC_CREATE_DELETE_SNAPSHOT,
 			csi.ControllerServiceCapability_RPC_LIST_SNAPSHOTS,
+			csi.ControllerServiceCapability_RPC_GET_CAPACITY,
 			csi.ControllerServiceCapability_RPC_EXPAND_VOLUME,
 			csi.ControllerServiceCapability_RPC_CLONE_VOLUME,
 			csi.ControllerServiceCapability_RPC_LIST_VOLUMES_PUBLISHED_NODES,

pkg/csi/cinder/openstack/openstack.go

@@ -70,6 +70,7 @@ type IOpenStack interface {
 	GetInstanceByID(instanceID string) (*servers.Server, error)
 	ExpandVolume(volumeID string, status string, size int) error
 	GetMaxVolLimit() int64
+	GetFreeQuotaStorageSpace() (int, error)
 	GetMetadataOpts() metadata.Opts
 	GetBlockStorageOpts() BlockStorageOpts
 }

pkg/csi/cinder/openstack/openstack_mock.go

@@ -460,6 +460,10 @@ func (_m *OpenStackMock) GetMaxVolLimit() int64 {
 	return 256
 }
 
+func (_m *OpenStackMock) GetFreeQuotaStorageSpace() (int, error) {
+	return 100, nil
+}
+
 func (_m *OpenStackMock) BackupsAreEnabled() (bool, error) {
 	return true, nil
 }

pkg/csi/cinder/openstack/openstack_volumes.go

@@ -23,6 +23,7 @@ import (
 	"time"
 
 	"github.com/gophercloud/gophercloud/v2/openstack"
+	"github.com/gophercloud/gophercloud/v2/openstack/blockstorage/v3/limits"
 	"github.com/gophercloud/gophercloud/v2/openstack/blockstorage/v3/volumes"
 	"github.com/gophercloud/gophercloud/v2/openstack/compute/v2/volumeattach"
 	"github.com/gophercloud/gophercloud/v2/pagination"
@@ -396,6 +397,22 @@ func (os *OpenStack) GetMaxVolLimit() int64 {
 	return defaultMaxVolAttachLimit
 }
 
+// GetFreeQuotaStorageSpace returns the tenant quota capacity of the block storage, in GB
+func (os *OpenStack) GetFreeQuotaStorageSpace() (int, error) {
+	mc := metrics.NewMetricContext("limits", "get")
+
+	res, err := limits.Get(context.TODO(), os.blockstorage).Extract()
+	if mc.ObserveRequest(err) != nil {
+		return 0, err
+	}
+
+	capacity := res.Absolute.MaxTotalVolumeGigabytes - res.Absolute.TotalGigabytesUsed
+	if capacity < 0 {
+		capacity = 0
+	}
+	return capacity, nil
+}
+
 // diskIsAttached queries if a volume is attached to a compute instance
 func (os *OpenStack) diskIsAttached(instanceID, volumeID string) (bool, error) {
 	volume, err := os.GetVolume(volumeID)

tests/sanity/cinder/fakecloud.go

@@ -332,6 +332,10 @@ func (cloud *cloud) GetMaxVolLimit() int64 {
 	return 256
 }
 
+func (cloud *cloud) GetFreeQuotaStorageSpace() (int, error) {
+	return 100, nil
+}
+
 func (cloud *cloud) GetMetadataOpts() metadata.Opts {
 	var m metadata.Opts
 	m.SearchOrder = fmt.Sprintf("%s,%s", "configDrive", "metadataService")