Merge pull request #99594 from cofyc/kep1845-api

Prioritizing nodes based on volume capacity: API changes
This commit is contained in:
Kubernetes Prow Robot 2021-07-01 15:35:51 -07:00 committed by GitHub
commit 25bbe2ebc5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 426 additions and 28 deletions

View File

@ -733,6 +733,16 @@ func TestCodecsEncodePluginConfig(t *testing.T) {
Args: runtime.RawExtension{
Object: &v1beta1.VolumeBindingArgs{
BindTimeoutSeconds: pointer.Int64Ptr(300),
Shape: []v1beta1.UtilizationShapePoint{
{
Utilization: 0,
Score: 0,
},
{
Utilization: 100,
Score: 10,
},
},
},
},
},
@ -804,6 +814,11 @@ profiles:
apiVersion: kubescheduler.config.k8s.io/v1beta1
bindTimeoutSeconds: 300
kind: VolumeBindingArgs
shape:
- score: 0
utilization: 0
- score: 10
utilization: 100
name: VolumeBinding
- args:
apiVersion: kubescheduler.config.k8s.io/v1beta1
@ -855,6 +870,16 @@ profiles:
Name: "VolumeBinding",
Args: &config.VolumeBindingArgs{
BindTimeoutSeconds: 300,
Shape: []config.UtilizationShapePoint{
{
Utilization: 0,
Score: 0,
},
{
Utilization: 100,
Score: 10,
},
},
},
},
{
@ -913,6 +938,11 @@ profiles:
apiVersion: kubescheduler.config.k8s.io/v1beta1
bindTimeoutSeconds: 300
kind: VolumeBindingArgs
shape:
- score: 0
utilization: 0
- score: 10
utilization: 100
name: VolumeBinding
- args:
apiVersion: kubescheduler.config.k8s.io/v1beta1
@ -945,6 +975,16 @@ profiles:
Args: runtime.RawExtension{
Object: &v1beta2.VolumeBindingArgs{
BindTimeoutSeconds: pointer.Int64Ptr(300),
Shape: []v1beta2.UtilizationShapePoint{
{
Utilization: 0,
Score: 0,
},
{
Utilization: 100,
Score: 10,
},
},
},
},
},
@ -1009,6 +1049,11 @@ profiles:
apiVersion: kubescheduler.config.k8s.io/v1beta2
bindTimeoutSeconds: 300
kind: VolumeBindingArgs
shape:
- score: 0
utilization: 0
- score: 10
utilization: 100
name: VolumeBinding
- args:
apiVersion: kubescheduler.config.k8s.io/v1beta2

View File

@ -214,6 +214,21 @@ type VolumeBindingArgs struct {
// Value must be non-negative integer. The value zero indicates no waiting.
// If this value is nil, the default value will be used.
BindTimeoutSeconds int64
// Shape specifies the points defining the score function shape, which is
// used to score nodes based on the utilization of statically provisioned
// PVs. The utilization is calculated by dividing the total requested
// storage of the pod by the total capacity of feasible PVs on each node.
// Each point contains utilization (ranges from 0 to 100) and its
// associated score (ranges from 0 to 10). You can turn the priority by
// specifying different scores for different utilization numbers.
// The default shape points are:
// 1) 0 for 0 utilization
// 2) 10 for 100 utilization
// All points must be sorted in increasing order by utilization.
// +featureGate=VolumeCapacityPriority
// +optional
Shape []UtilizationShapePoint
}
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

View File

@ -280,6 +280,18 @@ func SetDefaults_VolumeBindingArgs(obj *v1beta1.VolumeBindingArgs) {
if obj.BindTimeoutSeconds == nil {
obj.BindTimeoutSeconds = pointer.Int64Ptr(600)
}
if len(obj.Shape) == 0 && feature.DefaultFeatureGate.Enabled(features.VolumeCapacityPriority) {
obj.Shape = []v1beta1.UtilizationShapePoint{
{
Utilization: 0,
Score: 0,
},
{
Utilization: 100,
Score: int32(config.MaxCustomPriorityScore),
},
}
}
}
func SetDefaults_PodTopologySpreadArgs(obj *v1beta1.PodTopologySpreadArgs) {

View File

@ -701,6 +701,30 @@ func TestPluginArgsDefaults(t *testing.T) {
},
},
},
{
name: "VolumeBindingArgs empty, VolumeCapacityPriority disabled",
features: map[featuregate.Feature]bool{
features.VolumeCapacityPriority: false,
},
in: &v1beta1.VolumeBindingArgs{},
want: &v1beta1.VolumeBindingArgs{
BindTimeoutSeconds: pointer.Int64Ptr(600),
},
},
{
name: "VolumeBindingArgs empty, VolumeCapacityPriority enabled",
features: map[featuregate.Feature]bool{
features.VolumeCapacityPriority: true,
},
in: &v1beta1.VolumeBindingArgs{},
want: &v1beta1.VolumeBindingArgs{
BindTimeoutSeconds: pointer.Int64Ptr(600),
Shape: []v1beta1.UtilizationShapePoint{
{Utilization: 0, Score: 0},
{Utilization: 100, Score: 10},
},
},
},
}
for _, tc := range tests {
scheme := runtime.NewScheme()

View File

@ -916,6 +916,7 @@ func autoConvert_v1beta1_VolumeBindingArgs_To_config_VolumeBindingArgs(in *v1bet
if err := v1.Convert_Pointer_int64_To_int64(&in.BindTimeoutSeconds, &out.BindTimeoutSeconds, s); err != nil {
return err
}
out.Shape = *(*[]config.UtilizationShapePoint)(unsafe.Pointer(&in.Shape))
return nil
}
@ -928,6 +929,7 @@ func autoConvert_config_VolumeBindingArgs_To_v1beta1_VolumeBindingArgs(in *confi
if err := v1.Convert_int64_To_Pointer_int64(&in.BindTimeoutSeconds, &out.BindTimeoutSeconds, s); err != nil {
return err
}
out.Shape = *(*[]v1beta1.UtilizationShapePoint)(unsafe.Pointer(&in.Shape))
return nil
}

View File

@ -245,6 +245,18 @@ func SetDefaults_VolumeBindingArgs(obj *v1beta2.VolumeBindingArgs) {
if obj.BindTimeoutSeconds == nil {
obj.BindTimeoutSeconds = pointer.Int64Ptr(600)
}
if len(obj.Shape) == 0 && feature.DefaultFeatureGate.Enabled(features.VolumeCapacityPriority) {
obj.Shape = []v1beta2.UtilizationShapePoint{
{
Utilization: 0,
Score: 0,
},
{
Utilization: 100,
Score: int32(config.MaxCustomPriorityScore),
},
}
}
}
func SetDefaults_PodTopologySpreadArgs(obj *v1beta2.PodTopologySpreadArgs) {

View File

@ -675,6 +675,30 @@ func TestPluginArgsDefaults(t *testing.T) {
},
},
},
{
name: "VolumeBindingArgs empty, VolumeCapacityPriority disabled",
features: map[featuregate.Feature]bool{
features.VolumeCapacityPriority: false,
},
in: &v1beta2.VolumeBindingArgs{},
want: &v1beta2.VolumeBindingArgs{
BindTimeoutSeconds: pointer.Int64Ptr(600),
},
},
{
name: "VolumeBindingArgs empty, VolumeCapacityPriority enabled",
features: map[featuregate.Feature]bool{
features.VolumeCapacityPriority: true,
},
in: &v1beta2.VolumeBindingArgs{},
want: &v1beta2.VolumeBindingArgs{
BindTimeoutSeconds: pointer.Int64Ptr(600),
Shape: []v1beta2.UtilizationShapePoint{
{Utilization: 0, Score: 0},
{Utilization: 100, Score: 10},
},
},
},
}
for _, tc := range tests {
scheme := runtime.NewScheme()

View File

@ -815,6 +815,7 @@ func autoConvert_v1beta2_VolumeBindingArgs_To_config_VolumeBindingArgs(in *v1bet
if err := v1.Convert_Pointer_int64_To_int64(&in.BindTimeoutSeconds, &out.BindTimeoutSeconds, s); err != nil {
return err
}
out.Shape = *(*[]config.UtilizationShapePoint)(unsafe.Pointer(&in.Shape))
return nil
}
@ -827,6 +828,7 @@ func autoConvert_config_VolumeBindingArgs_To_v1beta2_VolumeBindingArgs(in *confi
if err := v1.Convert_int64_To_Pointer_int64(&in.BindTimeoutSeconds, &out.BindTimeoutSeconds, s); err != nil {
return err
}
out.Shape = *(*[]v1beta2.UtilizationShapePoint)(unsafe.Pointer(&in.Shape))
return nil
}

View File

@ -25,7 +25,9 @@ import (
"k8s.io/apimachinery/pkg/util/errors"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/validation/field"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/component-helpers/scheduling/corev1/nodeaffinity"
"k8s.io/kubernetes/pkg/features"
"k8s.io/kubernetes/pkg/scheduler/apis/config"
)
@ -294,13 +296,21 @@ func ValidateNodeAffinityArgs(path *field.Path, args *config.NodeAffinityArgs) e
// ValidateVolumeBindingArgs validates that VolumeBindingArgs are set correctly.
func ValidateVolumeBindingArgs(path *field.Path, args *config.VolumeBindingArgs) error {
var err error
var allErrs field.ErrorList
if args.BindTimeoutSeconds < 0 {
err = field.Invalid(path.Child("bindTimeoutSeconds"), args.BindTimeoutSeconds, "invalid BindTimeoutSeconds, should not be a negative value")
allErrs = append(allErrs, field.Invalid(path.Child("bindTimeoutSeconds"), args.BindTimeoutSeconds, "invalid BindTimeoutSeconds, should not be a negative value"))
}
return err
if utilfeature.DefaultFeatureGate.Enabled(features.VolumeCapacityPriority) {
allErrs = append(allErrs, validateFunctionShape(args.Shape, path.Child("shape"))...)
} else if args.Shape != nil {
// When the feature is off, return an error if the config is not nil.
// This prevents unexpected configuration from taking effect when the
// feature turns on in the future.
allErrs = append(allErrs, field.Invalid(path.Child("shape"), args.Shape, "unexpected field `shape`, remove it or turn on the feature gate VolumeCapacityPriority"))
}
return allErrs.ToAggregate()
}
func ValidateNodeResourcesFitArgs(path *field.Path, args *config.NodeResourcesFitArgs) error {

View File

@ -25,7 +25,12 @@ import (
"github.com/google/go-cmp/cmp/cmpopts"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/errors"
"k8s.io/apimachinery/pkg/util/validation/field"
"k8s.io/apiserver/pkg/util/feature"
"k8s.io/component-base/featuregate"
featuregatetesting "k8s.io/component-base/featuregate/testing"
"k8s.io/kubernetes/pkg/features"
"k8s.io/kubernetes/pkg/scheduler/apis/config"
)
@ -950,9 +955,10 @@ func TestValidateNodeAffinityArgs(t *testing.T) {
func TestValidateVolumeBindingArgs(t *testing.T) {
cases := []struct {
name string
args config.VolumeBindingArgs
wantErr error
name string
args config.VolumeBindingArgs
features map[featuregate.Feature]bool
wantErr error
}{
{
name: "zero is a valid config",
@ -971,14 +977,112 @@ func TestValidateVolumeBindingArgs(t *testing.T) {
args: config.VolumeBindingArgs{
BindTimeoutSeconds: -10,
},
wantErr: &field.Error{
Type: field.ErrorTypeInvalid,
Field: "bindTimeoutSeconds",
wantErr: errors.NewAggregate([]error{&field.Error{
Type: field.ErrorTypeInvalid,
Field: "bindTimeoutSeconds",
BadValue: int64(-10),
Detail: "invalid BindTimeoutSeconds, should not be a negative value",
}}),
},
{
name: "[VolumeCapacityPriority=off] shape should be nil when the feature is off",
features: map[featuregate.Feature]bool{
features.VolumeCapacityPriority: false,
},
args: config.VolumeBindingArgs{
BindTimeoutSeconds: 10,
Shape: nil,
},
},
{
name: "[VolumeCapacityPriority=off] error if the shape is not nil when the feature is off",
features: map[featuregate.Feature]bool{
features.VolumeCapacityPriority: false,
},
args: config.VolumeBindingArgs{
BindTimeoutSeconds: 10,
Shape: []config.UtilizationShapePoint{
{Utilization: 1, Score: 1},
{Utilization: 3, Score: 3},
},
},
wantErr: errors.NewAggregate([]error{&field.Error{
Type: field.ErrorTypeInvalid,
Field: "shape",
}}),
},
{
name: "[VolumeCapacityPriority=on] shape should not be empty",
features: map[featuregate.Feature]bool{
features.VolumeCapacityPriority: true,
},
args: config.VolumeBindingArgs{
BindTimeoutSeconds: 10,
Shape: []config.UtilizationShapePoint{},
},
wantErr: errors.NewAggregate([]error{&field.Error{
Type: field.ErrorTypeRequired,
Field: "shape",
}}),
},
{
name: "[VolumeCapacityPriority=on] shape points must be sorted in increasing order",
features: map[featuregate.Feature]bool{
features.VolumeCapacityPriority: true,
},
args: config.VolumeBindingArgs{
BindTimeoutSeconds: 10,
Shape: []config.UtilizationShapePoint{
{Utilization: 3, Score: 3},
{Utilization: 1, Score: 1},
},
},
wantErr: errors.NewAggregate([]error{&field.Error{
Type: field.ErrorTypeInvalid,
Field: "shape[1].utilization",
Detail: "Invalid value: 1: utilization values must be sorted in increasing order",
}}),
},
{
name: "[VolumeCapacityPriority=on] shape point: invalid utilization and score",
features: map[featuregate.Feature]bool{
features.VolumeCapacityPriority: true,
},
args: config.VolumeBindingArgs{
BindTimeoutSeconds: 10,
Shape: []config.UtilizationShapePoint{
{Utilization: -1, Score: 1},
{Utilization: 10, Score: -1},
{Utilization: 20, Score: 11},
{Utilization: 101, Score: 1},
},
},
wantErr: errors.NewAggregate([]error{
&field.Error{
Type: field.ErrorTypeInvalid,
Field: "shape[0].utilization",
},
&field.Error{
Type: field.ErrorTypeInvalid,
Field: "shape[1].score",
},
&field.Error{
Type: field.ErrorTypeInvalid,
Field: "shape[2].score",
},
&field.Error{
Type: field.ErrorTypeInvalid,
Field: "shape[3].utilization",
},
}),
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
for k, v := range tc.features {
defer featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, k, v)()
}
err := ValidateVolumeBindingArgs(nil, &tc.args)
if diff := cmp.Diff(tc.wantErr, err, ignoreBadValueDetail); diff != "" {
t.Errorf("ValidateVolumeBindingArgs returned err (-want,+got):\n%s", diff)

View File

@ -970,6 +970,11 @@ func (in *UtilizationShapePoint) DeepCopy() *UtilizationShapePoint {
func (in *VolumeBindingArgs) DeepCopyInto(out *VolumeBindingArgs) {
*out = *in
out.TypeMeta = in.TypeMeta
if in.Shape != nil {
in, out := &in.Shape, &out.Shape
*out = make([]UtilizationShapePoint, len(*in))
copy(*out, *in)
}
return
}

View File

@ -221,20 +221,6 @@ func (pl *VolumeBinding) Filter(ctx context.Context, cs *framework.CycleState, p
return nil
}
var (
// TODO (for alpha) make it configurable in config.VolumeBindingArgs
defaultShapePoint = []config.UtilizationShapePoint{
{
Utilization: 0,
Score: 0,
},
{
Utilization: 100,
Score: int32(config.MaxCustomPriorityScore),
},
}
)
// Score invoked at the score extension point.
func (pl *VolumeBinding) Score(ctx context.Context, cs *framework.CycleState, pod *v1.Pod, nodeName string) (int64, *framework.Status) {
if pl.scorer == nil {
@ -363,8 +349,8 @@ func New(plArgs runtime.Object, fh framework.Handle) (framework.Plugin, error) {
// build score function
var scorer volumeCapacityScorer
if utilfeature.DefaultFeatureGate.Enabled(features.VolumeCapacityPriority) {
shape := make(helper.FunctionShape, 0, len(defaultShapePoint))
for _, point := range defaultShapePoint {
shape := make(helper.FunctionShape, 0, len(args.Shape))
for _, point := range args.Shape {
shape = append(shape, helper.FunctionShapePoint{
Utilization: int64(point.Utilization),
Score: int64(point.Score) * (framework.MaxNodeScore / config.MaxCustomPriorityScore),

View File

@ -62,6 +62,17 @@ var (
},
VolumeBindingMode: &waitForFirstConsumer,
}
defaultShapePoint = []config.UtilizationShapePoint{
{
Utilization: 0,
Score: 0,
},
{
Utilization: 100,
Score: int32(config.MaxCustomPriorityScore),
},
}
)
func makeNode(name string) *v1.Node {
@ -178,6 +189,7 @@ func TestVolumeBinding(t *testing.T) {
pvcs []*v1.PersistentVolumeClaim
pvs []*v1.PersistentVolume
feature featuregate.Feature
args *config.VolumeBindingArgs
wantPreFilterStatus *framework.Status
wantStateAfterPreFilter *stateData
wantFilterStatus []*framework.Status
@ -524,6 +536,99 @@ func TestVolumeBinding(t *testing.T) {
0,
},
},
{
name: "zonal volumes with close capacity are preferred (custom shape)",
pod: makePod("pod-a", []string{"pvc-a"}),
nodes: []*v1.Node{
mergeNodeLabels(makeNode("zone-a-node-a"), map[string]string{
"topology.kubernetes.io/region": "region-a",
"topology.kubernetes.io/zone": "zone-a",
}),
mergeNodeLabels(makeNode("zone-a-node-b"), map[string]string{
"topology.kubernetes.io/region": "region-a",
"topology.kubernetes.io/zone": "zone-a",
}),
mergeNodeLabels(makeNode("zone-b-node-a"), map[string]string{
"topology.kubernetes.io/region": "region-b",
"topology.kubernetes.io/zone": "zone-b",
}),
mergeNodeLabels(makeNode("zone-b-node-b"), map[string]string{
"topology.kubernetes.io/region": "region-b",
"topology.kubernetes.io/zone": "zone-b",
}),
mergeNodeLabels(makeNode("zone-c-node-a"), map[string]string{
"topology.kubernetes.io/region": "region-c",
"topology.kubernetes.io/zone": "zone-c",
}),
mergeNodeLabels(makeNode("zone-c-node-b"), map[string]string{
"topology.kubernetes.io/region": "region-c",
"topology.kubernetes.io/zone": "zone-c",
}),
},
pvcs: []*v1.PersistentVolumeClaim{
setPVCRequestStorage(makePVC("pvc-a", "", waitSC.Name), resource.MustParse("50Gi")),
},
pvs: []*v1.PersistentVolume{
setPVNodeAffinity(setPVCapacity(makePV("pv-a-0", waitSC.Name), resource.MustParse("200Gi")), map[string][]string{
"topology.kubernetes.io/region": {"region-a"},
"topology.kubernetes.io/zone": {"zone-a"},
}),
setPVNodeAffinity(setPVCapacity(makePV("pv-a-1", waitSC.Name), resource.MustParse("200Gi")), map[string][]string{
"topology.kubernetes.io/region": {"region-a"},
"topology.kubernetes.io/zone": {"zone-a"},
}),
setPVNodeAffinity(setPVCapacity(makePV("pv-b-0", waitSC.Name), resource.MustParse("100Gi")), map[string][]string{
"topology.kubernetes.io/region": {"region-b"},
"topology.kubernetes.io/zone": {"zone-b"},
}),
setPVNodeAffinity(setPVCapacity(makePV("pv-b-1", waitSC.Name), resource.MustParse("100Gi")), map[string][]string{
"topology.kubernetes.io/region": {"region-b"},
"topology.kubernetes.io/zone": {"zone-b"},
}),
},
feature: features.VolumeCapacityPriority,
args: &config.VolumeBindingArgs{
BindTimeoutSeconds: 300,
Shape: []config.UtilizationShapePoint{
{
Utilization: 0,
Score: 0,
},
{
Utilization: 50,
Score: 3,
},
{
Utilization: 100,
Score: 5,
},
},
},
wantPreFilterStatus: nil,
wantStateAfterPreFilter: &stateData{
boundClaims: []*v1.PersistentVolumeClaim{},
claimsToBind: []*v1.PersistentVolumeClaim{
setPVCRequestStorage(makePVC("pvc-a", "", waitSC.Name), resource.MustParse("50Gi")),
},
podVolumesByNode: map[string]*scheduling.PodVolumes{},
},
wantFilterStatus: []*framework.Status{
nil,
nil,
nil,
nil,
framework.NewStatus(framework.UnschedulableAndUnresolvable, `node(s) didn't find available persistent volumes to bind`),
framework.NewStatus(framework.UnschedulableAndUnresolvable, `node(s) didn't find available persistent volumes to bind`),
},
wantScores: []int64{
15,
15,
30,
30,
0,
0,
},
},
}
for _, item := range table {
@ -543,9 +648,19 @@ func TestVolumeBinding(t *testing.T) {
if err != nil {
t.Fatal(err)
}
pl, err := New(&config.VolumeBindingArgs{
BindTimeoutSeconds: 300,
}, fh)
args := item.args
if args == nil {
// default args if the args is not specified in test cases
args = &config.VolumeBindingArgs{
BindTimeoutSeconds: 300,
}
if utilfeature.DefaultFeatureGate.Enabled(features.VolumeCapacityPriority) {
args.Shape = defaultShapePoint
}
}
pl, err := New(args, fh)
if err != nil {
t.Fatal(err)
}

View File

@ -226,6 +226,22 @@ type VolumeBindingArgs struct {
// Value must be non-negative integer. The value zero indicates no waiting.
// If this value is nil, the default value (600) will be used.
BindTimeoutSeconds *int64 `json:"bindTimeoutSeconds,omitempty"`
// Shape specifies the points defining the score function shape, which is
// used to score nodes based on the utilization of statically provisioned
// PVs. The utilization is calculated by dividing the total requested
// storage of the pod by the total capacity of feasible PVs on each node.
// Each point contains utilization (ranges from 0 to 100) and its
// associated score (ranges from 0 to 10). You can turn the priority by
// specifying different scores for different utilization numbers.
// The default shape points are:
// 1) 0 for 0 utilization
// 2) 10 for 100 utilization
// All points must be sorted in increasing order by utilization.
// +featureGate=VolumeCapacityPriority
// +optional
// +listType=atomic
Shape []UtilizationShapePoint `json:"shape,omitempty"`
}
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

View File

@ -727,6 +727,11 @@ func (in *VolumeBindingArgs) DeepCopyInto(out *VolumeBindingArgs) {
*out = new(int64)
**out = **in
}
if in.Shape != nil {
in, out := &in.Shape, &out.Shape
*out = make([]UtilizationShapePoint, len(*in))
copy(*out, *in)
}
return
}

View File

@ -142,6 +142,22 @@ type VolumeBindingArgs struct {
// Value must be non-negative integer. The value zero indicates no waiting.
// If this value is nil, the default value (600) will be used.
BindTimeoutSeconds *int64 `json:"bindTimeoutSeconds,omitempty"`
// Shape specifies the points defining the score function shape, which is
// used to score nodes based on the utilization of statically provisioned
// PVs. The utilization is calculated by dividing the total requested
// storage of the pod by the total capacity of feasible PVs on each node.
// Each point contains utilization (ranges from 0 to 100) and its
// associated score (ranges from 0 to 10). You can turn the priority by
// specifying different scores for different utilization numbers.
// The default shape points are:
// 1) 0 for 0 utilization
// 2) 10 for 100 utilization
// All points must be sorted in increasing order by utilization.
// +featureGate=VolumeCapacityPriority
// +optional
// +listType=atomic
Shape []UtilizationShapePoint `json:"shape,omitempty"`
}
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

View File

@ -508,6 +508,11 @@ func (in *VolumeBindingArgs) DeepCopyInto(out *VolumeBindingArgs) {
*out = new(int64)
**out = **in
}
if in.Shape != nil {
in, out := &in.Shape, &out.Shape
*out = make([]UtilizationShapePoint, len(*in))
copy(*out, *in)
}
return
}