mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-26 21:17:23 +00:00
Merge pull request #100537 from pohly/storage-capacity-e2e-test
storage e2e: verify CSIStorageCapacity publishing
This commit is contained in:
commit
e34046c81d
@ -144,7 +144,7 @@ func GetDriverTimeouts(driver TestDriver) *framework.TimeoutContext {
|
|||||||
// Capability represents a feature that a volume plugin supports
|
// Capability represents a feature that a volume plugin supports
|
||||||
type Capability string
|
type Capability string
|
||||||
|
|
||||||
// Constants related to capability
|
// Constants related to capabilities and behavior of the driver.
|
||||||
const (
|
const (
|
||||||
CapPersistence Capability = "persistence" // data is persisted across pod restarts
|
CapPersistence Capability = "persistence" // data is persisted across pod restarts
|
||||||
CapBlock Capability = "block" // raw block mode
|
CapBlock Capability = "block" // raw block mode
|
||||||
@ -166,6 +166,11 @@ const (
|
|||||||
CapVolumeLimits Capability = "volumeLimits" // support volume limits (can be *very* slow)
|
CapVolumeLimits Capability = "volumeLimits" // support volume limits (can be *very* slow)
|
||||||
CapSingleNodeVolume Capability = "singleNodeVolume" // support volume that can run on single node (like hostpath)
|
CapSingleNodeVolume Capability = "singleNodeVolume" // support volume that can run on single node (like hostpath)
|
||||||
CapTopology Capability = "topology" // support topology
|
CapTopology Capability = "topology" // support topology
|
||||||
|
|
||||||
|
// The driver publishes storage capacity information: when the storage class
|
||||||
|
// for dynamic provisioning exists, the driver is expected to provide
|
||||||
|
// capacity information for it.
|
||||||
|
CapCapacity Capability = "capacity"
|
||||||
)
|
)
|
||||||
|
|
||||||
// DriverInfo represents static information about a TestDriver.
|
// DriverInfo represents static information about a TestDriver.
|
||||||
|
@ -53,6 +53,7 @@ type migrationOpCheck struct {
|
|||||||
|
|
||||||
// BaseSuites is a list of storage test suites that work for in-tree and CSI drivers
|
// BaseSuites is a list of storage test suites that work for in-tree and CSI drivers
|
||||||
var BaseSuites = []func() storageframework.TestSuite{
|
var BaseSuites = []func() storageframework.TestSuite{
|
||||||
|
InitCapacityTestSuite,
|
||||||
InitVolumesTestSuite,
|
InitVolumesTestSuite,
|
||||||
InitVolumeIOTestSuite,
|
InitVolumeIOTestSuite,
|
||||||
InitVolumeModeTestSuite,
|
InitVolumeModeTestSuite,
|
||||||
|
384
test/e2e/storage/testsuites/capacity.go
Normal file
384
test/e2e/storage/testsuites/capacity.go
Normal file
@ -0,0 +1,384 @@
|
|||||||
|
/*
|
||||||
|
Copyright 2021 The Kubernetes Authors.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package testsuites
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/onsi/ginkgo"
|
||||||
|
"github.com/onsi/gomega"
|
||||||
|
"github.com/onsi/gomega/types"
|
||||||
|
|
||||||
|
storagev1 "k8s.io/api/storage/v1"
|
||||||
|
storagev1beta1 "k8s.io/api/storage/v1beta1"
|
||||||
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||||
|
"k8s.io/client-go/kubernetes"
|
||||||
|
"k8s.io/kubernetes/test/e2e/framework"
|
||||||
|
e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
|
||||||
|
e2evolume "k8s.io/kubernetes/test/e2e/framework/volume"
|
||||||
|
storageframework "k8s.io/kubernetes/test/e2e/storage/framework"
|
||||||
|
storageutils "k8s.io/kubernetes/test/e2e/storage/utils"
|
||||||
|
)
|
||||||
|
|
||||||
|
type capacityTestSuite struct {
|
||||||
|
tsInfo storageframework.TestSuiteInfo
|
||||||
|
}
|
||||||
|
|
||||||
|
// InitCustomCapacityTestSuite returns capacityTestSuite that implements TestSuite interface
|
||||||
|
// using custom test patterns
|
||||||
|
func InitCustomCapacityTestSuite(patterns []storageframework.TestPattern) storageframework.TestSuite {
|
||||||
|
return &capacityTestSuite{
|
||||||
|
tsInfo: storageframework.TestSuiteInfo{
|
||||||
|
Name: "capacity",
|
||||||
|
TestPatterns: patterns,
|
||||||
|
SupportedSizeRange: e2evolume.SizeRange{
|
||||||
|
Min: "1Mi",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// InitCapacityTestSuite returns capacityTestSuite that implements TestSuite interface\
|
||||||
|
// using test suite default patterns
|
||||||
|
func InitCapacityTestSuite() storageframework.TestSuite {
|
||||||
|
patterns := []storageframework.TestPattern{
|
||||||
|
storageframework.DefaultFsDynamicPV,
|
||||||
|
}
|
||||||
|
return InitCustomCapacityTestSuite(patterns)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *capacityTestSuite) GetTestSuiteInfo() storageframework.TestSuiteInfo {
|
||||||
|
return p.tsInfo
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *capacityTestSuite) SkipUnsupportedTests(driver storageframework.TestDriver, pattern storageframework.TestPattern) {
|
||||||
|
// Check preconditions.
|
||||||
|
if pattern.VolType != storageframework.DynamicPV {
|
||||||
|
e2eskipper.Skipf("Suite %q does not support %v", p.tsInfo.Name, pattern.VolType)
|
||||||
|
}
|
||||||
|
dInfo := driver.GetDriverInfo()
|
||||||
|
if !dInfo.Capabilities[storageframework.CapCapacity] {
|
||||||
|
e2eskipper.Skipf("Driver %s doesn't publish storage capacity -- skipping", dInfo.Name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *capacityTestSuite) DefineTests(driver storageframework.TestDriver, pattern storageframework.TestPattern) {
|
||||||
|
var (
|
||||||
|
dInfo = driver.GetDriverInfo()
|
||||||
|
dDriver storageframework.DynamicPVTestDriver
|
||||||
|
driverCleanup func()
|
||||||
|
sc *storagev1.StorageClass
|
||||||
|
)
|
||||||
|
|
||||||
|
// Beware that it also registers an AfterEach which renders f unusable. Any code using
|
||||||
|
// f must run inside an It or Context callback.
|
||||||
|
f := framework.NewFrameworkWithCustomTimeouts("capacity", storageframework.GetDriverTimeouts(driver))
|
||||||
|
|
||||||
|
init := func() {
|
||||||
|
dDriver, _ = driver.(storageframework.DynamicPVTestDriver)
|
||||||
|
// Now do the more expensive test initialization.
|
||||||
|
config, cleanup := driver.PrepareTest(f)
|
||||||
|
driverCleanup = cleanup
|
||||||
|
sc = dDriver.GetDynamicProvisionStorageClass(config, pattern.FsType)
|
||||||
|
if sc == nil {
|
||||||
|
e2eskipper.Skipf("Driver %q does not define Dynamic Provision StorageClass - skipping", dInfo.Name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
cleanup := func() {
|
||||||
|
err := storageutils.TryFunc(driverCleanup)
|
||||||
|
driverCleanup = nil
|
||||||
|
framework.ExpectNoError(err, "while cleaning up driver")
|
||||||
|
}
|
||||||
|
|
||||||
|
ginkgo.It("provides storage capacity information", func() {
|
||||||
|
init()
|
||||||
|
defer cleanup()
|
||||||
|
|
||||||
|
timeout := time.Minute
|
||||||
|
pollInterval := time.Second
|
||||||
|
matchSC := HaveCapacitiesForClass(sc.Name)
|
||||||
|
listAll := gomega.Eventually(func() (*storagev1beta1.CSIStorageCapacityList, error) {
|
||||||
|
return f.ClientSet.StorageV1beta1().CSIStorageCapacities("").List(context.Background(), metav1.ListOptions{})
|
||||||
|
}, timeout, pollInterval)
|
||||||
|
|
||||||
|
// If we have further information about what storage
|
||||||
|
// capacity information to expect from the driver,
|
||||||
|
// then we can make the check more specific. The baseline
|
||||||
|
// is that it provides some arbitrary capacity for the
|
||||||
|
// storage class.
|
||||||
|
matcher := matchSC
|
||||||
|
if len(dInfo.TopologyKeys) == 1 {
|
||||||
|
// We can construct topology segments by
|
||||||
|
// collecting all values for this one key and
|
||||||
|
// then expect one CSIStorageCapacity object
|
||||||
|
// per value for the storage class.
|
||||||
|
//
|
||||||
|
// Local storage on a node will be covered by
|
||||||
|
// this checking. A more complex approach for
|
||||||
|
// drivers with multiple keys might be
|
||||||
|
// possible, too, but is not currently
|
||||||
|
// implemented.
|
||||||
|
matcher = HaveCapacitiesForClassAndNodes(f.ClientSet, sc.Provisioner, sc.Name, dInfo.TopologyKeys[0])
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create storage class and wait for capacity information.
|
||||||
|
_, clearProvisionedStorageClass := SetupStorageClass(f.ClientSet, sc)
|
||||||
|
defer clearProvisionedStorageClass()
|
||||||
|
listAll.Should(MatchCapacities(matcher), "after creating storage class")
|
||||||
|
|
||||||
|
// Delete storage class again and wait for removal of storage capacity information.
|
||||||
|
clearProvisionedStorageClass()
|
||||||
|
listAll.ShouldNot(MatchCapacities(matchSC), "after deleting storage class")
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func formatCapacities(capacities []storagev1beta1.CSIStorageCapacity) []string {
|
||||||
|
lines := []string{}
|
||||||
|
for _, capacity := range capacities {
|
||||||
|
lines = append(lines, fmt.Sprintf(" %+v", capacity))
|
||||||
|
}
|
||||||
|
return lines
|
||||||
|
}
|
||||||
|
|
||||||
|
// MatchCapacities runs some kind of check against *storagev1beta1.CSIStorageCapacityList.
|
||||||
|
// In case of failure, all actual objects are appended to the failure message.
|
||||||
|
func MatchCapacities(match types.GomegaMatcher) types.GomegaMatcher {
|
||||||
|
return matchCSIStorageCapacities{match: match}
|
||||||
|
}
|
||||||
|
|
||||||
|
type matchCSIStorageCapacities struct {
|
||||||
|
match types.GomegaMatcher
|
||||||
|
}
|
||||||
|
|
||||||
|
var _ types.GomegaMatcher = matchCSIStorageCapacities{}
|
||||||
|
|
||||||
|
func (m matchCSIStorageCapacities) Match(actual interface{}) (success bool, err error) {
|
||||||
|
return m.match.Match(actual)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m matchCSIStorageCapacities) FailureMessage(actual interface{}) (message string) {
|
||||||
|
return m.match.FailureMessage(actual) + m.dump(actual)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m matchCSIStorageCapacities) NegatedFailureMessage(actual interface{}) (message string) {
|
||||||
|
return m.match.NegatedFailureMessage(actual) + m.dump(actual)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m matchCSIStorageCapacities) dump(actual interface{}) string {
|
||||||
|
capacities, ok := actual.(*storagev1beta1.CSIStorageCapacityList)
|
||||||
|
if !ok || capacities == nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
lines := []string{"\n\nall CSIStorageCapacity objects:"}
|
||||||
|
for _, capacity := range capacities.Items {
|
||||||
|
lines = append(lines, fmt.Sprintf("%+v", capacity))
|
||||||
|
}
|
||||||
|
return strings.Join(lines, "\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
// CapacityMatcher can be used to compose different matchers where one
|
||||||
|
// adds additional checks for CSIStorageCapacity objects already checked
|
||||||
|
// by another.
|
||||||
|
type CapacityMatcher interface {
|
||||||
|
types.GomegaMatcher
|
||||||
|
// MatchedCapacities returns all CSICapacityObjects which were
|
||||||
|
// found during the preceding Match call.
|
||||||
|
MatchedCapacities() []storagev1beta1.CSIStorageCapacity
|
||||||
|
}
|
||||||
|
|
||||||
|
// HaveCapacitiesForClass filters all storage capacity objects in a *storagev1beta1.CSIStorageCapacityList
|
||||||
|
// by storage class. Success is when when there is at least one.
|
||||||
|
func HaveCapacitiesForClass(scName string) CapacityMatcher {
|
||||||
|
return &haveCSIStorageCapacities{scName: scName}
|
||||||
|
}
|
||||||
|
|
||||||
|
type haveCSIStorageCapacities struct {
|
||||||
|
scName string
|
||||||
|
matchingCapacities []storagev1beta1.CSIStorageCapacity
|
||||||
|
}
|
||||||
|
|
||||||
|
var _ CapacityMatcher = &haveCSIStorageCapacities{}
|
||||||
|
|
||||||
|
func (h *haveCSIStorageCapacities) Match(actual interface{}) (success bool, err error) {
|
||||||
|
capacities, ok := actual.(*storagev1beta1.CSIStorageCapacityList)
|
||||||
|
if !ok {
|
||||||
|
return false, fmt.Errorf("expected *storagev1beta1.CSIStorageCapacityList, got: %T", actual)
|
||||||
|
}
|
||||||
|
h.matchingCapacities = nil
|
||||||
|
for _, capacity := range capacities.Items {
|
||||||
|
if capacity.StorageClassName == h.scName {
|
||||||
|
h.matchingCapacities = append(h.matchingCapacities, capacity)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return len(h.matchingCapacities) > 0, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *haveCSIStorageCapacities) MatchedCapacities() []storagev1beta1.CSIStorageCapacity {
|
||||||
|
return h.matchingCapacities
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *haveCSIStorageCapacities) FailureMessage(actual interface{}) (message string) {
|
||||||
|
return fmt.Sprintf("no CSIStorageCapacity objects for storage class %q", h.scName)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *haveCSIStorageCapacities) NegatedFailureMessage(actual interface{}) (message string) {
|
||||||
|
return fmt.Sprintf("CSIStorageCapacity objects for storage class %q:\n%s",
|
||||||
|
h.scName,
|
||||||
|
strings.Join(formatCapacities(h.matchingCapacities), "\n"),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// HaveCapacitiesForClassAndNodes matches objects by storage class name. It finds
|
||||||
|
// all nodes on which the driver runs and expects one object per node.
|
||||||
|
func HaveCapacitiesForClassAndNodes(client kubernetes.Interface, driverName, scName, topologyKey string) CapacityMatcher {
|
||||||
|
return &haveLocalStorageCapacities{
|
||||||
|
client: client,
|
||||||
|
driverName: driverName,
|
||||||
|
match: HaveCapacitiesForClass(scName),
|
||||||
|
topologyKey: topologyKey,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type haveLocalStorageCapacities struct {
|
||||||
|
client kubernetes.Interface
|
||||||
|
driverName string
|
||||||
|
match CapacityMatcher
|
||||||
|
topologyKey string
|
||||||
|
|
||||||
|
matchSuccess bool
|
||||||
|
expectedCapacities []storagev1beta1.CSIStorageCapacity
|
||||||
|
unexpectedCapacities []storagev1beta1.CSIStorageCapacity
|
||||||
|
missingTopologyValues []string
|
||||||
|
}
|
||||||
|
|
||||||
|
var _ CapacityMatcher = &haveLocalStorageCapacities{}
|
||||||
|
|
||||||
|
func (h *haveLocalStorageCapacities) Match(actual interface{}) (success bool, err error) {
|
||||||
|
h.expectedCapacities = nil
|
||||||
|
h.unexpectedCapacities = nil
|
||||||
|
h.missingTopologyValues = nil
|
||||||
|
|
||||||
|
// First check with underlying matcher.
|
||||||
|
success, err = h.match.Match(actual)
|
||||||
|
h.matchSuccess = success
|
||||||
|
if !success || err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Find all nodes on which the driver runs.
|
||||||
|
csiNodes, err := h.client.StorageV1().CSINodes().List(context.Background(), metav1.ListOptions{})
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
topologyValues := map[string]bool{}
|
||||||
|
for _, csiNode := range csiNodes.Items {
|
||||||
|
for _, driver := range csiNode.Spec.Drivers {
|
||||||
|
if driver.Name != h.driverName {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
node, err := h.client.CoreV1().Nodes().Get(context.Background(), csiNode.Name, metav1.GetOptions{})
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
value, ok := node.Labels[h.topologyKey]
|
||||||
|
if !ok || value == "" {
|
||||||
|
return false, fmt.Errorf("driver %q should run on node %q, but its topology label %q was not set",
|
||||||
|
h.driverName,
|
||||||
|
node.Name,
|
||||||
|
h.topologyKey)
|
||||||
|
}
|
||||||
|
topologyValues[value] = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(topologyValues) == 0 {
|
||||||
|
return false, fmt.Errorf("driver %q not running on any node", h.driverName)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Now check that for each topology value there is exactly one CSIStorageCapacity object.
|
||||||
|
remainingTopologyValues := map[string]bool{}
|
||||||
|
for value := range topologyValues {
|
||||||
|
remainingTopologyValues[value] = true
|
||||||
|
}
|
||||||
|
capacities := h.match.MatchedCapacities()
|
||||||
|
for _, capacity := range capacities {
|
||||||
|
if capacity.NodeTopology == nil ||
|
||||||
|
len(capacity.NodeTopology.MatchExpressions) > 0 ||
|
||||||
|
len(capacity.NodeTopology.MatchLabels) != 1 ||
|
||||||
|
!remainingTopologyValues[capacity.NodeTopology.MatchLabels[h.topologyKey]] {
|
||||||
|
h.unexpectedCapacities = append(h.unexpectedCapacities, capacity)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
remainingTopologyValues[capacity.NodeTopology.MatchLabels[h.topologyKey]] = false
|
||||||
|
h.expectedCapacities = append(h.expectedCapacities, capacity)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Success is when there were no unexpected capacities and enough expected ones.
|
||||||
|
for value, remaining := range remainingTopologyValues {
|
||||||
|
if remaining {
|
||||||
|
h.missingTopologyValues = append(h.missingTopologyValues, value)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return len(h.unexpectedCapacities) == 0 && len(h.missingTopologyValues) == 0, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *haveLocalStorageCapacities) MatchedCapacities() []storagev1beta1.CSIStorageCapacity {
|
||||||
|
return h.match.MatchedCapacities()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *haveLocalStorageCapacities) FailureMessage(actual interface{}) (message string) {
|
||||||
|
if !h.matchSuccess {
|
||||||
|
return h.match.FailureMessage(actual)
|
||||||
|
}
|
||||||
|
var lines []string
|
||||||
|
if len(h.unexpectedCapacities) != 0 {
|
||||||
|
lines = append(lines, "unexpected CSIStorageCapacity objects:")
|
||||||
|
lines = append(lines, formatCapacities(h.unexpectedCapacities)...)
|
||||||
|
}
|
||||||
|
if len(h.missingTopologyValues) != 0 {
|
||||||
|
lines = append(lines, fmt.Sprintf("no CSIStorageCapacity objects with topology key %q and values %v",
|
||||||
|
h.topologyKey, h.missingTopologyValues,
|
||||||
|
))
|
||||||
|
}
|
||||||
|
return strings.Join(lines, "\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *haveLocalStorageCapacities) NegatedFailureMessage(actual interface{}) (message string) {
|
||||||
|
if h.matchSuccess {
|
||||||
|
return h.match.NegatedFailureMessage(actual)
|
||||||
|
}
|
||||||
|
// It's not entirely clear whether negating this check is useful. Just dump all info that we have.
|
||||||
|
var lines []string
|
||||||
|
if len(h.expectedCapacities) != 0 {
|
||||||
|
lines = append(lines, "expected CSIStorageCapacity objects:")
|
||||||
|
lines = append(lines, formatCapacities(h.expectedCapacities)...)
|
||||||
|
}
|
||||||
|
if len(h.unexpectedCapacities) != 0 {
|
||||||
|
lines = append(lines, "unexpected CSIStorageCapacity objects:")
|
||||||
|
lines = append(lines, formatCapacities(h.unexpectedCapacities)...)
|
||||||
|
}
|
||||||
|
if len(h.missingTopologyValues) != 0 {
|
||||||
|
lines = append(lines, fmt.Sprintf("no CSIStorageCapacity objects with topology key %q and values %v",
|
||||||
|
h.topologyKey, h.missingTopologyValues,
|
||||||
|
))
|
||||||
|
}
|
||||||
|
return strings.Join(lines, "\n")
|
||||||
|
}
|
@ -337,7 +337,10 @@ func SetupStorageClass(
|
|||||||
framework.ExpectNoError(err)
|
framework.ExpectNoError(err)
|
||||||
clearComputedStorageClass = func() {
|
clearComputedStorageClass = func() {
|
||||||
framework.Logf("deleting storage class %s", computedStorageClass.Name)
|
framework.Logf("deleting storage class %s", computedStorageClass.Name)
|
||||||
framework.ExpectNoError(client.StorageV1().StorageClasses().Delete(context.TODO(), computedStorageClass.Name, metav1.DeleteOptions{}))
|
err := client.StorageV1().StorageClasses().Delete(context.TODO(), computedStorageClass.Name, metav1.DeleteOptions{})
|
||||||
|
if err != nil && !apierrors.IsNotFound(err) {
|
||||||
|
framework.ExpectNoError(err, "delete storage class")
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
|
Loading…
Reference in New Issue
Block a user