mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-11-13 06:04:02 +00:00
530ee716e3
In such cases when api server is overloaded and returns a lot of 429 (too many requests) errors - binding may take a lot of time to succeed due to retry policy implemented in rest client. In such events cache ttl for assumed pods wasn't big enough. In order to minimize probability of such errors ttl for assumed pods will be counted from the time when binding for particular pod is finished (either with error or success) Change-Id: Ib0122f8a76dc57c82f2c7c52497aad1bdd8be411
582 lines
16 KiB
Go
582 lines
16 KiB
Go
/*
|
|
Copyright 2015 The Kubernetes Authors.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package schedulercache
|
|
|
|
import (
|
|
"fmt"
|
|
"reflect"
|
|
"testing"
|
|
"time"
|
|
|
|
"k8s.io/kubernetes/pkg/api/resource"
|
|
"k8s.io/kubernetes/pkg/api/v1"
|
|
"k8s.io/kubernetes/pkg/labels"
|
|
priorityutil "k8s.io/kubernetes/plugin/pkg/scheduler/algorithm/priorities/util"
|
|
)
|
|
|
|
func deepEqualWithoutGeneration(t *testing.T, testcase int, actual, expected *NodeInfo) {
|
|
// Ignore generation field.
|
|
if actual != nil {
|
|
actual.generation = 0
|
|
}
|
|
if !reflect.DeepEqual(actual, expected) {
|
|
t.Errorf("#%d: node info get=%s, want=%s", testcase, actual, expected)
|
|
}
|
|
}
|
|
|
|
// TestAssumePodScheduled tests that after a pod is assumed, its information is aggregated
|
|
// on node level.
|
|
func TestAssumePodScheduled(t *testing.T) {
|
|
nodeName := "node"
|
|
testPods := []*v1.Pod{
|
|
makeBasePod(nodeName, "test", "100m", "500", []v1.ContainerPort{{HostPort: 80}}),
|
|
makeBasePod(nodeName, "test-1", "100m", "500", []v1.ContainerPort{{HostPort: 80}}),
|
|
makeBasePod(nodeName, "test-2", "200m", "1Ki", []v1.ContainerPort{{HostPort: 8080}}),
|
|
makeBasePod(nodeName, "test-nonzero", "", "", []v1.ContainerPort{{HostPort: 80}}),
|
|
}
|
|
|
|
tests := []struct {
|
|
pods []*v1.Pod
|
|
|
|
wNodeInfo *NodeInfo
|
|
}{{
|
|
pods: []*v1.Pod{testPods[0]},
|
|
wNodeInfo: &NodeInfo{
|
|
requestedResource: &Resource{
|
|
MilliCPU: 100,
|
|
Memory: 500,
|
|
},
|
|
nonzeroRequest: &Resource{
|
|
MilliCPU: 100,
|
|
Memory: 500,
|
|
},
|
|
allocatableResource: &Resource{},
|
|
pods: []*v1.Pod{testPods[0]},
|
|
},
|
|
}, {
|
|
pods: []*v1.Pod{testPods[1], testPods[2]},
|
|
wNodeInfo: &NodeInfo{
|
|
requestedResource: &Resource{
|
|
MilliCPU: 300,
|
|
Memory: 1524,
|
|
},
|
|
nonzeroRequest: &Resource{
|
|
MilliCPU: 300,
|
|
Memory: 1524,
|
|
},
|
|
allocatableResource: &Resource{},
|
|
pods: []*v1.Pod{testPods[1], testPods[2]},
|
|
},
|
|
}, { // test non-zero request
|
|
pods: []*v1.Pod{testPods[3]},
|
|
wNodeInfo: &NodeInfo{
|
|
requestedResource: &Resource{
|
|
MilliCPU: 0,
|
|
Memory: 0,
|
|
},
|
|
nonzeroRequest: &Resource{
|
|
MilliCPU: priorityutil.DefaultMilliCpuRequest,
|
|
Memory: priorityutil.DefaultMemoryRequest,
|
|
},
|
|
allocatableResource: &Resource{},
|
|
pods: []*v1.Pod{testPods[3]},
|
|
},
|
|
}}
|
|
|
|
for i, tt := range tests {
|
|
cache := newSchedulerCache(time.Second, time.Second, nil)
|
|
for _, pod := range tt.pods {
|
|
if err := cache.AssumePod(pod); err != nil {
|
|
t.Fatalf("AssumePod failed: %v", err)
|
|
}
|
|
}
|
|
n := cache.nodes[nodeName]
|
|
deepEqualWithoutGeneration(t, i, n, tt.wNodeInfo)
|
|
|
|
for _, pod := range tt.pods {
|
|
if err := cache.ForgetPod(pod); err != nil {
|
|
t.Fatalf("ForgetPod failed: %v", err)
|
|
}
|
|
}
|
|
if cache.nodes[nodeName] != nil {
|
|
t.Errorf("NodeInfo should be cleaned for %s", nodeName)
|
|
}
|
|
}
|
|
}
|
|
|
|
type testExpirePodStruct struct {
|
|
pod *v1.Pod
|
|
assumedTime time.Time
|
|
}
|
|
|
|
func assumeAndFinishBinding(cache *schedulerCache, pod *v1.Pod, assumedTime time.Time) error {
|
|
if err := cache.AssumePod(pod); err != nil {
|
|
return err
|
|
}
|
|
return cache.finishBinding(pod, assumedTime)
|
|
}
|
|
|
|
// TestExpirePod tests that assumed pods will be removed if expired.
|
|
// The removal will be reflected in node info.
|
|
func TestExpirePod(t *testing.T) {
|
|
nodeName := "node"
|
|
testPods := []*v1.Pod{
|
|
makeBasePod(nodeName, "test-1", "100m", "500", []v1.ContainerPort{{HostPort: 80}}),
|
|
makeBasePod(nodeName, "test-2", "200m", "1Ki", []v1.ContainerPort{{HostPort: 8080}}),
|
|
}
|
|
now := time.Now()
|
|
ttl := 10 * time.Second
|
|
tests := []struct {
|
|
pods []*testExpirePodStruct
|
|
cleanupTime time.Time
|
|
|
|
wNodeInfo *NodeInfo
|
|
}{{ // assumed pod would expires
|
|
pods: []*testExpirePodStruct{
|
|
{pod: testPods[0], assumedTime: now},
|
|
},
|
|
cleanupTime: now.Add(2 * ttl),
|
|
wNodeInfo: nil,
|
|
}, { // first one would expire, second one would not.
|
|
pods: []*testExpirePodStruct{
|
|
{pod: testPods[0], assumedTime: now},
|
|
{pod: testPods[1], assumedTime: now.Add(3 * ttl / 2)},
|
|
},
|
|
cleanupTime: now.Add(2 * ttl),
|
|
wNodeInfo: &NodeInfo{
|
|
requestedResource: &Resource{
|
|
MilliCPU: 200,
|
|
Memory: 1024,
|
|
},
|
|
nonzeroRequest: &Resource{
|
|
MilliCPU: 200,
|
|
Memory: 1024,
|
|
},
|
|
allocatableResource: &Resource{},
|
|
pods: []*v1.Pod{testPods[1]},
|
|
},
|
|
}}
|
|
|
|
for i, tt := range tests {
|
|
cache := newSchedulerCache(ttl, time.Second, nil)
|
|
|
|
for _, pod := range tt.pods {
|
|
if err := assumeAndFinishBinding(cache, pod.pod, pod.assumedTime); err != nil {
|
|
t.Fatalf("assumePod failed: %v", err)
|
|
}
|
|
}
|
|
// pods that have assumedTime + ttl < cleanupTime will get expired and removed
|
|
cache.cleanupAssumedPods(tt.cleanupTime)
|
|
n := cache.nodes[nodeName]
|
|
deepEqualWithoutGeneration(t, i, n, tt.wNodeInfo)
|
|
}
|
|
}
|
|
|
|
// TestAddPodWillConfirm tests that a pod being Add()ed will be confirmed if assumed.
|
|
// The pod info should still exist after manually expiring unconfirmed pods.
|
|
func TestAddPodWillConfirm(t *testing.T) {
|
|
nodeName := "node"
|
|
now := time.Now()
|
|
ttl := 10 * time.Second
|
|
|
|
testPods := []*v1.Pod{
|
|
makeBasePod(nodeName, "test-1", "100m", "500", []v1.ContainerPort{{HostPort: 80}}),
|
|
makeBasePod(nodeName, "test-2", "200m", "1Ki", []v1.ContainerPort{{HostPort: 8080}}),
|
|
}
|
|
tests := []struct {
|
|
podsToAssume []*v1.Pod
|
|
podsToAdd []*v1.Pod
|
|
|
|
wNodeInfo *NodeInfo
|
|
}{{ // two pod were assumed at same time. But first one is called Add() and gets confirmed.
|
|
podsToAssume: []*v1.Pod{testPods[0], testPods[1]},
|
|
podsToAdd: []*v1.Pod{testPods[0]},
|
|
wNodeInfo: &NodeInfo{
|
|
requestedResource: &Resource{
|
|
MilliCPU: 100,
|
|
Memory: 500,
|
|
},
|
|
nonzeroRequest: &Resource{
|
|
MilliCPU: 100,
|
|
Memory: 500,
|
|
},
|
|
allocatableResource: &Resource{},
|
|
pods: []*v1.Pod{testPods[0]},
|
|
},
|
|
}}
|
|
|
|
for i, tt := range tests {
|
|
cache := newSchedulerCache(ttl, time.Second, nil)
|
|
for _, podToAssume := range tt.podsToAssume {
|
|
if err := assumeAndFinishBinding(cache, podToAssume, now); err != nil {
|
|
t.Fatalf("assumePod failed: %v", err)
|
|
}
|
|
}
|
|
for _, podToAdd := range tt.podsToAdd {
|
|
if err := cache.AddPod(podToAdd); err != nil {
|
|
t.Fatalf("AddPod failed: %v", err)
|
|
}
|
|
}
|
|
cache.cleanupAssumedPods(now.Add(2 * ttl))
|
|
// check after expiration. confirmed pods shouldn't be expired.
|
|
n := cache.nodes[nodeName]
|
|
deepEqualWithoutGeneration(t, i, n, tt.wNodeInfo)
|
|
}
|
|
}
|
|
|
|
// TestAddPodAfterExpiration tests that a pod being Add()ed will be added back if expired.
|
|
func TestAddPodAfterExpiration(t *testing.T) {
|
|
nodeName := "node"
|
|
ttl := 10 * time.Second
|
|
basePod := makeBasePod(nodeName, "test", "100m", "500", []v1.ContainerPort{{HostPort: 80}})
|
|
tests := []struct {
|
|
pod *v1.Pod
|
|
|
|
wNodeInfo *NodeInfo
|
|
}{{
|
|
pod: basePod,
|
|
wNodeInfo: &NodeInfo{
|
|
requestedResource: &Resource{
|
|
MilliCPU: 100,
|
|
Memory: 500,
|
|
},
|
|
nonzeroRequest: &Resource{
|
|
MilliCPU: 100,
|
|
Memory: 500,
|
|
},
|
|
allocatableResource: &Resource{},
|
|
pods: []*v1.Pod{basePod},
|
|
},
|
|
}}
|
|
|
|
now := time.Now()
|
|
for i, tt := range tests {
|
|
cache := newSchedulerCache(ttl, time.Second, nil)
|
|
if err := assumeAndFinishBinding(cache, tt.pod, now); err != nil {
|
|
t.Fatalf("assumePod failed: %v", err)
|
|
}
|
|
cache.cleanupAssumedPods(now.Add(2 * ttl))
|
|
// It should be expired and removed.
|
|
n := cache.nodes[nodeName]
|
|
if n != nil {
|
|
t.Errorf("#%d: expecting nil node info, but get=%v", i, n)
|
|
}
|
|
if err := cache.AddPod(tt.pod); err != nil {
|
|
t.Fatalf("AddPod failed: %v", err)
|
|
}
|
|
// check after expiration. confirmed pods shouldn't be expired.
|
|
n = cache.nodes[nodeName]
|
|
deepEqualWithoutGeneration(t, i, n, tt.wNodeInfo)
|
|
}
|
|
}
|
|
|
|
// TestUpdatePod tests that a pod will be updated if added before.
|
|
func TestUpdatePod(t *testing.T) {
|
|
nodeName := "node"
|
|
ttl := 10 * time.Second
|
|
testPods := []*v1.Pod{
|
|
makeBasePod(nodeName, "test", "100m", "500", []v1.ContainerPort{{HostPort: 80}}),
|
|
makeBasePod(nodeName, "test", "200m", "1Ki", []v1.ContainerPort{{HostPort: 8080}}),
|
|
}
|
|
tests := []struct {
|
|
podsToAssume []*v1.Pod
|
|
podsToAdd []*v1.Pod
|
|
podsToUpdate []*v1.Pod
|
|
|
|
wNodeInfo []*NodeInfo
|
|
}{{ // add a pod and then update it twice
|
|
podsToAdd: []*v1.Pod{testPods[0]},
|
|
podsToUpdate: []*v1.Pod{testPods[0], testPods[1], testPods[0]},
|
|
wNodeInfo: []*NodeInfo{{
|
|
requestedResource: &Resource{
|
|
MilliCPU: 200,
|
|
Memory: 1024,
|
|
},
|
|
nonzeroRequest: &Resource{
|
|
MilliCPU: 200,
|
|
Memory: 1024,
|
|
},
|
|
allocatableResource: &Resource{},
|
|
pods: []*v1.Pod{testPods[1]},
|
|
}, {
|
|
requestedResource: &Resource{
|
|
MilliCPU: 100,
|
|
Memory: 500,
|
|
},
|
|
nonzeroRequest: &Resource{
|
|
MilliCPU: 100,
|
|
Memory: 500,
|
|
},
|
|
allocatableResource: &Resource{},
|
|
pods: []*v1.Pod{testPods[0]},
|
|
}},
|
|
}}
|
|
|
|
for _, tt := range tests {
|
|
cache := newSchedulerCache(ttl, time.Second, nil)
|
|
for _, podToAdd := range tt.podsToAdd {
|
|
if err := cache.AddPod(podToAdd); err != nil {
|
|
t.Fatalf("AddPod failed: %v", err)
|
|
}
|
|
}
|
|
|
|
for i := range tt.podsToUpdate {
|
|
if i == 0 {
|
|
continue
|
|
}
|
|
if err := cache.UpdatePod(tt.podsToUpdate[i-1], tt.podsToUpdate[i]); err != nil {
|
|
t.Fatalf("UpdatePod failed: %v", err)
|
|
}
|
|
// check after expiration. confirmed pods shouldn't be expired.
|
|
n := cache.nodes[nodeName]
|
|
deepEqualWithoutGeneration(t, i, n, tt.wNodeInfo[i-1])
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestExpireAddUpdatePod test the sequence that a pod is expired, added, then updated
|
|
func TestExpireAddUpdatePod(t *testing.T) {
|
|
nodeName := "node"
|
|
ttl := 10 * time.Second
|
|
testPods := []*v1.Pod{
|
|
makeBasePod(nodeName, "test", "100m", "500", []v1.ContainerPort{{HostPort: 80}}),
|
|
makeBasePod(nodeName, "test", "200m", "1Ki", []v1.ContainerPort{{HostPort: 8080}}),
|
|
}
|
|
tests := []struct {
|
|
podsToAssume []*v1.Pod
|
|
podsToAdd []*v1.Pod
|
|
podsToUpdate []*v1.Pod
|
|
|
|
wNodeInfo []*NodeInfo
|
|
}{{ // Pod is assumed, expired, and added. Then it would be updated twice.
|
|
podsToAssume: []*v1.Pod{testPods[0]},
|
|
podsToAdd: []*v1.Pod{testPods[0]},
|
|
podsToUpdate: []*v1.Pod{testPods[0], testPods[1], testPods[0]},
|
|
wNodeInfo: []*NodeInfo{{
|
|
requestedResource: &Resource{
|
|
MilliCPU: 200,
|
|
Memory: 1024,
|
|
},
|
|
nonzeroRequest: &Resource{
|
|
MilliCPU: 200,
|
|
Memory: 1024,
|
|
},
|
|
allocatableResource: &Resource{},
|
|
pods: []*v1.Pod{testPods[1]},
|
|
}, {
|
|
requestedResource: &Resource{
|
|
MilliCPU: 100,
|
|
Memory: 500,
|
|
},
|
|
nonzeroRequest: &Resource{
|
|
MilliCPU: 100,
|
|
Memory: 500,
|
|
},
|
|
allocatableResource: &Resource{},
|
|
pods: []*v1.Pod{testPods[0]},
|
|
}},
|
|
}}
|
|
|
|
now := time.Now()
|
|
for _, tt := range tests {
|
|
cache := newSchedulerCache(ttl, time.Second, nil)
|
|
for _, podToAssume := range tt.podsToAssume {
|
|
if err := assumeAndFinishBinding(cache, podToAssume, now); err != nil {
|
|
t.Fatalf("assumePod failed: %v", err)
|
|
}
|
|
}
|
|
cache.cleanupAssumedPods(now.Add(2 * ttl))
|
|
|
|
for _, podToAdd := range tt.podsToAdd {
|
|
if err := cache.AddPod(podToAdd); err != nil {
|
|
t.Fatalf("AddPod failed: %v", err)
|
|
}
|
|
}
|
|
|
|
for i := range tt.podsToUpdate {
|
|
if i == 0 {
|
|
continue
|
|
}
|
|
if err := cache.UpdatePod(tt.podsToUpdate[i-1], tt.podsToUpdate[i]); err != nil {
|
|
t.Fatalf("UpdatePod failed: %v", err)
|
|
}
|
|
// check after expiration. confirmed pods shouldn't be expired.
|
|
n := cache.nodes[nodeName]
|
|
deepEqualWithoutGeneration(t, i, n, tt.wNodeInfo[i-1])
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestRemovePod tests after added pod is removed, its information should also be subtracted.
|
|
func TestRemovePod(t *testing.T) {
|
|
nodeName := "node"
|
|
basePod := makeBasePod(nodeName, "test", "100m", "500", []v1.ContainerPort{{HostPort: 80}})
|
|
tests := []struct {
|
|
pod *v1.Pod
|
|
|
|
wNodeInfo *NodeInfo
|
|
}{{
|
|
pod: basePod,
|
|
wNodeInfo: &NodeInfo{
|
|
requestedResource: &Resource{
|
|
MilliCPU: 100,
|
|
Memory: 500,
|
|
},
|
|
nonzeroRequest: &Resource{
|
|
MilliCPU: 100,
|
|
Memory: 500,
|
|
},
|
|
allocatableResource: &Resource{},
|
|
pods: []*v1.Pod{basePod},
|
|
},
|
|
}}
|
|
|
|
for i, tt := range tests {
|
|
cache := newSchedulerCache(time.Second, time.Second, nil)
|
|
if err := cache.AddPod(tt.pod); err != nil {
|
|
t.Fatalf("AddPod failed: %v", err)
|
|
}
|
|
n := cache.nodes[nodeName]
|
|
deepEqualWithoutGeneration(t, i, n, tt.wNodeInfo)
|
|
|
|
if err := cache.RemovePod(tt.pod); err != nil {
|
|
t.Fatalf("RemovePod failed: %v", err)
|
|
}
|
|
|
|
n = cache.nodes[nodeName]
|
|
if n != nil {
|
|
t.Errorf("#%d: expecting pod deleted and nil node info, get=%s", i, n)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestForgetPod(t *testing.T) {
|
|
nodeName := "node"
|
|
basePod := makeBasePod(nodeName, "test", "100m", "500", []v1.ContainerPort{{HostPort: 80}})
|
|
tests := []struct {
|
|
pods []*v1.Pod
|
|
}{{
|
|
pods: []*v1.Pod{basePod},
|
|
}}
|
|
now := time.Now()
|
|
ttl := 10 * time.Second
|
|
|
|
for i, tt := range tests {
|
|
cache := newSchedulerCache(ttl, time.Second, nil)
|
|
for _, pod := range tt.pods {
|
|
if err := assumeAndFinishBinding(cache, pod, now); err != nil {
|
|
t.Fatalf("assumePod failed: %v", err)
|
|
}
|
|
}
|
|
for _, pod := range tt.pods {
|
|
if err := cache.ForgetPod(pod); err != nil {
|
|
t.Fatalf("ForgetPod failed: %v", err)
|
|
}
|
|
}
|
|
cache.cleanupAssumedPods(now.Add(2 * ttl))
|
|
if n := cache.nodes[nodeName]; n != nil {
|
|
t.Errorf("#%d: expecting pod deleted and nil node info, get=%s", i, n)
|
|
}
|
|
}
|
|
}
|
|
|
|
func BenchmarkList1kNodes30kPods(b *testing.B) {
|
|
cache := setupCacheOf1kNodes30kPods(b)
|
|
b.ResetTimer()
|
|
for n := 0; n < b.N; n++ {
|
|
cache.List(labels.Everything())
|
|
}
|
|
}
|
|
|
|
func BenchmarkExpire100Pods(b *testing.B) {
|
|
benchmarkExpire(b, 100)
|
|
}
|
|
|
|
func BenchmarkExpire1kPods(b *testing.B) {
|
|
benchmarkExpire(b, 1000)
|
|
}
|
|
|
|
func BenchmarkExpire10kPods(b *testing.B) {
|
|
benchmarkExpire(b, 10000)
|
|
}
|
|
|
|
func benchmarkExpire(b *testing.B, podNum int) {
|
|
now := time.Now()
|
|
for n := 0; n < b.N; n++ {
|
|
b.StopTimer()
|
|
cache := setupCacheWithAssumedPods(b, podNum, now)
|
|
b.StartTimer()
|
|
cache.cleanupAssumedPods(now.Add(2 * time.Second))
|
|
}
|
|
}
|
|
|
|
func makeBasePod(nodeName, objName, cpu, mem string, ports []v1.ContainerPort) *v1.Pod {
|
|
req := v1.ResourceList{}
|
|
if cpu != "" {
|
|
req = v1.ResourceList{
|
|
v1.ResourceCPU: resource.MustParse(cpu),
|
|
v1.ResourceMemory: resource.MustParse(mem),
|
|
}
|
|
}
|
|
return &v1.Pod{
|
|
ObjectMeta: v1.ObjectMeta{
|
|
Namespace: "node_info_cache_test",
|
|
Name: objName,
|
|
},
|
|
Spec: v1.PodSpec{
|
|
Containers: []v1.Container{{
|
|
Resources: v1.ResourceRequirements{
|
|
Requests: req,
|
|
},
|
|
Ports: ports,
|
|
}},
|
|
NodeName: nodeName,
|
|
},
|
|
}
|
|
}
|
|
|
|
func setupCacheOf1kNodes30kPods(b *testing.B) Cache {
|
|
cache := newSchedulerCache(time.Second, time.Second, nil)
|
|
for i := 0; i < 1000; i++ {
|
|
nodeName := fmt.Sprintf("node-%d", i)
|
|
for j := 0; j < 30; j++ {
|
|
objName := fmt.Sprintf("%s-pod-%d", nodeName, j)
|
|
pod := makeBasePod(nodeName, objName, "0", "0", nil)
|
|
|
|
if err := cache.AddPod(pod); err != nil {
|
|
b.Fatalf("AddPod failed: %v", err)
|
|
}
|
|
}
|
|
}
|
|
return cache
|
|
}
|
|
|
|
func setupCacheWithAssumedPods(b *testing.B, podNum int, assumedTime time.Time) *schedulerCache {
|
|
cache := newSchedulerCache(time.Second, time.Second, nil)
|
|
for i := 0; i < podNum; i++ {
|
|
nodeName := fmt.Sprintf("node-%d", i/10)
|
|
objName := fmt.Sprintf("%s-pod-%d", nodeName, i%10)
|
|
pod := makeBasePod(nodeName, objName, "0", "0", nil)
|
|
|
|
err := assumeAndFinishBinding(cache, pod, assumedTime)
|
|
if err != nil {
|
|
b.Fatalf("assumePod failed: %v", err)
|
|
}
|
|
}
|
|
return cache
|
|
}
|