mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-09-20 17:38:50 +00:00
Move scheduler fake artifacts to pkg/scheduler/testing
- move some fake artifacts from pkg/scheduler/core to pkg/scheduler/testing so it can be consumed by core as well as plugin testings
This commit is contained in:
@@ -5,17 +5,23 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library")
|
||||
go_library(
|
||||
name = "go_default_library",
|
||||
srcs = [
|
||||
"fake_extender.go",
|
||||
"fake_plugins.go",
|
||||
"framework_helpers.go",
|
||||
"workload_prep.go",
|
||||
"wrappers.go",
|
||||
],
|
||||
importpath = "k8s.io/kubernetes/pkg/scheduler/testing",
|
||||
deps = [
|
||||
"//pkg/api/v1/pod:go_default_library",
|
||||
"//pkg/scheduler/apis/config:go_default_library",
|
||||
"//pkg/scheduler/framework/v1alpha1:go_default_library",
|
||||
"//pkg/scheduler/util:go_default_library",
|
||||
"//staging/src/k8s.io/api/core/v1:go_default_library",
|
||||
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
|
||||
"//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
|
||||
"//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
|
||||
"//staging/src/k8s.io/kube-scheduler/extender/v1:go_default_library",
|
||||
],
|
||||
)
|
||||
|
||||
|
370
pkg/scheduler/testing/fake_extender.go
Normal file
370
pkg/scheduler/testing/fake_extender.go
Normal file
@@ -0,0 +1,370 @@
|
||||
/*
|
||||
Copyright 2020 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package testing
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
|
||||
v1 "k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
extenderv1 "k8s.io/kube-scheduler/extender/v1"
|
||||
podutil "k8s.io/kubernetes/pkg/api/v1/pod"
|
||||
framework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1"
|
||||
"k8s.io/kubernetes/pkg/scheduler/util"
|
||||
)
|
||||
|
||||
// FitPredicate is a function type which is used in fake extender.
|
||||
type FitPredicate func(pod *v1.Pod, node *v1.Node) (bool, error)
|
||||
|
||||
// PriorityFunc is a function type which is used in fake extender.
|
||||
type PriorityFunc func(pod *v1.Pod, nodes []*v1.Node) (*framework.NodeScoreList, error)
|
||||
|
||||
// PriorityConfig is used in fake extender to perform Prioritize function.
|
||||
type PriorityConfig struct {
|
||||
Function PriorityFunc
|
||||
Weight int64
|
||||
}
|
||||
|
||||
// ErrorPredicateExtender implements FitPredicate function to always return error.
|
||||
func ErrorPredicateExtender(pod *v1.Pod, node *v1.Node) (bool, error) {
|
||||
return false, fmt.Errorf("some error")
|
||||
}
|
||||
|
||||
// FalsePredicateExtender implements FitPredicate function to always return false.
|
||||
func FalsePredicateExtender(pod *v1.Pod, node *v1.Node) (bool, error) {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// TruePredicateExtender implements FitPredicate function to always return true.
|
||||
func TruePredicateExtender(pod *v1.Pod, node *v1.Node) (bool, error) {
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// Machine1PredicateExtender implements FitPredicate function to return true
|
||||
// when the given node's name is "machine1"; otherwise return false.
|
||||
func Machine1PredicateExtender(pod *v1.Pod, node *v1.Node) (bool, error) {
|
||||
if node.Name == "machine1" {
|
||||
return true, nil
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// Machine2PredicateExtender implements FitPredicate function to return true
|
||||
// when the given node's name is "machine2"; otherwise return false.
|
||||
func Machine2PredicateExtender(pod *v1.Pod, node *v1.Node) (bool, error) {
|
||||
if node.Name == "machine2" {
|
||||
return true, nil
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// ErrorPrioritizerExtender implements PriorityFunc function to always return error.
|
||||
func ErrorPrioritizerExtender(pod *v1.Pod, nodes []*v1.Node) (*framework.NodeScoreList, error) {
|
||||
return &framework.NodeScoreList{}, fmt.Errorf("some error")
|
||||
}
|
||||
|
||||
// Machine1PrioritizerExtender implements PriorityFunc function to give score 10
|
||||
// if the given node's name is "machine1"; otherwise score 1.
|
||||
func Machine1PrioritizerExtender(pod *v1.Pod, nodes []*v1.Node) (*framework.NodeScoreList, error) {
|
||||
result := framework.NodeScoreList{}
|
||||
for _, node := range nodes {
|
||||
score := 1
|
||||
if node.Name == "machine1" {
|
||||
score = 10
|
||||
}
|
||||
result = append(result, framework.NodeScore{Name: node.Name, Score: int64(score)})
|
||||
}
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
// Machine2PrioritizerExtender implements PriorityFunc function to give score 10
|
||||
// if the given node's name is "machine2"; otherwise score 1.
|
||||
func Machine2PrioritizerExtender(pod *v1.Pod, nodes []*v1.Node) (*framework.NodeScoreList, error) {
|
||||
result := framework.NodeScoreList{}
|
||||
for _, node := range nodes {
|
||||
score := 1
|
||||
if node.Name == "machine2" {
|
||||
score = 10
|
||||
}
|
||||
result = append(result, framework.NodeScore{Name: node.Name, Score: int64(score)})
|
||||
}
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
type machine2PrioritizerPlugin struct{}
|
||||
|
||||
// NewMachine2PrioritizerPlugin returns a factory function to build machine2PrioritizerPlugin.
|
||||
func NewMachine2PrioritizerPlugin() framework.PluginFactory {
|
||||
return func(_ runtime.Object, _ framework.FrameworkHandle) (framework.Plugin, error) {
|
||||
return &machine2PrioritizerPlugin{}, nil
|
||||
}
|
||||
}
|
||||
|
||||
// Name returns name of the plugin.
|
||||
func (pl *machine2PrioritizerPlugin) Name() string {
|
||||
return "Machine2Prioritizer"
|
||||
}
|
||||
|
||||
// Score return score 100 if the given nodeName is "machine2"; otherwise return score 10.
|
||||
func (pl *machine2PrioritizerPlugin) Score(_ context.Context, _ *framework.CycleState, _ *v1.Pod, nodeName string) (int64, *framework.Status) {
|
||||
score := 10
|
||||
if nodeName == "machine2" {
|
||||
score = 100
|
||||
}
|
||||
return int64(score), nil
|
||||
}
|
||||
|
||||
// ScoreExtensions returns nil.
|
||||
func (pl *machine2PrioritizerPlugin) ScoreExtensions() framework.ScoreExtensions {
|
||||
return nil
|
||||
}
|
||||
|
||||
// FakeExtender is a data struct which implements the Extender interface.
|
||||
type FakeExtender struct {
|
||||
Predicates []FitPredicate
|
||||
Prioritizers []PriorityConfig
|
||||
Weight int64
|
||||
NodeCacheCapable bool
|
||||
FilteredNodes []*v1.Node
|
||||
UnInterested bool
|
||||
Ignorable bool
|
||||
|
||||
// Cached node information for fake extender
|
||||
CachedNodeNameToInfo map[string]*framework.NodeInfo
|
||||
}
|
||||
|
||||
// Name returns name of the extender.
|
||||
func (f *FakeExtender) Name() string {
|
||||
return "FakeExtender"
|
||||
}
|
||||
|
||||
// IsIgnorable returns a bool value indicating whether internal errors can be ignored.
|
||||
func (f *FakeExtender) IsIgnorable() bool {
|
||||
return f.Ignorable
|
||||
}
|
||||
|
||||
// SupportsPreemption returns true indicating the extender supports preemption.
|
||||
func (f *FakeExtender) SupportsPreemption() bool {
|
||||
// Assume preempt verb is always defined.
|
||||
return true
|
||||
}
|
||||
|
||||
// ProcessPreemption implements the extender preempt function.
|
||||
func (f *FakeExtender) ProcessPreemption(
|
||||
pod *v1.Pod,
|
||||
nodeNameToVictims map[string]*extenderv1.Victims,
|
||||
nodeInfos framework.NodeInfoLister,
|
||||
) (map[string]*extenderv1.Victims, error) {
|
||||
nodeNameToVictimsCopy := map[string]*extenderv1.Victims{}
|
||||
// We don't want to change the original nodeNameToVictims
|
||||
for k, v := range nodeNameToVictims {
|
||||
// In real world implementation, extender's user should have their own way to get node object
|
||||
// by name if needed (e.g. query kube-apiserver etc).
|
||||
//
|
||||
// For test purpose, we just use node from parameters directly.
|
||||
nodeNameToVictimsCopy[k] = v
|
||||
}
|
||||
|
||||
for nodeName, victims := range nodeNameToVictimsCopy {
|
||||
// Try to do preemption on extender side.
|
||||
nodeInfo, _ := nodeInfos.Get(nodeName)
|
||||
extenderVictimPods, extenderPDBViolations, fits, err := f.selectVictimsOnNodeByExtender(pod, nodeInfo.Node())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// If it's unfit after extender's preemption, this node is unresolvable by preemption overall,
|
||||
// let's remove it from potential preemption nodes.
|
||||
if !fits {
|
||||
delete(nodeNameToVictimsCopy, nodeName)
|
||||
} else {
|
||||
// Append new victims to original victims
|
||||
nodeNameToVictimsCopy[nodeName].Pods = append(victims.Pods, extenderVictimPods...)
|
||||
nodeNameToVictimsCopy[nodeName].NumPDBViolations = victims.NumPDBViolations + int64(extenderPDBViolations)
|
||||
}
|
||||
}
|
||||
return nodeNameToVictimsCopy, nil
|
||||
}
|
||||
|
||||
// selectVictimsOnNodeByExtender checks the given nodes->pods map with predicates on extender's side.
|
||||
// Returns:
|
||||
// 1. More victim pods (if any) amended by preemption phase of extender.
|
||||
// 2. Number of violating victim (used to calculate PDB).
|
||||
// 3. Fits or not after preemption phase on extender's side.
|
||||
func (f *FakeExtender) selectVictimsOnNodeByExtender(pod *v1.Pod, node *v1.Node) ([]*v1.Pod, int, bool, error) {
|
||||
// If a extender support preemption but have no cached node info, let's run filter to make sure
|
||||
// default scheduler's decision still stand with given pod and node.
|
||||
if !f.NodeCacheCapable {
|
||||
fits, err := f.runPredicate(pod, node)
|
||||
if err != nil {
|
||||
return nil, 0, false, err
|
||||
}
|
||||
if !fits {
|
||||
return nil, 0, false, nil
|
||||
}
|
||||
return []*v1.Pod{}, 0, true, nil
|
||||
}
|
||||
|
||||
// Otherwise, as a extender support preemption and have cached node info, we will assume cachedNodeNameToInfo is available
|
||||
// and get cached node info by given node name.
|
||||
nodeInfoCopy := f.CachedNodeNameToInfo[node.GetName()].Clone()
|
||||
|
||||
var potentialVictims []*v1.Pod
|
||||
|
||||
removePod := func(rp *v1.Pod) {
|
||||
nodeInfoCopy.RemovePod(rp)
|
||||
}
|
||||
addPod := func(ap *v1.Pod) {
|
||||
nodeInfoCopy.AddPod(ap)
|
||||
}
|
||||
// As the first step, remove all the lower priority pods from the node and
|
||||
// check if the given pod can be scheduled.
|
||||
podPriority := podutil.GetPodPriority(pod)
|
||||
for _, p := range nodeInfoCopy.Pods {
|
||||
if podutil.GetPodPriority(p.Pod) < podPriority {
|
||||
potentialVictims = append(potentialVictims, p.Pod)
|
||||
removePod(p.Pod)
|
||||
}
|
||||
}
|
||||
sort.Slice(potentialVictims, func(i, j int) bool { return util.MoreImportantPod(potentialVictims[i], potentialVictims[j]) })
|
||||
|
||||
// If the new pod does not fit after removing all the lower priority pods,
|
||||
// we are almost done and this node is not suitable for preemption.
|
||||
fits, err := f.runPredicate(pod, nodeInfoCopy.Node())
|
||||
if err != nil {
|
||||
return nil, 0, false, err
|
||||
}
|
||||
if !fits {
|
||||
return nil, 0, false, nil
|
||||
}
|
||||
|
||||
var victims []*v1.Pod
|
||||
|
||||
// TODO(harry): handle PDBs in the future.
|
||||
numViolatingVictim := 0
|
||||
|
||||
reprievePod := func(p *v1.Pod) bool {
|
||||
addPod(p)
|
||||
fits, _ := f.runPredicate(pod, nodeInfoCopy.Node())
|
||||
if !fits {
|
||||
removePod(p)
|
||||
victims = append(victims, p)
|
||||
}
|
||||
return fits
|
||||
}
|
||||
|
||||
// For now, assume all potential victims to be non-violating.
|
||||
// Now we try to reprieve non-violating victims.
|
||||
for _, p := range potentialVictims {
|
||||
reprievePod(p)
|
||||
}
|
||||
|
||||
return victims, numViolatingVictim, true, nil
|
||||
}
|
||||
|
||||
// runPredicate run predicates of extender one by one for given pod and node.
|
||||
// Returns: fits or not.
|
||||
func (f *FakeExtender) runPredicate(pod *v1.Pod, node *v1.Node) (bool, error) {
|
||||
fits := true
|
||||
var err error
|
||||
for _, predicate := range f.Predicates {
|
||||
fits, err = predicate(pod, node)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if !fits {
|
||||
break
|
||||
}
|
||||
}
|
||||
return fits, nil
|
||||
}
|
||||
|
||||
// Filter implements the extender Filter function.
|
||||
func (f *FakeExtender) Filter(pod *v1.Pod, nodes []*v1.Node) ([]*v1.Node, extenderv1.FailedNodesMap, error) {
|
||||
var filtered []*v1.Node
|
||||
failedNodesMap := extenderv1.FailedNodesMap{}
|
||||
for _, node := range nodes {
|
||||
fits, err := f.runPredicate(pod, node)
|
||||
if err != nil {
|
||||
return []*v1.Node{}, extenderv1.FailedNodesMap{}, err
|
||||
}
|
||||
if fits {
|
||||
filtered = append(filtered, node)
|
||||
} else {
|
||||
failedNodesMap[node.Name] = "FakeExtender failed"
|
||||
}
|
||||
}
|
||||
|
||||
f.FilteredNodes = filtered
|
||||
if f.NodeCacheCapable {
|
||||
return filtered, failedNodesMap, nil
|
||||
}
|
||||
return filtered, failedNodesMap, nil
|
||||
}
|
||||
|
||||
// Prioritize implements the extender Prioritize function.
|
||||
func (f *FakeExtender) Prioritize(pod *v1.Pod, nodes []*v1.Node) (*extenderv1.HostPriorityList, int64, error) {
|
||||
result := extenderv1.HostPriorityList{}
|
||||
combinedScores := map[string]int64{}
|
||||
for _, prioritizer := range f.Prioritizers {
|
||||
weight := prioritizer.Weight
|
||||
if weight == 0 {
|
||||
continue
|
||||
}
|
||||
priorityFunc := prioritizer.Function
|
||||
prioritizedList, err := priorityFunc(pod, nodes)
|
||||
if err != nil {
|
||||
return &extenderv1.HostPriorityList{}, 0, err
|
||||
}
|
||||
for _, hostEntry := range *prioritizedList {
|
||||
combinedScores[hostEntry.Name] += hostEntry.Score * weight
|
||||
}
|
||||
}
|
||||
for host, score := range combinedScores {
|
||||
result = append(result, extenderv1.HostPriority{Host: host, Score: score})
|
||||
}
|
||||
return &result, f.Weight, nil
|
||||
}
|
||||
|
||||
// Bind implements the extender Bind function.
|
||||
func (f *FakeExtender) Bind(binding *v1.Binding) error {
|
||||
if len(f.FilteredNodes) != 0 {
|
||||
for _, node := range f.FilteredNodes {
|
||||
if node.Name == binding.Target.Name {
|
||||
f.FilteredNodes = nil
|
||||
return nil
|
||||
}
|
||||
}
|
||||
err := fmt.Errorf("Node %v not in filtered nodes %v", binding.Target.Name, f.FilteredNodes)
|
||||
f.FilteredNodes = nil
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// IsBinder returns true indicating the extender implements the Binder function.
|
||||
func (f *FakeExtender) IsBinder() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// IsInterested returns a bool true indicating whether extender
|
||||
func (f *FakeExtender) IsInterested(pod *v1.Pod) bool {
|
||||
return !f.UnInterested
|
||||
}
|
||||
|
||||
var _ framework.Extender = &FakeExtender{}
|
124
pkg/scheduler/testing/fake_plugins.go
Normal file
124
pkg/scheduler/testing/fake_plugins.go
Normal file
@@ -0,0 +1,124 @@
|
||||
/*
|
||||
Copyright 2020 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package testing
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync/atomic"
|
||||
|
||||
v1 "k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
framework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1"
|
||||
)
|
||||
|
||||
// ErrReasonFake is a fake error message denotes the filter function errored.
|
||||
const ErrReasonFake = "Nodes failed the fake plugin"
|
||||
|
||||
// FalseFilterPlugin is a filter plugin which always return Unschedulable when Filter function is called.
|
||||
type FalseFilterPlugin struct{}
|
||||
|
||||
// Name returns name of the plugin.
|
||||
func (pl *FalseFilterPlugin) Name() string {
|
||||
return "FalseFilter"
|
||||
}
|
||||
|
||||
// Filter invoked at the filter extension point.
|
||||
func (pl *FalseFilterPlugin) Filter(_ context.Context, _ *framework.CycleState, pod *v1.Pod, nodeInfo *framework.NodeInfo) *framework.Status {
|
||||
return framework.NewStatus(framework.Unschedulable, ErrReasonFake)
|
||||
}
|
||||
|
||||
// NewFalseFilterPlugin initializes a FalseFilterPlugin and returns it.
|
||||
func NewFalseFilterPlugin(_ runtime.Object, _ framework.FrameworkHandle) (framework.Plugin, error) {
|
||||
return &FalseFilterPlugin{}, nil
|
||||
}
|
||||
|
||||
// TrueFilterPlugin is a filter plugin which always return Success when Filter function is called.
|
||||
type TrueFilterPlugin struct{}
|
||||
|
||||
// Name returns name of the plugin.
|
||||
func (pl *TrueFilterPlugin) Name() string {
|
||||
return "TrueFilter"
|
||||
}
|
||||
|
||||
// Filter invoked at the filter extension point.
|
||||
func (pl *TrueFilterPlugin) Filter(_ context.Context, _ *framework.CycleState, pod *v1.Pod, nodeInfo *framework.NodeInfo) *framework.Status {
|
||||
return nil
|
||||
}
|
||||
|
||||
// NewTrueFilterPlugin initializes a TrueFilterPlugin and returns it.
|
||||
func NewTrueFilterPlugin(_ runtime.Object, _ framework.FrameworkHandle) (framework.Plugin, error) {
|
||||
return &TrueFilterPlugin{}, nil
|
||||
}
|
||||
|
||||
// FakeFilterPlugin is a test filter plugin to record how many times its Filter() function have
|
||||
// been called, and it returns different 'Code' depending on its internal 'failedNodeReturnCodeMap'.
|
||||
type FakeFilterPlugin struct {
|
||||
NumFilterCalled int32
|
||||
FailedNodeReturnCodeMap map[string]framework.Code
|
||||
}
|
||||
|
||||
// Name returns name of the plugin.
|
||||
func (pl *FakeFilterPlugin) Name() string {
|
||||
return "FakeFilter"
|
||||
}
|
||||
|
||||
// Filter invoked at the filter extension point.
|
||||
func (pl *FakeFilterPlugin) Filter(_ context.Context, _ *framework.CycleState, pod *v1.Pod, nodeInfo *framework.NodeInfo) *framework.Status {
|
||||
atomic.AddInt32(&pl.NumFilterCalled, 1)
|
||||
|
||||
if returnCode, ok := pl.FailedNodeReturnCodeMap[nodeInfo.Node().Name]; ok {
|
||||
return framework.NewStatus(returnCode, fmt.Sprintf("injecting failure for pod %v", pod.Name))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// NewFakeFilterPlugin initializes a fakeFilterPlugin and returns it.
|
||||
func NewFakeFilterPlugin(failedNodeReturnCodeMap map[string]framework.Code) framework.PluginFactory {
|
||||
return func(_ runtime.Object, _ framework.FrameworkHandle) (framework.Plugin, error) {
|
||||
return &FakeFilterPlugin{
|
||||
FailedNodeReturnCodeMap: failedNodeReturnCodeMap,
|
||||
}, nil
|
||||
}
|
||||
}
|
||||
|
||||
// MatchFilterPlugin is a filter plugin which return Success when the evaluated pod and node
|
||||
// have the same name; otherwise return Unschedulable.
|
||||
type MatchFilterPlugin struct{}
|
||||
|
||||
// Name returns name of the plugin.
|
||||
func (pl *MatchFilterPlugin) Name() string {
|
||||
return "MatchFilter"
|
||||
}
|
||||
|
||||
// Filter invoked at the filter extension point.
|
||||
func (pl *MatchFilterPlugin) Filter(_ context.Context, _ *framework.CycleState, pod *v1.Pod, nodeInfo *framework.NodeInfo) *framework.Status {
|
||||
node := nodeInfo.Node()
|
||||
if node == nil {
|
||||
return framework.NewStatus(framework.Error, "node not found")
|
||||
}
|
||||
if pod.Name == node.Name {
|
||||
return nil
|
||||
}
|
||||
return framework.NewStatus(framework.Unschedulable, ErrReasonFake)
|
||||
}
|
||||
|
||||
// NewMatchFilterPlugin initializes a MatchFilterPlugin and returns it.
|
||||
func NewMatchFilterPlugin(_ runtime.Object, _ framework.FrameworkHandle) (framework.Plugin, error) {
|
||||
return &MatchFilterPlugin{}, nil
|
||||
}
|
Reference in New Issue
Block a user