github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-31 07:20:13 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #21272 from deads2k/add-priority-restmapper

Browse Source 
Auto commit by PR queue bot
This commit is contained in:
k8s-merge-robot 2016-02-25 07:26:41 -08:00
commit cbf5dc1228
14 changed files with 756 additions and 113 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
hack/test-cmd.sh
View File

@ -702,7 +702,7 @@ __EOF__
# Command: autoscale rc "frontend"
kubectl autoscale -f hack/testdata/frontend-controller.yaml --save-config "${kube_flags[@]}" --max=2
# Post-Condition: hpa "frontend" has configuration annotation
[[ "$(kubectl get hpa frontend -o yaml "${kube_flags[@]}" | grep kubectl.kubernetes.io/last-applied-configuration)" ]]
[[ "$(kubectl get hpa.extensions frontend -o yaml "${kube_flags[@]}" | grep kubectl.kubernetes.io/last-applied-configuration)" ]]
# Clean up
kubectl delete rc,hpa frontend "${kube_flags[@]}"
@ -1191,10 +1191,10 @@ __EOF__
kube::test::get_object_assert deployment "{{range.items}}{{$id_field}}:{{end}}" 'nginx-deployment:'
# autoscale 2~3 pods, default CPU utilization (80%)
kubectl-with-retry autoscale deployment nginx-deployment "${kube_flags[@]}" --min=2 --max=3
kube::test::get_object_assert 'hpa nginx-deployment' "{{$hpa_min_field}} {{$hpa_max_field}} {{$hpa_cpu_field}}" '2 3 80'
kube::test::get_object_assert 'hpa.extensions nginx-deployment' "{{$hpa_min_field}} {{$hpa_max_field}} {{$hpa_cpu_field}}" '2 3 80'
# Clean up
kubectl delete hpa nginx-deployment "${kube_flags[@]}"
kubectl delete deployment nginx-deployment "${kube_flags[@]}"
kubectl delete deployment.extensions nginx-deployment "${kube_flags[@]}"
### Rollback a deployment
# Pre-condition: no deployment exists
@ -1209,7 +1209,7 @@ __EOF__
kube::test::get_object_assert deployment "{{range.items}}{{$deployment_image_field}}:{{end}}" 'nginx:'
# Update the deployment (revision 2)
kubectl apply -f hack/testdata/deployment-revision2.yaml "${kube_flags[@]}"
kube::test::get_object_assert deployment "{{range.items}}{{$deployment_image_field}}:{{end}}" 'nginx:latest:'
kube::test::get_object_assert deployment.extensions "{{range.items}}{{$deployment_image_field}}:{{end}}" 'nginx:latest:'
# Rollback to revision 1
kubectl rollout undo deployment nginx-deployment --to-revision=1 "${kube_flags[@]}"
sleep 1

139
pkg/api/meta/multirestmapper.go
View File

@ -21,6 +21,7 @@ import (
"strings"
"k8s.io/kubernetes/pkg/api/unversioned"
utilerrors "k8s.io/kubernetes/pkg/util/errors"
)
// MultiRESTMapper is a wrapper for multiple RESTMappers.
@ -50,72 +51,144 @@ func (m MultiRESTMapper) ResourceSingularizer(resource string) (singular string,
}
func (m MultiRESTMapper) ResourcesFor(resource unversioned.GroupVersionResource) ([]unversioned.GroupVersionResource, error) {
allGVRs := []unversioned.GroupVersionResource{}
for _, t := range m {
gvrs, err := t.ResourcesFor(resource)
// ignore "no match" errors, but any other error percolates back up
if !IsNoResourceMatchError(err) {
return gvrs, err
if IsNoResourceMatchError(err) {
continue
}
if err != nil {
return nil, err
}
// walk the existing values to de-dup
for _, curr := range gvrs {
found := false
for _, existing := range allGVRs {
if curr == existing {
found = true
break
}
}
if !found {
allGVRs = append(allGVRs, curr)
}
}
}
return nil, &NoResourceMatchError{PartialResource: resource}
if len(allGVRs) == 0 {
return nil, &NoResourceMatchError{PartialResource: resource}
}
return allGVRs, nil
}
// KindsFor provides the Kind mappings for the REST resources. This implementation supports multiple REST schemas and returns
// the first match.
func (m MultiRESTMapper) KindsFor(resource unversioned.GroupVersionResource) (gvk []unversioned.GroupVersionKind, err error) {
allGVKs := []unversioned.GroupVersionKind{}
for _, t := range m {
gvks, err := t.KindsFor(resource)
// ignore "no match" errors, but any other error percolates back up
if !IsNoResourceMatchError(err) {
return gvks, err
if IsNoResourceMatchError(err) {
continue
}
if err != nil {
return nil, err
}
// walk the existing values to de-dup
for _, curr := range gvks {
found := false
for _, existing := range allGVKs {
if curr == existing {
found = true
break
}
}
if !found {
allGVKs = append(allGVKs, curr)
}
}
}
return nil, &NoResourceMatchError{PartialResource: resource}
if len(allGVKs) == 0 {
return nil, &NoResourceMatchError{PartialResource: resource}
}
return allGVKs, nil
}
func (m MultiRESTMapper) ResourceFor(resource unversioned.GroupVersionResource) (unversioned.GroupVersionResource, error) {
for _, t := range m {
gvr, err := t.ResourceFor(resource)
// ignore "no match" errors, but any other error percolates back up
if !IsNoResourceMatchError(err) {
return gvr, err
}
resources, err := m.ResourcesFor(resource)
if err != nil {
return unversioned.GroupVersionResource{}, err
}
return unversioned.GroupVersionResource{}, &NoResourceMatchError{PartialResource: resource}
if len(resources) == 1 {
return resources[0], nil
}
return unversioned.GroupVersionResource{}, &AmbiguousResourceError{PartialResource: resource, MatchingResources: resources}
}
// KindsFor provides the Kind mapping for the REST resources. This implementation supports multiple REST schemas and returns
// the first match.
func (m MultiRESTMapper) KindFor(resource unversioned.GroupVersionResource) (unversioned.GroupVersionKind, error) {
for _, t := range m {
gvk, err := t.KindFor(resource)
// ignore "no match" errors, but any other error percolates back up
if !IsNoResourceMatchError(err) {
return gvk, err
}
kinds, err := m.KindsFor(resource)
if err != nil {
return unversioned.GroupVersionKind{}, err
}
return unversioned.GroupVersionKind{}, &NoResourceMatchError{PartialResource: resource}
if len(kinds) == 1 {
return kinds[0], nil
}
return unversioned.GroupVersionKind{}, &AmbiguousResourceError{PartialResource: resource, MatchingKinds: kinds}
}
// RESTMapping provides the REST mapping for the resource based on the
// kind and version. This implementation supports multiple REST schemas and
// return the first match.
func (m MultiRESTMapper) RESTMapping(gk unversioned.GroupKind, versions ...string) (mapping *RESTMapping, err error) {
func (m MultiRESTMapper) RESTMapping(gk unversioned.GroupKind, versions ...string) (*RESTMapping, error) {
allMappings := []*RESTMapping{}
errors := []error{}
for _, t := range m {
mapping, err = t.RESTMapping(gk, versions...)
if err == nil {
return
currMapping, err := t.RESTMapping(gk, versions...)
// ignore "no match" errors, but any other error percolates back up
if IsNoResourceMatchError(err) {
continue
}
if err != nil {
errors = append(errors, err)
continue
}
allMappings = append(allMappings, currMapping)
}
return
// if we got exactly one mapping, then use it even if other requested failed
if len(allMappings) == 1 {
return allMappings[0], nil
}
if len(errors) > 0 {
return nil, utilerrors.NewAggregate(errors)
}
if len(allMappings) == 0 {
return nil, fmt.Errorf("no match found for %v in %v", gk, versions)
}
return nil, fmt.Errorf("multiple matches found for %v in %v", gk, versions)
}
// AliasesForResource finds the first alias response for the provided mappers.
func (m MultiRESTMapper) AliasesForResource(alias string) (aliases []string, ok bool) {
func (m MultiRESTMapper) AliasesForResource(alias string) ([]string, bool) {
allAliases := []string{}
handled := false
for _, t := range m {
if aliases, ok = t.AliasesForResource(alias); ok {
return
if currAliases, currOk := t.AliasesForResource(alias); currOk {
allAliases = append(allAliases, currAliases...)
handled = true
}
}
return nil, false
return allAliases, handled
}

88
pkg/api/meta/multirestmapper_test.go
View File

@ -24,7 +24,7 @@ import (
"k8s.io/kubernetes/pkg/api/unversioned"
)
func TestMultiRESTMapperResourceForErrorHandling(t *testing.T) {
func TestMultiRESTMapperResourceFor(t *testing.T) {
tcs := []struct {
name string
@ -49,7 +49,7 @@ func TestMultiRESTMapperResourceForErrorHandling(t *testing.T) {
},
{
name: "accept first failure",
mapper: MultiRESTMapper{fixedRESTMapper{err: errors.New("fail on this")}, fixedRESTMapper{resourceFor: unversioned.GroupVersionResource{Resource: "unused"}}},
mapper: MultiRESTMapper{fixedRESTMapper{err: errors.New("fail on this")}, fixedRESTMapper{resourcesFor: []unversioned.GroupVersionResource{{Resource: "unused"}}}},
input: unversioned.GroupVersionResource{Resource: "foo"},
result: unversioned.GroupVersionResource{},
err: errors.New("fail on this"),
@ -61,13 +61,19 @@ func TestMultiRESTMapperResourceForErrorHandling(t *testing.T) {
if e, a := tc.result, actualResult; e != a {
t.Errorf("%s: expected %v, got %v", tc.name, e, a)
}
if e, a := tc.err.Error(), actualErr.Error(); e != a {
t.Errorf("%s: expected %v, got %v", tc.name, e, a)
switch {
case tc.err == nil && actualErr == nil:
case tc.err == nil:
t.Errorf("%s: unexpected error: %v", tc.name, actualErr)
case actualErr == nil:
t.Errorf("%s: expected error: %v got nil", tc.name, tc.err)
case tc.err.Error() != actualErr.Error():
t.Errorf("%s: expected %v, got %v", tc.name, tc.err, actualErr)
}
}
}
func TestMultiRESTMapperResourcesForErrorHandling(t *testing.T) {
func TestMultiRESTMapperResourcesFor(t *testing.T) {
tcs := []struct {
name string
@ -97,6 +103,24 @@ func TestMultiRESTMapperResourcesForErrorHandling(t *testing.T) {
result: nil,
err: errors.New("fail on this"),
},
{
name: "union and dedup",
mapper: MultiRESTMapper{
fixedRESTMapper{resourcesFor: []unversioned.GroupVersionResource{{Resource: "dupe"}, {Resource: "first"}}},
fixedRESTMapper{resourcesFor: []unversioned.GroupVersionResource{{Resource: "dupe"}, {Resource: "second"}}},
},
input: unversioned.GroupVersionResource{Resource: "foo"},
result: []unversioned.GroupVersionResource{{Resource: "dupe"}, {Resource: "first"}, {Resource: "second"}},
},
{
name: "skip not and continue",
mapper: MultiRESTMapper{
fixedRESTMapper{err: &NoResourceMatchError{PartialResource: unversioned.GroupVersionResource{Resource: "IGNORE_THIS"}}},
fixedRESTMapper{resourcesFor: []unversioned.GroupVersionResource{{Resource: "first"}, {Resource: "second"}}},
},
input: unversioned.GroupVersionResource{Resource: "foo"},
result: []unversioned.GroupVersionResource{{Resource: "first"}, {Resource: "second"}},
},
}
for _, tc := range tcs {
@ -104,13 +128,19 @@ func TestMultiRESTMapperResourcesForErrorHandling(t *testing.T) {
if e, a := tc.result, actualResult; !reflect.DeepEqual(e, a) {
t.Errorf("%s: expected %v, got %v", tc.name, e, a)
}
if e, a := tc.err.Error(), actualErr.Error(); e != a {
t.Errorf("%s: expected %v, got %v", tc.name, e, a)
switch {
case tc.err == nil && actualErr == nil:
case tc.err == nil:
t.Errorf("%s: unexpected error: %v", tc.name, actualErr)
case actualErr == nil:
t.Errorf("%s: expected error: %v got nil", tc.name, tc.err)
case tc.err.Error() != actualErr.Error():
t.Errorf("%s: expected %v, got %v", tc.name, tc.err, actualErr)
}
}
}
func TestMultiRESTMapperKindsForErrorHandling(t *testing.T) {
func TestMultiRESTMapperKindsFor(t *testing.T) {
tcs := []struct {
name string
@ -140,6 +170,24 @@ func TestMultiRESTMapperKindsForErrorHandling(t *testing.T) {
result: nil,
err: errors.New("fail on this"),
},
{
name: "union and dedup",
mapper: MultiRESTMapper{
fixedRESTMapper{kindsFor: []unversioned.GroupVersionKind{{Kind: "dupe"}, {Kind: "first"}}},
fixedRESTMapper{kindsFor: []unversioned.GroupVersionKind{{Kind: "dupe"}, {Kind: "second"}}},
},
input: unversioned.GroupVersionResource{Resource: "foo"},
result: []unversioned.GroupVersionKind{{Kind: "dupe"}, {Kind: "first"}, {Kind: "second"}},
},
{
name: "skip not and continue",
mapper: MultiRESTMapper{
fixedRESTMapper{err: &NoResourceMatchError{PartialResource: unversioned.GroupVersionResource{Resource: "IGNORE_THIS"}}},
fixedRESTMapper{kindsFor: []unversioned.GroupVersionKind{{Kind: "first"}, {Kind: "second"}}},
},
input: unversioned.GroupVersionResource{Resource: "foo"},
result: []unversioned.GroupVersionKind{{Kind: "first"}, {Kind: "second"}},
},
}
for _, tc := range tcs {
@ -147,13 +195,19 @@ func TestMultiRESTMapperKindsForErrorHandling(t *testing.T) {
if e, a := tc.result, actualResult; !reflect.DeepEqual(e, a) {
t.Errorf("%s: expected %v, got %v", tc.name, e, a)
}
if e, a := tc.err.Error(), actualErr.Error(); e != a {
t.Errorf("%s: expected %v, got %v", tc.name, e, a)
switch {
case tc.err == nil && actualErr == nil:
case tc.err == nil:
t.Errorf("%s: unexpected error: %v", tc.name, actualErr)
case actualErr == nil:
t.Errorf("%s: expected error: %v got nil", tc.name, tc.err)
case tc.err.Error() != actualErr.Error():
t.Errorf("%s: expected %v, got %v", tc.name, tc.err, actualErr)
}
}
}
func TestMultiRESTMapperKindForErrorHandling(t *testing.T) {
func TestMultiRESTMapperKindFor(t *testing.T) {
tcs := []struct {
name string
@ -178,7 +232,7 @@ func TestMultiRESTMapperKindForErrorHandling(t *testing.T) {
},
{
name: "accept first failure",
mapper: MultiRESTMapper{fixedRESTMapper{err: errors.New("fail on this")}, fixedRESTMapper{kindFor: unversioned.GroupVersionKind{Kind: "unused"}}},
mapper: MultiRESTMapper{fixedRESTMapper{err: errors.New("fail on this")}, fixedRESTMapper{kindsFor: []unversioned.GroupVersionKind{{Kind: "unused"}}}},
input: unversioned.GroupVersionResource{Resource: "foo"},
result: unversioned.GroupVersionKind{},
err: errors.New("fail on this"),
@ -190,8 +244,14 @@ func TestMultiRESTMapperKindForErrorHandling(t *testing.T) {
if e, a := tc.result, actualResult; e != a {
t.Errorf("%s: expected %v, got %v", tc.name, e, a)
}
if e, a := tc.err.Error(), actualErr.Error(); e != a {
t.Errorf("%s: expected %v, got %v", tc.name, e, a)
switch {
case tc.err == nil && actualErr == nil:
case tc.err == nil:
t.Errorf("%s: unexpected error: %v", tc.name, actualErr)
case actualErr == nil:
t.Errorf("%s: expected error: %v got nil", tc.name, tc.err)
case tc.err.Error() != actualErr.Error():
t.Errorf("%s: expected %v, got %v", tc.name, tc.err, actualErr)
}
}
}

173
pkg/api/meta/priority.go Normal file
View File

@ -0,0 +1,173 @@
/*
Copyright 2016 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package meta
import (
"fmt"
"k8s.io/kubernetes/pkg/api/unversioned"
)
const (
AnyGroup = "*"
AnyVersion = "*"
AnyResource = "*"
AnyKind = "*"
)
// PriorityRESTMapper is a wrapper for automatically choosing a particular Resource or Kind
// when multiple matches are possible
type PriorityRESTMapper struct {
// Delegate is the RESTMapper to use to locate all the Kind and Resource matches
Delegate RESTMapper
// ResourcePriority is a list of priority patterns to apply to matching resources.
// The list of all matching resources is narrowed based on the patterns until only one remains.
// A pattern with no matches is skipped. A pattern with more than one match uses its
// matches as the list to continue matching against.
ResourcePriority []unversioned.GroupVersionResource
// KindPriority is a list of priority patterns to apply to matching kinds.
// The list of all matching kinds is narrowed based on the patterns until only one remains.
// A pattern with no matches is skipped. A pattern with more than one match uses its
// matches as the list to continue matching against.
KindPriority []unversioned.GroupVersionKind
}
func (m PriorityRESTMapper) String() string {
return fmt.Sprintf("PriorityRESTMapper{\n\t%v\n\t%v\n\t%v\n}", m.ResourcePriority, m.KindPriority, m.Delegate)
}
// ResourceFor finds all resources, then passes them through the ResourcePriority patterns to find a single matching hit.
func (m PriorityRESTMapper) ResourceFor(partiallySpecifiedResource unversioned.GroupVersionResource) (unversioned.GroupVersionResource, error) {
originalGVRs, err := m.Delegate.ResourcesFor(partiallySpecifiedResource)
if err != nil {
return unversioned.GroupVersionResource{}, err
}
if len(originalGVRs) == 1 {
return originalGVRs[0], nil
}
remainingGVRs := append([]unversioned.GroupVersionResource{}, originalGVRs...)
for _, pattern := range m.ResourcePriority {
matchedGVRs := []unversioned.GroupVersionResource{}
for _, gvr := range remainingGVRs {
if resourceMatches(pattern, gvr) {
matchedGVRs = append(matchedGVRs, gvr)
}
}
switch len(matchedGVRs) {
case 0:
// if you have no matches, then nothing matched this pattern just move to the next
continue
case 1:
// one match, return
return matchedGVRs[0], nil
default:
// more than one match, use the matched hits as the list moving to the next pattern.
// this way you can have a series of selection criteria
remainingGVRs = matchedGVRs
}
}
return unversioned.GroupVersionResource{}, &AmbiguousResourceError{PartialResource: partiallySpecifiedResource, MatchingResources: originalGVRs}
}
// KindFor finds all kinds, then passes them through the KindPriority patterns to find a single matching hit.
func (m PriorityRESTMapper) KindFor(partiallySpecifiedResource unversioned.GroupVersionResource) (unversioned.GroupVersionKind, error) {
originalGVKs, err := m.Delegate.KindsFor(partiallySpecifiedResource)
if err != nil {
return unversioned.GroupVersionKind{}, err
}
if len(originalGVKs) == 1 {
return originalGVKs[0], nil
}
remainingGVKs := append([]unversioned.GroupVersionKind{}, originalGVKs...)
for _, pattern := range m.KindPriority {
matchedGVKs := []unversioned.GroupVersionKind{}
for _, gvr := range remainingGVKs {
if kindMatches(pattern, gvr) {
matchedGVKs = append(matchedGVKs, gvr)
}
}
switch len(matchedGVKs) {
case 0:
// if you have no matches, then nothing matched this pattern just move to the next
continue
case 1:
// one match, return
return matchedGVKs[0], nil
default:
// more than one match, use the matched hits as the list moving to the next pattern.
// this way you can have a series of selection criteria
remainingGVKs = matchedGVKs
}
}
return unversioned.GroupVersionKind{}, &AmbiguousResourceError{PartialResource: partiallySpecifiedResource, MatchingKinds: originalGVKs}
}
func resourceMatches(pattern unversioned.GroupVersionResource, resource unversioned.GroupVersionResource) bool {
if pattern.Group != AnyGroup && pattern.Group != resource.Group {
return false
}
if pattern.Version != AnyVersion && pattern.Version != resource.Version {
return false
}
if pattern.Resource != AnyResource && pattern.Resource != resource.Resource {
return false
}
return true
}
func kindMatches(pattern unversioned.GroupVersionKind, kind unversioned.GroupVersionKind) bool {
if pattern.Group != AnyGroup && pattern.Group != kind.Group {
return false
}
if pattern.Version != AnyVersion && pattern.Version != kind.Version {
return false
}
if pattern.Kind != AnyKind && pattern.Kind != kind.Kind {
return false
}
return true
}
func (m PriorityRESTMapper) RESTMapping(gk unversioned.GroupKind, versions ...string) (mapping *RESTMapping, err error) {
return m.Delegate.RESTMapping(gk, versions...)
}
func (m PriorityRESTMapper) AliasesForResource(alias string) (aliases []string, ok bool) {
return m.Delegate.AliasesForResource(alias)
}
func (m PriorityRESTMapper) ResourceSingularizer(resource string) (singular string, err error) {
return m.Delegate.ResourceSingularizer(resource)
}
func (m PriorityRESTMapper) ResourcesFor(partiallySpecifiedResource unversioned.GroupVersionResource) ([]unversioned.GroupVersionResource, error) {
return m.Delegate.ResourcesFor(partiallySpecifiedResource)
}
func (m PriorityRESTMapper) KindsFor(partiallySpecifiedResource unversioned.GroupVersionResource) (gvk []unversioned.GroupVersionKind, err error) {
return m.Delegate.KindsFor(partiallySpecifiedResource)
}

206
pkg/api/meta/priority_test.go Normal file
View File

@ -0,0 +1,206 @@
/*
Copyright 2016 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package meta
import (
"strings"
"testing"
"k8s.io/kubernetes/pkg/api/unversioned"
)
func TestPriorityRESTMapperResourceForErrorHandling(t *testing.T) {
tcs := []struct {
name string
delegate RESTMapper
resourcePatterns []unversioned.GroupVersionResource
result unversioned.GroupVersionResource
err string
}{
{
name: "single hit",
delegate: fixedRESTMapper{resourcesFor: []unversioned.GroupVersionResource{{Resource: "single-hit"}}},
result: unversioned.GroupVersionResource{Resource: "single-hit"},
},
{
name: "ambiguous match",
delegate: fixedRESTMapper{resourcesFor: []unversioned.GroupVersionResource{
{Group: "one", Version: "a", Resource: "first"},
{Group: "two", Version: "b", Resource: "second"},
}},
err: "matches multiple resources",
},
{
name: "group selection",
delegate: fixedRESTMapper{resourcesFor: []unversioned.GroupVersionResource{
{Group: "one", Version: "a", Resource: "first"},
{Group: "two", Version: "b", Resource: "second"},
}},
resourcePatterns: []unversioned.GroupVersionResource{
{Group: "one", Version: AnyVersion, Resource: AnyResource},
},
result: unversioned.GroupVersionResource{Group: "one", Version: "a", Resource: "first"},
},
{
name: "empty match continues",
delegate: fixedRESTMapper{resourcesFor: []unversioned.GroupVersionResource{
{Group: "one", Version: "a", Resource: "first"},
{Group: "two", Version: "b", Resource: "second"},
}},
resourcePatterns: []unversioned.GroupVersionResource{
{Group: "fail", Version: AnyVersion, Resource: AnyResource},
{Group: "one", Version: AnyVersion, Resource: AnyResource},
},
result: unversioned.GroupVersionResource{Group: "one", Version: "a", Resource: "first"},
},
{
name: "group followed by version selection",
delegate: fixedRESTMapper{resourcesFor: []unversioned.GroupVersionResource{
{Group: "one", Version: "a", Resource: "first"},
{Group: "two", Version: "b", Resource: "second"},
{Group: "one", Version: "c", Resource: "third"},
}},
resourcePatterns: []unversioned.GroupVersionResource{
{Group: "one", Version: AnyVersion, Resource: AnyResource},
{Group: AnyGroup, Version: "a", Resource: AnyResource},
},
result: unversioned.GroupVersionResource{Group: "one", Version: "a", Resource: "first"},
},
{
name: "resource selection",
delegate: fixedRESTMapper{resourcesFor: []unversioned.GroupVersionResource{
{Group: "one", Version: "a", Resource: "first"},
{Group: "one", Version: "a", Resource: "second"},
}},
resourcePatterns: []unversioned.GroupVersionResource{
{Group: AnyGroup, Version: AnyVersion, Resource: "second"},
},
result: unversioned.GroupVersionResource{Group: "one", Version: "a", Resource: "second"},
},
}
for _, tc := range tcs {
mapper := PriorityRESTMapper{Delegate: tc.delegate, ResourcePriority: tc.resourcePatterns}
actualResult, actualErr := mapper.ResourceFor(unversioned.GroupVersionResource{})
if e, a := tc.result, actualResult; e != a {
t.Errorf("%s: expected %v, got %v", tc.name, e, a)
}
if len(tc.err) == 0 && actualErr == nil {
continue
}
if len(tc.err) > 0 && actualErr == nil {
t.Errorf("%s: missing expected err: %v", tc.name, tc.err)
continue
}
if !strings.Contains(actualErr.Error(), tc.err) {
t.Errorf("%s: expected %v, got %v", tc.name, tc.err, actualErr)
}
}
}
func TestPriorityRESTMapperKindForErrorHandling(t *testing.T) {
tcs := []struct {
name string
delegate RESTMapper
kindPatterns []unversioned.GroupVersionKind
result unversioned.GroupVersionKind
err string
}{
{
name: "single hit",
delegate: fixedRESTMapper{kindsFor: []unversioned.GroupVersionKind{{Kind: "single-hit"}}},
result: unversioned.GroupVersionKind{Kind: "single-hit"},
},
{
name: "ambiguous match",
delegate: fixedRESTMapper{kindsFor: []unversioned.GroupVersionKind{
{Group: "one", Version: "a", Kind: "first"},
{Group: "two", Version: "b", Kind: "second"},
}},
err: "matches multiple kinds",
},
{
name: "group selection",
delegate: fixedRESTMapper{kindsFor: []unversioned.GroupVersionKind{
{Group: "one", Version: "a", Kind: "first"},
{Group: "two", Version: "b", Kind: "second"},
}},
kindPatterns: []unversioned.GroupVersionKind{
{Group: "one", Version: AnyVersion, Kind: AnyKind},
},
result: unversioned.GroupVersionKind{Group: "one", Version: "a", Kind: "first"},
},
{
name: "empty match continues",
delegate: fixedRESTMapper{kindsFor: []unversioned.GroupVersionKind{
{Group: "one", Version: "a", Kind: "first"},
{Group: "two", Version: "b", Kind: "second"},
}},
kindPatterns: []unversioned.GroupVersionKind{
{Group: "fail", Version: AnyVersion, Kind: AnyKind},
{Group: "one", Version: AnyVersion, Kind: AnyKind},
},
result: unversioned.GroupVersionKind{Group: "one", Version: "a", Kind: "first"},
},
{
name: "group followed by version selection",
delegate: fixedRESTMapper{kindsFor: []unversioned.GroupVersionKind{
{Group: "one", Version: "a", Kind: "first"},
{Group: "two", Version: "b", Kind: "second"},
{Group: "one", Version: "c", Kind: "third"},
}},
kindPatterns: []unversioned.GroupVersionKind{
{Group: "one", Version: AnyVersion, Kind: AnyKind},
{Group: AnyGroup, Version: "a", Kind: AnyKind},
},
result: unversioned.GroupVersionKind{Group: "one", Version: "a", Kind: "first"},
},
{
name: "kind selection",
delegate: fixedRESTMapper{kindsFor: []unversioned.GroupVersionKind{
{Group: "one", Version: "a", Kind: "first"},
{Group: "one", Version: "a", Kind: "second"},
}},
kindPatterns: []unversioned.GroupVersionKind{
{Group: AnyGroup, Version: AnyVersion, Kind: "second"},
},
result: unversioned.GroupVersionKind{Group: "one", Version: "a", Kind: "second"},
},
}
for _, tc := range tcs {
mapper := PriorityRESTMapper{Delegate: tc.delegate, KindPriority: tc.kindPatterns}
actualResult, actualErr := mapper.KindFor(unversioned.GroupVersionResource{})
if e, a := tc.result, actualResult; e != a {
t.Errorf("%s: expected %v, got %v", tc.name, e, a)
}
if len(tc.err) == 0 && actualErr == nil {
continue
}
if len(tc.err) > 0 && actualErr == nil {
t.Errorf("%s: missing expected err: %v", tc.name, tc.err)
continue
}
if !strings.Contains(actualErr.Error(), tc.err) {
t.Errorf("%s: expected %v, got %v", tc.name, tc.err, actualErr)
}
}
}

20
pkg/api/meta/restmapper.go
View File

@ -24,7 +24,6 @@ import (
"k8s.io/kubernetes/pkg/api/unversioned"
"k8s.io/kubernetes/pkg/runtime"
"k8s.io/kubernetes/pkg/util/sets"
)
// Implements RESTScope interface
@ -330,23 +329,8 @@ func (m *DefaultRESTMapper) KindFor(resource unversioned.GroupVersionResource) (
if err != nil {
return unversioned.GroupVersionKind{}, err
}
// TODO for each group, choose the most preferred (first) version. This keeps us consistent with code today.
// eventually, we'll need a RESTMapper that is aware of what's available server-side and deconflicts that with
// user preferences
oneKindPerGroup := []unversioned.GroupVersionKind{}
groupsAdded := sets.String{}
for _, kind := range kinds {
if groupsAdded.Has(kind.Group) {
continue
}
oneKindPerGroup = append(oneKindPerGroup, kind)
groupsAdded.Insert(kind.Group)
}
if len(oneKindPerGroup) == 1 {
return oneKindPerGroup[0], nil
if len(kinds) == 1 {
return kinds[0], nil
}
return unversioned.GroupVersionKind{}, &AmbiguousResourceError{PartialResource: resource, MatchingKinds: kinds}

11
pkg/api/unversioned/group_version.go
View File

@ -46,6 +46,17 @@ func (gr *GroupResource) String() string {
return gr.Resource + "." + gr.Group
}
// ParseGroupResource turns "resource.group" string into a GroupResource struct. Empty strings are allowed
// for each field.
func ParseGroupResource(gr string) GroupResource {
s := strings.SplitN(gr, ".", 2)
if len(s) == 1 {
return GroupResource{Resource: s[0]}
}
return GroupResource{Group: s[1], Resource: s[0]}
}
// GroupVersionResource unambiguously identifies a resource. It doesn't anonymously include GroupVersion
// to avoid automatic coersion. It doesn't use a GroupVersion to avoid custom marshalling
//

93
pkg/apimachinery/registered/registered.go
View File

@ -24,8 +24,11 @@ import (
"strings"
"github.com/golang/glog"
"k8s.io/kubernetes/pkg/api/meta"
"k8s.io/kubernetes/pkg/api/unversioned"
"k8s.io/kubernetes/pkg/apimachinery"
"k8s.io/kubernetes/pkg/util/sets"
)
var (
@ -43,8 +46,8 @@ var (
// envRequestedVersions represents the versions requested via the
// KUBE_API_VERSIONS environment variable. The install package of each group
// checks this list before add their versions to the latest package and
// Scheme.
envRequestedVersions = map[unversioned.GroupVersion]struct{}{}
// Scheme. This list is small and order matters, so represent as a slice
envRequestedVersions = []unversioned.GroupVersion{}
)
func init() {
@ -58,7 +61,7 @@ func init() {
glog.Fatalf("invalid api version: %s in KUBE_API_VERSIONS: %s.",
version, os.Getenv("KUBE_API_VERSIONS"))
}
envRequestedVersions[gv] = struct{}{}
envRequestedVersions = append(envRequestedVersions, gv)
}
}
}
@ -104,8 +107,12 @@ func IsAllowedVersion(v unversioned.GroupVersion) bool {
if len(envRequestedVersions) == 0 {
return true
}
_, found := envRequestedVersions[v]
return found
for _, envGV := range envRequestedVersions {
if v == envGV {
return true
}
}
return false
}
// IsEnabledVersion returns if a version is enabled.
@ -167,6 +174,80 @@ func GroupOrDie(group string) *apimachinery.GroupMeta {
return &groupMetaCopy
}
// RESTMapper returns a union RESTMapper of all known types with priorities chosen in the following order:
// 1. if KUBE_API_VERSIONS is specified, then KUBE_API_VERSIONS in order, OR
// 1. legacy kube group preferred version, extensions preferred version, metrics perferred version, legacy
// kube any version, extensions any version, metrics any version, all other groups alphabetical preferred version,
// all other groups alphabetical.
func RESTMapper(versionPatterns ...unversioned.GroupVersion) meta.RESTMapper {
unionMapper := meta.MultiRESTMapper{}
for enabledVersion := range enabledVersions {
groupMeta := groupMetaMap[enabledVersion.Group]
unionMapper = append(unionMapper, groupMeta.RESTMapper)
}
if len(versionPatterns) != 0 {
resourcePriority := []unversioned.GroupVersionResource{}
kindPriority := []unversioned.GroupVersionKind{}
for _, versionPriority := range versionPatterns {
resourcePriority = append(resourcePriority, versionPriority.WithResource(meta.AnyResource))
kindPriority = append(kindPriority, versionPriority.WithKind(meta.AnyKind))
}
return meta.PriorityRESTMapper{Delegate: unionMapper, ResourcePriority: resourcePriority, KindPriority: kindPriority}
}
if len(envRequestedVersions) != 0 {
resourcePriority := []unversioned.GroupVersionResource{}
kindPriority := []unversioned.GroupVersionKind{}
for _, versionPriority := range envRequestedVersions {
resourcePriority = append(resourcePriority, versionPriority.WithResource(meta.AnyResource))
kindPriority = append(kindPriority, versionPriority.WithKind(meta.AnyKind))
}
return meta.PriorityRESTMapper{Delegate: unionMapper, ResourcePriority: resourcePriority, KindPriority: kindPriority}
}
prioritizedGroups := []string{"", "extensions", "metrics"}
resourcePriority, kindPriority := prioritiesForGroups(prioritizedGroups...)
prioritizedGroupsSet := sets.NewString(prioritizedGroups...)
remainingGroups := sets.String{}
for enabledVersion := range enabledVersions {
if !prioritizedGroupsSet.Has(enabledVersion.Group) {
remainingGroups.Insert(enabledVersion.Group)
}
}
remainingResourcePriority, remainingKindPriority := prioritiesForGroups(remainingGroups.List()...)
resourcePriority = append(resourcePriority, remainingResourcePriority...)
kindPriority = append(kindPriority, remainingKindPriority...)
return meta.PriorityRESTMapper{Delegate: unionMapper, ResourcePriority: resourcePriority, KindPriority: kindPriority}
}
// prioritiesForGroups returns the resource and kind priorities for a PriorityRESTMapper, preferring the preferred version of each group first,
// then any non-preferred version of the group second.
func prioritiesForGroups(groups ...string) ([]unversioned.GroupVersionResource, []unversioned.GroupVersionKind) {
resourcePriority := []unversioned.GroupVersionResource{}
kindPriority := []unversioned.GroupVersionKind{}
for _, group := range groups {
availableVersions := EnabledVersionsForGroup(group)
if len(availableVersions) > 0 {
resourcePriority = append(resourcePriority, availableVersions[0].WithResource(meta.AnyResource))
kindPriority = append(kindPriority, availableVersions[0].WithKind(meta.AnyKind))
}
}
for _, group := range groups {
resourcePriority = append(resourcePriority, unversioned.GroupVersionResource{Group: group, Version: meta.AnyVersion, Resource: meta.AnyResource})
kindPriority = append(kindPriority, unversioned.GroupVersionKind{Group: group, Version: meta.AnyVersion, Kind: meta.AnyKind})
}
return resourcePriority, kindPriority
}
// AllPreferredGroupVersions returns the preferred versions of all registered
// groups in the form of "group1/version1,group2/version2,..."
func AllPreferredGroupVersions() string {
@ -185,7 +266,7 @@ func AllPreferredGroupVersions() string {
// the KUBE_API_VERSIONS environment variable, but not enabled.
func ValidateEnvRequestedVersions() []unversioned.GroupVersion {
var missingVersions []unversioned.GroupVersion
for v := range envRequestedVersions {
for _, v := range envRequestedVersions {
if _, found := enabledVersions[v]; !found {
missingVersions = append(missingVersions, v)
}

3
pkg/client/unversioned/testclient/testclient.go
View File

@ -25,6 +25,7 @@ import (
"k8s.io/kubernetes/pkg/api"
"k8s.io/kubernetes/pkg/api/unversioned"
"k8s.io/kubernetes/pkg/api/v1"
"k8s.io/kubernetes/pkg/apimachinery/registered"
client "k8s.io/kubernetes/pkg/client/unversioned"
"k8s.io/kubernetes/pkg/runtime"
"k8s.io/kubernetes/pkg/version"
@ -41,7 +42,7 @@ func NewSimpleFake(objects ...runtime.Object) *Fake {
}
fakeClient := &Fake{}
fakeClient.AddReactor("*", "*", ObjectReaction(o, api.RESTMapper))
fakeClient.AddReactor("*", "*", ObjectReaction(o, registered.RESTMapper()))
fakeClient.AddWatchReactor("*", DefaultWatchReactor(watch.NewFake(), nil))

22
pkg/kubectl/cmd/cmd_test.go
View File

@ -180,7 +180,16 @@ func NewTestFactory() (*cmdutil.Factory, *testFactory, runtime.Codec) {
}
return &cmdutil.Factory{
Object: func() (meta.RESTMapper, runtime.ObjectTyper) {
return t.Mapper, t.Typer
priorityRESTMapper := meta.PriorityRESTMapper{
Delegate: t.Mapper,
ResourcePriority: []unversioned.GroupVersionResource{
{Group: meta.AnyGroup, Version: "v1", Resource: meta.AnyResource},
},
KindPriority: []unversioned.GroupVersionKind{
{Group: meta.AnyGroup, Version: "v1", Kind: meta.AnyKind},
},
}
return priorityRESTMapper, t.Typer
},
ClientForMapping: func(*meta.RESTMapping) (resource.RESTClient, error) {
return t.Client, t.Err
@ -212,7 +221,16 @@ func NewTestFactory() (*cmdutil.Factory, *testFactory, runtime.Codec) {
func NewMixedFactory(apiClient resource.RESTClient) (*cmdutil.Factory, *testFactory, runtime.Codec) {
f, t, c := NewTestFactory()
f.Object = func() (meta.RESTMapper, runtime.ObjectTyper) {
return meta.MultiRESTMapper{t.Mapper, testapi.Default.RESTMapper()}, runtime.MultiObjectTyper{t.Typer, api.Scheme}
priorityRESTMapper := meta.PriorityRESTMapper{
Delegate: meta.MultiRESTMapper{t.Mapper, testapi.Default.RESTMapper()},
ResourcePriority: []unversioned.GroupVersionResource{
{Group: meta.AnyGroup, Version: "v1", Resource: meta.AnyResource},
},
KindPriority: []unversioned.GroupVersionKind{
{Group: meta.AnyGroup, Version: "v1", Kind: meta.AnyKind},
},
}
return priorityRESTMapper, runtime.MultiObjectTyper{t.Typer, api.Scheme}
}
f.ClientForMapping = func(m *meta.RESTMapping) (resource.RESTClient, error) {
if m.ObjectConvertor == api.Scheme {

2
pkg/kubectl/cmd/explain.go
View File

@ -79,7 +79,7 @@ func RunExplain(f *cmdutil.Factory, out io.Writer, cmd *cobra.Command, args []st
}
// TODO: We should deduce the group for a resource by discovering the supported resources at server.
gvk, err := mapper.KindFor(unversioned.GroupVersionResource{Resource: inModel})
gvk, err := mapper.KindFor(unversioned.ParseGroupResource(inModel).WithVersion(""))
if err != nil {
return err
}

20
pkg/kubectl/cmd/util/factory.go
View File

@ -43,6 +43,7 @@ import (
"k8s.io/kubernetes/pkg/apis/autoscaling"
"k8s.io/kubernetes/pkg/apis/batch"
"k8s.io/kubernetes/pkg/apis/extensions"
"k8s.io/kubernetes/pkg/apis/metrics"
clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
client "k8s.io/kubernetes/pkg/client/unversioned"
"k8s.io/kubernetes/pkg/client/unversioned/clientcmd"
@ -202,7 +203,24 @@ func NewFactory(optionalClientConfig clientcmd.ClientConfig) *Factory {
cmdApiVersion = *cfg.GroupVersion
}
return kubectl.OutputVersionMapper{RESTMapper: mapper, OutputVersions: []unversioned.GroupVersion{cmdApiVersion}}, api.Scheme
outputRESTMapper := kubectl.OutputVersionMapper{RESTMapper: mapper, OutputVersions: []unversioned.GroupVersion{cmdApiVersion}}
// eventually this should allow me choose a group priority based on the order of the discovery doc, for now hardcode a given order
priorityRESTMapper := meta.PriorityRESTMapper{
Delegate: outputRESTMapper,
ResourcePriority: []unversioned.GroupVersionResource{
{Group: api.GroupName, Version: meta.AnyVersion, Resource: meta.AnyResource},
{Group: extensions.GroupName, Version: meta.AnyVersion, Resource: meta.AnyResource},
{Group: metrics.GroupName, Version: meta.AnyVersion, Resource: meta.AnyResource},
},
KindPriority: []unversioned.GroupVersionKind{
{Group: api.GroupName, Version: meta.AnyVersion, Kind: meta.AnyKind},
{Group: extensions.GroupName, Version: meta.AnyVersion, Kind: meta.AnyKind},
{Group: metrics.GroupName, Version: meta.AnyVersion, Kind: meta.AnyKind},
},
}
return priorityRESTMapper, api.Scheme
},
Client: func() (*client.Client, error) {
return clients.ClientForVersion(nil)

80
pkg/kubectl/kubectl.go
View File

@ -26,7 +26,6 @@ import (
"k8s.io/kubernetes/pkg/api"
"k8s.io/kubernetes/pkg/api/meta"
"k8s.io/kubernetes/pkg/api/unversioned"
"k8s.io/kubernetes/pkg/apis/extensions"
)
const kubectlAnnotationPrefix = "kubectl.kubernetes.io/"
@ -75,53 +74,72 @@ func (m OutputVersionMapper) RESTMapping(gk unversioned.GroupKind, versions ...s
}
// ShortcutExpander is a RESTMapper that can be used for Kubernetes
// resources.
// resources. It expands the resource first, then invokes the wrapped RESTMapper
type ShortcutExpander struct {
meta.RESTMapper
RESTMapper meta.RESTMapper
}
var _ meta.RESTMapper = &ShortcutExpander{}
// KindFor implements meta.RESTMapper. It expands the resource first, then invokes the wrapped
// mapper.
func (e ShortcutExpander) KindFor(resource unversioned.GroupVersionResource) (unversioned.GroupVersionKind, error) {
resource = expandResourceShortcut(resource)
return e.RESTMapper.KindFor(resource)
return e.RESTMapper.KindFor(expandResourceShortcut(resource))
}
func (e ShortcutExpander) KindsFor(resource unversioned.GroupVersionResource) ([]unversioned.GroupVersionKind, error) {
return e.RESTMapper.KindsFor(expandResourceShortcut(resource))
}
func (e ShortcutExpander) ResourcesFor(resource unversioned.GroupVersionResource) ([]unversioned.GroupVersionResource, error) {
return e.RESTMapper.ResourcesFor(expandResourceShortcut(resource))
}
func (e ShortcutExpander) ResourceFor(resource unversioned.GroupVersionResource) (unversioned.GroupVersionResource, error) {
return e.RESTMapper.ResourceFor(expandResourceShortcut(resource))
}
// ResourceSingularizer expands the named resource and then singularizes it.
func (e ShortcutExpander) ResourceSingularizer(resource string) (string, error) {
return e.RESTMapper.ResourceSingularizer(expandResourceShortcut(unversioned.GroupVersionResource{Resource: resource}).Resource)
}
func (e ShortcutExpander) RESTMapping(gk unversioned.GroupKind, versions ...string) (*meta.RESTMapping, error) {
return e.RESTMapper.RESTMapping(gk, versions...)
}
func (e ShortcutExpander) AliasesForResource(resource string) ([]string, bool) {
return e.RESTMapper.AliasesForResource(expandResourceShortcut(unversioned.GroupVersionResource{Resource: resource}).Resource)
}
// shortForms is the list of short names to their expanded names
var shortForms = map[string]string{
// Please keep this alphabetized
// If you add an entry here, please also take a look at pkg/kubectl/cmd/cmd.go
// and add an entry to valid_resources when appropriate.
"cs": "componentstatuses",
"ds": "daemonsets",
"ep": "endpoints",
"ev": "events",
"hpa": "horizontalpodautoscalers",
"ing": "ingresses",
"limits": "limitranges",
"no": "nodes",
"ns": "namespaces",
"po": "pods",
"psp": "podSecurityPolicies",
"pvc": "persistentvolumeclaims",
"pv": "persistentvolumes",
"quota": "resourcequotas",
"rc": "replicationcontrollers",
"rs": "replicasets",
"svc": "services",
}
// expandResourceShortcut will return the expanded version of resource
// (something that a pkg/api/meta.RESTMapper can understand), if it is
// indeed a shortcut. Otherwise, will return resource unmodified.
func expandResourceShortcut(resource unversioned.GroupVersionResource) unversioned.GroupVersionResource {
shortForms := map[string]unversioned.GroupVersionResource{
// Please keep this alphabetized
// If you add an entry here, please also take a look at pkg/kubectl/cmd/cmd.go
// and add an entry to valid_resources when appropriate.
"cs": api.SchemeGroupVersion.WithResource("componentstatuses"),
"ds": extensions.SchemeGroupVersion.WithResource("daemonsets"),
"ep": api.SchemeGroupVersion.WithResource("endpoints"),
"ev": api.SchemeGroupVersion.WithResource("events"),
"hpa": extensions.SchemeGroupVersion.WithResource("horizontalpodautoscalers"),
"ing": extensions.SchemeGroupVersion.WithResource("ingresses"),
"limits": api.SchemeGroupVersion.WithResource("limitranges"),
"no": api.SchemeGroupVersion.WithResource("nodes"),
"ns": api.SchemeGroupVersion.WithResource("namespaces"),
"po": api.SchemeGroupVersion.WithResource("pods"),
"psp": api.SchemeGroupVersion.WithResource("podSecurityPolicies"),
"pvc": api.SchemeGroupVersion.WithResource("persistentvolumeclaims"),
"pv": api.SchemeGroupVersion.WithResource("persistentvolumes"),
"quota": api.SchemeGroupVersion.WithResource("resourcequotas"),
"rc": api.SchemeGroupVersion.WithResource("replicationcontrollers"),
"rs": extensions.SchemeGroupVersion.WithResource("replicasets"),
"svc": api.SchemeGroupVersion.WithResource("services"),
}
if expanded, ok := shortForms[resource.Resource]; ok {
return expanded
// don't change the group or version that's already been specified
resource.Resource = expanded
}
return resource
}

4
pkg/kubectl/resource/builder.go
View File

@ -440,7 +440,7 @@ func (b *Builder) resourceMappings() ([]*meta.RESTMapping, error) {
}
mappings := []*meta.RESTMapping{}
for _, r := range b.resources {
gvk, err := b.mapper.KindFor(unversioned.GroupVersionResource{Resource: r})
gvk, err := b.mapper.KindFor(unversioned.ParseGroupResource(r).WithVersion(""))
if err != nil {
return nil, err
}
@ -460,7 +460,7 @@ func (b *Builder) resourceTupleMappings() (map[string]*meta.RESTMapping, error)
if _, ok := mappings[r.Resource]; ok {
continue
}
gvk, err := b.mapper.KindFor(unversioned.GroupVersionResource{Resource: r.Resource})
gvk, err := b.mapper.KindFor(unversioned.ParseGroupResource(r.Resource).WithVersion(""))
if err != nil {
return nil, err
}