Add percentage of used resources to node metrics.

This commit is contained in:
mksalawa 2016-08-17 14:40:18 +02:00
parent 5898f87722
commit 566af82be3
5 changed files with 187 additions and 87 deletions

View File

@ -25,6 +25,8 @@ import (
"github.com/renstrom/dedent"
"github.com/spf13/cobra"
"k8s.io/kubernetes/pkg/api"
"k8s.io/kubernetes/pkg/labels"
)
// TopNodeOptions contains all the options for running the top-node cli command.
@ -95,6 +97,12 @@ func (o *TopNodeOptions) Validate() error {
if len(o.ResourceName) > 0 && len(o.Selector) > 0 {
return errors.New("only one of NAME or --selector can be provided")
}
if len(o.Selector) > 0 {
_, err := labels.Parse(o.Selector)
if err != nil {
return err
}
}
return nil
}
@ -103,5 +111,36 @@ func (o TopNodeOptions) RunTopNode() error {
if err != nil {
return err
}
return o.Printer.PrintNodeMetrics(metrics)
selector := labels.Everything()
if len(o.Selector) > 0 {
selector, err = labels.Parse(o.Selector)
if err != nil {
return err
}
}
var nodes []api.Node
if len(o.ResourceName) > 0 {
node, err := o.Client.Nodes().Get(o.ResourceName)
if err != nil {
return err
}
nodes = append(nodes, *node)
} else {
nodeList, err := o.Client.Nodes().List(api.ListOptions{
LabelSelector: selector,
})
if err != nil {
return err
}
nodes = append(nodes, nodeList.Items...)
}
allocatable := make(map[string]api.ResourceList)
for _, n := range nodes {
allocatable[n.Name] = n.Status.Allocatable
}
return o.Printer.PrintNodeMetrics(metrics, allocatable)
}

View File

@ -23,31 +23,40 @@ import (
"strings"
"testing"
"k8s.io/kubernetes/pkg/api"
"k8s.io/kubernetes/pkg/api/unversioned"
"k8s.io/kubernetes/pkg/client/restclient"
"k8s.io/kubernetes/pkg/client/unversioned/fake"
"net/url"
)
const (
apiPrefix = "api"
apiVersion = "v1"
)
func TestTopNodeAllMetrics(t *testing.T) {
initTestErrorHandler(t)
metrics := testNodeMetricsData()
expectedPath := fmt.Sprintf("%s/%s/nodes", baseMetricsAddress, metricsApiVersion)
metrics, nodes := testNodeMetricsData()
expectedMetricsPath := fmt.Sprintf("%s/%s/nodes", baseMetricsAddress, metricsApiVersion)
expectedNodePath := fmt.Sprintf("/%s/%s/nodes", apiPrefix, apiVersion)
f, tf, _, ns := NewAPIFactory()
f, tf, codec, ns := NewAPIFactory()
tf.Printer = &testPrinter{}
tf.Client = &fake.RESTClient{
NegotiatedSerializer: ns,
Client: fake.CreateHTTPClient(func(req *http.Request) (*http.Response, error) {
switch p, m := req.URL.Path, req.Method; {
case p == expectedPath && m == "GET":
case p == expectedMetricsPath && m == "GET":
body, err := marshallBody(metrics)
if err != nil {
t.Errorf("unexpected error: %v", err)
}
return &http.Response{StatusCode: 200, Header: defaultHeader(), Body: body}, nil
case p == expectedNodePath && m == "GET":
return &http.Response{StatusCode: 200, Header: defaultHeader(), Body: objBody(codec, nodes)}, nil
default:
t.Fatalf("unexpected request: %#v\nGot URL: %#v\nExpected path: %#v", req, req.URL, expectedPath)
t.Fatalf("unexpected request: %#v\nGot URL: %#v\nExpected path: %#v", req, req.URL, expectedMetricsPath)
return nil, nil
}
}),
@ -70,12 +79,14 @@ func TestTopNodeAllMetrics(t *testing.T) {
func TestTopNodeWithNameMetrics(t *testing.T) {
initTestErrorHandler(t)
metrics := testNodeMetricsData()
metrics, nodes := testNodeMetricsData()
expectedMetrics := metrics[0]
expectedNode := &nodes.Items[0]
nonExpectedMetrics := metrics[1:]
expectedPath := fmt.Sprintf("%s/%s/nodes/%s", baseMetricsAddress, metricsApiVersion, expectedMetrics.Name)
expectedNodePath := fmt.Sprintf("/%s/%s/nodes/%s", apiPrefix, apiVersion, expectedMetrics.Name)
f, tf, _, ns := NewAPIFactory()
f, tf, codec, ns := NewAPIFactory()
tf.Printer = &testPrinter{}
tf.Client = &fake.RESTClient{
NegotiatedSerializer: ns,
@ -87,6 +98,8 @@ func TestTopNodeWithNameMetrics(t *testing.T) {
t.Errorf("unexpected error: %v", err)
}
return &http.Response{StatusCode: 200, Header: defaultHeader(), Body: body}, nil
case p == expectedNodePath && m == "GET":
return &http.Response{StatusCode: 200, Header: defaultHeader(), Body: objBody(codec, expectedNode)}, nil
default:
t.Fatalf("unexpected request: %#v\nGot URL: %#v\nExpected path: %#v", req, req.URL, expectedPath)
return nil, nil
@ -114,14 +127,19 @@ func TestTopNodeWithNameMetrics(t *testing.T) {
func TestTopNodeWithLabelSelectorMetrics(t *testing.T) {
initTestErrorHandler(t)
metrics := testNodeMetricsData()
metrics, nodes := testNodeMetricsData()
expectedMetrics := metrics[0:1]
expectedNodes := &api.NodeList{
ListMeta: nodes.ListMeta,
Items: nodes.Items[0:1],
}
nonExpectedMetrics := metrics[1:]
label := "key=value"
expectedPath := fmt.Sprintf("%s/%s/nodes", baseMetricsAddress, metricsApiVersion)
expectedQuery := fmt.Sprintf("labelSelector=%s", url.QueryEscape(label))
expectedNodePath := fmt.Sprintf("/%s/%s/nodes", apiPrefix, apiVersion)
f, tf, _, ns := NewAPIFactory()
f, tf, codec, ns := NewAPIFactory()
tf.Printer = &testPrinter{}
tf.Client = &fake.RESTClient{
NegotiatedSerializer: ns,
@ -133,6 +151,8 @@ func TestTopNodeWithLabelSelectorMetrics(t *testing.T) {
t.Errorf("unexpected error: %v", err)
}
return &http.Response{StatusCode: 200, Header: defaultHeader(), Body: body}, nil
case p == expectedNodePath && m == "GET":
return &http.Response{StatusCode: 200, Header: defaultHeader(), Body: objBody(codec, expectedNodes)}, nil
default:
t.Fatalf("unexpected request: %#v\nGot URL: %#v\nExpected path: %#v", req, req.URL, expectedPath)
return nil, nil

View File

@ -24,9 +24,10 @@ import (
"time"
metrics_api "k8s.io/heapster/metrics/apis/metrics/v1alpha1"
"k8s.io/kubernetes/pkg/api"
"k8s.io/kubernetes/pkg/api/resource"
"k8s.io/kubernetes/pkg/api/unversioned"
api "k8s.io/kubernetes/pkg/api/v1"
v1 "k8s.io/kubernetes/pkg/api/v1"
"testing"
)
@ -56,93 +57,121 @@ func marshallBody(metrics interface{}) (io.ReadCloser, error) {
return ioutil.NopCloser(bytes.NewReader(result)), nil
}
func testNodeMetricsData() []metrics_api.NodeMetrics {
return []metrics_api.NodeMetrics{
func testNodeMetricsData() ([]metrics_api.NodeMetrics, *api.NodeList) {
metrics := []metrics_api.NodeMetrics{
{
ObjectMeta: api.ObjectMeta{Name: "node1", ResourceVersion: "10"},
ObjectMeta: v1.ObjectMeta{Name: "node1", ResourceVersion: "10"},
Window: unversioned.Duration{Duration: time.Minute},
Usage: api.ResourceList{
api.ResourceCPU: *resource.NewMilliQuantity(1, resource.DecimalSI),
api.ResourceMemory: *resource.NewQuantity(2*(1024*1024), resource.DecimalSI),
api.ResourceStorage: *resource.NewQuantity(3*(1024*1024), resource.DecimalSI),
Usage: v1.ResourceList{
v1.ResourceCPU: *resource.NewMilliQuantity(1, resource.DecimalSI),
v1.ResourceMemory: *resource.NewQuantity(2*(1024*1024), resource.DecimalSI),
v1.ResourceStorage: *resource.NewQuantity(3*(1024*1024), resource.DecimalSI),
},
},
{
ObjectMeta: api.ObjectMeta{Name: "node2", ResourceVersion: "11"},
ObjectMeta: v1.ObjectMeta{Name: "node2", ResourceVersion: "11"},
Window: unversioned.Duration{Duration: time.Minute},
Usage: api.ResourceList{
api.ResourceCPU: *resource.NewMilliQuantity(5, resource.DecimalSI),
api.ResourceMemory: *resource.NewQuantity(6*(1024*1024), resource.DecimalSI),
api.ResourceStorage: *resource.NewQuantity(7*(1024*1024), resource.DecimalSI),
Usage: v1.ResourceList{
v1.ResourceCPU: *resource.NewMilliQuantity(5, resource.DecimalSI),
v1.ResourceMemory: *resource.NewQuantity(6*(1024*1024), resource.DecimalSI),
v1.ResourceStorage: *resource.NewQuantity(7*(1024*1024), resource.DecimalSI),
},
},
}
nodes := &api.NodeList{
ListMeta: unversioned.ListMeta{
ResourceVersion: "15",
},
Items: []api.Node{
{
ObjectMeta: api.ObjectMeta{Name: "node1", ResourceVersion: "10"},
Status: api.NodeStatus{
Allocatable: api.ResourceList{
api.ResourceCPU: *resource.NewMilliQuantity(10, resource.DecimalSI),
api.ResourceMemory: *resource.NewQuantity(20*(1024*1024), resource.DecimalSI),
api.ResourceStorage: *resource.NewQuantity(30*(1024*1024), resource.DecimalSI),
},
},
},
{
ObjectMeta: api.ObjectMeta{Name: "node2", ResourceVersion: "11"},
Status: api.NodeStatus{
Allocatable: api.ResourceList{
api.ResourceCPU: *resource.NewMilliQuantity(50, resource.DecimalSI),
api.ResourceMemory: *resource.NewQuantity(60*(1024*1024), resource.DecimalSI),
api.ResourceStorage: *resource.NewQuantity(70*(1024*1024), resource.DecimalSI),
},
},
},
},
}
return metrics, nodes
}
func testPodMetricsData() []metrics_api.PodMetrics {
return []metrics_api.PodMetrics{
{
ObjectMeta: api.ObjectMeta{Name: "pod1", Namespace: "test", ResourceVersion: "10"},
ObjectMeta: v1.ObjectMeta{Name: "pod1", Namespace: "test", ResourceVersion: "10"},
Window: unversioned.Duration{Duration: time.Minute},
Containers: []metrics_api.ContainerMetrics{
{
Name: "container1-1",
Usage: api.ResourceList{
api.ResourceCPU: *resource.NewMilliQuantity(1, resource.DecimalSI),
api.ResourceMemory: *resource.NewQuantity(2*(1024*1024), resource.DecimalSI),
api.ResourceStorage: *resource.NewQuantity(3*(1024*1024), resource.DecimalSI),
Usage: v1.ResourceList{
v1.ResourceCPU: *resource.NewMilliQuantity(1, resource.DecimalSI),
v1.ResourceMemory: *resource.NewQuantity(2*(1024*1024), resource.DecimalSI),
v1.ResourceStorage: *resource.NewQuantity(3*(1024*1024), resource.DecimalSI),
},
},
{
Name: "container1-2",
Usage: api.ResourceList{
api.ResourceCPU: *resource.NewMilliQuantity(4, resource.DecimalSI),
api.ResourceMemory: *resource.NewQuantity(5*(1024*1024), resource.DecimalSI),
api.ResourceStorage: *resource.NewQuantity(6*(1024*1024), resource.DecimalSI),
Usage: v1.ResourceList{
v1.ResourceCPU: *resource.NewMilliQuantity(4, resource.DecimalSI),
v1.ResourceMemory: *resource.NewQuantity(5*(1024*1024), resource.DecimalSI),
v1.ResourceStorage: *resource.NewQuantity(6*(1024*1024), resource.DecimalSI),
},
},
},
},
{
ObjectMeta: api.ObjectMeta{Name: "pod2", Namespace: "test", ResourceVersion: "11"},
ObjectMeta: v1.ObjectMeta{Name: "pod2", Namespace: "test", ResourceVersion: "11"},
Window: unversioned.Duration{Duration: time.Minute},
Containers: []metrics_api.ContainerMetrics{
{
Name: "container2-1",
Usage: api.ResourceList{
api.ResourceCPU: *resource.NewMilliQuantity(7, resource.DecimalSI),
api.ResourceMemory: *resource.NewQuantity(8*(1024*1024), resource.DecimalSI),
api.ResourceStorage: *resource.NewQuantity(9*(1024*1024), resource.DecimalSI),
Usage: v1.ResourceList{
v1.ResourceCPU: *resource.NewMilliQuantity(7, resource.DecimalSI),
v1.ResourceMemory: *resource.NewQuantity(8*(1024*1024), resource.DecimalSI),
v1.ResourceStorage: *resource.NewQuantity(9*(1024*1024), resource.DecimalSI),
},
},
{
Name: "container2-2",
Usage: api.ResourceList{
api.ResourceCPU: *resource.NewMilliQuantity(10, resource.DecimalSI),
api.ResourceMemory: *resource.NewQuantity(11*(1024*1024), resource.DecimalSI),
api.ResourceStorage: *resource.NewQuantity(12*(1024*1024), resource.DecimalSI),
Usage: v1.ResourceList{
v1.ResourceCPU: *resource.NewMilliQuantity(10, resource.DecimalSI),
v1.ResourceMemory: *resource.NewQuantity(11*(1024*1024), resource.DecimalSI),
v1.ResourceStorage: *resource.NewQuantity(12*(1024*1024), resource.DecimalSI),
},
},
{
Name: "container2-3",
Usage: api.ResourceList{
api.ResourceCPU: *resource.NewMilliQuantity(13, resource.DecimalSI),
api.ResourceMemory: *resource.NewQuantity(14*(1024*1024), resource.DecimalSI),
api.ResourceStorage: *resource.NewQuantity(15*(1024*1024), resource.DecimalSI),
Usage: v1.ResourceList{
v1.ResourceCPU: *resource.NewMilliQuantity(13, resource.DecimalSI),
v1.ResourceMemory: *resource.NewQuantity(14*(1024*1024), resource.DecimalSI),
v1.ResourceStorage: *resource.NewQuantity(15*(1024*1024), resource.DecimalSI),
},
},
},
},
{
ObjectMeta: api.ObjectMeta{Name: "pod3", Namespace: "test", ResourceVersion: "12"},
ObjectMeta: v1.ObjectMeta{Name: "pod3", Namespace: "test", ResourceVersion: "12"},
Window: unversioned.Duration{Duration: time.Minute},
Containers: []metrics_api.ContainerMetrics{
{
Name: "container3-1",
Usage: api.ResourceList{
api.ResourceCPU: *resource.NewMilliQuantity(7, resource.DecimalSI),
api.ResourceMemory: *resource.NewQuantity(8*(1024*1024), resource.DecimalSI),
api.ResourceStorage: *resource.NewQuantity(9*(1024*1024), resource.DecimalSI),
Usage: v1.ResourceList{
v1.ResourceCPU: *resource.NewMilliQuantity(7, resource.DecimalSI),
v1.ResourceMemory: *resource.NewQuantity(8*(1024*1024), resource.DecimalSI),
v1.ResourceStorage: *resource.NewQuantity(9*(1024*1024), resource.DecimalSI),
},
},
},

View File

@ -45,7 +45,7 @@ var (
)
type HeapsterMetricsClient struct {
Client *client.Client
*client.Client
HeapsterNamespace string
HeapsterScheme string
HeapsterService string
@ -66,7 +66,7 @@ func DefaultHeapsterMetricsClient(client *client.Client) *HeapsterMetricsClient
return NewHeapsterMetricsClient(client, DefaultHeapsterNamespace, DefaultHeapsterScheme, DefaultHeapsterService, DefaultHeapsterPort)
}
func PodMetricsUrl(namespace string, name string) (string, error) {
func podMetricsUrl(namespace string, name string) (string, error) {
errs := validation.ValidateNamespaceName(namespace, false)
if len(errs) > 0 {
message := fmt.Sprintf("invalid namespace: %s - %v", namespace, errs)
@ -82,7 +82,7 @@ func PodMetricsUrl(namespace string, name string) (string, error) {
return fmt.Sprintf("%s/namespaces/%s/pods/%s", MetricsRoot, namespace, name), nil
}
func NodeMetricsUrl(name string) (string, error) {
func nodeMetricsUrl(name string) (string, error) {
if len(name) > 0 {
errs := validation.ValidateNodeName(name, false)
if len(errs) > 0 {
@ -95,7 +95,7 @@ func NodeMetricsUrl(name string) (string, error) {
func (cli *HeapsterMetricsClient) GetNodeMetrics(nodeName string, selector string) ([]metrics_api.NodeMetrics, error) {
params := map[string]string{"labelSelector": selector}
path, err := NodeMetricsUrl(nodeName)
path, err := nodeMetricsUrl(nodeName)
if err != nil {
return []metrics_api.NodeMetrics{}, err
}
@ -139,7 +139,7 @@ func (cli *HeapsterMetricsClient) GetPodMetrics(namespace string, podName string
params := map[string]string{"labelSelector": selector}
allMetrics := make([]metrics_api.PodMetrics, 0)
for _, ns := range namespaces {
path, err := PodMetricsUrl(ns, podName)
path, err := podMetricsUrl(ns, podName)
if err != nil {
return []metrics_api.PodMetrics{}, err
}
@ -167,7 +167,7 @@ func (cli *HeapsterMetricsClient) GetPodMetrics(namespace string, podName string
}
func GetHeapsterMetrics(cli *HeapsterMetricsClient, path string, params map[string]string) ([]byte, error) {
return cli.Client.Services(cli.HeapsterNamespace).
return cli.Services(cli.HeapsterNamespace).
ProxyGet(cli.HeapsterScheme, cli.HeapsterService, cli.HeapsterPort, path, params).
DoRaw()
}

View File

@ -19,30 +19,28 @@ package metricsutil
import (
"fmt"
"io"
"time"
metrics_api "k8s.io/heapster/metrics/apis/metrics/v1alpha1"
"k8s.io/kubernetes/pkg/api"
"k8s.io/kubernetes/pkg/api/resource"
"k8s.io/kubernetes/pkg/api/v1"
"k8s.io/kubernetes/pkg/kubectl"
)
var (
MeasuredResources = []v1.ResourceName{
v1.ResourceCPU,
v1.ResourceMemory,
v1.ResourceStorage,
MeasuredResources = []api.ResourceName{
api.ResourceCPU,
api.ResourceMemory,
}
NodeColumns = []string{"NAME", "CPU (cores)", "MEMORY (bytes)", "STORAGE (bytes)", "TIMESTAMP"}
PodColumns = []string{"NAME", "CPU (cores)", "MEMORY (bytes)", "STORAGE (bytes)", "TIMESTAMP"}
NodeColumns = []string{"NAME", "CPU(cores)", "CPU%", "MEMORY(bytes)", "MEMORY%"}
PodColumns = []string{"NAME", "CPU(cores)", "MEMORY(bytes)"}
NamespaceColumn = "NAMESPACE"
PodColumn = "POD"
)
type ResourceMetricsInfo struct {
Name string
Metrics v1.ResourceList
Timestamp string
Metrics api.ResourceList
Available api.ResourceList
}
type TopCmdPrinter struct {
@ -53,7 +51,7 @@ func NewTopCmdPrinter(out io.Writer) *TopCmdPrinter {
return &TopCmdPrinter{out: out}
}
func (printer *TopCmdPrinter) PrintNodeMetrics(metrics []metrics_api.NodeMetrics) error {
func (printer *TopCmdPrinter) PrintNodeMetrics(metrics []metrics_api.NodeMetrics, availableResources map[string]api.ResourceList) error {
if len(metrics) == 0 {
return nil
}
@ -61,11 +59,16 @@ func (printer *TopCmdPrinter) PrintNodeMetrics(metrics []metrics_api.NodeMetrics
defer w.Flush()
printColumnNames(w, NodeColumns)
var usage api.ResourceList
for _, m := range metrics {
err := api.Scheme.Convert(&m.Usage, &usage, nil)
if err != nil {
return err
}
printMetricsLine(w, &ResourceMetricsInfo{
Name: m.Name,
Metrics: m.Usage,
Timestamp: m.Timestamp.Time.Format(time.RFC1123Z),
Metrics: usage,
Available: availableResources[m.Name],
})
}
return nil
@ -86,7 +89,10 @@ func (printer *TopCmdPrinter) PrintPodMetrics(metrics []metrics_api.PodMetrics,
}
printColumnNames(w, PodColumns)
for _, m := range metrics {
printSinglePodMetrics(w, &m, printContainers, withNamespace)
err := printSinglePodMetrics(w, &m, printContainers, withNamespace)
if err != nil {
return err
}
}
return nil
}
@ -98,18 +104,23 @@ func printColumnNames(out io.Writer, names []string) {
fmt.Fprint(out, "\n")
}
func printSinglePodMetrics(out io.Writer, m *metrics_api.PodMetrics, printContainersOnly bool, withNamespace bool) {
containers := make(map[string]v1.ResourceList)
podMetrics := make(v1.ResourceList)
func printSinglePodMetrics(out io.Writer, m *metrics_api.PodMetrics, printContainersOnly bool, withNamespace bool) error {
containers := make(map[string]api.ResourceList)
podMetrics := make(api.ResourceList)
for _, res := range MeasuredResources {
podMetrics[res], _ = resource.ParseQuantity("0")
}
var usage api.ResourceList
for _, c := range m.Containers {
containers[c.Name] = c.Usage
err := api.Scheme.Convert(&c.Usage, &usage, nil)
if err != nil {
return err
}
containers[c.Name] = usage
if !printContainersOnly {
for _, res := range MeasuredResources {
quantity := podMetrics[res]
quantity.Add(c.Usage[res])
quantity.Add(usage[res])
podMetrics[res] = quantity
}
}
@ -123,7 +134,7 @@ func printSinglePodMetrics(out io.Writer, m *metrics_api.PodMetrics, printContai
printMetricsLine(out, &ResourceMetricsInfo{
Name: contName,
Metrics: containers[contName],
Timestamp: m.Timestamp.Time.Format(time.RFC1123Z),
Available: api.ResourceList{},
})
}
} else {
@ -133,15 +144,15 @@ func printSinglePodMetrics(out io.Writer, m *metrics_api.PodMetrics, printContai
printMetricsLine(out, &ResourceMetricsInfo{
Name: m.Name,
Metrics: podMetrics,
Timestamp: m.Timestamp.Time.Format(time.RFC1123Z),
Available: api.ResourceList{},
})
}
return nil
}
func printMetricsLine(out io.Writer, metrics *ResourceMetricsInfo) {
printValue(out, metrics.Name)
printAllResourceUsages(out, metrics.Metrics)
printValue(out, metrics.Timestamp)
printAllResourceUsages(out, metrics)
fmt.Fprint(out, "\n")
}
@ -149,23 +160,24 @@ func printValue(out io.Writer, value interface{}) {
fmt.Fprintf(out, "%v\t", value)
}
func printAllResourceUsages(out io.Writer, usage v1.ResourceList) {
func printAllResourceUsages(out io.Writer, metrics *ResourceMetricsInfo) {
for _, res := range MeasuredResources {
quantity := usage[res]
quantity := metrics.Metrics[res]
printSingleResourceUsage(out, res, quantity)
fmt.Fprint(out, "\t")
if available, found := metrics.Available[res]; found {
fraction := float64(quantity.MilliValue()) / float64(available.MilliValue()) * 100
fmt.Fprintf(out, "%d%%\t", int64(fraction))
}
}
}
func printSingleResourceUsage(out io.Writer, resourceType v1.ResourceName, quantity resource.Quantity) {
func printSingleResourceUsage(out io.Writer, resourceType api.ResourceName, quantity resource.Quantity) {
switch resourceType {
case v1.ResourceCPU:
case api.ResourceCPU:
fmt.Fprintf(out, "%vm", quantity.MilliValue())
case v1.ResourceMemory:
case api.ResourceMemory:
fmt.Fprintf(out, "%vMi", quantity.Value()/(1024*1024))
case v1.ResourceStorage:
// TODO: Change it after storage metrics collection is finished.
fmt.Fprint(out, "-")
default:
fmt.Fprintf(out, "%v", quantity.Value())
}