From 394c9452e7d41e9402b7e9be8a0f6987e5d6c875 Mon Sep 17 00:00:00 2001 From: Daniel Smith Date: Thu, 4 Sep 2014 16:50:57 -0700 Subject: [PATCH] Start numeric type for resources --- pkg/api/resource/quantity.go | 255 ++++++++++++++++++++++++++++++ pkg/api/resource/quantity_test.go | 221 ++++++++++++++++++++++++++ pkg/api/resource/suffix.go | 132 ++++++++++++++++ 3 files changed, 608 insertions(+) create mode 100644 pkg/api/resource/quantity.go create mode 100644 pkg/api/resource/quantity_test.go create mode 100644 pkg/api/resource/suffix.go diff --git a/pkg/api/resource/quantity.go b/pkg/api/resource/quantity.go new file mode 100644 index 00000000000..57e1a192b82 --- /dev/null +++ b/pkg/api/resource/quantity.go @@ -0,0 +1,255 @@ +/* +Copyright 2014 Google Inc. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package resource + +import ( + "errors" + "math/big" + "regexp" + "strings" + + "speter.net/go/exp/math/dec/inf" +) + +// Format lists the three possible formattings of a quantity. +type Format string + +const ( + DecimalExponent = Format("DecExponent") + BinarySI = Format("BinSI") + DecimalSI = Format("DecSI") +) + +// Quantity is a fixed-point representation of a number. +// It provides convenient marshaling/unmarshaling in JSON and YAML, +// in addition to String() and Int64() accessors. +// +// The serialization format is: +// +// ::= | +// ::= | . +// ::= "+" | "-" +// ::= | +// ::= 0 | 1 | ... | 9 +// ::= | | +// ::= i | Ki | Mi | Gi | Ti | Pi | Ei +// ::= m | "" | k | M | G | T | P | E +// ::= "e" | "E" +// (Where digits is always a multiple of 3) +// (Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.) +// +// No matter which of the three exponent forms is used, no quantity may represent +// a number less than .001m or greater than 2^63-1 in magnitude. Numbers that exceed +// a bound will be capped at that bound. (E.g.: 0.0001m will be treated as 0.001m.) +// This may be extended in the future if we require larger or smaller quantities. +// +// Numbers with binary suffixes may not have any fractional part. +// +// Quantities will be serialized in the same format that they were parsed from. +// Before serializing, Quantity will be put in "canonical form". +// This means that Exponent will be adjusted up or down (with a +// corresponding increase or decrease in Mantissa) until one of the +// following is true: +// a. Binary SI mode: Mantissa mod 1024 is nonzero. +// Examples: 1Gi 300Mi 6Ki 1001Gi +// Non-canonical: 1024Gi (Should be 1Ti) +// b. Decimal SI mode: exponent is greater than 3 and one of Mantissa's three least +// significant digits is nonzero. +// Examples: 1G 300M 3K 1001G +// Non-canonical: 1000G (Should be 1T) +// c. Decimal SI mode: exponent is less than or equal to zero, and Mantissa has no more +// than three nonzero decimals. If any decimals are nonzero, three +// decimals will be emitted. +// Examples: 5 123.450 1.001 0.045 +// Non-canonical: 1m (should be 0.001) +// d. Decimal exponent mode: as for decimal SI mode, but using the corresponding +// "e6" or "E6" form. +// +// The sign will be omitted unless the number is negative. +// +// Note that the quantity will NEVER be represented by a floating point number. That is +// the whole point of this exercise. +// +// Non-canonical values will still parse as long as they are well formed, +// but will be re-emitted in their canonical form. (So always use canonical +// form, or don't diff.) +// +// This format is intended to make it difficult to use these numbers without +// writing some sort of special handling code in the hopes that that will +// cause implementors to also use a fixed point implementation. +type Quantity struct { + Amount *inf.Dec + Format +} + +const ( + splitREString = "^([+-]?[0123456789.]+)([eEimkKMGTP]*[-+]?[0123456789]*)$" +) + +var ( + // splitRE is used to get the various parts of a number. + splitRE = regexp.MustCompile(splitREString) + + ErrFormatWrong = errors.New("quantities must match the regular expression '" + splitREString + "'") + ErrNumeric = errors.New("unable to parse numeric part of quantity") + ErrSuffix = errors.New("unable to parse quantity's suffix") + ErrFractionalBinary = errors.New("numbers with binary-style SI suffixes can't have fractional parts") +) + +// ParseQuantity turns str into a Quantity, or returns an error. +func ParseQuantity(str string) (*Quantity, error) { + parts := splitRE.FindStringSubmatch(strings.TrimSpace(str)) + if len(parts) != 3 { + return nil, ErrFormatWrong + } + + amount := new(inf.Dec) + if _, ok := amount.SetString(parts[1]); !ok { + return nil, ErrNumeric + } + + base, exponent, format, ok := quantitySuffixer.interpret(suffix(parts[2])) + if !ok { + return nil, ErrSuffix + } + + // So that no one but us has to think about suffixes, remove it. + if base == 10 { + amount.SetScale(amount.Scale() + inf.Scale(-exponent)) + } else if base == 2 { + // Detect fractional parts by rounding. There's probably + // a better way to do this. + if rounded := new(inf.Dec).Round(amount, 0, inf.RoundFloor); rounded.Cmp(amount) != 0 { + return nil, ErrFractionalBinary + } + if exponent < 0 { + return nil, ErrFractionalBinary + } + // exponent will always be a multiple of 10. + dec1024 := inf.NewDec(1024, 0) + for exponent > 0 { + amount.Mul(amount, dec1024) + exponent -= 10 + } + } + + return &Quantity{amount, format}, nil +} + +var ( + // Commonly needed big.Ints-- treat as read only! + ten = big.NewInt(10) + zero = big.NewInt(0) + thousand = big.NewInt(1000) + ten24 = big.NewInt(1024) + + minAllowed = inf.NewDec(1, 6) + maxAllowed = inf.NewDec(999, -18) +) + +// removeFactors divides in a loop; the return values have the property that +// d == result * factor ^ times +// d may be modified in place. +// If d == 0, then the return values will be (0, 0) +func removeFactors(d, factor *big.Int) (result *big.Int, times int) { + q := big.NewInt(0) + m := big.NewInt(0) + for d.Cmp(zero) != 0 { + q.DivMod(d, factor, m) + if m.Cmp(zero) != 0 { + break + } + times++ + d, q = q, d + } + return d, times +} + +// Canonicalize returns the canonical form of q and its suffix (see comment on Quantity). +func (q *Quantity) Canonicalize() (string, suffix) { + mantissa := q.Amount.UnscaledBig() + exponent := int(-q.Amount.Scale()) + amount := big.NewInt(0).Set(mantissa) + + switch q.Format { + case DecimalExponent, DecimalSI: + // move all factors of 10 into the exponent for easy reasoning + amount, times := removeFactors(amount, ten) + exponent += times + + // make sure exponent is a multiple of 3 + for exponent%3 != 0 { + amount.Mul(amount, ten) + exponent-- + } + + absAmount := big.NewInt(0).Abs(amount) + + // Canonical form has three decimal digits. + if absAmount.Cmp(thousand) >= 0 { + // Unless that would cause an exponent of 3-- 111.111e3 is silly. + if exponent != 0 { + suffix, _ := quantitySuffixer.construct(10, exponent+3, q.Format) + number := inf.NewDecBig(amount, 3).String() + return number, suffix + } + } + suffix, _ := quantitySuffixer.construct(10, exponent, q.Format) + number := amount.String() + return number, suffix + case BinarySI: + // Apply the (base-10) shift. This will lose any fractional + // part, which is intentional. + for exponent < 0 { + amount.Mul(amount, ten) + exponent++ + } + for exponent > 0 { + amount.Mul(amount, ten) + exponent-- + } + + amount, exponent := removeFactors(amount, ten24) + suffix, _ := quantitySuffixer.construct(2, exponent*10, q.Format) + number := amount.String() + return number, suffix + } + return "0", "" +} + +// String formats the Quantity as a string. +func (q *Quantity) String() string { + number, suffix := q.Canonicalize() + return number + string(suffix) +} + +// MarshalJSON implements the json.Marshaller interface. +func (q Quantity) MarshalJSON() ([]byte, error) { + return []byte(`"` + q.String() + `"`), nil +} + +// UnmarshalJSON implements the json.Unmarshaller interface. +func (q *Quantity) UnmarshalJSON(value []byte) error { + str := string(value) + parsed, err := ParseQuantity(strings.Trim(str, `"`)) + if err != nil { + return err + } + // This copy is safe because parsed will not be referred to again. + *q = *parsed + return nil +} diff --git a/pkg/api/resource/quantity_test.go b/pkg/api/resource/quantity_test.go new file mode 100644 index 00000000000..b51edc1ba56 --- /dev/null +++ b/pkg/api/resource/quantity_test.go @@ -0,0 +1,221 @@ +/* +Copyright 2014 Google Inc. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package resource + +import ( + //"reflect" + "testing" + + "speter.net/go/exp/math/dec/inf" +) + +func dec(i int64, exponent int) *inf.Dec { + // See the below test-- scale is the negative of an exponent. + return inf.NewDec(i, inf.Scale(-exponent)) +} + +func TestDec(t *testing.T) { + table := []struct { + got *inf.Dec + expect string + }{ + {dec(1, 0), "1"}, + {dec(1, 1), "10"}, + {dec(5, 2), "500"}, + {dec(8, 3), "8000"}, + {dec(2, 0), "2"}, + {dec(1, -1), "0.1"}, + {dec(3, -2), "0.03"}, + {dec(4, -3), "0.004"}, + } + + for _, item := range table { + if e, a := item.expect, item.got.String(); e != a { + t.Errorf("expected %v, got %v", e, a) + } + } +} + +func TestQuantityParse(t *testing.T) { + table := []struct { + input string + expect Quantity + }{ + {"0", Quantity{dec(0, 0), DecimalSI}}, + // Binary suffixes + {"9i", Quantity{dec(9, 0), BinarySI}}, + {"8Ki", Quantity{dec(8*1024, 0), BinarySI}}, + {"7Mi", Quantity{dec(7*1024*1024, 0), BinarySI}}, + {"6Gi", Quantity{dec(6*1024*1024*1024, 0), BinarySI}}, + {"5Ti", Quantity{dec(5*1024*1024*1024*1024, 0), BinarySI}}, + {"4Pi", Quantity{dec(4*1024*1024*1024*1024*1024, 0), BinarySI}}, + {"3Ei", Quantity{dec(3*1024*1024*1024*1024*1024*1024, 0), BinarySI}}, + + {"10Ti", Quantity{dec(10*1024*1024*1024*1024, 0), BinarySI}}, + {"100Ti", Quantity{dec(100*1024*1024*1024*1024, 0), BinarySI}}, + + // Decimal suffixes + {"3m", Quantity{dec(3, -3), DecimalSI}}, + {"9", Quantity{dec(9, 0), DecimalSI}}, + {"8k", Quantity{dec(8, 3), DecimalSI}}, + {"7M", Quantity{dec(7, 6), DecimalSI}}, + {"6G", Quantity{dec(6, 9), DecimalSI}}, + {"5T", Quantity{dec(5, 12), DecimalSI}}, + {"40T", Quantity{dec(4, 13), DecimalSI}}, + {"300T", Quantity{dec(3, 14), DecimalSI}}, + {"2P", Quantity{dec(2, 15), DecimalSI}}, + {"1E", Quantity{dec(1, 18), DecimalSI}}, + + // Decimal exponents + {"1E-3", Quantity{dec(1, -3), DecimalExponent}}, + {"1e3", Quantity{dec(1, 3), DecimalExponent}}, + {"1E6", Quantity{dec(1, 6), DecimalExponent}}, + {"1e9", Quantity{dec(1, 9), DecimalExponent}}, + {"1E12", Quantity{dec(1, 12), DecimalExponent}}, + {"1e15", Quantity{dec(1, 15), DecimalExponent}}, + {"1E18", Quantity{dec(1, 18), DecimalExponent}}, + + // Nonstandard but still parsable + {"1e14", Quantity{dec(1, 14), DecimalExponent}}, + {"1e13", Quantity{dec(1, 13), DecimalExponent}}, + {"1e3", Quantity{dec(1, 3), DecimalExponent}}, + {"100.035k", Quantity{dec(100035, 0), DecimalSI}}, + + // Things that look like floating point + {"0.001", Quantity{dec(1, -3), DecimalSI}}, + {"0.0005", Quantity{dec(5, -4), DecimalSI}}, + {"0.005", Quantity{dec(5, -3), DecimalSI}}, + {"0.05", Quantity{dec(5, -2), DecimalSI}}, + {"0.5", Quantity{dec(5, -1), DecimalSI}}, + {"0.00050", Quantity{dec(5, -4), DecimalSI}}, + {"0.00500", Quantity{dec(5, -3), DecimalSI}}, + {"0.05000", Quantity{dec(5, -2), DecimalSI}}, + {"0.50000", Quantity{dec(5, -1), DecimalSI}}, + {"0.5e-3", Quantity{dec(5, -4), DecimalExponent}}, + {"0.5e-2", Quantity{dec(5, -3), DecimalExponent}}, + {"0.5e-1", Quantity{dec(5, -2), DecimalExponent}}, + {"0.5e0", Quantity{dec(5, -1), DecimalExponent}}, + {"10.035M", Quantity{dec(10035, 3), DecimalSI}}, + + {"1.1E-3", Quantity{dec(11, -4), DecimalExponent}}, + {"1.2e3", Quantity{dec(12, 2), DecimalExponent}}, + {"1.3E6", Quantity{dec(13, 5), DecimalExponent}}, + {"1.40e9", Quantity{dec(14, 8), DecimalExponent}}, + {"1.53E12", Quantity{dec(153, 10), DecimalExponent}}, + {"1.6e15", Quantity{dec(16, 14), DecimalExponent}}, + {"1.7E18", Quantity{dec(17, 17), DecimalExponent}}, + + {"3.001m", Quantity{dec(3001, -6), DecimalSI}}, + {"9.01", Quantity{dec(901, -2), DecimalSI}}, + {"8.1k", Quantity{dec(81, 2), DecimalSI}}, + {"7.123456M", Quantity{dec(7123456, 0), DecimalSI}}, + {"6.987654321G", Quantity{dec(6987654321, 0), DecimalSI}}, + {"5.444T", Quantity{dec(5444, 9), DecimalSI}}, + {"40.1T", Quantity{dec(401, 11), DecimalSI}}, + {"300.2T", Quantity{dec(3002, 11), DecimalSI}}, + {"2.5P", Quantity{dec(25, 14), DecimalSI}}, + {"1.01E", Quantity{dec(101, 16), DecimalSI}}, + + // Things that saturate + //{"0.1m", + //{"9Ei", + //{"9223372036854775807Ki", + //{"12E", + } + + for _, item := range table { + got, err := ParseQuantity(item.input) + if err != nil { + t.Errorf("%v: unexpected error: %v", item.input, err) + continue + } + if e, a := item.expect.Amount, got.Amount; e.Cmp(a) != 0 { + t.Errorf("%v: expected %v, got %v", item.input, e, a) + } + if e, a := item.expect.Format, got.Format; e != a { + t.Errorf("%v: expected %#v, got %#v", item.input, e, a) + } + } + + invalid := []string{ + "1.1.M", + "1+1.0M", + "0.1mi", + "0.1am", + "0.0001i", + "100.035Ki", + "0.5Mi", + "0.05Gi", + "0.5Ti", + "0.005i", + "0.05i", + "0.5i", + "aoeu", + } + for _, item := range invalid { + _, err := ParseQuantity(item) + if err == nil { + t.Errorf("%v parsed unexpectedly", item) + } + } +} + +func TestQuantityString(t *testing.T) { + table := []struct { + in Quantity + expect string + }{ + {Quantity{dec(1024*1024*1024, 0), BinarySI}, "1Gi"}, + {Quantity{dec(300*1024*1024, 0), BinarySI}, "300Mi"}, + {Quantity{dec(6*1024, 0), BinarySI}, "6Ki"}, + {Quantity{dec(1001*1024*1024*1024, 0), BinarySI}, "1001Gi"}, + {Quantity{dec(1024*1024*1024*1024, 0), BinarySI}, "1Ti"}, + {Quantity{dec(5, 0), BinarySI}, "5i"}, + {Quantity{dec(1, 9), DecimalSI}, "1G"}, + {Quantity{dec(1000, 6), DecimalSI}, "1G"}, + {Quantity{dec(1000000, 3), DecimalSI}, "1G"}, + {Quantity{dec(1000000000, 0), DecimalSI}, "1G"}, + {Quantity{dec(1, -3), DecimalSI}, "1m"}, + {Quantity{dec(80, -3), DecimalSI}, "80m"}, + {Quantity{dec(1080, -3), DecimalSI}, "1.080"}, + {Quantity{dec(108, -2), DecimalSI}, "1.080"}, + {Quantity{dec(10800, -4), DecimalSI}, "1.080"}, + {Quantity{dec(300, 6), DecimalSI}, "300M"}, + {Quantity{dec(1, 12), DecimalSI}, "1T"}, + {Quantity{dec(3, 3), DecimalSI}, "3k"}, + {Quantity{dec(0, 0), DecimalSI}, "0"}, + {Quantity{dec(0, 0), BinarySI}, "0i"}, + {Quantity{dec(1, 9), DecimalExponent}, "1e9"}, + {Quantity{dec(1, -3), DecimalExponent}, "1e-3"}, + {Quantity{dec(80, -3), DecimalExponent}, "80e-3"}, + {Quantity{dec(300, 6), DecimalExponent}, "300e6"}, + {Quantity{dec(1, 12), DecimalExponent}, "1e12"}, + {Quantity{dec(1, 3), DecimalExponent}, "1e3"}, + {Quantity{dec(3, 3), DecimalExponent}, "3e3"}, + {Quantity{dec(3, 3), DecimalSI}, "3k"}, + {Quantity{dec(0, 0), DecimalExponent}, "0"}, + + {Quantity{dec(-1080, -3), DecimalSI}, "-1.080"}, + {Quantity{dec(-80*1024, 0), BinarySI}, "-80Ki"}, + } + for _, item := range table { + got := item.in.String() + if e, a := item.expect, got; e != a { + t.Errorf("%#v: expected %v, got %v", item.in, e, a) + } + } +} diff --git a/pkg/api/resource/suffix.go b/pkg/api/resource/suffix.go new file mode 100644 index 00000000000..818df999d50 --- /dev/null +++ b/pkg/api/resource/suffix.go @@ -0,0 +1,132 @@ +/* +Copyright 2014 Google Inc. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package resource + +import ( + "strconv" +) + +type suffix string + +// suffixer can interpret and construct suffixes. +type suffixer interface { + interpret(suffix) (base, exponent int, fmt Format, ok bool) + construct(base, exponent int, fmt Format) (s suffix, ok bool) +} + +// quantitySuffixer handles suffixes for all three formats that quantity +// can handle. +var quantitySuffixer = newSuffixer() + +type bePair struct { + base, exponent int +} + +type listSuffixer struct { + suffixToBE map[suffix]bePair + beToSuffix map[bePair]suffix +} + +func (ls *listSuffixer) addSuffix(s suffix, pair bePair) { + if ls.suffixToBE == nil { + ls.suffixToBE = map[suffix]bePair{} + } + if ls.beToSuffix == nil { + ls.beToSuffix = map[bePair]suffix{} + } + ls.suffixToBE[s] = pair + ls.beToSuffix[pair] = s +} + +func (ls *listSuffixer) lookup(s suffix) (base, exponent int, ok bool) { + pair, ok := ls.suffixToBE[s] + if !ok { + return 0, 0, false + } + return pair.base, pair.exponent, true +} + +func (ls *listSuffixer) construct(base, exponent int) (s suffix, ok bool) { + s, ok = ls.beToSuffix[bePair{base, exponent}] + return +} + +type suffixHandler struct { + decSuffixes listSuffixer + binSuffixes listSuffixer +} + +func newSuffixer() suffixer { + sh := &suffixHandler{} + + sh.binSuffixes.addSuffix("i", bePair{2, 0}) + sh.binSuffixes.addSuffix("Ki", bePair{2, 10}) + sh.binSuffixes.addSuffix("Mi", bePair{2, 20}) + sh.binSuffixes.addSuffix("Gi", bePair{2, 30}) + sh.binSuffixes.addSuffix("Ti", bePair{2, 40}) + sh.binSuffixes.addSuffix("Pi", bePair{2, 50}) + sh.binSuffixes.addSuffix("Ei", bePair{2, 60}) + + sh.decSuffixes.addSuffix("m", bePair{10, -3}) + sh.decSuffixes.addSuffix("", bePair{10, 0}) + sh.decSuffixes.addSuffix("k", bePair{10, 3}) + sh.decSuffixes.addSuffix("M", bePair{10, 6}) + sh.decSuffixes.addSuffix("G", bePair{10, 9}) + sh.decSuffixes.addSuffix("T", bePair{10, 12}) + sh.decSuffixes.addSuffix("P", bePair{10, 15}) + sh.decSuffixes.addSuffix("E", bePair{10, 18}) + + return sh +} + +func (sh *suffixHandler) construct(base, exponent int, fmt Format) (s suffix, ok bool) { + switch fmt { + case DecimalSI: + return sh.decSuffixes.construct(base, exponent) + case BinarySI: + return sh.binSuffixes.construct(base, exponent) + case DecimalExponent: + if base != 10 { + return "", false + } + if exponent == 0 { + return "", true + } + return suffix("e" + strconv.FormatInt(int64(exponent), 10)), true + } + return "", false +} + +func (sh *suffixHandler) interpret(suffix suffix) (base, exponent int, fmt Format, ok bool) { + // Try lookup tables first + if b, e, ok := sh.decSuffixes.lookup(suffix); ok { + return b, e, DecimalSI, true + } + if b, e, ok := sh.binSuffixes.lookup(suffix); ok { + return b, e, BinarySI, true + } + + if len(suffix) > 1 && suffix[0] == 'E' || suffix[0] == 'e' { + parsed, err := strconv.ParseInt(string(suffix[1:]), 10, 64) + if err != nil { + return 0, 0, DecimalExponent, false + } + return 10, int(parsed), DecimalExponent, true + } + + return 0, 0, DecimalExponent, false +}