mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-08-16 07:13:53 +00:00
Improve precision of Quantity.AsApproximateFloat64
This improves the precision of Quantity.AsApproximateFloat64, by way of example: decQuantity(7*1024*1024, -1, BinarySI) Before: 734003.2000000001, After: 734003.2 Co-Authored-By: Bo Sunesen <7479047+bosunesen@users.noreply.github.com>
This commit is contained in:
parent
a7702cb47b
commit
75ba5a9651
@ -20,7 +20,6 @@ import (
|
|||||||
"bytes"
|
"bytes"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"math"
|
|
||||||
"math/big"
|
"math/big"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
@ -464,20 +463,29 @@ func (q *Quantity) CanonicalizeBytes(out []byte) (result, suffix []byte) {
|
|||||||
// lose precision. If the value of the quantity is outside the range of a float64
|
// lose precision. If the value of the quantity is outside the range of a float64
|
||||||
// +Inf/-Inf will be returned.
|
// +Inf/-Inf will be returned.
|
||||||
func (q *Quantity) AsApproximateFloat64() float64 {
|
func (q *Quantity) AsApproximateFloat64() float64 {
|
||||||
var base float64
|
infDec := q.AsDec()
|
||||||
var exponent int
|
|
||||||
if q.d.Dec != nil {
|
var absScale int64
|
||||||
base, _ = big.NewFloat(0).SetInt(q.d.Dec.UnscaledBig()).Float64()
|
if infDec.Scale() < 0 {
|
||||||
exponent = int(-q.d.Dec.Scale())
|
absScale = int64(-infDec.Scale())
|
||||||
} else {
|
} else {
|
||||||
base = float64(q.i.value)
|
absScale = int64(infDec.Scale())
|
||||||
exponent = int(q.i.scale)
|
|
||||||
}
|
}
|
||||||
if exponent == 0 {
|
pow10AbsScale := big.NewInt(10)
|
||||||
return base
|
pow10AbsScale = pow10AbsScale.Exp(pow10AbsScale, big.NewInt(absScale), nil)
|
||||||
|
|
||||||
|
var resultBigFloat *big.Float
|
||||||
|
if infDec.Scale() < 0 {
|
||||||
|
resultBigInt := new(big.Int).Mul(infDec.UnscaledBig(), pow10AbsScale)
|
||||||
|
resultBigFloat = new(big.Float).SetInt(resultBigInt)
|
||||||
|
} else {
|
||||||
|
pow10AbsScaleFloat := new(big.Float).SetInt(pow10AbsScale)
|
||||||
|
resultBigFloat = new(big.Float).SetInt(infDec.UnscaledBig())
|
||||||
|
resultBigFloat = resultBigFloat.Quo(resultBigFloat, pow10AbsScaleFloat)
|
||||||
}
|
}
|
||||||
|
|
||||||
return base * math.Pow10(exponent)
|
result, _ := resultBigFloat.Float64()
|
||||||
|
return result
|
||||||
}
|
}
|
||||||
|
|
||||||
// AsInt64 returns a representation of the current value as an int64 if a fast conversion
|
// AsInt64 returns a representation of the current value as an int64 if a fast conversion
|
||||||
|
@ -1313,7 +1313,7 @@ func TestQuantityAsApproximateFloat64(t *testing.T) {
|
|||||||
{decQuantity(7*1024*1024, 1, BinarySI), (7 * 1024 * 1024) * 10},
|
{decQuantity(7*1024*1024, 1, BinarySI), (7 * 1024 * 1024) * 10},
|
||||||
{decQuantity(7*1024*1024, 4, BinarySI), (7 * 1024 * 1024) * 10000},
|
{decQuantity(7*1024*1024, 4, BinarySI), (7 * 1024 * 1024) * 10000},
|
||||||
{decQuantity(7*1024*1024, 8, BinarySI), (7 * 1024 * 1024) * 100000000},
|
{decQuantity(7*1024*1024, 8, BinarySI), (7 * 1024 * 1024) * 100000000},
|
||||||
{decQuantity(7*1024*1024, -1, BinarySI), (7 * 1024 * 1024) * math.Pow10(-1)}, // '* Pow10' and '/ float(10)' do not round the same way
|
{decQuantity(7*1024*1024, -1, BinarySI), (7 * 1024 * 1024) / float64(10)},
|
||||||
{decQuantity(7*1024*1024, -8, BinarySI), (7 * 1024 * 1024) / float64(100000000)},
|
{decQuantity(7*1024*1024, -8, BinarySI), (7 * 1024 * 1024) / float64(100000000)},
|
||||||
|
|
||||||
{decQuantity(1024, 0, DecimalSI), 1024},
|
{decQuantity(1024, 0, DecimalSI), 1024},
|
||||||
@ -1322,7 +1322,7 @@ func TestQuantityAsApproximateFloat64(t *testing.T) {
|
|||||||
{decQuantity(7*1024*1024, 1, DecimalSI), (7 * 1024 * 1024) * 10},
|
{decQuantity(7*1024*1024, 1, DecimalSI), (7 * 1024 * 1024) * 10},
|
||||||
{decQuantity(7*1024*1024, 4, DecimalSI), (7 * 1024 * 1024) * 10000},
|
{decQuantity(7*1024*1024, 4, DecimalSI), (7 * 1024 * 1024) * 10000},
|
||||||
{decQuantity(7*1024*1024, 8, DecimalSI), (7 * 1024 * 1024) * 100000000},
|
{decQuantity(7*1024*1024, 8, DecimalSI), (7 * 1024 * 1024) * 100000000},
|
||||||
{decQuantity(7*1024*1024, -1, DecimalSI), (7 * 1024 * 1024) * math.Pow10(-1)}, // '* Pow10' and '/ float(10)' do not round the same way
|
{decQuantity(7*1024*1024, -1, DecimalSI), (7 * 1024 * 1024) / float64(10)},
|
||||||
{decQuantity(7*1024*1024, -8, DecimalSI), (7 * 1024 * 1024) / float64(100000000)},
|
{decQuantity(7*1024*1024, -8, DecimalSI), (7 * 1024 * 1024) / float64(100000000)},
|
||||||
|
|
||||||
{decQuantity(1024, 0, DecimalExponent), 1024},
|
{decQuantity(1024, 0, DecimalExponent), 1024},
|
||||||
@ -1331,7 +1331,7 @@ func TestQuantityAsApproximateFloat64(t *testing.T) {
|
|||||||
{decQuantity(7*1024*1024, 1, DecimalExponent), (7 * 1024 * 1024) * 10},
|
{decQuantity(7*1024*1024, 1, DecimalExponent), (7 * 1024 * 1024) * 10},
|
||||||
{decQuantity(7*1024*1024, 4, DecimalExponent), (7 * 1024 * 1024) * 10000},
|
{decQuantity(7*1024*1024, 4, DecimalExponent), (7 * 1024 * 1024) * 10000},
|
||||||
{decQuantity(7*1024*1024, 8, DecimalExponent), (7 * 1024 * 1024) * 100000000},
|
{decQuantity(7*1024*1024, 8, DecimalExponent), (7 * 1024 * 1024) * 100000000},
|
||||||
{decQuantity(7*1024*1024, -1, DecimalExponent), (7 * 1024 * 1024) * math.Pow10(-1)}, // '* Pow10' and '/ float(10)' do not round the same way
|
{decQuantity(7*1024*1024, -1, DecimalExponent), (7 * 1024 * 1024) / float64(10)},
|
||||||
{decQuantity(7*1024*1024, -8, DecimalExponent), (7 * 1024 * 1024) / float64(100000000)},
|
{decQuantity(7*1024*1024, -8, DecimalExponent), (7 * 1024 * 1024) / float64(100000000)},
|
||||||
|
|
||||||
// very large numbers
|
// very large numbers
|
||||||
@ -1344,11 +1344,11 @@ func TestQuantityAsApproximateFloat64(t *testing.T) {
|
|||||||
{decQuantity(-12, 500, DecimalSI), math.Inf(-1)},
|
{decQuantity(-12, 500, DecimalSI), math.Inf(-1)},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, item := range table {
|
for i, item := range table {
|
||||||
t.Run(fmt.Sprintf("%s %s", item.in.Format, item.in.String()), func(t *testing.T) {
|
t.Run(fmt.Sprintf("%s %s", item.in.Format, item.in.String()), func(t *testing.T) {
|
||||||
out := item.in.AsApproximateFloat64()
|
out := item.in.AsApproximateFloat64()
|
||||||
if out != item.out {
|
if out != item.out {
|
||||||
t.Fatalf("expected %v, got %v", item.out, out)
|
t.Fatalf("test %d expected %v, got %v", i+1, item.out, out)
|
||||||
}
|
}
|
||||||
if item.in.d.Dec != nil {
|
if item.in.d.Dec != nil {
|
||||||
if i, ok := item.in.AsInt64(); ok {
|
if i, ok := item.in.AsInt64(); ok {
|
||||||
|
Loading…
Reference in New Issue
Block a user