Improve precision of Quantity.AsApproximateFloat64

This improves the precision of Quantity.AsApproximateFloat64, by way of example:

decQuantity(7*1024*1024, -1, BinarySI)
Before: 734003.2000000001,
After: 734003.2

Co-Authored-By: Bo Sunesen <7479047+bosunesen@users.noreply.github.com>
This commit is contained in:
Lucy Sweet 2023-08-11 16:28:31 +02:00 committed by Tim Hockin
parent a7702cb47b
commit 75ba5a9651
No known key found for this signature in database
2 changed files with 24 additions and 16 deletions

View File

@ -20,7 +20,6 @@ import (
"bytes" "bytes"
"errors" "errors"
"fmt" "fmt"
"math"
"math/big" "math/big"
"strconv" "strconv"
"strings" "strings"
@ -464,20 +463,29 @@ func (q *Quantity) CanonicalizeBytes(out []byte) (result, suffix []byte) {
// lose precision. If the value of the quantity is outside the range of a float64 // lose precision. If the value of the quantity is outside the range of a float64
// +Inf/-Inf will be returned. // +Inf/-Inf will be returned.
func (q *Quantity) AsApproximateFloat64() float64 { func (q *Quantity) AsApproximateFloat64() float64 {
var base float64 infDec := q.AsDec()
var exponent int
if q.d.Dec != nil { var absScale int64
base, _ = big.NewFloat(0).SetInt(q.d.Dec.UnscaledBig()).Float64() if infDec.Scale() < 0 {
exponent = int(-q.d.Dec.Scale()) absScale = int64(-infDec.Scale())
} else { } else {
base = float64(q.i.value) absScale = int64(infDec.Scale())
exponent = int(q.i.scale)
} }
if exponent == 0 { pow10AbsScale := big.NewInt(10)
return base pow10AbsScale = pow10AbsScale.Exp(pow10AbsScale, big.NewInt(absScale), nil)
var resultBigFloat *big.Float
if infDec.Scale() < 0 {
resultBigInt := new(big.Int).Mul(infDec.UnscaledBig(), pow10AbsScale)
resultBigFloat = new(big.Float).SetInt(resultBigInt)
} else {
pow10AbsScaleFloat := new(big.Float).SetInt(pow10AbsScale)
resultBigFloat = new(big.Float).SetInt(infDec.UnscaledBig())
resultBigFloat = resultBigFloat.Quo(resultBigFloat, pow10AbsScaleFloat)
} }
return base * math.Pow10(exponent) result, _ := resultBigFloat.Float64()
return result
} }
// AsInt64 returns a representation of the current value as an int64 if a fast conversion // AsInt64 returns a representation of the current value as an int64 if a fast conversion

View File

@ -1313,7 +1313,7 @@ func TestQuantityAsApproximateFloat64(t *testing.T) {
{decQuantity(7*1024*1024, 1, BinarySI), (7 * 1024 * 1024) * 10}, {decQuantity(7*1024*1024, 1, BinarySI), (7 * 1024 * 1024) * 10},
{decQuantity(7*1024*1024, 4, BinarySI), (7 * 1024 * 1024) * 10000}, {decQuantity(7*1024*1024, 4, BinarySI), (7 * 1024 * 1024) * 10000},
{decQuantity(7*1024*1024, 8, BinarySI), (7 * 1024 * 1024) * 100000000}, {decQuantity(7*1024*1024, 8, BinarySI), (7 * 1024 * 1024) * 100000000},
{decQuantity(7*1024*1024, -1, BinarySI), (7 * 1024 * 1024) * math.Pow10(-1)}, // '* Pow10' and '/ float(10)' do not round the same way {decQuantity(7*1024*1024, -1, BinarySI), (7 * 1024 * 1024) / float64(10)},
{decQuantity(7*1024*1024, -8, BinarySI), (7 * 1024 * 1024) / float64(100000000)}, {decQuantity(7*1024*1024, -8, BinarySI), (7 * 1024 * 1024) / float64(100000000)},
{decQuantity(1024, 0, DecimalSI), 1024}, {decQuantity(1024, 0, DecimalSI), 1024},
@ -1322,7 +1322,7 @@ func TestQuantityAsApproximateFloat64(t *testing.T) {
{decQuantity(7*1024*1024, 1, DecimalSI), (7 * 1024 * 1024) * 10}, {decQuantity(7*1024*1024, 1, DecimalSI), (7 * 1024 * 1024) * 10},
{decQuantity(7*1024*1024, 4, DecimalSI), (7 * 1024 * 1024) * 10000}, {decQuantity(7*1024*1024, 4, DecimalSI), (7 * 1024 * 1024) * 10000},
{decQuantity(7*1024*1024, 8, DecimalSI), (7 * 1024 * 1024) * 100000000}, {decQuantity(7*1024*1024, 8, DecimalSI), (7 * 1024 * 1024) * 100000000},
{decQuantity(7*1024*1024, -1, DecimalSI), (7 * 1024 * 1024) * math.Pow10(-1)}, // '* Pow10' and '/ float(10)' do not round the same way {decQuantity(7*1024*1024, -1, DecimalSI), (7 * 1024 * 1024) / float64(10)},
{decQuantity(7*1024*1024, -8, DecimalSI), (7 * 1024 * 1024) / float64(100000000)}, {decQuantity(7*1024*1024, -8, DecimalSI), (7 * 1024 * 1024) / float64(100000000)},
{decQuantity(1024, 0, DecimalExponent), 1024}, {decQuantity(1024, 0, DecimalExponent), 1024},
@ -1331,7 +1331,7 @@ func TestQuantityAsApproximateFloat64(t *testing.T) {
{decQuantity(7*1024*1024, 1, DecimalExponent), (7 * 1024 * 1024) * 10}, {decQuantity(7*1024*1024, 1, DecimalExponent), (7 * 1024 * 1024) * 10},
{decQuantity(7*1024*1024, 4, DecimalExponent), (7 * 1024 * 1024) * 10000}, {decQuantity(7*1024*1024, 4, DecimalExponent), (7 * 1024 * 1024) * 10000},
{decQuantity(7*1024*1024, 8, DecimalExponent), (7 * 1024 * 1024) * 100000000}, {decQuantity(7*1024*1024, 8, DecimalExponent), (7 * 1024 * 1024) * 100000000},
{decQuantity(7*1024*1024, -1, DecimalExponent), (7 * 1024 * 1024) * math.Pow10(-1)}, // '* Pow10' and '/ float(10)' do not round the same way {decQuantity(7*1024*1024, -1, DecimalExponent), (7 * 1024 * 1024) / float64(10)},
{decQuantity(7*1024*1024, -8, DecimalExponent), (7 * 1024 * 1024) / float64(100000000)}, {decQuantity(7*1024*1024, -8, DecimalExponent), (7 * 1024 * 1024) / float64(100000000)},
// very large numbers // very large numbers
@ -1344,11 +1344,11 @@ func TestQuantityAsApproximateFloat64(t *testing.T) {
{decQuantity(-12, 500, DecimalSI), math.Inf(-1)}, {decQuantity(-12, 500, DecimalSI), math.Inf(-1)},
} }
for _, item := range table { for i, item := range table {
t.Run(fmt.Sprintf("%s %s", item.in.Format, item.in.String()), func(t *testing.T) { t.Run(fmt.Sprintf("%s %s", item.in.Format, item.in.String()), func(t *testing.T) {
out := item.in.AsApproximateFloat64() out := item.in.AsApproximateFloat64()
if out != item.out { if out != item.out {
t.Fatalf("expected %v, got %v", item.out, out) t.Fatalf("test %d expected %v, got %v", i+1, item.out, out)
} }
if item.in.d.Dec != nil { if item.in.d.Dec != nil {
if i, ok := item.in.AsInt64(); ok { if i, ok := item.in.AsInt64(); ok {