added image partitioning support

This commit is contained in:
Giovanni Bartolomeo 2024-09-24 15:09:05 +02:00
parent f0bd0f6899
commit 17fada712d
511 changed files with 37085 additions and 21383 deletions

BIN
.DS_Store vendored Normal file

Binary file not shown.

2
buildnrun.sh Executable file
View File

@ -0,0 +1,2 @@
sudo docker build -t distribution/distribution:edge .
sudo docker run -d -p 10500:5000 -v $PWD/FS/PATH:/var/lib/registry --restart always --name registry distribution/distribution:edge

64
go.mod
View File

@ -3,8 +3,8 @@ module github.com/distribution/distribution/v3
go 1.22.5
require (
cloud.google.com/go/storage v1.30.1
github.com/AdaLogics/go-fuzz-headers v0.0.0-20221103172237-443f56ff4ba8
cloud.google.com/go/storage v1.39.1
github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.6.0
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0
@ -19,51 +19,56 @@ require (
github.com/gorilla/handlers v1.5.2
github.com/gorilla/mux v1.8.1
github.com/hashicorp/golang-lru/arc/v2 v2.0.5
github.com/klauspost/compress v1.17.4
github.com/klauspost/compress v1.17.8
github.com/mitchellh/mapstructure v1.5.0
github.com/opencontainers/go-digest v1.0.0
github.com/opencontainers/image-spec v1.1.0
github.com/redis/go-redis/extra/redisotel/v9 v9.0.5
github.com/redis/go-redis/v9 v9.1.0
github.com/sirupsen/logrus v1.9.3
github.com/spf13/cobra v1.8.0
github.com/spf13/cobra v1.8.1
github.com/stretchr/testify v1.9.0
go.opentelemetry.io/contrib/exporters/autoexport v0.46.1
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1
go.opentelemetry.io/otel v1.21.0
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.50.0
go.opentelemetry.io/otel v1.25.0
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.21.0
go.opentelemetry.io/otel/sdk v1.21.0
go.opentelemetry.io/otel/trace v1.21.0
go.opentelemetry.io/otel/sdk v1.25.0
go.opentelemetry.io/otel/trace v1.25.0
golang.org/x/crypto v0.24.0
golang.org/x/net v0.26.0
golang.org/x/oauth2 v0.11.0
google.golang.org/api v0.126.0
golang.org/x/oauth2 v0.18.0
google.golang.org/api v0.170.0
gopkg.in/yaml.v2 v2.4.0
)
require (
cloud.google.com/go v0.110.7 // indirect
cloud.google.com/go/compute v1.23.0 // indirect
github.com/giobart/2dfs-builder v0.0.1
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.50.0 // indirect
golang.org/x/time v0.5.0 // indirect
)
require (
cloud.google.com/go v0.112.2 // indirect
cloud.google.com/go/compute v1.25.1 // indirect
cloud.google.com/go/compute/metadata v0.2.3 // indirect
cloud.google.com/go/iam v1.1.1 // indirect
cloud.google.com/go/iam v1.1.7 // indirect
github.com/Azure/azure-sdk-for-go/sdk/internal v1.8.0 // indirect
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/cenkalti/backoff/v4 v4.2.1 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/cyphar/filepath-securejoin v0.2.4 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/go-logr/logr v1.3.0 // indirect
github.com/go-logr/logr v1.4.1 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/golang-jwt/jwt/v5 v5.2.1 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/google/s2a-go v0.1.4 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect
github.com/googleapis/gax-go/v2 v2.11.0 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect
github.com/golang/protobuf v1.5.4 // indirect
github.com/google/s2a-go v0.1.7 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
github.com/googleapis/gax-go/v2 v2.12.3 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 // indirect
github.com/hashicorp/golang-lru/v2 v2.0.5 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
@ -80,23 +85,22 @@ require (
go.opencensus.io v0.24.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.44.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.44.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.25.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.21.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.21.0 // indirect
go.opentelemetry.io/otel/exporters/prometheus v0.44.0 // indirect
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v0.44.0 // indirect
go.opentelemetry.io/otel/metric v1.21.0 // indirect
go.opentelemetry.io/otel/metric v1.25.0 // indirect
go.opentelemetry.io/otel/sdk/metric v1.21.0 // indirect
go.opentelemetry.io/proto/otlp v1.0.0 // indirect
go.opentelemetry.io/proto/otlp v1.2.0 // indirect
golang.org/x/sync v0.7.0
golang.org/x/sys v0.21.0 // indirect
golang.org/x/sys v0.22.0 // indirect
golang.org/x/text v0.16.0 // indirect
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect
google.golang.org/grpc v1.59.0 // indirect
google.golang.org/appengine v1.6.8 // indirect
google.golang.org/genproto v0.0.0-20240415180920-8c6c420018be // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20240415180920-8c6c420018be // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240415180920-8c6c420018be // indirect
google.golang.org/grpc v1.63.2 // indirect
google.golang.org/protobuf v1.33.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)

175
go.sum
View File

@ -1,17 +1,16 @@
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
cloud.google.com/go v0.110.7 h1:rJyC7nWRg2jWGZ4wSJ5nY65GTdYJkg0cd/uXb+ACI6o=
cloud.google.com/go v0.110.7/go.mod h1:+EYjdK8e5RME/VY/qLCAtuyALQ9q67dvuum8i+H5xsI=
cloud.google.com/go/compute v1.23.0 h1:tP41Zoavr8ptEqaW6j+LQOnyBBhO7OkOMAGrgLopTwY=
cloud.google.com/go/compute v1.23.0/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM=
cloud.google.com/go v0.112.2 h1:ZaGT6LiG7dBzi6zNOvVZwacaXlmf3lRqnC4DQzqyRQw=
cloud.google.com/go v0.112.2/go.mod h1:iEqjp//KquGIJV/m+Pk3xecgKNhV+ry+vVTsy4TbDms=
cloud.google.com/go/compute v1.25.1 h1:ZRpHJedLtTpKgr3RV1Fx23NuaAEN1Zfx9hw1u4aJdjU=
cloud.google.com/go/compute v1.25.1/go.mod h1:oopOIR53ly6viBYxaDhBfJwzUAxf1zE//uf3IB011ls=
cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY=
cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA=
cloud.google.com/go/iam v1.1.1 h1:lW7fzj15aVIXYHREOqjRBV9PsH0Z6u8Y46a1YGvQP4Y=
cloud.google.com/go/iam v1.1.1/go.mod h1:A5avdyVL2tCppe4unb0951eI9jreack+RJ0/d+KUZOU=
cloud.google.com/go/storage v1.30.1 h1:uOdMxAs8HExqBlnLtnQyP0YkvbiDpdGShGKtx6U/oNM=
cloud.google.com/go/storage v1.30.1/go.mod h1:NfxhC0UJE1aXSx7CIIbCf7y9HKT7BiccwkR7+P7gN8E=
github.com/AdaLogics/go-fuzz-headers v0.0.0-20221103172237-443f56ff4ba8 h1:d+pBUmsteW5tM87xmVXHZ4+LibHRFn40SPAoZJOg2ak=
github.com/AdaLogics/go-fuzz-headers v0.0.0-20221103172237-443f56ff4ba8/go.mod h1:i9fr2JpcEcY/IHEvzCM3qXUZYOQHgR89dt4es1CgMhc=
cloud.google.com/go/iam v1.1.7 h1:z4VHOhwKLF/+UYXAJDFwGtNF0b6gjsW1Pk9Ml0U/IoM=
cloud.google.com/go/iam v1.1.7/go.mod h1:J4PMPg8TtyurAUvSmPj8FF3EDgY1SPRZxcUGrn7WXGA=
cloud.google.com/go/storage v1.39.1 h1:MvraqHKhogCOTXTlct/9C3K3+Uy2jBmFYb3/Sp6dVtY=
cloud.google.com/go/storage v1.39.1/go.mod h1:xK6xZmxZmo+fyP7+DEF6FhNc24/JAe95OLyOHCXFH1o=
github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 h1:bvDV9vkmnHYOMsOr4WLk+Vo07yKIzd94sVoIqshQ4bU=
github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8=
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1 h1:E+OJmp2tPvt1W+amx48v1eqbjDYsgN+RzP4q16yV5eM=
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1/go.mod h1:a6xsAQUZg+VsS3TJ05SRp524Hs4pZ/AeFSr5ENf0Yjo=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.6.0 h1:U2rTu3Ef+7w9FHKIAXM6ZyqF3UOWJZ12zIm8zECAFfg=
@ -25,7 +24,6 @@ github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2/go.mod h1:wP83
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
github.com/aws/aws-sdk-go v1.48.10 h1:0LIFG3wp2Dt6PsxKWCg1Y1xRrn2vZnW5/gWdgaBalKg=
github.com/aws/aws-sdk-go v1.48.10/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
@ -42,23 +40,13 @@ github.com/bsm/gomega v1.26.0/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H
github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM=
github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI=
github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs=
github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/cyphar/filepath-securejoin v0.2.3/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4=
github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg=
github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4=
github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@ -73,20 +61,23 @@ github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHz
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/giobart/2dfs-builder v0.0.0-20240731100503-ec9efc17b420 h1:YyHyVFBzXP6oq9EVNzO3SDZuQBgjdDV706RsmHx1bQw=
github.com/giobart/2dfs-builder v0.0.0-20240731100503-ec9efc17b420/go.mod h1:Bq+a7chqJp+JtYOwRYa97hTMt2+bVHeltZ2FlNl/jaU=
github.com/giobart/2dfs-builder v0.0.0-20240917153712-1066c1c6b748 h1:rwQy+L6E0ru1lwJRXkyv7xcwPz6IEZtQXvJ4sG/kzCI=
github.com/giobart/2dfs-builder v0.0.0-20240917153712-1066c1c6b748/go.mod h1:Bq+a7chqJp+JtYOwRYa97hTMt2+bVHeltZ2FlNl/jaU=
github.com/giobart/2dfs-builder v0.0.1 h1:TC27nyVOY5BKIUhUY+LVyfGYcDAqk7k1+/yrKFxmHf0=
github.com/giobart/2dfs-builder v0.0.1/go.mod h1:Bq+a7chqJp+JtYOwRYa97hTMt2+bVHeltZ2FlNl/jaU=
github.com/go-jose/go-jose/v4 v4.0.2 h1:R3l3kkBds16bO7ZFAEEcofK0MkrAJt3jlJznWZG0nvk=
github.com/go-jose/go-jose/v4 v4.0.2/go.mod h1:WVf9LFMHh/QVrmqrOfqun0C45tMe3RoiKJMPvgWwLfY=
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY=
github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ=
github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
@ -95,8 +86,6 @@ github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a
github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk=
github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/glog v1.1.2 h1:DVjP2PbBOzHyzA+dn3WhHIq4NdVu3Q+pvivFICf/7fo=
github.com/golang/glog v1.1.2/go.mod h1:zR+okUeTbrL6EL3xHUDxZuEtGv04p5shwip1+mL/rLQ=
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@ -104,19 +93,17 @@ github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfb
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
@ -129,22 +116,21 @@ github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw=
github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk=
github.com/google/s2a-go v0.1.4 h1:1kZ/sQM3srePvKs3tXAvQzo66XfcReoqFpIpIccE7Oc=
github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A=
github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o=
github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw=
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/googleapis/enterprise-certificate-proxy v0.2.3 h1:yk9/cqRKtT9wXZSsRH9aurXEpJX+U6FLtpYTdC3R06k=
github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k=
github.com/googleapis/gax-go/v2 v2.11.0 h1:9V9PWXEsWnPpQhu/PeQIkS4eGzMlTLGgt80cUUI8Ki4=
github.com/googleapis/gax-go/v2 v2.11.0/go.mod h1:DxmR61SGKkGLa2xigwuZIQpkCI2S5iydzRfb3peWZJI=
github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs=
github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0=
github.com/googleapis/gax-go/v2 v2.12.3 h1:5/zPPDvw8Q1SuXjrqrZslrqT7dL/uJT2CQii/cLCKqA=
github.com/googleapis/gax-go/v2 v2.12.3/go.mod h1:AKloxT6GtNbaLm8QTNSidHUVsHYcBHwWRvkNFJUQcS4=
github.com/gorilla/handlers v1.5.2 h1:cLTUSsNkgcwhgRqvCNmdbRWG0A3N4F+M2nWKdScwyEE=
github.com/gorilla/handlers v1.5.2/go.mod h1:dX+xVpaxdSw+q0Qek8SSsl3dfMk3jNddUkMzo0GtH0w=
github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=
github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rHAxPBD8KFhJpmcqms=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 h1:/c3QmbOGMGTOumP2iT/rCwB7b0QDGLKzqOmktBjT+Is=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1/go.mod h1:5SN9VR2LTsRFsrEC6FHgRbTWrTHu6tqPeKxEQv15giM=
github.com/hashicorp/golang-lru/arc/v2 v2.0.5 h1:l2zaLDubNhW4XO3LnliVj0GXO3+/CGNJAg1dcN2Fpfw=
github.com/hashicorp/golang-lru/arc/v2 v2.0.5/go.mod h1:ny6zBSQZi2JxIeYcv7kt2sH2PXJtirBN7RDhRpxPkxU=
github.com/hashicorp/golang-lru/v2 v2.0.5 h1:wW7h1TG88eUIJ2i69gaE3uNVtEPIagzhGvHgwfx2Vm4=
@ -158,15 +144,12 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfC
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4=
github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM=
github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU=
github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
@ -216,16 +199,14 @@ github.com/redis/go-redis/extra/redisotel/v9 v9.0.5/go.mod h1:WZjPDy7VNzn77AAfnA
github.com/redis/go-redis/v9 v9.0.5/go.mod h1:WqMKv5vnQbRuZstUwxQI195wHy+t4PuXDOjzMvcuQHk=
github.com/redis/go-redis/v9 v9.1.0 h1:137FnGdk+EQdCbye1FW+qOEcY5S+SpY9T0NiuqvtfMY=
github.com/redis/go-redis/v9 v9.1.0/go.mod h1:urWj3He21Dj5k4TK1y59xH8Uj6ATueP8AH1cY3lZl4c=
github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0=
github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho=
github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM=
github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y=
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@ -234,7 +215,6 @@ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSS
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
@ -246,16 +226,18 @@ go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
go.opentelemetry.io/contrib/exporters/autoexport v0.46.1 h1:ysCfPZB9AjUlMa1UHYup3c9dAOCMQX/6sxSfPBUoxHw=
go.opentelemetry.io/contrib/exporters/autoexport v0.46.1/go.mod h1:ha0aiYm+DOPsLHjh0zoQ8W8sLT+LJ58J3j47lGpSLrU=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 h1:aFJWCqJMNjENlcleuuOkGAPH82y0yULBScfXcIEdS24=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1/go.mod h1:sEGXWArGqc3tVa+ekntsN65DmVbVeW+7lTKTjZF3/Fo=
go.opentelemetry.io/otel v1.21.0 h1:hzLeKBZEL7Okw2mGzZ0cc4k/A7Fta0uoPgaJCr8fsFc=
go.opentelemetry.io/otel v1.21.0/go.mod h1:QZzNPQPm1zLX4gZK4cMi+71eaorMSGT3A4znnUvNNEo=
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.50.0 h1:zvpPXY7RfYAGSdYQLjp6zxdJNSYD/+FFoCTQN9IPxBs=
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.50.0/go.mod h1:BMn8NB1vsxTljvuorms2hyOs8IBuuBEq0pl7ltOfy30=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.50.0 h1:cEPbyTSEHlQR89XVlyo78gqluF8Y3oMeBkXGWzQsfXY=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.50.0/go.mod h1:DKdbWcT4GH1D0Y3Sqt/PFXt2naRKDWtU+eE6oLdFNA8=
go.opentelemetry.io/otel v1.25.0 h1:gldB5FfhRl7OJQbUHt/8s0a7cE8fbsPAtdpRaApKy4k=
go.opentelemetry.io/otel v1.25.0/go.mod h1:Wa2ds5NOXEMkCmUou1WA7ZBfLTHWIsp034OVD7AO+Vg=
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.44.0 h1:jd0+5t/YynESZqsSyPz+7PAFdEop0dlN0+PkyHYo8oI=
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.44.0/go.mod h1:U707O40ee1FpQGyhvqnzmCJm1Wh6OX6GGBVn0E6Uyyk=
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.44.0 h1:bflGWrfYyuulcdxf14V6n9+CoQcu5SAAdHmDPAJnlps=
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.44.0/go.mod h1:qcTO4xHAxZLaLxPd60TdE88rxtItPHgHWqOhOGRr0as=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0 h1:cl5P5/GIfFh4t6xyruOgJP5QiA1pw4fYYdv6nc6CBWw=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0/go.mod h1:zgBdWWAu7oEEMC06MMKc5NLbA/1YDXV1sMpSqEeLQLg=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.25.0 h1:dT33yIHtmsqpixFsSQPwNeY5drM9wTcoL8h0FWF4oGM=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.25.0/go.mod h1:h95q0LBGh7hlAC08X2DhSeyIG02YQ0UyioTCVAqRPmc=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.21.0 h1:tIqheXEFWAZ7O8A7m+J0aPTmpJN3YQ7qetUAdkkkKpk=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.21.0/go.mod h1:nUeKExfxAQVbiVFn32YXpXZZHZ61Cc3s3Rn1pDBGAb0=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.21.0 h1:digkEZCJWobwBqMwC0cwCq8/wkkRy/OowZg5OArWZrM=
@ -266,24 +248,22 @@ go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v0.44.0 h1:dEZWPjVN22urgY
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v0.44.0/go.mod h1:sTt30Evb7hJB/gEk27qLb1+l9n4Tb8HvHkR0Wx3S6CU=
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.21.0 h1:VhlEQAPp9R1ktYfrPk5SOryw1e9LDDTZCbIPFrho0ec=
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.21.0/go.mod h1:kB3ufRbfU+CQ4MlUcqtW8Z7YEOBeK2DJ6CmR5rYYF3E=
go.opentelemetry.io/otel/metric v1.21.0 h1:tlYWfeo+Bocx5kLEloTjbcDwBuELRrIFxwdQ36PlJu4=
go.opentelemetry.io/otel/metric v1.21.0/go.mod h1:o1p3CA8nNHW8j5yuQLdc1eeqEaPfzug24uvsyIEJRWM=
go.opentelemetry.io/otel/sdk v1.21.0 h1:FTt8qirL1EysG6sTQRZ5TokkU8d0ugCj8htOgThZXQ8=
go.opentelemetry.io/otel/sdk v1.21.0/go.mod h1:Nna6Yv7PWTdgJHVRD9hIYywQBRx7pbox6nwBnZIxl/E=
go.opentelemetry.io/otel/metric v1.25.0 h1:LUKbS7ArpFL/I2jJHdJcqMGxkRdxpPHE0VU/D4NuEwA=
go.opentelemetry.io/otel/metric v1.25.0/go.mod h1:rkDLUSd2lC5lq2dFNrX9LGAbINP5B7WBkC78RXCpH5s=
go.opentelemetry.io/otel/sdk v1.25.0 h1:PDryEJPC8YJZQSyLY5eqLeafHtG+X7FWnf3aXMtxbqo=
go.opentelemetry.io/otel/sdk v1.25.0/go.mod h1:oFgzCM2zdsxKzz6zwpTZYLLQsFwc+K0daArPdIhuxkw=
go.opentelemetry.io/otel/sdk/metric v1.21.0 h1:smhI5oD714d6jHE6Tie36fPx4WDFIg+Y6RfAY4ICcR0=
go.opentelemetry.io/otel/sdk/metric v1.21.0/go.mod h1:FJ8RAsoPGv/wYMgBdUJXOm+6pzFY3YdljnXtv1SBE8Q=
go.opentelemetry.io/otel/trace v1.21.0 h1:WD9i5gzvoUPuXIXH24ZNBudiarZDKuekPqi/E8fpfLc=
go.opentelemetry.io/otel/trace v1.21.0/go.mod h1:LGbsEB0f9LGjN+OZaQQ26sohbOmiMR+BaslueVtS/qQ=
go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I=
go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM=
go.opentelemetry.io/otel/trace v1.25.0 h1:tqukZGLwQYRIFtSQM2u2+yfMVTgGVeqRLPUYx1Dq6RM=
go.opentelemetry.io/otel/trace v1.25.0/go.mod h1:hCCs70XM/ljO+BeQkyFnbK28SBIJ/Emuha+ccrCRT7I=
go.opentelemetry.io/proto/otlp v1.2.0 h1:pVeZGk7nXDC9O2hncA6nHldxEjm6LByfA2aN8IOkz94=
go.opentelemetry.io/proto/otlp v1.2.0/go.mod h1:gGpR8txAl5M03pDhMC79G6SdqNV26naRm/KDsgaHD8A=
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI=
golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@ -294,24 +274,19 @@ golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ=
golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.11.0 h1:vPL4xzxBM4niKCW6g9whtaWVXTJf1U5e4aZxxFx/gbU=
golang.org/x/oauth2 v0.11.0/go.mod h1:LdF7O/8bLR/qWK9DrpXmbHLTouvRHK0SgJl0GmDBchk=
golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI=
golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi8=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@ -325,29 +300,25 @@ golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5h
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws=
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
@ -357,35 +328,30 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk=
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
google.golang.org/api v0.126.0 h1:q4GJq+cAdMAC7XP7njvQ4tvohGLiSlytuL4BQxbIZ+o=
google.golang.org/api v0.126.0/go.mod h1:mBwVAtz+87bEN6CbA1GtZPDOqY2R5ONPqJeIlvyo4Aw=
golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU=
golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90=
google.golang.org/api v0.170.0 h1:zMaruDePM88zxZBG+NG8+reALO2rfLhe/JShitLyT48=
google.golang.org/api v0.170.0/go.mod h1:/xql9M2btF85xac/VAm4PsLMTLVGUOpq4BE9R8jyNy8=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM=
google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d h1:VBu5YqKPv6XiJ199exd8Br+Aetz+o08F+PLMnwJQHAY=
google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d/go.mod h1:yZTlhN0tQnXo3h00fuXNCxJdLdIdnVFVBaRJ5LWBbw4=
google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d h1:DoPTO70H+bcDXcd39vOqb2viZxgqeBeSGtZ55yZU4/Q=
google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d/go.mod h1:KjSP20unUpOx5kyQUFa7k4OJg0qeJ7DEZflGDu2p6Bk=
google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d h1:uvYuEyMHKNt+lT4K3bN6fGswmK8qSvcreM3BwjDh+y4=
google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M=
google.golang.org/genproto v0.0.0-20240415180920-8c6c420018be h1:g4aX8SUFA8V5F4LrSY5EclyGYw1OZN4HS1jTyjB9ZDc=
google.golang.org/genproto v0.0.0-20240415180920-8c6c420018be/go.mod h1:FeSdT5fk+lkxatqJP38MsUicGqHax5cLtmy/6TAuxO4=
google.golang.org/genproto/googleapis/api v0.0.0-20240415180920-8c6c420018be h1:Zz7rLWqp0ApfsR/l7+zSHhY3PMiH2xqgxlfYfAfNpoU=
google.golang.org/genproto/googleapis/api v0.0.0-20240415180920-8c6c420018be/go.mod h1:dvdCTIoAGbkWbcIKBniID56/7XHTt6WfxXNMxuziJ+w=
google.golang.org/genproto/googleapis/rpc v0.0.0-20240415180920-8c6c420018be h1:LG9vZxsWGOmUKieR8wPAUR3u3MpnYFQZROPIMaXh7/A=
google.golang.org/genproto/googleapis/rpc v0.0.0-20240415180920-8c6c420018be/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0=
google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ=
google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk=
google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98=
google.golang.org/grpc v1.63.2 h1:MUeiw1B2maTVZthpU5xvASfTh3LDbxHd6IJ6QQVU+xM=
google.golang.org/grpc v1.63.2/go.mod h1:WAX/8DgncnokcFUldAxq7GeB5DXHDbMF+lLvDomNkRA=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
@ -404,13 +370,10 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

View File

@ -129,11 +129,7 @@ func (m *DeserializedImageIndex) UnmarshalJSON(b []byte) error {
// MarshalJSON returns the contents of canonical. If canonical is empty,
// marshals the inner contents.
func (m *DeserializedImageIndex) MarshalJSON() ([]byte, error) {
if len(m.canonical) > 0 {
return m.canonical, nil
}
return nil, errors.New("JSON representation not initialized in DeserializedImageIndex")
return json.Marshal(m.ImageIndex)
}
// Payload returns the raw content of the manifest list. The contents can be

75
manifest/tdfs/manifest.go Normal file
View File

@ -0,0 +1,75 @@
package tdfs
import (
"fmt"
"github.com/distribution/distribution/v3"
tdfs "github.com/giobart/2dfs-builder/filesystem"
"github.com/opencontainers/go-digest"
)
const (
// MediaTypeForeignLayer is the mediaType used for layers that must be
// downloaded from foreign URLs.
MediaTypeTdfsLayer = "application/vnd.oci.image.layer.v1.2dfs.field"
)
const (
defaultSchemaVersion = 1
defaultMediaType = MediaTypeTdfsLayer
)
func init() {
if err := distribution.RegisterManifestSchema(defaultMediaType, unmarshalTdfs); err != nil {
panic(fmt.Sprintf("Unable to register manifest: %s", err))
}
}
func unmarshalTdfs(b []byte) (distribution.Manifest, distribution.Descriptor, error) {
m := &DeserializedTdfsManifest{}
field, err := tdfs.GetField().Unmarshal(string(b))
if err != nil {
return nil, distribution.Descriptor{}, err
}
m.canonical = b
m.MediaType = MediaTypeTdfsLayer
m.Field = field
return m, distribution.Descriptor{
Digest: digest.FromBytes(b),
Size: int64(len(b)),
MediaType: defaultMediaType,
}, nil
}
// Manifest defines a schema2 manifest.
type TdfsManifest struct {
MediaType string
Field tdfs.Field
}
// References returns the descriptors of this manifests references.
func (m TdfsManifest) References() []distribution.Descriptor {
return []distribution.Descriptor{}
}
// Target returns the target of this manifest.
func (m TdfsManifest) Target() distribution.Descriptor {
return distribution.Descriptor{}
}
// DeserializedManifest wraps Manifest with a copy of the original JSON.
// It satisfies the distribution.Manifest interface.
type DeserializedTdfsManifest struct {
TdfsManifest
// canonical is the canonical byte representation of the Manifest.
canonical []byte
}
// Payload returns the raw content of the manifest. The contents can be used to
// calculate the content identifier.
func (m DeserializedTdfsManifest) Payload() (string, []byte, error) {
return m.MediaType, m.canonical, nil
}

173
manifest/tdfs/tdfsutils.go Normal file
View File

@ -0,0 +1,173 @@
package tdfs
import (
"context"
"encoding/json"
"fmt"
"log"
"regexp"
"strconv"
"strings"
"github.com/opencontainers/go-digest"
v1 "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/distribution/distribution/v3"
"github.com/distribution/distribution/v3/manifest/ocischema"
tdfsfilesystem "github.com/giobart/2dfs-builder/filesystem"
)
type Partition struct {
x1 int
y1 int
x2 int
y2 int
}
const (
//semantic tag partition init char
partitionInit = `--`
//semantic tag partition split char
partitionSplitChar = `.`
//semantic partition regex patter
semanticTagPattern = partitionInit + `\d+\` + partitionSplitChar + `\d+\` + partitionSplitChar + `\d+\` + partitionSplitChar + `\d+`
)
// CheckTagPartitions checks if the tag contains semantic partitions and returns the tag and the partitions
func CheckTagPartitions(tag string) (string, []Partition) {
partitions := []Partition{}
onlyTag := tag
re := regexp.MustCompile(semanticTagPattern)
matches := re.FindAllString(tag, -1)
if len(matches) > 0 {
onlyTag = strings.Split(tag, partitionInit)[0]
//semantic tag with partition
log.Default().Printf("Semantic tag with partition detected %s\n", tag)
for _, match := range matches {
part, err := parsePartition(strings.Replace(match, partitionInit, "", -1))
if err != nil {
log.Default().Printf("[WARNING] Invalid partition %s, skipping...\n", match)
continue
}
partitions = append(partitions, part)
log.Default().Printf("[PARTITIONING...] Added partition %s \n", match)
}
}
return onlyTag, partitions
}
func parsePartition(p string) (Partition, error) {
parts := strings.Split(p, partitionSplitChar)
result := Partition{}
if len(parts) != 4 {
return result, fmt.Errorf("invalid partition %s", p)
}
var err error
result.x1, err = strconv.Atoi(parts[0])
if err != nil {
return result, err
}
result.y1, err = strconv.Atoi(parts[1])
if err != nil {
return result, err
}
result.x2, err = strconv.Atoi(parts[2])
if err != nil {
return result, err
}
result.y2, err = strconv.Atoi(parts[3])
if err != nil {
return result, err
}
return result, nil
}
func ConvertTdfsManifestToOciManifest(ctx context.Context, tdfsManifest *ocischema.DeserializedManifest, blobService distribution.BlobService, partitions []Partition) (distribution.Manifest, error) {
log.Default().Printf("Converting TDFS manifest to OCI manifest\n")
newLayers := []distribution.Descriptor{}
partitionAllotment := []tdfsfilesystem.Allotment{}
layerConfigBlob, err := blobService.Get(ctx, tdfsManifest.Config.Digest)
if err != nil {
log.Default().Printf("Error getting config %s\n", tdfsManifest.Config.Digest)
return nil, err
}
var config v1.Image = v1.Image{}
err = json.Unmarshal(layerConfigBlob, &config)
if err != nil {
log.Default().Printf("Error unmarshalling config %s\n", tdfsManifest.Config.Digest)
return nil, err
}
//select partitions
for _, layer := range tdfsManifest.Layers {
if layer.MediaType == MediaTypeTdfsLayer {
log.Default().Printf("Converting layer %s\n", layer.Digest)
layerContent, err := blobService.Get(ctx, layer.Digest)
if err != nil {
log.Default().Printf("Error getting layer %s\n", layer.Digest)
return nil, err
}
field, err := tdfsfilesystem.GetField().Unmarshal(string(layerContent))
if err != nil {
log.Default().Printf("Error unmarshalling layer %s\n", layer.Digest)
return nil, err
}
if len(partitionAllotment) == 0 {
if field != nil {
for allotment := range field.IterateAllotments() {
//skip empty allotments
if allotment.Digest == "" {
continue
}
for _, p := range partitions {
if allotment.Row >= p.x1 && allotment.Row <= p.x2 && allotment.Col >= p.y1 && allotment.Col <= p.y2 {
partitionAllotment = append(partitionAllotment, allotment)
//TODO remove duplicated
}
}
}
}
}
} else {
newLayers = append(newLayers, layer)
}
}
//create new layers
if len(partitionAllotment) > 0 {
//adding partitioned layers
for _, p := range partitionAllotment {
blob, err := blobService.Stat(ctx, digest.Digest(fmt.Sprintf("sha256:%s", p.Digest)))
if err != nil {
return nil, err
}
fmt.Printf("Partition %s [CREATING]\n", p.Digest)
newLayers = append(newLayers, distribution.Descriptor{
MediaType: "application/vnd.oci.image.layer.v1.tar+gzip",
Digest: digest.Digest(fmt.Sprintf("sha256:%s", p.Digest)),
Size: blob.Size,
})
config.RootFS.DiffIDs = append(config.RootFS.DiffIDs, digest.Digest(fmt.Sprintf("sha256:%s", p.DiffID)))
}
}
newConfig, err := json.Marshal(config)
if err != nil {
return nil, err
}
//create new manifest
manifestBuilder := ocischema.NewManifestBuilder(blobService, newConfig, tdfsManifest.Annotations)
manifestBuilder.SetMediaType(v1.MediaTypeImageManifest)
for _, layer := range newLayers {
manifestBuilder.AppendReference(layer)
}
return manifestBuilder.Build(ctx)
}
func ConvertPartitionedIndexToOciIndex(tdfsManifest *ocischema.DeserializedImageIndex) ([]byte, error) {
log.Default().Printf("Converting partitioned index to OCI index\n")
return tdfsManifest.MarshalJSON()
}

View File

@ -3,11 +3,14 @@ package handlers
import (
"bytes"
"fmt"
"log"
"mime"
"net/http"
"strings"
"sync"
"github.com/distribution/distribution/v3/manifest/tdfs"
"github.com/distribution/distribution/v3"
"github.com/distribution/distribution/v3/internal/dcontext"
"github.com/distribution/distribution/v3/manifest/manifestlist"
@ -73,14 +76,16 @@ type manifestHandler struct {
*Context
// One of tag or digest gets set, depending on what is present in context.
Tag string
Digest digest.Digest
Tag string
Partitions []tdfs.Partition
Digest digest.Digest
}
// GetManifest fetches the image manifest from the storage backend, if it exists.
func (imh *manifestHandler) GetManifest(w http.ResponseWriter, r *http.Request) {
dcontext.GetLogger(imh).Debug("GetImageManifest")
manifests, err := imh.Repository.Manifests(imh)
blobstore := imh.Repository.Blobs(imh)
if err != nil {
imh.Errors = append(imh.Errors, err)
return
@ -117,6 +122,14 @@ func (imh *manifestHandler) GetManifest(w http.ResponseWriter, r *http.Request)
if imh.Tag != "" {
tags := imh.Repository.Tags(imh)
// Remove semantical partitioning if one provided in the tag
tag, partitions := tdfs.CheckTagPartitions(imh.Tag)
if len(partitions) > 0 {
imh.Tag = tag
imh.Partitions = partitions
}
desc, err := tags.Get(imh, imh.Tag)
if err != nil {
if _, ok := err.(distribution.ErrTagUnknown); ok {
@ -127,6 +140,7 @@ func (imh *manifestHandler) GetManifest(w http.ResponseWriter, r *http.Request)
return
}
imh.Digest = desc.Digest
}
if etagMatch(r, imh.Digest.String()) {
@ -211,6 +225,57 @@ func (imh *manifestHandler) GetManifest(w http.ResponseWriter, r *http.Request)
return
}
// perform 2dfs partitioning if index requested and partitions provided
if partitionedOciManifest, ok := manifest.(*ocischema.DeserializedImageIndex); ok && len(imh.Partitions) > 0 {
log.Default().Printf("Partitioning index %s\n", imh.Digest)
//for each manifest in the index, partition it and upload it to the store
for i, manifestDescriptor := range partitionedOciManifest.Manifests {
submanifest, err := manifests.Get(imh, manifestDescriptor.Digest, options...)
if err != nil {
if _, ok := err.(distribution.ErrManifestUnknownRevision); ok {
imh.Errors = append(imh.Errors, errcode.ErrorCodeManifestUnknown.WithDetail(err))
} else {
imh.Errors = append(imh.Errors, errcode.ErrorCodeUnknown.WithDetail(err))
}
return
}
if ociSubManifest, isOci := submanifest.(*ocischema.DeserializedManifest); isOci {
partitioned, err := tdfs.ConvertTdfsManifestToOciManifest(imh, ociSubManifest, blobstore, imh.Partitions)
if err != nil {
imh.Errors = append(imh.Errors, errcode.ErrorCodeUnknown.WithDetail(err))
return
}
_, b, _ := partitioned.Payload()
//upload new menifest
digest, err := manifests.Put(imh, partitioned)
if err != nil {
imh.Errors = append(imh.Errors, errcode.ErrorCodeUnknown.WithDetail(err))
return
}
manifestDescriptor.Digest = digest
manifestDescriptor.Size = int64(len(b))
partitionedOciManifest.Manifests[i] = manifestDescriptor
}
}
imh.Digest = digest.FromBytes(p)
bytes, err := tdfs.ConvertPartitionedIndexToOciIndex(partitionedOciManifest)
if err != nil {
imh.Errors = append(imh.Errors, errcode.ErrorCodeUnknown.WithDetail(err))
return
}
//return the index with partitiond manifests
p = bytes
}
w.Header().Set("Content-Type", ct)
w.Header().Set("Content-Length", fmt.Sprint(len(p)))
w.Header().Set("Docker-Content-Digest", imh.Digest.String())

View File

@ -15,4 +15,4 @@
package internal
// Version is the current tagged release of the library.
const Version = "1.23.0"
const Version = "1.25.1"

View File

@ -1,6 +1,48 @@
# Changes
## [1.1.7](https://github.com/googleapis/google-cloud-go/compare/iam/v1.1.6...iam/v1.1.7) (2024-03-14)
### Bug Fixes
* **iam:** Update protobuf dep to v1.33.0 ([30b038d](https://github.com/googleapis/google-cloud-go/commit/30b038d8cac0b8cd5dd4761c87f3f298760dd33a))
## [1.1.6](https://github.com/googleapis/google-cloud-go/compare/iam/v1.1.5...iam/v1.1.6) (2024-01-30)
### Bug Fixes
* **iam:** Enable universe domain resolution options ([fd1d569](https://github.com/googleapis/google-cloud-go/commit/fd1d56930fa8a747be35a224611f4797b8aeb698))
## [1.1.5](https://github.com/googleapis/google-cloud-go/compare/iam/v1.1.4...iam/v1.1.5) (2023-11-01)
### Bug Fixes
* **iam:** Bump google.golang.org/api to v0.149.0 ([8d2ab9f](https://github.com/googleapis/google-cloud-go/commit/8d2ab9f320a86c1c0fab90513fc05861561d0880))
## [1.1.4](https://github.com/googleapis/google-cloud-go/compare/iam/v1.1.3...iam/v1.1.4) (2023-10-26)
### Bug Fixes
* **iam:** Update grpc-go to v1.59.0 ([81a97b0](https://github.com/googleapis/google-cloud-go/commit/81a97b06cb28b25432e4ece595c55a9857e960b7))
## [1.1.3](https://github.com/googleapis/google-cloud-go/compare/iam/v1.1.2...iam/v1.1.3) (2023-10-12)
### Bug Fixes
* **iam:** Update golang.org/x/net to v0.17.0 ([174da47](https://github.com/googleapis/google-cloud-go/commit/174da47254fefb12921bbfc65b7829a453af6f5d))
## [1.1.2](https://github.com/googleapis/google-cloud-go/compare/iam/v1.1.1...iam/v1.1.2) (2023-08-08)
### Documentation
* **iam:** Minor formatting ([b4349cc](https://github.com/googleapis/google-cloud-go/commit/b4349cc507870ff8629bbc07de578b63bb889626))
## [1.1.1](https://github.com/googleapis/google-cloud-go/compare/iam/v1.1.0...iam/v1.1.1) (2023-06-20)

View File

@ -14,8 +14,8 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.30.0
// protoc v4.23.2
// protoc-gen-go v1.32.0
// protoc v4.25.2
// source: google/iam/v1/iam_policy.proto
package iampb

View File

@ -14,8 +14,8 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.30.0
// protoc v4.23.2
// protoc-gen-go v1.32.0
// protoc v4.25.2
// source: google/iam/v1/options.proto
package iampb

View File

@ -1,4 +1,4 @@
// Copyright 2022 Google LLC
// Copyright 2023 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@ -14,8 +14,8 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.30.0
// protoc v4.23.2
// protoc-gen-go v1.32.0
// protoc v4.25.2
// source: google/iam/v1/policy.proto
package iampb
@ -219,6 +219,8 @@ func (AuditConfigDelta_Action) EnumDescriptor() ([]byte, []int) {
//
// **JSON example:**
//
// ```
//
// {
// "bindings": [
// {
@ -247,8 +249,12 @@ func (AuditConfigDelta_Action) EnumDescriptor() ([]byte, []int) {
// "version": 3
// }
//
// ```
//
// **YAML example:**
//
// ```
//
// bindings:
// - members:
// - user:mike@example.com
@ -266,6 +272,8 @@ func (AuditConfigDelta_Action) EnumDescriptor() ([]byte, []int) {
// etag: BwWWja0YfJA=
// version: 3
//
// ```
//
// For a description of IAM and its features, see the
// [IAM documentation](https://cloud.google.com/iam/docs/).
type Policy struct {
@ -281,11 +289,11 @@ type Policy struct {
// Any operation that affects conditional role bindings must specify version
// `3`. This requirement applies to the following operations:
//
// * Getting a policy that includes a conditional role binding
// * Adding a conditional role binding to a policy
// * Changing a conditional role binding in a policy
// * Removing any role binding, with or without a condition, from a policy
// that includes conditions
// - Getting a policy that includes a conditional role binding
// - Adding a conditional role binding to a policy
// - Changing a conditional role binding in a policy
// - Removing any role binding, with or without a condition, from a policy
// that includes conditions
//
// **Important:** If you use IAM Conditions, you must include the `etag` field
// whenever you call `setIamPolicy`. If you omit this field, then IAM allows
@ -396,50 +404,46 @@ type Binding struct {
// Role that is assigned to the list of `members`, or principals.
// For example, `roles/viewer`, `roles/editor`, or `roles/owner`.
Role string `protobuf:"bytes,1,opt,name=role,proto3" json:"role,omitempty"`
// Specifies the principals requesting access for a Cloud Platform resource.
// Specifies the principals requesting access for a Google Cloud resource.
// `members` can have the following values:
//
// * `allUsers`: A special identifier that represents anyone who is
// on the internet; with or without a Google account.
// - `allUsers`: A special identifier that represents anyone who is
// on the internet; with or without a Google account.
//
// * `allAuthenticatedUsers`: A special identifier that represents anyone
// who is authenticated with a Google account or a service account.
// - `allAuthenticatedUsers`: A special identifier that represents anyone
// who is authenticated with a Google account or a service account.
//
// * `user:{emailid}`: An email address that represents a specific Google
// account. For example, `alice@example.com` .
// - `user:{emailid}`: An email address that represents a specific Google
// account. For example, `alice@example.com` .
//
// - `serviceAccount:{emailid}`: An email address that represents a service
// account. For example, `my-other-app@appspot.gserviceaccount.com`.
//
// * `serviceAccount:{emailid}`: An email address that represents a service
// account. For example, `my-other-app@appspot.gserviceaccount.com`.
// - `group:{emailid}`: An email address that represents a Google group.
// For example, `admins@example.com`.
//
// * `group:{emailid}`: An email address that represents a Google group.
// For example, `admins@example.com`.
// - `deleted:user:{emailid}?uid={uniqueid}`: An email address (plus unique
// identifier) representing a user that has been recently deleted. For
// example, `alice@example.com?uid=123456789012345678901`. If the user is
// recovered, this value reverts to `user:{emailid}` and the recovered user
// retains the role in the binding.
//
// * `deleted:user:{emailid}?uid={uniqueid}`: An email address (plus unique
// identifier) representing a user that has been recently deleted. For
// example, `alice@example.com?uid=123456789012345678901`. If the user is
// recovered, this value reverts to `user:{emailid}` and the recovered user
// retains the role in the binding.
//
// * `deleted:serviceAccount:{emailid}?uid={uniqueid}`: An email address (plus
// unique identifier) representing a service account that has been recently
// deleted. For example,
// `my-other-app@appspot.gserviceaccount.com?uid=123456789012345678901`.
// If the service account is undeleted, this value reverts to
// `serviceAccount:{emailid}` and the undeleted service account retains the
// role in the binding.
//
// * `deleted:group:{emailid}?uid={uniqueid}`: An email address (plus unique
// identifier) representing a Google group that has been recently
// deleted. For example, `admins@example.com?uid=123456789012345678901`. If
// the group is recovered, this value reverts to `group:{emailid}` and the
// recovered group retains the role in the binding.
//
//
// * `domain:{domain}`: The G Suite domain (primary) that represents all the
// users of that domain. For example, `google.com` or `example.com`.
// - `deleted:serviceAccount:{emailid}?uid={uniqueid}`: An email address (plus
// unique identifier) representing a service account that has been recently
// deleted. For example,
// `my-other-app@appspot.gserviceaccount.com?uid=123456789012345678901`.
// If the service account is undeleted, this value reverts to
// `serviceAccount:{emailid}` and the undeleted service account retains the
// role in the binding.
//
// - `deleted:group:{emailid}?uid={uniqueid}`: An email address (plus unique
// identifier) representing a Google group that has been recently
// deleted. For example, `admins@example.com?uid=123456789012345678901`. If
// the group is recovered, this value reverts to `group:{emailid}` and the
// recovered group retains the role in the binding.
//
// - `domain:{domain}`: The G Suite domain (primary) that represents all the
// users of that domain. For example, `google.com` or `example.com`.
Members []string `protobuf:"bytes,2,rep,name=members,proto3" json:"members,omitempty"`
// The condition that is associated with this binding.
//
@ -558,8 +562,8 @@ func (x *Binding) GetCondition() *expr.Expr {
// }
//
// For sampleservice, this policy enables DATA_READ, DATA_WRITE and ADMIN_READ
// logging. It also exempts jose@example.com from DATA_READ logging, and
// aliya@example.com from DATA_WRITE logging.
// logging. It also exempts `jose@example.com` from DATA_READ logging, and
// `aliya@example.com` from DATA_WRITE logging.
type AuditConfig struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
@ -770,7 +774,7 @@ type BindingDelta struct {
// For example, `roles/viewer`, `roles/editor`, or `roles/owner`.
// Required
Role string `protobuf:"bytes,2,opt,name=role,proto3" json:"role,omitempty"`
// A single identity requesting access for a Cloud Platform resource.
// A single identity requesting access for a Google Cloud resource.
// Follows the same format of Binding.members.
// Required
Member string `protobuf:"bytes,3,opt,name=member,proto3" json:"member,omitempty"`

View File

@ -29,6 +29,26 @@
"release_level": "stable",
"library_type": "GAPIC_AUTO"
},
"cloud.google.com/go/ai/generativelanguage/apiv1": {
"api_shortname": "generativelanguage",
"distribution_name": "cloud.google.com/go/ai/generativelanguage/apiv1",
"description": "Generative Language API",
"language": "go",
"client_library_type": "generated",
"client_documentation": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/ai/latest/generativelanguage/apiv1",
"release_level": "preview",
"library_type": "GAPIC_AUTO"
},
"cloud.google.com/go/ai/generativelanguage/apiv1beta": {
"api_shortname": "generativelanguage",
"distribution_name": "cloud.google.com/go/ai/generativelanguage/apiv1beta",
"description": "Generative Language API",
"language": "go",
"client_library_type": "generated",
"client_documentation": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/ai/latest/generativelanguage/apiv1beta",
"release_level": "preview",
"library_type": "GAPIC_AUTO"
},
"cloud.google.com/go/ai/generativelanguage/apiv1beta2": {
"api_shortname": "generativelanguage",
"distribution_name": "cloud.google.com/go/ai/generativelanguage/apiv1beta2",
@ -89,6 +109,16 @@
"release_level": "preview",
"library_type": "GAPIC_AUTO"
},
"cloud.google.com/go/alloydb/connectors/apiv1": {
"api_shortname": "connectors",
"distribution_name": "cloud.google.com/go/alloydb/connectors/apiv1",
"description": "AlloyDB connectors",
"language": "go",
"client_library_type": "generated",
"client_documentation": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/alloydb/latest/connectors/apiv1",
"release_level": "preview",
"library_type": "GAPIC_AUTO"
},
"cloud.google.com/go/alloydb/connectors/apiv1alpha": {
"api_shortname": "connectors",
"distribution_name": "cloud.google.com/go/alloydb/connectors/apiv1alpha",
@ -99,6 +129,16 @@
"release_level": "preview",
"library_type": "GAPIC_AUTO"
},
"cloud.google.com/go/alloydb/connectors/apiv1beta": {
"api_shortname": "connectors",
"distribution_name": "cloud.google.com/go/alloydb/connectors/apiv1beta",
"description": "AlloyDB connectors",
"language": "go",
"client_library_type": "generated",
"client_documentation": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/alloydb/latest/connectors/apiv1beta",
"release_level": "preview",
"library_type": "GAPIC_AUTO"
},
"cloud.google.com/go/analytics/admin/apiv1alpha": {
"api_shortname": "analyticsadmin",
"distribution_name": "cloud.google.com/go/analytics/admin/apiv1alpha",
@ -136,7 +176,7 @@
"language": "go",
"client_library_type": "generated",
"client_documentation": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/apigeeregistry/latest/apiv1",
"release_level": "preview",
"release_level": "stable",
"library_type": "GAPIC_AUTO"
},
"cloud.google.com/go/apikeys/apiv2": {
@ -159,6 +199,46 @@
"release_level": "stable",
"library_type": "GAPIC_AUTO"
},
"cloud.google.com/go/apphub/apiv1": {
"api_shortname": "apphub",
"distribution_name": "cloud.google.com/go/apphub/apiv1",
"description": "App Hub API",
"language": "go",
"client_library_type": "generated",
"client_documentation": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/apphub/latest/apiv1",
"release_level": "preview",
"library_type": "GAPIC_AUTO"
},
"cloud.google.com/go/apps/events/subscriptions/apiv1": {
"api_shortname": "workspaceevents",
"distribution_name": "cloud.google.com/go/apps/events/subscriptions/apiv1",
"description": "Google Workspace Events API",
"language": "go",
"client_library_type": "generated",
"client_documentation": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/apps/latest/events/subscriptions/apiv1",
"release_level": "preview",
"library_type": "GAPIC_AUTO"
},
"cloud.google.com/go/apps/meet/apiv2": {
"api_shortname": "meet",
"distribution_name": "cloud.google.com/go/apps/meet/apiv2",
"description": "Google Meet API",
"language": "go",
"client_library_type": "generated",
"client_documentation": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/apps/latest/meet/apiv2",
"release_level": "preview",
"library_type": "GAPIC_AUTO"
},
"cloud.google.com/go/apps/meet/apiv2beta": {
"api_shortname": "meet",
"distribution_name": "cloud.google.com/go/apps/meet/apiv2beta",
"description": "Google Meet API",
"language": "go",
"client_library_type": "generated",
"client_documentation": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/apps/latest/meet/apiv2beta",
"release_level": "preview",
"library_type": "GAPIC_AUTO"
},
"cloud.google.com/go/area120/tables/apiv1alpha1": {
"api_shortname": "area120tables",
"distribution_name": "cloud.google.com/go/area120/tables/apiv1alpha1",
@ -349,6 +429,26 @@
"release_level": "stable",
"library_type": "GAPIC_AUTO"
},
"cloud.google.com/go/bigquery/biglake/apiv1": {
"api_shortname": "biglake",
"distribution_name": "cloud.google.com/go/bigquery/biglake/apiv1",
"description": "BigLake API",
"language": "go",
"client_library_type": "generated",
"client_documentation": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/bigquery/latest/biglake/apiv1",
"release_level": "stable",
"library_type": "GAPIC_AUTO"
},
"cloud.google.com/go/bigquery/biglake/apiv1alpha1": {
"api_shortname": "biglake",
"distribution_name": "cloud.google.com/go/bigquery/biglake/apiv1alpha1",
"description": "BigLake API",
"language": "go",
"client_library_type": "generated",
"client_documentation": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/bigquery/latest/biglake/apiv1alpha1",
"release_level": "preview",
"library_type": "GAPIC_AUTO"
},
"cloud.google.com/go/bigquery/connection/apiv1": {
"api_shortname": "bigqueryconnection",
"distribution_name": "cloud.google.com/go/bigquery/connection/apiv1",
@ -569,6 +669,26 @@
"release_level": "stable",
"library_type": "GAPIC_AUTO"
},
"cloud.google.com/go/cloudcontrolspartner/apiv1": {
"api_shortname": "cloudcontrolspartner",
"distribution_name": "cloud.google.com/go/cloudcontrolspartner/apiv1",
"description": "Cloud Controls Partner API",
"language": "go",
"client_library_type": "generated",
"client_documentation": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/cloudcontrolspartner/latest/apiv1",
"release_level": "preview",
"library_type": "GAPIC_AUTO"
},
"cloud.google.com/go/cloudcontrolspartner/apiv1beta": {
"api_shortname": "cloudcontrolspartner",
"distribution_name": "cloud.google.com/go/cloudcontrolspartner/apiv1beta",
"description": "Cloud Controls Partner API",
"language": "go",
"client_library_type": "generated",
"client_documentation": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/cloudcontrolspartner/latest/apiv1beta",
"release_level": "preview",
"library_type": "GAPIC_AUTO"
},
"cloud.google.com/go/clouddms/apiv1": {
"api_shortname": "datamigration",
"distribution_name": "cloud.google.com/go/clouddms/apiv1",
@ -579,6 +699,26 @@
"release_level": "stable",
"library_type": "GAPIC_AUTO"
},
"cloud.google.com/go/cloudprofiler/apiv2": {
"api_shortname": "cloudprofiler",
"distribution_name": "cloud.google.com/go/cloudprofiler/apiv2",
"description": "Cloud Profiler API",
"language": "go",
"client_library_type": "generated",
"client_documentation": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/cloudprofiler/latest/apiv2",
"release_level": "preview",
"library_type": "GAPIC_AUTO"
},
"cloud.google.com/go/cloudquotas/apiv1": {
"api_shortname": "cloudquotas",
"distribution_name": "cloud.google.com/go/cloudquotas/apiv1",
"description": "Cloud Quotas API",
"language": "go",
"client_library_type": "generated",
"client_documentation": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/cloudquotas/latest/apiv1",
"release_level": "preview",
"library_type": "GAPIC_AUTO"
},
"cloud.google.com/go/cloudtasks/apiv2": {
"api_shortname": "cloudtasks",
"distribution_name": "cloud.google.com/go/cloudtasks/apiv2",
@ -659,6 +799,16 @@
"release_level": "preview",
"library_type": "GAPIC_AUTO"
},
"cloud.google.com/go/config/apiv1": {
"api_shortname": "config",
"distribution_name": "cloud.google.com/go/config/apiv1",
"description": "Infrastructure Manager API",
"language": "go",
"client_library_type": "generated",
"client_documentation": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/config/latest/apiv1",
"release_level": "preview",
"library_type": "GAPIC_AUTO"
},
"cloud.google.com/go/contactcenterinsights/apiv1": {
"api_shortname": "contactcenterinsights",
"distribution_name": "cloud.google.com/go/contactcenterinsights/apiv1",
@ -685,7 +835,7 @@
"description": "Container Analysis API",
"language": "go",
"client_library_type": "generated",
"client_documentation": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/latest/containeranalysis/apiv1beta1",
"client_documentation": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/containeranalysis/latest/apiv1beta1",
"release_level": "preview",
"library_type": "GAPIC_AUTO"
},
@ -909,6 +1059,16 @@
"release_level": "stable",
"library_type": "GAPIC_AUTO"
},
"cloud.google.com/go/discoveryengine/apiv1alpha": {
"api_shortname": "discoveryengine",
"distribution_name": "cloud.google.com/go/discoveryengine/apiv1alpha",
"description": "Discovery Engine API",
"language": "go",
"client_library_type": "generated",
"client_documentation": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/discoveryengine/latest/apiv1alpha",
"release_level": "preview",
"library_type": "GAPIC_AUTO"
},
"cloud.google.com/go/discoveryengine/apiv1beta": {
"api_shortname": "discoveryengine",
"distribution_name": "cloud.google.com/go/discoveryengine/apiv1beta",
@ -922,7 +1082,7 @@
"cloud.google.com/go/dlp/apiv2": {
"api_shortname": "dlp",
"distribution_name": "cloud.google.com/go/dlp/apiv2",
"description": "Cloud Data Loss Prevention (DLP) API",
"description": "Sensitive Data Protection (DLP)",
"language": "go",
"client_library_type": "generated",
"client_documentation": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/dlp/latest/apiv2",
@ -969,6 +1129,16 @@
"release_level": "stable",
"library_type": "GAPIC_AUTO"
},
"cloud.google.com/go/edgenetwork/apiv1": {
"api_shortname": "edgenetwork",
"distribution_name": "cloud.google.com/go/edgenetwork/apiv1",
"description": "Distributed Cloud Edge Network API",
"language": "go",
"client_library_type": "generated",
"client_documentation": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/edgenetwork/latest/apiv1",
"release_level": "preview",
"library_type": "GAPIC_AUTO"
},
"cloud.google.com/go/errorreporting": {
"api_shortname": "clouderrorreporting",
"distribution_name": "cloud.google.com/go/errorreporting",
@ -1049,6 +1219,16 @@
"release_level": "stable",
"library_type": "GAPIC_AUTO"
},
"cloud.google.com/go/firestore/apiv1/admin": {
"api_shortname": "firestore",
"distribution_name": "cloud.google.com/go/firestore/apiv1/admin",
"description": "Cloud Firestore API",
"language": "go",
"client_library_type": "generated",
"client_documentation": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/firestore/latest/apiv1/admin",
"release_level": "stable",
"library_type": "GAPIC_AUTO"
},
"cloud.google.com/go/functions/apiv1": {
"api_shortname": "cloudfunctions",
"distribution_name": "cloud.google.com/go/functions/apiv1",
@ -1249,6 +1429,16 @@
"release_level": "preview",
"library_type": "GAPIC_AUTO"
},
"cloud.google.com/go/language/apiv2": {
"api_shortname": "language",
"distribution_name": "cloud.google.com/go/language/apiv2",
"description": "Cloud Natural Language API",
"language": "go",
"client_library_type": "generated",
"client_documentation": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/language/latest/apiv2",
"release_level": "stable",
"library_type": "GAPIC_AUTO"
},
"cloud.google.com/go/lifesciences/apiv2beta": {
"api_shortname": "lifesciences",
"distribution_name": "cloud.google.com/go/lifesciences/apiv2beta",
@ -1309,6 +1499,26 @@
"release_level": "stable",
"library_type": "GAPIC_AUTO"
},
"cloud.google.com/go/maps/fleetengine/apiv1": {
"api_shortname": "fleetengine",
"distribution_name": "cloud.google.com/go/maps/fleetengine/apiv1",
"description": "Local Rides and Deliveries API",
"language": "go",
"client_library_type": "generated",
"client_documentation": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/maps/latest/fleetengine/apiv1",
"release_level": "stable",
"library_type": "GAPIC_AUTO"
},
"cloud.google.com/go/maps/fleetengine/delivery/apiv1": {
"api_shortname": "fleetengine",
"distribution_name": "cloud.google.com/go/maps/fleetengine/delivery/apiv1",
"description": "Last Mile Fleet Solution Delivery API",
"language": "go",
"client_library_type": "generated",
"client_documentation": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/maps/latest/fleetengine/delivery/apiv1",
"release_level": "stable",
"library_type": "GAPIC_AUTO"
},
"cloud.google.com/go/maps/mapsplatformdatasets/apiv1alpha": {
"api_shortname": "mapsplatformdatasets",
"distribution_name": "cloud.google.com/go/maps/mapsplatformdatasets/apiv1alpha",
@ -1509,6 +1719,16 @@
"release_level": "preview",
"library_type": "GAPIC_AUTO"
},
"cloud.google.com/go/notebooks/apiv2": {
"api_shortname": "notebooks",
"distribution_name": "cloud.google.com/go/notebooks/apiv2",
"description": "Notebooks API",
"language": "go",
"client_library_type": "generated",
"client_documentation": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/notebooks/latest/apiv2",
"release_level": "stable",
"library_type": "GAPIC_AUTO"
},
"cloud.google.com/go/optimization/apiv1": {
"api_shortname": "cloudoptimization",
"distribution_name": "cloud.google.com/go/optimization/apiv1",
@ -1609,6 +1829,16 @@
"release_level": "preview",
"library_type": "GAPIC_AUTO"
},
"cloud.google.com/go/parallelstore/apiv1beta": {
"api_shortname": "parallelstore",
"distribution_name": "cloud.google.com/go/parallelstore/apiv1beta",
"description": "Parallelstore API",
"language": "go",
"client_library_type": "generated",
"client_documentation": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/parallelstore/latest/apiv1beta",
"release_level": "preview",
"library_type": "GAPIC_AUTO"
},
"cloud.google.com/go/phishingprotection/apiv1beta1": {
"api_shortname": "phishingprotection",
"distribution_name": "cloud.google.com/go/phishingprotection/apiv1beta1",
@ -1626,7 +1856,7 @@
"language": "go",
"client_library_type": "generated",
"client_documentation": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/policysimulator/latest/apiv1",
"release_level": "preview",
"release_level": "stable",
"library_type": "GAPIC_AUTO"
},
"cloud.google.com/go/policytroubleshooter/apiv1": {
@ -1639,6 +1869,16 @@
"release_level": "stable",
"library_type": "GAPIC_AUTO"
},
"cloud.google.com/go/policytroubleshooter/iam/apiv3": {
"api_shortname": "policytroubleshooter",
"distribution_name": "cloud.google.com/go/policytroubleshooter/iam/apiv3",
"description": "Policy Troubleshooter API",
"language": "go",
"client_library_type": "generated",
"client_documentation": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/policytroubleshooter/latest/iam/apiv3",
"release_level": "stable",
"library_type": "GAPIC_AUTO"
},
"cloud.google.com/go/privatecatalog/apiv1beta1": {
"api_shortname": "cloudprivatecatalog",
"distribution_name": "cloud.google.com/go/privatecatalog/apiv1beta1",
@ -1779,6 +2019,16 @@
"release_level": "preview",
"library_type": "GAPIC_AUTO"
},
"cloud.google.com/go/redis/cluster/apiv1": {
"api_shortname": "redis",
"distribution_name": "cloud.google.com/go/redis/cluster/apiv1",
"description": "Google Cloud Memorystore for Redis API",
"language": "go",
"client_library_type": "generated",
"client_documentation": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/redis/latest/cluster/apiv1",
"release_level": "preview",
"library_type": "GAPIC_AUTO"
},
"cloud.google.com/go/resourcemanager/apiv2": {
"api_shortname": "cloudresourcemanager",
"distribution_name": "cloud.google.com/go/resourcemanager/apiv2",
@ -1889,6 +2139,26 @@
"release_level": "stable",
"library_type": "GAPIC_AUTO"
},
"cloud.google.com/go/secretmanager/apiv1beta2": {
"api_shortname": "secretmanager",
"distribution_name": "cloud.google.com/go/secretmanager/apiv1beta2",
"description": "Secret Manager API",
"language": "go",
"client_library_type": "generated",
"client_documentation": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/secretmanager/latest/apiv1beta2",
"release_level": "preview",
"library_type": "GAPIC_AUTO"
},
"cloud.google.com/go/securesourcemanager/apiv1": {
"api_shortname": "securesourcemanager",
"distribution_name": "cloud.google.com/go/securesourcemanager/apiv1",
"description": "Secure Source Manager API",
"language": "go",
"client_library_type": "generated",
"client_documentation": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/securesourcemanager/latest/apiv1",
"release_level": "preview",
"library_type": "GAPIC_AUTO"
},
"cloud.google.com/go/security/privateca/apiv1": {
"api_shortname": "privateca",
"distribution_name": "cloud.google.com/go/security/privateca/apiv1",
@ -1939,6 +2209,16 @@
"release_level": "preview",
"library_type": "GAPIC_AUTO"
},
"cloud.google.com/go/securitycenter/apiv2": {
"api_shortname": "securitycenter",
"distribution_name": "cloud.google.com/go/securitycenter/apiv2",
"description": "Security Command Center API",
"language": "go",
"client_library_type": "generated",
"client_documentation": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/securitycenter/latest/apiv2",
"release_level": "preview",
"library_type": "GAPIC_AUTO"
},
"cloud.google.com/go/securitycenter/settings/apiv1beta1": {
"api_shortname": "securitycenter",
"distribution_name": "cloud.google.com/go/securitycenter/settings/apiv1beta1",
@ -1949,6 +2229,26 @@
"release_level": "preview",
"library_type": "GAPIC_AUTO"
},
"cloud.google.com/go/securitycentermanagement/apiv1": {
"api_shortname": "securitycentermanagement",
"distribution_name": "cloud.google.com/go/securitycentermanagement/apiv1",
"description": "Security Center Management API",
"language": "go",
"client_library_type": "generated",
"client_documentation": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/securitycentermanagement/latest/apiv1",
"release_level": "preview",
"library_type": "GAPIC_AUTO"
},
"cloud.google.com/go/securityposture/apiv1": {
"api_shortname": "securityposture",
"distribution_name": "cloud.google.com/go/securityposture/apiv1",
"description": "Security Posture API",
"language": "go",
"client_library_type": "generated",
"client_documentation": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/securityposture/latest/apiv1",
"release_level": "preview",
"library_type": "GAPIC_AUTO"
},
"cloud.google.com/go/servicecontrol/apiv1": {
"api_shortname": "servicecontrol",
"distribution_name": "cloud.google.com/go/servicecontrol/apiv1",
@ -1979,6 +2279,16 @@
"release_level": "preview",
"library_type": "GAPIC_AUTO"
},
"cloud.google.com/go/servicehealth/apiv1": {
"api_shortname": "servicehealth",
"distribution_name": "cloud.google.com/go/servicehealth/apiv1",
"description": "Service Health API",
"language": "go",
"client_library_type": "generated",
"client_documentation": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/servicehealth/latest/apiv1",
"release_level": "preview",
"library_type": "GAPIC_AUTO"
},
"cloud.google.com/go/servicemanagement/apiv1": {
"api_shortname": "servicemanagement",
"distribution_name": "cloud.google.com/go/servicemanagement/apiv1",
@ -2009,6 +2319,26 @@
"release_level": "stable",
"library_type": "GAPIC_AUTO"
},
"cloud.google.com/go/shopping/css/apiv1": {
"api_shortname": "css",
"distribution_name": "cloud.google.com/go/shopping/css/apiv1",
"description": "CSS API",
"language": "go",
"client_library_type": "generated",
"client_documentation": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/shopping/latest/css/apiv1",
"release_level": "preview",
"library_type": "GAPIC_AUTO"
},
"cloud.google.com/go/shopping/merchant/inventories/apiv1beta": {
"api_shortname": "merchantapi",
"distribution_name": "cloud.google.com/go/shopping/merchant/inventories/apiv1beta",
"description": "Merchant API",
"language": "go",
"client_library_type": "generated",
"client_documentation": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/shopping/latest/merchant/inventories/apiv1beta",
"release_level": "preview",
"library_type": "GAPIC_AUTO"
},
"cloud.google.com/go/spanner": {
"api_shortname": "spanner",
"distribution_name": "cloud.google.com/go/spanner",
@ -2049,6 +2379,16 @@
"release_level": "stable",
"library_type": "GAPIC_AUTO"
},
"cloud.google.com/go/spanner/executor/apiv1": {
"api_shortname": "spanner-cloud-executor",
"distribution_name": "cloud.google.com/go/spanner/executor/apiv1",
"description": "Cloud Spanner Executor test API",
"language": "go",
"client_library_type": "generated",
"client_documentation": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/spanner/latest/executor/apiv1",
"release_level": "preview",
"library_type": "GAPIC_AUTO"
},
"cloud.google.com/go/speech/apiv1": {
"api_shortname": "speech",
"distribution_name": "cloud.google.com/go/speech/apiv1",
@ -2089,6 +2429,16 @@
"release_level": "stable",
"library_type": "GAPIC_MANUAL"
},
"cloud.google.com/go/storage/control/apiv2": {
"api_shortname": "storage",
"distribution_name": "cloud.google.com/go/storage/control/apiv2",
"description": "Cloud Storage API",
"language": "go",
"client_library_type": "generated",
"client_documentation": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/storage/latest/control/apiv2",
"release_level": "preview",
"library_type": "GAPIC_AUTO"
},
"cloud.google.com/go/storage/internal/apiv2": {
"api_shortname": "storage",
"distribution_name": "cloud.google.com/go/storage/internal/apiv2",
@ -2149,6 +2499,16 @@
"release_level": "preview",
"library_type": "GAPIC_AUTO"
},
"cloud.google.com/go/telcoautomation/apiv1": {
"api_shortname": "telcoautomation",
"distribution_name": "cloud.google.com/go/telcoautomation/apiv1",
"description": "Telco Automation API",
"language": "go",
"client_library_type": "generated",
"client_documentation": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/telcoautomation/latest/apiv1",
"release_level": "preview",
"library_type": "GAPIC_AUTO"
},
"cloud.google.com/go/texttospeech/apiv1": {
"api_shortname": "texttospeech",
"distribution_name": "cloud.google.com/go/texttospeech/apiv1",
@ -2215,8 +2575,8 @@
"description": "Video Stitcher API",
"language": "go",
"client_library_type": "generated",
"client_documentation": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/latest/video/stitcher/apiv1",
"release_level": "preview",
"client_documentation": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/video/latest/stitcher/apiv1",
"release_level": "stable",
"library_type": "GAPIC_AUTO"
},
"cloud.google.com/go/video/transcoder/apiv1": {
@ -2279,6 +2639,16 @@
"release_level": "preview",
"library_type": "GAPIC_AUTO"
},
"cloud.google.com/go/visionai/apiv1": {
"api_shortname": "visionai",
"distribution_name": "cloud.google.com/go/visionai/apiv1",
"description": "Vision AI API",
"language": "go",
"client_library_type": "generated",
"client_documentation": "https://cloud.google.com/go/docs/reference/cloud.google.com/go/visionai/latest/apiv1",
"release_level": "stable",
"library_type": "GAPIC_AUTO"
},
"cloud.google.com/go/vmmigration/apiv1": {
"api_shortname": "vmmigration",
"distribution_name": "cloud.google.com/go/vmmigration/apiv1",

View File

@ -16,35 +16,120 @@ package trace
import (
"context"
"errors"
"fmt"
"os"
"strings"
"sync"
"go.opencensus.io/trace"
"golang.org/x/xerrors"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
ottrace "go.opentelemetry.io/otel/trace"
"google.golang.org/api/googleapi"
"google.golang.org/genproto/googleapis/rpc/code"
"google.golang.org/grpc/status"
)
// StartSpan adds a span to the trace with the given name.
const (
// TelemetryPlatformTracingOpenCensus is the value to which the environment
// variable GOOGLE_API_GO_EXPERIMENTAL_TELEMETRY_PLATFORM_TRACING should be
// set to enable OpenCensus tracing.
TelemetryPlatformTracingOpenCensus = "opencensus"
// TelemetryPlatformTracingOpenCensus is the value to which the environment
// variable GOOGLE_API_GO_EXPERIMENTAL_TELEMETRY_PLATFORM_TRACING should be
// set to enable OpenTelemetry tracing.
TelemetryPlatformTracingOpenTelemetry = "opentelemetry"
// TelemetryPlatformTracingOpenCensus is the name of the environment
// variable that can be set to change the default tracing from OpenCensus
// to OpenTelemetry.
TelemetryPlatformTracingVar = "GOOGLE_API_GO_EXPERIMENTAL_TELEMETRY_PLATFORM_TRACING"
// OpenTelemetryTracerName is the name given to the OpenTelemetry Tracer
// when it is obtained from the OpenTelemetry TracerProvider.
OpenTelemetryTracerName = "cloud.google.com/go"
)
var (
// openTelemetryTracingEnabledMu guards access to openTelemetryTracingEnabled field
openTelemetryTracingEnabledMu = sync.RWMutex{}
// openTelemetryTracingEnabled is true if the environment variable
// GOOGLE_API_GO_EXPERIMENTAL_TELEMETRY_PLATFORM_TRACING is set to the
// case-insensitive value "opentelemetry".
openTelemetryTracingEnabled bool = strings.EqualFold(strings.TrimSpace(
os.Getenv(TelemetryPlatformTracingVar)), TelemetryPlatformTracingOpenTelemetry)
)
// SetOpenTelemetryTracingEnabledField programmatically sets the value provided by GOOGLE_API_GO_EXPERIMENTAL_TELEMETRY_PLATFORM_TRACING for the purpose of unit testing.
// Do not invoke it directly. Intended for use only in unit tests. Restore original value after each test.
func SetOpenTelemetryTracingEnabledField(enabled bool) {
openTelemetryTracingEnabledMu.Lock()
defer openTelemetryTracingEnabledMu.Unlock()
openTelemetryTracingEnabled = enabled
}
// IsOpenCensusTracingEnabled returns true if the environment variable
// GOOGLE_API_GO_EXPERIMENTAL_TELEMETRY_PLATFORM_TRACING is NOT set to the
// case-insensitive value "opentelemetry".
func IsOpenCensusTracingEnabled() bool {
return !IsOpenTelemetryTracingEnabled()
}
// IsOpenTelemetryTracingEnabled returns true if the environment variable
// GOOGLE_API_GO_EXPERIMENTAL_TELEMETRY_PLATFORM_TRACING is set to the
// case-insensitive value "opentelemetry".
func IsOpenTelemetryTracingEnabled() bool {
openTelemetryTracingEnabledMu.RLock()
defer openTelemetryTracingEnabledMu.RUnlock()
return openTelemetryTracingEnabled
}
// StartSpan adds a span to the trace with the given name. If IsOpenCensusTracingEnabled
// returns true, the span will be an OpenCensus span. If IsOpenTelemetryTracingEnabled
// returns true, the span will be an OpenTelemetry span. Set the environment variable
// GOOGLE_API_GO_EXPERIMENTAL_TELEMETRY_PLATFORM_TRACING to the case-insensitive
// value "opentelemetry" before loading the package to use OpenTelemetry tracing.
// The default will remain OpenCensus until May 29, 2024, at which time the default will
// switch to "opentelemetry" and explicitly setting the environment variable to
// "opencensus" will be required to continue using OpenCensus tracing.
func StartSpan(ctx context.Context, name string) context.Context {
ctx, _ = trace.StartSpan(ctx, name)
if IsOpenTelemetryTracingEnabled() {
ctx, _ = otel.GetTracerProvider().Tracer(OpenTelemetryTracerName).Start(ctx, name)
} else {
ctx, _ = trace.StartSpan(ctx, name)
}
return ctx
}
// EndSpan ends a span with the given error.
// EndSpan ends a span with the given error. If IsOpenCensusTracingEnabled
// returns true, the span will be an OpenCensus span. If IsOpenTelemetryTracingEnabled
// returns true, the span will be an OpenTelemetry span. Set the environment variable
// GOOGLE_API_GO_EXPERIMENTAL_TELEMETRY_PLATFORM_TRACING to the case-insensitive
// value "opentelemetry" before loading the package to use OpenTelemetry tracing.
// The default will remain OpenCensus until May 29, 2024, at which time the default will
// switch to "opentelemetry" and explicitly setting the environment variable to
// "opencensus" will be required to continue using OpenCensus tracing.
func EndSpan(ctx context.Context, err error) {
span := trace.FromContext(ctx)
if err != nil {
span.SetStatus(toStatus(err))
if IsOpenTelemetryTracingEnabled() {
span := ottrace.SpanFromContext(ctx)
if err != nil {
span.SetStatus(codes.Error, toOpenTelemetryStatusDescription(err))
span.RecordError(err)
}
span.End()
} else {
span := trace.FromContext(ctx)
if err != nil {
span.SetStatus(toStatus(err))
}
span.End()
}
span.End()
}
// toStatus interrogates an error and converts it to an appropriate
// OpenCensus status.
// toStatus converts an error to an equivalent OpenCensus status.
func toStatus(err error) trace.Status {
var err2 *googleapi.Error
if ok := xerrors.As(err, &err2); ok {
if ok := errors.As(err, &err2); ok {
return trace.Status{Code: httpStatusCodeToOCCode(err2.Code), Message: err2.Message}
} else if s, ok := status.FromError(err); ok {
return trace.Status{Code: int32(s.Code()), Message: s.Message()}
@ -53,6 +138,18 @@ func toStatus(err error) trace.Status {
}
}
// toOpenTelemetryStatus converts an error to an equivalent OpenTelemetry status description.
func toOpenTelemetryStatusDescription(err error) string {
var err2 *googleapi.Error
if ok := errors.As(err, &err2); ok {
return err2.Message
} else if s, ok := status.FromError(err); ok {
return s.Message()
} else {
return err.Error()
}
}
// TODO(deklerk): switch to using OpenCensus function when it becomes available.
// Reference: https://github.com/googleapis/googleapis/blob/26b634d2724ac5dd30ae0b0cbfb01f07f2e4050e/google/rpc/code.proto
func httpStatusCodeToOCCode(httpStatusCode int) int32 {
@ -86,10 +183,33 @@ func httpStatusCodeToOCCode(httpStatusCode int) int32 {
}
}
// TODO: (odeke-em): perhaps just pass around spans due to the cost
// incurred from using trace.FromContext(ctx) yet we could avoid
// throwing away the work done by ctx, span := trace.StartSpan.
// TracePrintf retrieves the current OpenCensus or OpenTelemetry span from context, then:
// * calls Span.Annotatef if OpenCensus is enabled; or
// * calls Span.AddEvent if OpenTelemetry is enabled.
//
// If IsOpenCensusTracingEnabled returns true, the expected span must be an
// OpenCensus span. If IsOpenTelemetryTracingEnabled returns true, the expected
// span must be an OpenTelemetry span. Set the environment variable
// GOOGLE_API_GO_EXPERIMENTAL_TELEMETRY_PLATFORM_TRACING to the case-insensitive
// value "opentelemetry" before loading the package to use OpenTelemetry tracing.
// The default will remain OpenCensus until May 29, 2024, at which time the default will
// switch to "opentelemetry" and explicitly setting the environment variable to
// "opencensus" will be required to continue using OpenCensus tracing.
func TracePrintf(ctx context.Context, attrMap map[string]interface{}, format string, args ...interface{}) {
if IsOpenTelemetryTracingEnabled() {
attrs := otAttrs(attrMap)
ottrace.SpanFromContext(ctx).AddEvent(fmt.Sprintf(format, args...), ottrace.WithAttributes(attrs...))
} else {
attrs := ocAttrs(attrMap)
// TODO: (odeke-em): perhaps just pass around spans due to the cost
// incurred from using trace.FromContext(ctx) yet we could avoid
// throwing away the work done by ctx, span := trace.StartSpan.
trace.FromContext(ctx).Annotatef(attrs, format, args...)
}
}
// ocAttrs converts a generic map to OpenCensus attributes.
func ocAttrs(attrMap map[string]interface{}) []trace.Attribute {
var attrs []trace.Attribute
for k, v := range attrMap {
var a trace.Attribute
@ -107,5 +227,27 @@ func TracePrintf(ctx context.Context, attrMap map[string]interface{}, format str
}
attrs = append(attrs, a)
}
trace.FromContext(ctx).Annotatef(attrs, format, args...)
return attrs
}
// otAttrs converts a generic map to OpenTelemetry attributes.
func otAttrs(attrMap map[string]interface{}) []attribute.KeyValue {
var attrs []attribute.KeyValue
for k, v := range attrMap {
var a attribute.KeyValue
switch v := v.(type) {
case string:
a = attribute.Key(k).String(v)
case bool:
a = attribute.Key(k).Bool(v)
case int:
a = attribute.Key(k).Int(v)
case int64:
a = attribute.Key(k).Int64(v)
default:
a = attribute.Key(k).String(fmt.Sprintf("%#v", v))
}
attrs = append(attrs, a)
}
return attrs
}

View File

@ -1,6 +1,159 @@
# Changes
## [1.39.1](https://github.com/googleapis/google-cloud-go/compare/storage/v1.39.0...storage/v1.39.1) (2024-03-11)
### Bug Fixes
* **storage:** Add object validation case and test ([#9521](https://github.com/googleapis/google-cloud-go/issues/9521)) ([386bef3](https://github.com/googleapis/google-cloud-go/commit/386bef319b4678beaa926ddfe4edef190f11b68d))
## [1.39.0](https://github.com/googleapis/google-cloud-go/compare/storage/v1.38.0...storage/v1.39.0) (2024-02-29)
### Features
* **storage:** Make it possible to disable Content-Type sniffing ([#9431](https://github.com/googleapis/google-cloud-go/issues/9431)) ([0676670](https://github.com/googleapis/google-cloud-go/commit/067667058c06689b64401be11858d84441584039))
## [1.38.0](https://github.com/googleapis/google-cloud-go/compare/storage/v1.37.0...storage/v1.38.0) (2024-02-12)
### Features
* **storage:** Support auto-detection of access ID for external_account creds ([#9208](https://github.com/googleapis/google-cloud-go/issues/9208)) ([b958d44](https://github.com/googleapis/google-cloud-go/commit/b958d44589f2b6b226ea3bef23829ac75a0aa6a6))
* **storage:** Support custom hostname for VirtualHostedStyle SignedURLs ([#9348](https://github.com/googleapis/google-cloud-go/issues/9348)) ([7eec40e](https://github.com/googleapis/google-cloud-go/commit/7eec40e4cf82c53e5bf02bd2c14e0b25043da6d0))
* **storage:** Support universe domains ([#9344](https://github.com/googleapis/google-cloud-go/issues/9344)) ([29a7498](https://github.com/googleapis/google-cloud-go/commit/29a7498b8eb0d00fdb5acd7ee8ce0e5a2a8c11ce))
### Bug Fixes
* **storage:** Fix v4 url signing for hosts that specify ports ([#9347](https://github.com/googleapis/google-cloud-go/issues/9347)) ([f127b46](https://github.com/googleapis/google-cloud-go/commit/f127b4648f861c1ba44f41a280a62652620c04c2))
### Documentation
* **storage:** Indicate that gRPC is incompatible with universe domains ([#9386](https://github.com/googleapis/google-cloud-go/issues/9386)) ([e8bd85b](https://github.com/googleapis/google-cloud-go/commit/e8bd85bbce12d5f7ab87fa49d166a6a0d84bd12d))
## [1.37.0](https://github.com/googleapis/google-cloud-go/compare/storage/v1.36.0...storage/v1.37.0) (2024-01-24)
### Features
* **storage:** Add maxAttempts RetryOption ([#9215](https://github.com/googleapis/google-cloud-go/issues/9215)) ([e348cc5](https://github.com/googleapis/google-cloud-go/commit/e348cc5340e127b530e8ee4664fd995e6f038b2c))
* **storage:** Support IncludeFoldersAsPrefixes ([#9211](https://github.com/googleapis/google-cloud-go/issues/9211)) ([98c9d71](https://github.com/googleapis/google-cloud-go/commit/98c9d7157306de5134547a67c084c248484c9a51))
### Bug Fixes
* **storage:** Migrate deprecated proto dep ([#9232](https://github.com/googleapis/google-cloud-go/issues/9232)) ([ebbb610](https://github.com/googleapis/google-cloud-go/commit/ebbb610e0f58035fd01ad7893971382d8bbd092f)), refs [#9189](https://github.com/googleapis/google-cloud-go/issues/9189)
## [1.36.0](https://github.com/googleapis/google-cloud-go/compare/storage/v1.35.1...storage/v1.36.0) (2023-12-14)
### Features
* **storage:** Add object retention feature ([#9072](https://github.com/googleapis/google-cloud-go/issues/9072)) ([16ecfd1](https://github.com/googleapis/google-cloud-go/commit/16ecfd150ff1982f03d207a80a82e934d1013874))
### Bug Fixes
* **storage:** Do not inhibit the dead code elimination. ([#8543](https://github.com/googleapis/google-cloud-go/issues/8543)) ([ca2493f](https://github.com/googleapis/google-cloud-go/commit/ca2493f43c299bbaed5f7e5b70f66cc763ff9802))
* **storage:** Set flush and get_state to false on the last write in gRPC ([#9013](https://github.com/googleapis/google-cloud-go/issues/9013)) ([c1e9fe5](https://github.com/googleapis/google-cloud-go/commit/c1e9fe5f4166a71e55814ccf126926ec0e0e7945))
## [1.35.1](https://github.com/googleapis/google-cloud-go/compare/storage/v1.35.0...storage/v1.35.1) (2023-11-09)
### Bug Fixes
* **storage:** Rename aux.go to auxiliary.go fixing windows build ([ba23673](https://github.com/googleapis/google-cloud-go/commit/ba23673da7707c31292e4aa29d65b7ac1446d4a6))
## [1.35.0](https://github.com/googleapis/google-cloud-go/compare/storage/v1.34.1...storage/v1.35.0) (2023-11-09)
### Features
* **storage:** Change gRPC writes to use bi-directional streams ([#8930](https://github.com/googleapis/google-cloud-go/issues/8930)) ([3e23a36](https://github.com/googleapis/google-cloud-go/commit/3e23a364b1a20c4fda7aef257e4136586ec769a4))
## [1.34.1](https://github.com/googleapis/google-cloud-go/compare/storage/v1.34.0...storage/v1.34.1) (2023-11-01)
### Bug Fixes
* **storage:** Bump google.golang.org/api to v0.149.0 ([8d2ab9f](https://github.com/googleapis/google-cloud-go/commit/8d2ab9f320a86c1c0fab90513fc05861561d0880))
## [1.34.0](https://github.com/googleapis/google-cloud-go/compare/storage/v1.33.0...storage/v1.34.0) (2023-10-31)
### Features
* **storage/internal:** Add match_glob field to ListObjectsRequest ([#8618](https://github.com/googleapis/google-cloud-go/issues/8618)) ([e9ae601](https://github.com/googleapis/google-cloud-go/commit/e9ae6018983ae09781740e4ff939e6e365863dbb))
* **storage/internal:** Add terminal_storage_class fields to Autoclass message ([57fc1a6](https://github.com/googleapis/google-cloud-go/commit/57fc1a6de326456eb68ef25f7a305df6636ed386))
* **storage/internal:** Adds the RestoreObject operation ([56ce871](https://github.com/googleapis/google-cloud-go/commit/56ce87195320634b07ae0b012efcc5f2b3813fb0))
* **storage:** Support autoclass v2.1 ([#8721](https://github.com/googleapis/google-cloud-go/issues/8721)) ([fe1e195](https://github.com/googleapis/google-cloud-go/commit/fe1e19590a252c6adc6ca6c51a69b6e561e143b8))
* **storage:** Support MatchGlob for gRPC ([#8670](https://github.com/googleapis/google-cloud-go/issues/8670)) ([3df0287](https://github.com/googleapis/google-cloud-go/commit/3df0287f88d5e2c4526e9e6b8dc2a4ca54f88918)), refs [#7727](https://github.com/googleapis/google-cloud-go/issues/7727)
### Bug Fixes
* **storage:** Drop stream reference after closing it for gRPC writes ([#8872](https://github.com/googleapis/google-cloud-go/issues/8872)) ([525abde](https://github.com/googleapis/google-cloud-go/commit/525abdee433864d4d456f1f1fff5599017b557ff))
* **storage:** Update golang.org/x/net to v0.17.0 ([174da47](https://github.com/googleapis/google-cloud-go/commit/174da47254fefb12921bbfc65b7829a453af6f5d))
* **storage:** Update grpc-go to v1.56.3 ([343cea8](https://github.com/googleapis/google-cloud-go/commit/343cea8c43b1e31ae21ad50ad31d3b0b60143f8c))
* **storage:** Update grpc-go to v1.59.0 ([81a97b0](https://github.com/googleapis/google-cloud-go/commit/81a97b06cb28b25432e4ece595c55a9857e960b7))
## [1.33.0](https://github.com/googleapis/google-cloud-go/compare/storage/v1.32.0...storage/v1.33.0) (2023-09-07)
### Features
* **storage:** Export gRPC client constructor ([#8509](https://github.com/googleapis/google-cloud-go/issues/8509)) ([1a928ae](https://github.com/googleapis/google-cloud-go/commit/1a928ae205f2325cb5206304af4d609dc3c1447a))
## [1.32.0](https://github.com/googleapis/google-cloud-go/compare/storage/v1.31.0...storage/v1.32.0) (2023-08-15)
### Features
* **storage:** Add support for custom headers ([#8294](https://github.com/googleapis/google-cloud-go/issues/8294)) ([313fd4a](https://github.com/googleapis/google-cloud-go/commit/313fd4a60380d36c5ecaead3e968dbc84d044a0b))
* **storage:** Add trace span to Writer ([#8375](https://github.com/googleapis/google-cloud-go/issues/8375)) ([f7ac85b](https://github.com/googleapis/google-cloud-go/commit/f7ac85bec2806d351529714bd7744a91a9fdefdd)), refs [#6144](https://github.com/googleapis/google-cloud-go/issues/6144)
* **storage:** Support single-shot uploads in gRPC ([#8348](https://github.com/googleapis/google-cloud-go/issues/8348)) ([7de4a7d](https://github.com/googleapis/google-cloud-go/commit/7de4a7da31ab279a343b1592b15a126cda03e5e7)), refs [#7798](https://github.com/googleapis/google-cloud-go/issues/7798)
* **storage:** Trace span covers life of a Reader ([#8390](https://github.com/googleapis/google-cloud-go/issues/8390)) ([8de30d7](https://github.com/googleapis/google-cloud-go/commit/8de30d752eec2fed2ea4c127482d3e213f9050e2))
### Bug Fixes
* **storage:** Fix AllObjects condition in gRPC ([#8184](https://github.com/googleapis/google-cloud-go/issues/8184)) ([2b99e4f](https://github.com/googleapis/google-cloud-go/commit/2b99e4f39be20fe21e8bc5c1ec1c0e758222c46e)), refs [#6205](https://github.com/googleapis/google-cloud-go/issues/6205)
* **storage:** Fix gRPC generation/condition issues ([#8396](https://github.com/googleapis/google-cloud-go/issues/8396)) ([ca68ff5](https://github.com/googleapis/google-cloud-go/commit/ca68ff54b680732b59b223655070d0f6abccefee))
* **storage:** Same method name and Trace Span name ([#8150](https://github.com/googleapis/google-cloud-go/issues/8150)) ([e277213](https://github.com/googleapis/google-cloud-go/commit/e2772133896bb94097b5d1f090f1bcafd136f2ed))
* **storage:** Update gRPC retry codes ([#8202](https://github.com/googleapis/google-cloud-go/issues/8202)) ([afdf772](https://github.com/googleapis/google-cloud-go/commit/afdf772fc6a90b3010eee9d70ab65e22e276f53f))
## [1.31.0](https://github.com/googleapis/google-cloud-go/compare/storage/v1.30.1...storage/v1.31.0) (2023-06-27)
### Features
* **storage/internal:** Add ctype=CORD for ChecksummedData.content ([ca94e27](https://github.com/googleapis/google-cloud-go/commit/ca94e2724f9e2610b46aefd0a3b5ddc06102e91b))
* **storage:** Add support for MatchGlob ([#8097](https://github.com/googleapis/google-cloud-go/issues/8097)) ([9426a5a](https://github.com/googleapis/google-cloud-go/commit/9426a5a45d4c2fd07f84261f6d602680e79cdc48)), refs [#7727](https://github.com/googleapis/google-cloud-go/issues/7727) [#7728](https://github.com/googleapis/google-cloud-go/issues/7728)
* **storage:** Respect WithEndpoint for SignedURLs and PostPolicy ([#8113](https://github.com/googleapis/google-cloud-go/issues/8113)) ([f918f23](https://github.com/googleapis/google-cloud-go/commit/f918f23a3cda4fbc8d709e32b914ead8b735d664))
* **storage:** Update all direct dependencies ([b340d03](https://github.com/googleapis/google-cloud-go/commit/b340d030f2b52a4ce48846ce63984b28583abde6))
### Bug Fixes
* **storage:** Fix CreateBucket logic for gRPC ([#8165](https://github.com/googleapis/google-cloud-go/issues/8165)) ([8424e7e](https://github.com/googleapis/google-cloud-go/commit/8424e7e145a117c91006318fa924a8b2643c1c7e)), refs [#8162](https://github.com/googleapis/google-cloud-go/issues/8162)
* **storage:** Fix reads with "./" in object names [XML] ([#8017](https://github.com/googleapis/google-cloud-go/issues/8017)) ([6b7b21f](https://github.com/googleapis/google-cloud-go/commit/6b7b21f8a334b6ad3a25e1f66ae1265b4d1f0995))
* **storage:** Fix routing header for writes ([#8159](https://github.com/googleapis/google-cloud-go/issues/8159)) ([42a59f5](https://github.com/googleapis/google-cloud-go/commit/42a59f5a23ab9b4743ab032ad92304922c801d93)), refs [#8142](https://github.com/googleapis/google-cloud-go/issues/8142) [#8143](https://github.com/googleapis/google-cloud-go/issues/8143) [#8144](https://github.com/googleapis/google-cloud-go/issues/8144) [#8145](https://github.com/googleapis/google-cloud-go/issues/8145) [#8149](https://github.com/googleapis/google-cloud-go/issues/8149)
* **storage:** REST query UpdateMask bug ([df52820](https://github.com/googleapis/google-cloud-go/commit/df52820b0e7721954809a8aa8700b93c5662dc9b))
* **storage:** Update grpc to v1.55.0 ([1147ce0](https://github.com/googleapis/google-cloud-go/commit/1147ce02a990276ca4f8ab7a1ab65c14da4450ef))
### Documentation
* **storage/internal:** Clarifications about behavior of DeleteObject RPC ([3f1ed9c](https://github.com/googleapis/google-cloud-go/commit/3f1ed9c63fb115f47607a3ab478842fe5ba0df11))
* **storage/internal:** Clarified the behavior of supplying bucket.name field in CreateBucket to reflect actual implementation ([ebae64d](https://github.com/googleapis/google-cloud-go/commit/ebae64d53397ec5dfe851f098754eaa1f5df7cb1))
* **storage/internal:** Revert ChecksummedData message definition not to specify ctype=CORD, because it would be a breaking change. ([ef61e47](https://github.com/googleapis/google-cloud-go/commit/ef61e4799280a355b960da8ae240ceb2efbe71ac))
* **storage/internal:** Update routing annotations for CancelResumableWriteRequest and QueryWriteStatusRequest ([4900851](https://github.com/googleapis/google-cloud-go/commit/49008518e168fe6f7891b907d6fc14eecdef758c))
* **storage/internal:** Updated ChecksummedData message definition to specify ctype=CORD, and removed incorrect earlier attempt that set that annotation in the ReadObjectResponse message definition ([ef61e47](https://github.com/googleapis/google-cloud-go/commit/ef61e4799280a355b960da8ae240ceb2efbe71ac))
* **storage:** WithXMLReads should mention XML instead of JSON API ([#7881](https://github.com/googleapis/google-cloud-go/issues/7881)) ([36f56c8](https://github.com/googleapis/google-cloud-go/commit/36f56c80c456ca74ffc03df76844ce15980ced82))
## [1.30.1](https://github.com/googleapis/google-cloud-go/compare/storage/v1.30.0...storage/v1.30.1) (2023-03-21)

View File

@ -20,7 +20,7 @@ import (
"reflect"
"cloud.google.com/go/internal/trace"
storagepb "cloud.google.com/go/storage/internal/apiv2/stubs"
"cloud.google.com/go/storage/internal/apiv2/storagepb"
raw "google.golang.org/api/storage/v1"
)

View File

@ -27,7 +27,7 @@ import (
"cloud.google.com/go/compute/metadata"
"cloud.google.com/go/internal/optional"
"cloud.google.com/go/internal/trace"
storagepb "cloud.google.com/go/storage/internal/apiv2/stubs"
"cloud.google.com/go/storage/internal/apiv2/storagepb"
"google.golang.org/api/googleapi"
"google.golang.org/api/iamcredentials/v1"
"google.golang.org/api/iterator"
@ -41,13 +41,14 @@ import (
// BucketHandle provides operations on a Google Cloud Storage bucket.
// Use Client.Bucket to get a handle.
type BucketHandle struct {
c *Client
name string
acl ACLHandle
defaultObjectACL ACLHandle
conds *BucketConditions
userProject string // project for Requester Pays buckets
retry *retryConfig
c *Client
name string
acl ACLHandle
defaultObjectACL ACLHandle
conds *BucketConditions
userProject string // project for Requester Pays buckets
retry *retryConfig
enableObjectRetention *bool
}
// Bucket returns a BucketHandle, which provides operations on the named bucket.
@ -85,7 +86,8 @@ func (b *BucketHandle) Create(ctx context.Context, projectID string, attrs *Buck
defer func() { trace.EndSpan(ctx, err) }()
o := makeStorageOpts(true, b.retry, b.userProject)
if _, err := b.c.tc.CreateBucket(ctx, projectID, b.name, attrs, o...); err != nil {
if _, err := b.c.tc.CreateBucket(ctx, projectID, b.name, attrs, b.enableObjectRetention, o...); err != nil {
return err
}
return nil
@ -152,7 +154,7 @@ func (b *BucketHandle) Attrs(ctx context.Context) (attrs *BucketAttrs, err error
// Update updates a bucket's attributes.
func (b *BucketHandle) Update(ctx context.Context, uattrs BucketAttrsToUpdate) (attrs *BucketAttrs, err error) {
ctx = trace.StartSpan(ctx, "cloud.google.com/go/storage.Bucket.Create")
ctx = trace.StartSpan(ctx, "cloud.google.com/go/storage.Bucket.Update")
defer func() { trace.EndSpan(ctx, err) }()
isIdempotent := b.conds != nil && b.conds.MetagenerationMatch != 0
@ -173,12 +175,18 @@ func (b *BucketHandle) Update(ctx context.Context, uattrs BucketAttrsToUpdate) (
// [Overview of access control]: https://cloud.google.com/storage/docs/accesscontrol#signed_urls_query_string_authentication
// [automatic detection of credentials]: https://pkg.go.dev/cloud.google.com/go/storage#hdr-Credential_requirements_for_signing
func (b *BucketHandle) SignedURL(object string, opts *SignedURLOptions) (string, error) {
if opts.GoogleAccessID != "" && (opts.SignBytes != nil || len(opts.PrivateKey) > 0) {
return SignedURL(b.name, object, opts)
}
// Make a copy of opts so we don't modify the pointer parameter.
newopts := opts.clone()
if newopts.Hostname == "" {
// Extract the correct host from the readhost set on the client
newopts.Hostname = b.c.xmlHost
}
if opts.GoogleAccessID != "" && (opts.SignBytes != nil || len(opts.PrivateKey) > 0) {
return SignedURL(b.name, object, newopts)
}
if newopts.GoogleAccessID == "" {
id, err := b.detectDefaultGoogleAccessID()
if err != nil {
@ -215,12 +223,18 @@ func (b *BucketHandle) SignedURL(object string, opts *SignedURLOptions) (string,
//
// [automatic detection of credentials]: https://pkg.go.dev/cloud.google.com/go/storage#hdr-Credential_requirements_for_signing
func (b *BucketHandle) GenerateSignedPostPolicyV4(object string, opts *PostPolicyV4Options) (*PostPolicyV4, error) {
if opts.GoogleAccessID != "" && (opts.SignRawBytes != nil || opts.SignBytes != nil || len(opts.PrivateKey) > 0) {
return GenerateSignedPostPolicyV4(b.name, object, opts)
}
// Make a copy of opts so we don't modify the pointer parameter.
newopts := opts.clone()
if newopts.Hostname == "" {
// Extract the correct host from the readhost set on the client
newopts.Hostname = b.c.xmlHost
}
if opts.GoogleAccessID != "" && (opts.SignRawBytes != nil || opts.SignBytes != nil || len(opts.PrivateKey) > 0) {
return GenerateSignedPostPolicyV4(b.name, object, newopts)
}
if newopts.GoogleAccessID == "" {
id, err := b.detectDefaultGoogleAccessID()
if err != nil {
@ -261,18 +275,24 @@ func (b *BucketHandle) detectDefaultGoogleAccessID() (string, error) {
err := json.Unmarshal(b.c.creds.JSON, &sa)
if err != nil {
returnErr = err
} else if sa.CredType == "impersonated_service_account" {
start, end := strings.LastIndex(sa.SAImpersonationURL, "/"), strings.LastIndex(sa.SAImpersonationURL, ":")
if end <= start {
returnErr = errors.New("error parsing impersonated service account credentials")
} else {
return sa.SAImpersonationURL[start+1 : end], nil
}
} else if sa.CredType == "service_account" && sa.ClientEmail != "" {
return sa.ClientEmail, nil
} else {
returnErr = errors.New("unable to parse credentials; only service_account and impersonated_service_account credentials are supported")
switch sa.CredType {
case "impersonated_service_account", "external_account":
start, end := strings.LastIndex(sa.SAImpersonationURL, "/"), strings.LastIndex(sa.SAImpersonationURL, ":")
if end <= start {
returnErr = errors.New("error parsing external or impersonated service account credentials")
} else {
return sa.SAImpersonationURL[start+1 : end], nil
}
case "service_account":
if sa.ClientEmail != "" {
return sa.ClientEmail, nil
}
returnErr = errors.New("empty service account client email")
default:
returnErr = errors.New("unable to parse credentials; only service_account, external_account and impersonated_service_account credentials are supported")
}
}
}
@ -288,7 +308,7 @@ func (b *BucketHandle) detectDefaultGoogleAccessID() (string, error) {
}
}
return "", fmt.Errorf("storage: unable to detect default GoogleAccessID: %w. Please provide the GoogleAccessID or use a supported means for autodetecting it (see https://pkg.go.dev/cloud.google.com/go/storage#hdr-Credential_requirements_for_[BucketHandle.SignedURL]_and_[BucketHandle.GenerateSignedPostPolicyV4])", returnErr)
return "", fmt.Errorf("storage: unable to detect default GoogleAccessID: %w. Please provide the GoogleAccessID or use a supported means for autodetecting it (see https://pkg.go.dev/cloud.google.com/go/storage#hdr-Credential_requirements_for_signing)", returnErr)
}
func (b *BucketHandle) defaultSignBytesFunc(email string) func([]byte) ([]byte, error) {
@ -450,6 +470,15 @@ type BucketAttrs struct {
// allows for the automatic selection of the best storage class
// based on object access patterns.
Autoclass *Autoclass
// ObjectRetentionMode reports whether individual objects in the bucket can
// be configured with a retention policy. An empty value means that object
// retention is disabled.
// This field is read-only. Object retention can be enabled only by creating
// a bucket with SetObjectRetention set to true on the BucketHandle. It
// cannot be modified once the bucket is created.
// ObjectRetention cannot be configured or reported through the gRPC API.
ObjectRetentionMode string
}
// BucketPolicyOnly is an alias for UniformBucketLevelAccess.
@ -728,6 +757,13 @@ type Autoclass struct {
// If Autoclass is enabled when the bucket is created, the ToggleTime
// is set to the bucket creation time. This field is read-only.
ToggleTime time.Time
// TerminalStorageClass: The storage class that objects in the bucket
// eventually transition to if they are not read for a certain length of
// time. Valid values are NEARLINE and ARCHIVE.
TerminalStorageClass string
// TerminalStorageClassUpdateTime represents the time of the most recent
// update to "TerminalStorageClass".
TerminalStorageClassUpdateTime time.Time
}
func newBucket(b *raw.Bucket) (*BucketAttrs, error) {
@ -738,6 +774,7 @@ func newBucket(b *raw.Bucket) (*BucketAttrs, error) {
if err != nil {
return nil, err
}
return &BucketAttrs{
Name: b.Name,
Location: b.Location,
@ -752,6 +789,7 @@ func newBucket(b *raw.Bucket) (*BucketAttrs, error) {
RequesterPays: b.Billing != nil && b.Billing.RequesterPays,
Lifecycle: toLifecycle(b.Lifecycle),
RetentionPolicy: rp,
ObjectRetentionMode: toBucketObjectRetention(b.ObjectRetention),
CORS: toCORS(b.Cors),
Encryption: toBucketEncryption(b.Encryption),
Logging: toBucketLogging(b.Logging),
@ -921,8 +959,6 @@ func (ua *BucketAttrsToUpdate) toProtoBucket() *storagepb.Bucket {
return &storagepb.Bucket{}
}
// TODO(cathyo): Handle labels. Pending b/230510191.
var v *storagepb.Bucket_Versioning
if ua.VersioningEnabled != nil {
v = &storagepb.Bucket_Versioning{Enabled: optional.ToBool(ua.VersioningEnabled)}
@ -996,6 +1032,7 @@ func (ua *BucketAttrsToUpdate) toProtoBucket() *storagepb.Bucket {
IamConfig: bktIAM,
Rpo: ua.RPO.String(),
Autoclass: ua.Autoclass.toProtoAutoclass(),
Labels: ua.setLabels,
}
}
@ -1230,9 +1267,11 @@ func (ua *BucketAttrsToUpdate) toRawBucket() *raw.Bucket {
}
if ua.Autoclass != nil {
rb.Autoclass = &raw.BucketAutoclass{
Enabled: ua.Autoclass.Enabled,
ForceSendFields: []string{"Enabled"},
Enabled: ua.Autoclass.Enabled,
TerminalStorageClass: ua.Autoclass.TerminalStorageClass,
ForceSendFields: []string{"Enabled"},
}
rb.ForceSendFields = append(rb.ForceSendFields, "Autoclass")
}
if ua.PredefinedACL != "" {
// Clear ACL or the call will fail.
@ -1264,7 +1303,9 @@ func (ua *BucketAttrsToUpdate) toRawBucket() *raw.Bucket {
}
// If returns a new BucketHandle that applies a set of preconditions.
// Preconditions already set on the BucketHandle are ignored.
// Preconditions already set on the BucketHandle are ignored. The supplied
// BucketConditions must have exactly one field set to a non-zero value;
// otherwise an error will be returned from any operation on the BucketHandle.
// Operations on the new handle will return an error if the preconditions are not
// satisfied. The only valid preconditions for buckets are MetagenerationMatch
// and MetagenerationNotMatch.
@ -1326,6 +1367,17 @@ func (b *BucketHandle) LockRetentionPolicy(ctx context.Context) error {
return b.c.tc.LockBucketRetentionPolicy(ctx, b.name, b.conds, o...)
}
// SetObjectRetention returns a new BucketHandle that will enable object retention
// on bucket creation. To enable object retention, you must use the returned
// handle to create the bucket. This has no effect on an already existing bucket.
// ObjectRetention is not enabled by default.
// ObjectRetention cannot be configured through the gRPC API.
func (b *BucketHandle) SetObjectRetention(enable bool) *BucketHandle {
b2 := *b
b2.enableObjectRetention = &enable
return &b2
}
// applyBucketConds modifies the provided call using the conditions in conds.
// call is something that quacks like a *raw.WhateverCall.
func applyBucketConds(method string, conds *BucketConditions, call interface{}) error {
@ -1338,11 +1390,11 @@ func applyBucketConds(method string, conds *BucketConditions, call interface{})
cval := reflect.ValueOf(call)
switch {
case conds.MetagenerationMatch != 0:
if !setConditionField(cval, "IfMetagenerationMatch", conds.MetagenerationMatch) {
if !setIfMetagenerationMatch(cval, conds.MetagenerationMatch) {
return fmt.Errorf("storage: %s: ifMetagenerationMatch not supported", method)
}
case conds.MetagenerationNotMatch != 0:
if !setConditionField(cval, "IfMetagenerationNotMatch", conds.MetagenerationNotMatch) {
if !setIfMetagenerationNotMatch(cval, conds.MetagenerationNotMatch) {
return fmt.Errorf("storage: %s: ifMetagenerationNotMatch not supported", method)
}
}
@ -1425,6 +1477,13 @@ func toRetentionPolicyFromProto(rp *storagepb.Bucket_RetentionPolicy) *Retention
}
}
func toBucketObjectRetention(or *raw.BucketObjectRetention) string {
if or == nil {
return ""
}
return or.Mode
}
func toRawCORS(c []CORS) []*raw.BucketCors {
var out []*raw.BucketCors
for _, v := range c {
@ -1545,7 +1604,6 @@ func toProtoLifecycle(l Lifecycle) *storagepb.Bucket_Lifecycle {
// doc states "format: int32"), so the client types used int64,
// but the proto uses int32 so we have a potentially lossy
// conversion.
AgeDays: proto.Int32(int32(r.Condition.AgeInDays)),
DaysSinceCustomTime: proto.Int32(int32(r.Condition.DaysSinceCustomTime)),
DaysSinceNoncurrentTime: proto.Int32(int32(r.Condition.DaysSinceNoncurrentTime)),
MatchesPrefix: r.Condition.MatchesPrefix,
@ -1555,7 +1613,11 @@ func toProtoLifecycle(l Lifecycle) *storagepb.Bucket_Lifecycle {
},
}
// TODO(#6205): This may not be needed for gRPC
// Only set AgeDays in the proto if it is non-zero, or if the user has set
// Condition.AllObjects.
if r.Condition.AgeInDays != 0 {
rr.Condition.AgeDays = proto.Int32(int32(r.Condition.AgeInDays))
}
if r.Condition.AllObjects {
rr.Condition.AgeDays = proto.Int32(0)
}
@ -1654,8 +1716,8 @@ func toLifecycleFromProto(rl *storagepb.Bucket_Lifecycle) Lifecycle {
},
}
// TODO(#6205): This may not be needed for gRPC
if rr.GetCondition().GetAgeDays() == 0 {
// Only set Condition.AllObjects if AgeDays is zero, not if it is nil.
if rr.GetCondition().AgeDays != nil && rr.GetCondition().GetAgeDays() == 0 {
r.Condition.AllObjects = true
}
@ -1938,9 +2000,10 @@ func (a *Autoclass) toRawAutoclass() *raw.BucketAutoclass {
if a == nil {
return nil
}
// Excluding read only field ToggleTime.
// Excluding read only fields ToggleTime and TerminalStorageClassUpdateTime.
return &raw.BucketAutoclass{
Enabled: a.Enabled,
Enabled: a.Enabled,
TerminalStorageClass: a.TerminalStorageClass,
}
}
@ -1948,27 +2011,34 @@ func (a *Autoclass) toProtoAutoclass() *storagepb.Bucket_Autoclass {
if a == nil {
return nil
}
// Excluding read only field ToggleTime.
return &storagepb.Bucket_Autoclass{
// Excluding read only fields ToggleTime and TerminalStorageClassUpdateTime.
ba := &storagepb.Bucket_Autoclass{
Enabled: a.Enabled,
}
if a.TerminalStorageClass != "" {
ba.TerminalStorageClass = &a.TerminalStorageClass
}
return ba
}
func toAutoclassFromRaw(a *raw.BucketAutoclass) *Autoclass {
if a == nil || a.ToggleTime == "" {
return nil
}
// Return Autoclass.ToggleTime only if parsed with a valid value.
ac := &Autoclass{
Enabled: a.Enabled,
TerminalStorageClass: a.TerminalStorageClass,
}
// Return ToggleTime and TSCUpdateTime only if parsed with valid values.
t, err := time.Parse(time.RFC3339, a.ToggleTime)
if err != nil {
return &Autoclass{
Enabled: a.Enabled,
}
if err == nil {
ac.ToggleTime = t
}
return &Autoclass{
Enabled: a.Enabled,
ToggleTime: t,
ut, err := time.Parse(time.RFC3339, a.TerminalStorageClassUpdateTime)
if err == nil {
ac.TerminalStorageClassUpdateTime = ut
}
return ac
}
func toAutoclassFromProto(a *storagepb.Bucket_Autoclass) *Autoclass {
@ -1976,8 +2046,10 @@ func toAutoclassFromProto(a *storagepb.Bucket_Autoclass) *Autoclass {
return nil
}
return &Autoclass{
Enabled: a.GetEnabled(),
ToggleTime: a.GetToggleTime().AsTime(),
Enabled: a.GetEnabled(),
ToggleTime: a.GetToggleTime().AsTime(),
TerminalStorageClass: a.GetTerminalStorageClass(),
TerminalStorageClassUpdateTime: a.GetTerminalStorageClassUpdateTime().AsTime(),
}
}

View File

@ -44,7 +44,7 @@ type storageClient interface {
// Top-level methods.
GetServiceAccount(ctx context.Context, project string, opts ...storageOption) (string, error)
CreateBucket(ctx context.Context, project, bucket string, attrs *BucketAttrs, opts ...storageOption) (*BucketAttrs, error)
CreateBucket(ctx context.Context, project, bucket string, attrs *BucketAttrs, enableObjectRetention *bool, opts ...storageOption) (*BucketAttrs, error)
ListBuckets(ctx context.Context, project string, opts ...storageOption) *BucketIterator
Close() error
@ -60,7 +60,7 @@ type storageClient interface {
DeleteObject(ctx context.Context, bucket, object string, gen int64, conds *Conditions, opts ...storageOption) error
GetObject(ctx context.Context, bucket, object string, gen int64, encryptionKey []byte, conds *Conditions, opts ...storageOption) (*ObjectAttrs, error)
UpdateObject(ctx context.Context, bucket, object string, uattrs *ObjectAttrsToUpdate, gen int64, encryptionKey []byte, conds *Conditions, opts ...storageOption) (*ObjectAttrs, error)
UpdateObject(ctx context.Context, params *updateObjectParams, opts ...storageOption) (*ObjectAttrs, error)
// Default Object ACL methods.
@ -254,6 +254,9 @@ type openWriterParams struct {
// attrs - see `Writer.ObjectAttrs`.
// Required.
attrs *ObjectAttrs
// forceEmptyContentType - Disables auto-detect of Content-Type
// Optional.
forceEmptyContentType bool
// conds - see `Writer.o.conds`.
// Optional.
conds *Conditions
@ -291,6 +294,15 @@ type newRangeReaderParams struct {
readCompressed bool // Use accept-encoding: gzip. Only works for HTTP currently.
}
type updateObjectParams struct {
bucket, object string
uattrs *ObjectAttrsToUpdate
gen int64
encryptionKey []byte
conds *Conditions
overrideRetention *bool
}
type composeObjectRequest struct {
dstBucket string
dstObject destinationObject

View File

@ -320,6 +320,45 @@ client (using [Client.SetRetry]). For example:
// Handle err.
}
# Sending Custom Headers
You can add custom headers to any API call made by this package by using
[callctx.SetHeaders] on the context which is passed to the method. For example,
to add a [custom audit logging] header:
ctx := context.Background()
ctx = callctx.SetHeaders(ctx, "x-goog-custom-audit-<key>", "<value>")
// Use client as usual with the context and the additional headers will be sent.
client.Bucket("my-bucket").Attrs(ctx)
# Experimental gRPC API
This package includes support for the Cloud Storage gRPC API, which is currently
in preview. This implementation uses gRPC rather than the current JSON & XML
APIs to make requests to Cloud Storage. Kindly contact the Google Cloud Storage gRPC
team at gcs-grpc-contact@google.com with a list of GCS buckets you would like to
allowlist to access this API. The Go Storage gRPC library is not yet generally
available, so it may be subject to breaking changes.
To create a client which will use gRPC, use the alternate constructor:
ctx := context.Background()
client, err := storage.NewGRPCClient(ctx)
if err != nil {
// TODO: Handle error.
}
// Use client as usual.
If the application is running within GCP, users may get better performance by
enabling Google Direct Access (enabling requests to skip some proxy steps). To enable,
set the environment variable `GOOGLE_CLOUD_ENABLE_DIRECT_PATH_XDS=true` and add
the following side-effect imports to your application:
import (
_ "google.golang.org/grpc/balancer/rls"
_ "google.golang.org/grpc/xds/googledirectpath"
)
[Cloud Storage IAM docs]: https://cloud.google.com/storage/docs/access-control/iam
[XML POST Object docs]: https://cloud.google.com/storage/docs/xml-api/post-object
[Cloud Storage retry docs]: https://cloud.google.com/storage/docs/retry-strategy
@ -327,5 +366,6 @@ client (using [Client.SetRetry]). For example:
[gcloud using application default credentials]: https://cloud.google.com/sdk/gcloud/reference/auth/application-default/login
[impersonation enabled]: https://cloud.google.com/sdk/gcloud/reference#--impersonate-service-account
[IAM Service Account Credentials API]: https://console.developers.google.com/apis/api/iamcredentials.googleapis.com/overview
[custom audit logging]: https://cloud.google.com/storage/docs/audit-logging#add-custom-metadata
*/
package storage // import "cloud.google.com/go/storage"

File diff suppressed because it is too large Load Diff

View File

@ -20,14 +20,12 @@ import (
"fmt"
"time"
storagepb "cloud.google.com/go/storage/internal/apiv2/stubs"
"cloud.google.com/go/storage/internal/apiv2/storagepb"
"google.golang.org/api/iterator"
raw "google.golang.org/api/storage/v1"
)
// HMACState is the state of the HMAC key.
//
// This type is EXPERIMENTAL and subject to change or removal without notice.
type HMACState string
const (
@ -50,8 +48,6 @@ const (
//
// HMAC keys are used to authenticate signed access to objects. To enable HMAC key
// authentication, please visit https://cloud.google.com/storage/docs/migrating.
//
// This type is EXPERIMENTAL and subject to change or removal without notice.
type HMACKey struct {
// The HMAC's secret key.
Secret string
@ -85,8 +81,6 @@ type HMACKey struct {
}
// HMACKeyHandle helps provide access and management for HMAC keys.
//
// This type is EXPERIMENTAL and subject to change or removal without notice.
type HMACKeyHandle struct {
projectID string
accessID string
@ -95,8 +89,6 @@ type HMACKeyHandle struct {
}
// HMACKeyHandle creates a handle that will be used for HMACKey operations.
//
// This method is EXPERIMENTAL and subject to change or removal without notice.
func (c *Client) HMACKeyHandle(projectID, accessID string) *HMACKeyHandle {
return &HMACKeyHandle{
projectID: projectID,
@ -111,8 +103,6 @@ func (c *Client) HMACKeyHandle(projectID, accessID string) *HMACKeyHandle {
//
// Options such as UserProjectForHMACKeys can be used to set the
// userProject to be billed against for operations.
//
// This method is EXPERIMENTAL and subject to change or removal without notice.
func (hkh *HMACKeyHandle) Get(ctx context.Context, opts ...HMACKeyOption) (*HMACKey, error) {
desc := new(hmacKeyDesc)
for _, opt := range opts {
@ -128,8 +118,6 @@ func (hkh *HMACKeyHandle) Get(ctx context.Context, opts ...HMACKeyOption) (*HMAC
// Delete invokes an RPC to delete the key referenced by accessID, on Google Cloud Storage.
// Only inactive HMAC keys can be deleted.
// After deletion, a key cannot be used to authenticate requests.
//
// This method is EXPERIMENTAL and subject to change or removal without notice.
func (hkh *HMACKeyHandle) Delete(ctx context.Context, opts ...HMACKeyOption) error {
desc := new(hmacKeyDesc)
for _, opt := range opts {
@ -187,8 +175,6 @@ func toHMACKeyFromProto(pbmd *storagepb.HmacKeyMetadata) *HMACKey {
}
// CreateHMACKey invokes an RPC for Google Cloud Storage to create a new HMACKey.
//
// This method is EXPERIMENTAL and subject to change or removal without notice.
func (c *Client) CreateHMACKey(ctx context.Context, projectID, serviceAccountEmail string, opts ...HMACKeyOption) (*HMACKey, error) {
if projectID == "" {
return nil, errors.New("storage: expecting a non-blank projectID")
@ -208,8 +194,6 @@ func (c *Client) CreateHMACKey(ctx context.Context, projectID, serviceAccountEma
}
// HMACKeyAttrsToUpdate defines the attributes of an HMACKey that will be updated.
//
// This type is EXPERIMENTAL and subject to change or removal without notice.
type HMACKeyAttrsToUpdate struct {
// State is required and must be either StateActive or StateInactive.
State HMACState
@ -219,8 +203,6 @@ type HMACKeyAttrsToUpdate struct {
}
// Update mutates the HMACKey referred to by accessID.
//
// This method is EXPERIMENTAL and subject to change or removal without notice.
func (h *HMACKeyHandle) Update(ctx context.Context, au HMACKeyAttrsToUpdate, opts ...HMACKeyOption) (*HMACKey, error) {
if au.State != Active && au.State != Inactive {
return nil, fmt.Errorf("storage: invalid state %q for update, must be either %q or %q", au.State, Active, Inactive)
@ -240,8 +222,6 @@ func (h *HMACKeyHandle) Update(ctx context.Context, au HMACKeyAttrsToUpdate, opt
// An HMACKeysIterator is an iterator over HMACKeys.
//
// Note: This iterator is not safe for concurrent operations without explicit synchronization.
//
// This type is EXPERIMENTAL and subject to change or removal without notice.
type HMACKeysIterator struct {
ctx context.Context
raw *raw.ProjectsHmacKeysService
@ -257,8 +237,6 @@ type HMACKeysIterator struct {
// ListHMACKeys returns an iterator for listing HMACKeys.
//
// Note: This iterator is not safe for concurrent operations without explicit synchronization.
//
// This method is EXPERIMENTAL and subject to change or removal without notice.
func (c *Client) ListHMACKeys(ctx context.Context, projectID string, opts ...HMACKeyOption) *HMACKeysIterator {
desc := new(hmacKeyDesc)
for _, opt := range opts {
@ -274,8 +252,6 @@ func (c *Client) ListHMACKeys(ctx context.Context, projectID string, opts ...HMA
// calls will return iterator.Done.
//
// Note: This iterator is not safe for concurrent operations without explicit synchronization.
//
// This method is EXPERIMENTAL and subject to change or removal without notice.
func (it *HMACKeysIterator) Next() (*HMACKey, error) {
if err := it.nextFunc(); err != nil {
return nil, err
@ -290,8 +266,6 @@ func (it *HMACKeysIterator) Next() (*HMACKey, error) {
// PageInfo supports pagination. See the google.golang.org/api/iterator package for details.
//
// Note: This iterator is not safe for concurrent operations without explicit synchronization.
//
// This method is EXPERIMENTAL and subject to change or removal without notice.
func (it *HMACKeysIterator) PageInfo() *iterator.PageInfo { return it.pageInfo }
func (it *HMACKeysIterator) fetch(pageSize int, pageToken string) (token string, err error) {
@ -315,12 +289,11 @@ func (it *HMACKeysIterator) fetch(pageSize int, pageToken string) (token string,
call = call.MaxResults(int64(pageSize))
}
ctx := it.ctx
var resp *raw.HmacKeysMetadata
err = run(it.ctx, func() error {
err = run(it.ctx, func(ctx context.Context) error {
resp, err = call.Context(ctx).Do()
return err
}, it.retry, true, setRetryHeaderHTTP(call))
}, it.retry, true)
if err != nil {
return "", err
}
@ -345,8 +318,6 @@ type hmacKeyDesc struct {
}
// HMACKeyOption configures the behavior of HMACKey related methods and actions.
//
// This interface is EXPERIMENTAL and subject to change or removal without notice.
type HMACKeyOption interface {
withHMACKeyDesc(*hmacKeyDesc)
}
@ -362,8 +333,6 @@ func (hkdf hmacKeyDescFunc) withHMACKeyDesc(hkd *hmacKeyDesc) {
//
// Only one service account email can be used as a filter, so if multiple
// of these options are applied, the last email to be set will be used.
//
// This option is EXPERIMENTAL and subject to change or removal without notice.
func ForHMACKeyServiceAccountEmail(serviceAccountEmail string) HMACKeyOption {
return hmacKeyDescFunc(func(hkd *hmacKeyDesc) {
hkd.forServiceAccountEmail = serviceAccountEmail
@ -371,8 +340,6 @@ func ForHMACKeyServiceAccountEmail(serviceAccountEmail string) HMACKeyOption {
}
// ShowDeletedHMACKeys will also list keys whose state is "DELETED".
//
// This option is EXPERIMENTAL and subject to change or removal without notice.
func ShowDeletedHMACKeys() HMACKeyOption {
return hmacKeyDescFunc(func(hkd *hmacKeyDesc) {
hkd.showDeletedKeys = true
@ -383,8 +350,6 @@ func ShowDeletedHMACKeys() HMACKeyOption {
// if userProjectID is non-empty.
//
// Note: This is a noop right now and only provided for API compatibility.
//
// This option is EXPERIMENTAL and subject to change or removal without notice.
func UserProjectForHMACKeys(userProjectID string) HMACKeyOption {
return hmacKeyDescFunc(func(hkd *hmacKeyDesc) {
hkd.userProjectID = userProjectID

View File

@ -32,6 +32,7 @@ import (
"cloud.google.com/go/iam/apiv1/iampb"
"cloud.google.com/go/internal/optional"
"cloud.google.com/go/internal/trace"
"github.com/googleapis/gax-go/v2/callctx"
"golang.org/x/oauth2/google"
"google.golang.org/api/googleapi"
"google.golang.org/api/iterator"
@ -44,12 +45,10 @@ import (
// httpStorageClient is the HTTP-JSON API implementation of the transport-agnostic
// storageClient interface.
//
// This is an experimental API and not intended for public use.
type httpStorageClient struct {
creds *google.Credentials
hc *http.Client
readHost string
xmlHost string
raw *raw.Service
scheme string
settings *settings
@ -58,8 +57,6 @@ type httpStorageClient struct {
// newHTTPStorageClient initializes a new storageClient that uses the HTTP-JSON
// Storage API.
//
// This is an experimental API and not intended for public use.
func newHTTPStorageClient(ctx context.Context, opts ...storageOption) (storageClient, error) {
s := initSettings(opts...)
o := s.clientOption
@ -77,9 +74,10 @@ func newHTTPStorageClient(ctx context.Context, opts ...storageOption) (storageCl
// Prepend default options to avoid overriding options passed by the user.
o = append([]option.ClientOption{option.WithScopes(ScopeFullControl, "https://www.googleapis.com/auth/cloud-platform"), option.WithUserAgent(userAgent)}, o...)
o = append(o, internaloption.WithDefaultEndpoint("https://storage.googleapis.com/storage/v1/"))
o = append(o, internaloption.WithDefaultMTLSEndpoint("https://storage.mtls.googleapis.com/storage/v1/"))
o = append(o, internaloption.WithDefaultEndpointTemplate("https://storage.UNIVERSE_DOMAIN/storage/v1/"),
internaloption.WithDefaultMTLSEndpoint("https://storage.mtls.googleapis.com/storage/v1/"),
internaloption.WithDefaultUniverseDomain("googleapis.com"),
)
// Don't error out here. The user may have passed in their own HTTP
// client which does not auth with ADC or other common conventions.
c, err := transport.Creds(ctx, o...)
@ -123,7 +121,7 @@ func newHTTPStorageClient(ctx context.Context, opts ...storageOption) (storageCl
if err != nil {
return nil, fmt.Errorf("storage client: %w", err)
}
// Update readHost and scheme with the chosen endpoint.
// Update xmlHost and scheme with the chosen endpoint.
u, err := url.Parse(ep)
if err != nil {
return nil, fmt.Errorf("supplied endpoint %q is not valid: %w", ep, err)
@ -132,7 +130,7 @@ func newHTTPStorageClient(ctx context.Context, opts ...storageOption) (storageCl
return &httpStorageClient{
creds: creds,
hc: hc,
readHost: u.Host,
xmlHost: u.Host,
raw: rawService,
scheme: u.Scheme,
settings: s,
@ -151,18 +149,18 @@ func (c *httpStorageClient) GetServiceAccount(ctx context.Context, project strin
s := callSettings(c.settings, opts...)
call := c.raw.Projects.ServiceAccount.Get(project)
var res *raw.ServiceAccount
err := run(ctx, func() error {
err := run(ctx, func(ctx context.Context) error {
var err error
res, err = call.Context(ctx).Do()
return err
}, s.retry, s.idempotent, setRetryHeaderHTTP(call))
}, s.retry, s.idempotent)
if err != nil {
return "", err
}
return res.EmailAddress, nil
}
func (c *httpStorageClient) CreateBucket(ctx context.Context, project, bucket string, attrs *BucketAttrs, opts ...storageOption) (*BucketAttrs, error) {
func (c *httpStorageClient) CreateBucket(ctx context.Context, project, bucket string, attrs *BucketAttrs, enableObjectRetention *bool, opts ...storageOption) (*BucketAttrs, error) {
s := callSettings(c.settings, opts...)
var bkt *raw.Bucket
if attrs != nil {
@ -184,15 +182,18 @@ func (c *httpStorageClient) CreateBucket(ctx context.Context, project, bucket st
if attrs != nil && attrs.PredefinedDefaultObjectACL != "" {
req.PredefinedDefaultObjectAcl(attrs.PredefinedDefaultObjectACL)
}
if enableObjectRetention != nil {
req.EnableObjectRetention(*enableObjectRetention)
}
var battrs *BucketAttrs
err := run(ctx, func() error {
err := run(ctx, func(ctx context.Context) error {
b, err := req.Context(ctx).Do()
if err != nil {
return err
}
battrs, err = newBucket(b)
return err
}, s.retry, s.idempotent, setRetryHeaderHTTP(req))
}, s.retry, s.idempotent)
return battrs, err
}
@ -213,10 +214,10 @@ func (c *httpStorageClient) ListBuckets(ctx context.Context, project string, opt
req.MaxResults(int64(pageSize))
}
var resp *raw.Buckets
err = run(it.ctx, func() error {
resp, err = req.Context(it.ctx).Do()
err = run(it.ctx, func(ctx context.Context) error {
resp, err = req.Context(ctx).Do()
return err
}, s.retry, s.idempotent, setRetryHeaderHTTP(req))
}, s.retry, s.idempotent)
if err != nil {
return "", err
}
@ -251,7 +252,7 @@ func (c *httpStorageClient) DeleteBucket(ctx context.Context, bucket string, con
req.UserProject(s.userProject)
}
return run(ctx, func() error { return req.Context(ctx).Do() }, s.retry, s.idempotent, setRetryHeaderHTTP(req))
return run(ctx, func(ctx context.Context) error { return req.Context(ctx).Do() }, s.retry, s.idempotent)
}
func (c *httpStorageClient) GetBucket(ctx context.Context, bucket string, conds *BucketConditions, opts ...storageOption) (*BucketAttrs, error) {
@ -267,10 +268,10 @@ func (c *httpStorageClient) GetBucket(ctx context.Context, bucket string, conds
}
var resp *raw.Bucket
err = run(ctx, func() error {
err = run(ctx, func(ctx context.Context) error {
resp, err = req.Context(ctx).Do()
return err
}, s.retry, s.idempotent, setRetryHeaderHTTP(req))
}, s.retry, s.idempotent)
var e *googleapi.Error
if ok := errors.As(err, &e); ok && e.Code == http.StatusNotFound {
@ -301,10 +302,10 @@ func (c *httpStorageClient) UpdateBucket(ctx context.Context, bucket string, uat
}
var rawBucket *raw.Bucket
err = run(ctx, func() error {
err = run(ctx, func(ctx context.Context) error {
rawBucket, err = req.Context(ctx).Do()
return err
}, s.retry, s.idempotent, setRetryHeaderHTTP(req))
}, s.retry, s.idempotent)
if err != nil {
return nil, err
}
@ -320,10 +321,10 @@ func (c *httpStorageClient) LockBucketRetentionPolicy(ctx context.Context, bucke
}
req := c.raw.Buckets.LockRetentionPolicy(bucket, metageneration)
return run(ctx, func() error {
return run(ctx, func(ctx context.Context) error {
_, err := req.Context(ctx).Do()
return err
}, s.retry, s.idempotent, setRetryHeaderHTTP(req))
}, s.retry, s.idempotent)
}
func (c *httpStorageClient) ListObjects(ctx context.Context, bucket string, q *Query, opts ...storageOption) *ObjectIterator {
s := callSettings(c.settings, opts...)
@ -347,6 +348,8 @@ func (c *httpStorageClient) ListObjects(ctx context.Context, bucket string, q *Q
req.EndOffset(it.query.EndOffset)
req.Versions(it.query.Versions)
req.IncludeTrailingDelimiter(it.query.IncludeTrailingDelimiter)
req.MatchGlob(it.query.MatchGlob)
req.IncludeFoldersAsPrefixes(it.query.IncludeFoldersAsPrefixes)
if selection := it.query.toFieldSelection(); selection != "" {
req.Fields("nextPageToken", googleapi.Field(selection))
}
@ -359,10 +362,10 @@ func (c *httpStorageClient) ListObjects(ctx context.Context, bucket string, q *Q
}
var resp *raw.Objects
var err error
err = run(it.ctx, func() error {
resp, err = req.Context(it.ctx).Do()
err = run(it.ctx, func(ctx context.Context) error {
resp, err = req.Context(ctx).Do()
return err
}, s.retry, s.idempotent, setRetryHeaderHTTP(req))
}, s.retry, s.idempotent)
if err != nil {
var e *googleapi.Error
if ok := errors.As(err, &e); ok && e.Code == http.StatusNotFound {
@ -397,7 +400,7 @@ func (c *httpStorageClient) DeleteObject(ctx context.Context, bucket, object str
if s.userProject != "" {
req.UserProject(s.userProject)
}
err := run(ctx, func() error { return req.Context(ctx).Do() }, s.retry, s.idempotent, setRetryHeaderHTTP(req))
err := run(ctx, func(ctx context.Context) error { return req.Context(ctx).Do() }, s.retry, s.idempotent)
var e *googleapi.Error
if ok := errors.As(err, &e); ok && e.Code == http.StatusNotFound {
return ErrObjectNotExist
@ -419,10 +422,10 @@ func (c *httpStorageClient) GetObject(ctx context.Context, bucket, object string
}
var obj *raw.Object
var err error
err = run(ctx, func() error {
err = run(ctx, func(ctx context.Context) error {
obj, err = req.Context(ctx).Do()
return err
}, s.retry, s.idempotent, setRetryHeaderHTTP(req))
}, s.retry, s.idempotent)
var e *googleapi.Error
if ok := errors.As(err, &e); ok && e.Code == http.StatusNotFound {
return nil, ErrObjectNotExist
@ -433,7 +436,8 @@ func (c *httpStorageClient) GetObject(ctx context.Context, bucket, object string
return newObject(obj), nil
}
func (c *httpStorageClient) UpdateObject(ctx context.Context, bucket, object string, uattrs *ObjectAttrsToUpdate, gen int64, encryptionKey []byte, conds *Conditions, opts ...storageOption) (*ObjectAttrs, error) {
func (c *httpStorageClient) UpdateObject(ctx context.Context, params *updateObjectParams, opts ...storageOption) (*ObjectAttrs, error) {
uattrs := params.uattrs
s := callSettings(c.settings, opts...)
var attrs ObjectAttrs
@ -498,11 +502,21 @@ func (c *httpStorageClient) UpdateObject(ctx context.Context, bucket, object str
// we don't append to nullFields here.
forceSendFields = append(forceSendFields, "Acl")
}
rawObj := attrs.toRawObject(bucket)
if uattrs.Retention != nil {
// For ObjectRetention it's an error to send empty fields.
// Instead we send a null as the user's intention is to remove.
if uattrs.Retention.Mode == "" && uattrs.Retention.RetainUntil.IsZero() {
nullFields = append(nullFields, "Retention")
} else {
attrs.Retention = uattrs.Retention
forceSendFields = append(forceSendFields, "Retention")
}
}
rawObj := attrs.toRawObject(params.bucket)
rawObj.ForceSendFields = forceSendFields
rawObj.NullFields = nullFields
call := c.raw.Objects.Patch(bucket, object, rawObj).Projection("full").Context(ctx)
if err := applyConds("Update", gen, conds, call); err != nil {
call := c.raw.Objects.Patch(params.bucket, params.object, rawObj).Projection("full")
if err := applyConds("Update", params.gen, params.conds, call); err != nil {
return nil, err
}
if s.userProject != "" {
@ -511,12 +525,17 @@ func (c *httpStorageClient) UpdateObject(ctx context.Context, bucket, object str
if uattrs.PredefinedACL != "" {
call.PredefinedAcl(uattrs.PredefinedACL)
}
if err := setEncryptionHeaders(call.Header(), encryptionKey, false); err != nil {
if err := setEncryptionHeaders(call.Header(), params.encryptionKey, false); err != nil {
return nil, err
}
if params.overrideRetention != nil {
call.OverrideUnlockedRetention(*params.overrideRetention)
}
var obj *raw.Object
var err error
err = run(ctx, func() error { obj, err = call.Do(); return err }, s.retry, s.idempotent, setRetryHeaderHTTP(call))
err = run(ctx, func(ctx context.Context) error { obj, err = call.Context(ctx).Do(); return err }, s.retry, s.idempotent)
var e *googleapi.Error
if errors.As(err, &e) && e.Code == http.StatusNotFound {
return nil, ErrObjectNotExist
@ -533,7 +552,7 @@ func (c *httpStorageClient) DeleteDefaultObjectACL(ctx context.Context, bucket s
s := callSettings(c.settings, opts...)
req := c.raw.DefaultObjectAccessControls.Delete(bucket, string(entity))
configureACLCall(ctx, s.userProject, req)
return run(ctx, func() error { return req.Context(ctx).Do() }, s.retry, s.idempotent, setRetryHeaderHTTP(req))
return run(ctx, func(ctx context.Context) error { return req.Context(ctx).Do() }, s.retry, s.idempotent)
}
func (c *httpStorageClient) ListDefaultObjectACLs(ctx context.Context, bucket string, opts ...storageOption) ([]ACLRule, error) {
@ -542,10 +561,10 @@ func (c *httpStorageClient) ListDefaultObjectACLs(ctx context.Context, bucket st
var err error
req := c.raw.DefaultObjectAccessControls.List(bucket)
configureACLCall(ctx, s.userProject, req)
err = run(ctx, func() error {
acls, err = req.Do()
err = run(ctx, func(ctx context.Context) error {
acls, err = req.Context(ctx).Do()
return err
}, s.retry, true, setRetryHeaderHTTP(req))
}, s.retry, true)
if err != nil {
return nil, err
}
@ -562,14 +581,13 @@ func (c *httpStorageClient) UpdateDefaultObjectACL(ctx context.Context, bucket s
Entity: string(entity),
Role: string(role),
}
var req setRequest
var err error
req = c.raw.DefaultObjectAccessControls.Update(bucket, string(entity), acl)
req := c.raw.DefaultObjectAccessControls.Update(bucket, string(entity), acl)
configureACLCall(ctx, s.userProject, req)
return run(ctx, func() error {
_, err = req.Do()
return run(ctx, func(ctx context.Context) error {
_, err = req.Context(ctx).Do()
return err
}, s.retry, s.idempotent, setRetryHeaderHTTP(req))
}, s.retry, s.idempotent)
}
// Bucket ACL methods.
@ -578,7 +596,7 @@ func (c *httpStorageClient) DeleteBucketACL(ctx context.Context, bucket string,
s := callSettings(c.settings, opts...)
req := c.raw.BucketAccessControls.Delete(bucket, string(entity))
configureACLCall(ctx, s.userProject, req)
return run(ctx, func() error { return req.Context(ctx).Do() }, s.retry, s.idempotent, setRetryHeaderHTTP(req))
return run(ctx, func(ctx context.Context) error { return req.Context(ctx).Do() }, s.retry, s.idempotent)
}
func (c *httpStorageClient) ListBucketACLs(ctx context.Context, bucket string, opts ...storageOption) ([]ACLRule, error) {
@ -587,10 +605,10 @@ func (c *httpStorageClient) ListBucketACLs(ctx context.Context, bucket string, o
var err error
req := c.raw.BucketAccessControls.List(bucket)
configureACLCall(ctx, s.userProject, req)
err = run(ctx, func() error {
acls, err = req.Do()
err = run(ctx, func(ctx context.Context) error {
acls, err = req.Context(ctx).Do()
return err
}, s.retry, true, setRetryHeaderHTTP(req))
}, s.retry, true)
if err != nil {
return nil, err
}
@ -607,10 +625,10 @@ func (c *httpStorageClient) UpdateBucketACL(ctx context.Context, bucket string,
req := c.raw.BucketAccessControls.Update(bucket, string(entity), acl)
configureACLCall(ctx, s.userProject, req)
var err error
return run(ctx, func() error {
_, err = req.Do()
return run(ctx, func(ctx context.Context) error {
_, err = req.Context(ctx).Do()
return err
}, s.retry, s.idempotent, setRetryHeaderHTTP(req))
}, s.retry, s.idempotent)
}
// configureACLCall sets the context, user project and headers on the apiary library call.
@ -630,7 +648,7 @@ func (c *httpStorageClient) DeleteObjectACL(ctx context.Context, bucket, object
s := callSettings(c.settings, opts...)
req := c.raw.ObjectAccessControls.Delete(bucket, object, string(entity))
configureACLCall(ctx, s.userProject, req)
return run(ctx, func() error { return req.Context(ctx).Do() }, s.retry, s.idempotent, setRetryHeaderHTTP(req))
return run(ctx, func(ctx context.Context) error { return req.Context(ctx).Do() }, s.retry, s.idempotent)
}
// ListObjectACLs retrieves object ACL entries. By default, it operates on the latest generation of this object.
@ -641,10 +659,10 @@ func (c *httpStorageClient) ListObjectACLs(ctx context.Context, bucket, object s
var err error
req := c.raw.ObjectAccessControls.List(bucket, object)
configureACLCall(ctx, s.userProject, req)
err = run(ctx, func() error {
acls, err = req.Do()
err = run(ctx, func(ctx context.Context) error {
acls, err = req.Context(ctx).Do()
return err
}, s.retry, s.idempotent, setRetryHeaderHTTP(req))
}, s.retry, s.idempotent)
if err != nil {
return nil, err
}
@ -663,14 +681,13 @@ func (c *httpStorageClient) UpdateObjectACL(ctx context.Context, bucket, object
Entity: string(entity),
Role: string(role),
}
var req setRequest
var err error
req = c.raw.ObjectAccessControls.Update(bucket, object, string(entity), acl)
req := c.raw.ObjectAccessControls.Update(bucket, object, string(entity), acl)
configureACLCall(ctx, s.userProject, req)
return run(ctx, func() error {
_, err = req.Do()
return run(ctx, func(ctx context.Context) error {
_, err = req.Context(ctx).Do()
return err
}, s.retry, s.idempotent, setRetryHeaderHTTP(req))
}, s.retry, s.idempotent)
}
// Media operations.
@ -694,7 +711,7 @@ func (c *httpStorageClient) ComposeObject(ctx context.Context, req *composeObjec
rawReq.SourceObjects = append(rawReq.SourceObjects, srcObj)
}
call := c.raw.Objects.Compose(req.dstBucket, req.dstObject.name, rawReq).Context(ctx)
call := c.raw.Objects.Compose(req.dstBucket, req.dstObject.name, rawReq)
if err := applyConds("ComposeFrom destination", defaultGen, req.dstObject.conds, call); err != nil {
return nil, err
}
@ -711,9 +728,9 @@ func (c *httpStorageClient) ComposeObject(ctx context.Context, req *composeObjec
setClientHeader(call.Header())
var err error
retryCall := func() error { obj, err = call.Do(); return err }
retryCall := func(ctx context.Context) error { obj, err = call.Context(ctx).Do(); return err }
if err := run(ctx, retryCall, s.retry, s.idempotent, setRetryHeaderHTTP(call)); err != nil {
if err := run(ctx, retryCall, s.retry, s.idempotent); err != nil {
return nil, err
}
return newObject(obj), nil
@ -723,7 +740,7 @@ func (c *httpStorageClient) RewriteObject(ctx context.Context, req *rewriteObjec
rawObject := req.dstObject.attrs.toRawObject("")
call := c.raw.Objects.Rewrite(req.srcObject.bucket, req.srcObject.name, req.dstObject.bucket, req.dstObject.name, rawObject)
call.Context(ctx).Projection("full")
call.Projection("full")
if req.token != "" {
call.RewriteToken(req.token)
}
@ -759,9 +776,9 @@ func (c *httpStorageClient) RewriteObject(ctx context.Context, req *rewriteObjec
var err error
setClientHeader(call.Header())
retryCall := func() error { res, err = call.Do(); return err }
retryCall := func(ctx context.Context) error { res, err = call.Context(ctx).Do(); return err }
if err := run(ctx, retryCall, s.retry, s.idempotent, setRetryHeaderHTTP(call)); err != nil {
if err := run(ctx, retryCall, s.retry, s.idempotent); err != nil {
return nil, err
}
@ -790,9 +807,10 @@ func (c *httpStorageClient) NewRangeReader(ctx context.Context, params *newRange
func (c *httpStorageClient) newRangeReaderXML(ctx context.Context, params *newRangeReaderParams, s *settings) (r *Reader, err error) {
u := &url.URL{
Scheme: c.scheme,
Host: c.readHost,
Path: fmt.Sprintf("/%s/%s", params.bucket, params.object),
Scheme: c.scheme,
Host: c.xmlHost,
Path: fmt.Sprintf("/%s/%s", params.bucket, params.object),
RawPath: fmt.Sprintf("/%s/%s", params.bucket, url.PathEscape(params.object)),
}
verb := "GET"
if params.length == 0 {
@ -802,7 +820,6 @@ func (c *httpStorageClient) newRangeReaderXML(ctx context.Context, params *newRa
if err != nil {
return nil, err
}
req = req.WithContext(ctx)
if s.userProject != "" {
req.Header.Set("X-Goog-User-Project", s.userProject)
@ -812,8 +829,17 @@ func (c *httpStorageClient) newRangeReaderXML(ctx context.Context, params *newRa
return nil, err
}
// Set custom headers passed in via the context. This is only required for XML;
// for gRPC & JSON this is handled in the GAPIC and Apiary layers respectively.
ctxHeaders := callctx.HeadersFromContext(ctx)
for k, vals := range ctxHeaders {
for _, v := range vals {
req.Header.Add(k, v)
}
}
reopen := readerReopen(ctx, req.Header, params, s,
func() (*http.Response, error) { return c.hc.Do(req) },
func(ctx context.Context) (*http.Response, error) { return c.hc.Do(req.WithContext(ctx)) },
func() error { return setConditionsHeaders(req.Header, params.conds) },
func() { req.URL.RawQuery = fmt.Sprintf("generation=%d", params.gen) })
@ -828,7 +854,6 @@ func (c *httpStorageClient) newRangeReaderJSON(ctx context.Context, params *newR
call := c.raw.Objects.Get(params.bucket, params.object)
setClientHeader(call.Header())
call.Context(ctx)
call.Projection("full")
if s.userProject != "" {
@ -839,7 +864,7 @@ func (c *httpStorageClient) newRangeReaderJSON(ctx context.Context, params *newR
return nil, err
}
reopen := readerReopen(ctx, call.Header(), params, s, func() (*http.Response, error) { return call.Download() },
reopen := readerReopen(ctx, call.Header(), params, s, func(ctx context.Context) (*http.Response, error) { return call.Context(ctx).Download() },
func() error { return applyConds("NewReader", params.gen, params.conds, call) },
func() { call.Generation(params.gen) })
@ -860,7 +885,7 @@ func (c *httpStorageClient) OpenWriter(params *openWriterParams, opts ...storage
mediaOpts := []googleapi.MediaOption{
googleapi.ChunkSize(params.chunkSize),
}
if c := attrs.ContentType; c != "" {
if c := attrs.ContentType; c != "" || params.forceEmptyContentType {
mediaOpts = append(mediaOpts, googleapi.ContentType(c))
}
if params.chunkRetryDeadline != 0 {
@ -949,11 +974,11 @@ func (c *httpStorageClient) GetIamPolicy(ctx context.Context, resource string, v
call.UserProject(s.userProject)
}
var rp *raw.Policy
err := run(ctx, func() error {
err := run(ctx, func(ctx context.Context) error {
var err error
rp, err = call.Context(ctx).Do()
return err
}, s.retry, s.idempotent, setRetryHeaderHTTP(call))
}, s.retry, s.idempotent)
if err != nil {
return nil, err
}
@ -970,10 +995,10 @@ func (c *httpStorageClient) SetIamPolicy(ctx context.Context, resource string, p
call.UserProject(s.userProject)
}
return run(ctx, func() error {
return run(ctx, func(ctx context.Context) error {
_, err := call.Context(ctx).Do()
return err
}, s.retry, s.idempotent, setRetryHeaderHTTP(call))
}, s.retry, s.idempotent)
}
func (c *httpStorageClient) TestIamPermissions(ctx context.Context, resource string, permissions []string, opts ...storageOption) ([]string, error) {
@ -984,11 +1009,11 @@ func (c *httpStorageClient) TestIamPermissions(ctx context.Context, resource str
call.UserProject(s.userProject)
}
var res *raw.TestIamPermissionsResponse
err := run(ctx, func() error {
err := run(ctx, func(ctx context.Context) error {
var err error
res, err = call.Context(ctx).Do()
return err
}, s.retry, s.idempotent, setRetryHeaderHTTP(call))
}, s.retry, s.idempotent)
if err != nil {
return nil, err
}
@ -1006,10 +1031,10 @@ func (c *httpStorageClient) GetHMACKey(ctx context.Context, project, accessID st
var metadata *raw.HmacKeyMetadata
var err error
if err := run(ctx, func() error {
if err := run(ctx, func(ctx context.Context) error {
metadata, err = call.Context(ctx).Do()
return err
}, s.retry, s.idempotent, setRetryHeaderHTTP(call)); err != nil {
}, s.retry, s.idempotent); err != nil {
return nil, err
}
hk := &raw.HmacKey{
@ -1046,10 +1071,10 @@ func (c *httpStorageClient) ListHMACKeys(ctx context.Context, project, serviceAc
}
var resp *raw.HmacKeysMetadata
err = run(it.ctx, func() error {
resp, err = call.Context(it.ctx).Do()
err = run(it.ctx, func(ctx context.Context) error {
resp, err = call.Context(ctx).Do()
return err
}, s.retry, s.idempotent, setRetryHeaderHTTP(call))
}, s.retry, s.idempotent)
if err != nil {
return "", err
}
@ -1091,10 +1116,10 @@ func (c *httpStorageClient) UpdateHMACKey(ctx context.Context, project, serviceA
var metadata *raw.HmacKeyMetadata
var err error
if err := run(ctx, func() error {
if err := run(ctx, func(ctx context.Context) error {
metadata, err = call.Context(ctx).Do()
return err
}, s.retry, s.idempotent, setRetryHeaderHTTP(call)); err != nil {
}, s.retry, s.idempotent); err != nil {
return nil, err
}
hk := &raw.HmacKey{
@ -1111,11 +1136,11 @@ func (c *httpStorageClient) CreateHMACKey(ctx context.Context, project, serviceA
}
var hk *raw.HmacKey
if err := run(ctx, func() error {
if err := run(ctx, func(ctx context.Context) error {
h, err := call.Context(ctx).Do()
hk = h
return err
}, s.retry, s.idempotent, setRetryHeaderHTTP(call)); err != nil {
}, s.retry, s.idempotent); err != nil {
return nil, err
}
return toHMACKeyFromRaw(hk, true)
@ -1127,9 +1152,9 @@ func (c *httpStorageClient) DeleteHMACKey(ctx context.Context, project string, a
if s.userProject != "" {
call = call.UserProject(s.userProject)
}
return run(ctx, func() error {
return run(ctx, func(ctx context.Context) error {
return call.Context(ctx).Do()
}, s.retry, s.idempotent, setRetryHeaderHTTP(call))
}, s.retry, s.idempotent)
}
// Notification methods.
@ -1148,10 +1173,10 @@ func (c *httpStorageClient) ListNotifications(ctx context.Context, bucket string
call.UserProject(s.userProject)
}
var res *raw.Notifications
err = run(ctx, func() error {
err = run(ctx, func(ctx context.Context) error {
res, err = call.Context(ctx).Do()
return err
}, s.retry, true, setRetryHeaderHTTP(call))
}, s.retry, true)
if err != nil {
return nil, err
}
@ -1168,10 +1193,10 @@ func (c *httpStorageClient) CreateNotification(ctx context.Context, bucket strin
call.UserProject(s.userProject)
}
var rn *raw.Notification
err = run(ctx, func() error {
err = run(ctx, func(ctx context.Context) error {
rn, err = call.Context(ctx).Do()
return err
}, s.retry, s.idempotent, setRetryHeaderHTTP(call))
}, s.retry, s.idempotent)
if err != nil {
return nil, err
}
@ -1187,9 +1212,9 @@ func (c *httpStorageClient) DeleteNotification(ctx context.Context, bucket strin
if s.userProject != "" {
call.UserProject(s.userProject)
}
return run(ctx, func() error {
return run(ctx, func(ctx context.Context) error {
return call.Context(ctx).Do()
}, s.retry, s.idempotent, setRetryHeaderHTTP(call))
}, s.retry, s.idempotent)
}
type httpReader struct {
@ -1238,7 +1263,7 @@ func setRangeReaderHeaders(h http.Header, params *newRangeReaderParams) error {
// readerReopen initiates a Read with offset and length, assuming we
// have already read seen bytes.
func readerReopen(ctx context.Context, header http.Header, params *newRangeReaderParams, s *settings,
doDownload func() (*http.Response, error), applyConditions func() error, setGeneration func()) func(int64) (*http.Response, error) {
doDownload func(context.Context) (*http.Response, error), applyConditions func() error, setGeneration func()) func(int64) (*http.Response, error) {
return func(seen int64) (*http.Response, error) {
// If the context has already expired, return immediately without making a
// call.
@ -1265,8 +1290,8 @@ func readerReopen(ctx context.Context, header http.Header, params *newRangeReade
var err error
var res *http.Response
err = run(ctx, func() error {
res, err = doDownload()
err = run(ctx, func(ctx context.Context) error {
res, err = doDownload(ctx)
if err != nil {
var e *googleapi.Error
if errors.As(err, &e) {
@ -1320,7 +1345,7 @@ func readerReopen(ctx context.Context, header http.Header, params *newRangeReade
params.gen = gen64
}
return nil
}, s.retry, s.idempotent, setRetryHeaderHTTP(nil))
}, s.retry, s.idempotent)
if err != nil {
return nil, err
}
@ -1373,6 +1398,8 @@ func parseReadResponse(res *http.Response, params *newRangeReaderParams, reopen
remain := res.ContentLength
body := res.Body
// If the user requested zero bytes, explicitly close and remove the request
// body.
if params.length == 0 {
remain = 0
body.Close()

View File

@ -0,0 +1,210 @@
// Copyright 2024 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Code generated by protoc-gen-go_gapic. DO NOT EDIT.
package storage
import (
storagepb "cloud.google.com/go/storage/internal/apiv2/storagepb"
"google.golang.org/api/iterator"
)
// BucketIterator manages a stream of *storagepb.Bucket.
type BucketIterator struct {
items []*storagepb.Bucket
pageInfo *iterator.PageInfo
nextFunc func() error
// Response is the raw response for the current page.
// It must be cast to the RPC response type.
// Calling Next() or InternalFetch() updates this value.
Response interface{}
// InternalFetch is for use by the Google Cloud Libraries only.
// It is not part of the stable interface of this package.
//
// InternalFetch returns results from a single call to the underlying RPC.
// The number of results is no greater than pageSize.
// If there are no more results, nextPageToken is empty and err is nil.
InternalFetch func(pageSize int, pageToken string) (results []*storagepb.Bucket, nextPageToken string, err error)
}
// PageInfo supports pagination. See the google.golang.org/api/iterator package for details.
func (it *BucketIterator) PageInfo() *iterator.PageInfo {
return it.pageInfo
}
// Next returns the next result. Its second return value is iterator.Done if there are no more
// results. Once Next returns Done, all subsequent calls will return Done.
func (it *BucketIterator) Next() (*storagepb.Bucket, error) {
var item *storagepb.Bucket
if err := it.nextFunc(); err != nil {
return item, err
}
item = it.items[0]
it.items = it.items[1:]
return item, nil
}
func (it *BucketIterator) bufLen() int {
return len(it.items)
}
func (it *BucketIterator) takeBuf() interface{} {
b := it.items
it.items = nil
return b
}
// HmacKeyMetadataIterator manages a stream of *storagepb.HmacKeyMetadata.
type HmacKeyMetadataIterator struct {
items []*storagepb.HmacKeyMetadata
pageInfo *iterator.PageInfo
nextFunc func() error
// Response is the raw response for the current page.
// It must be cast to the RPC response type.
// Calling Next() or InternalFetch() updates this value.
Response interface{}
// InternalFetch is for use by the Google Cloud Libraries only.
// It is not part of the stable interface of this package.
//
// InternalFetch returns results from a single call to the underlying RPC.
// The number of results is no greater than pageSize.
// If there are no more results, nextPageToken is empty and err is nil.
InternalFetch func(pageSize int, pageToken string) (results []*storagepb.HmacKeyMetadata, nextPageToken string, err error)
}
// PageInfo supports pagination. See the google.golang.org/api/iterator package for details.
func (it *HmacKeyMetadataIterator) PageInfo() *iterator.PageInfo {
return it.pageInfo
}
// Next returns the next result. Its second return value is iterator.Done if there are no more
// results. Once Next returns Done, all subsequent calls will return Done.
func (it *HmacKeyMetadataIterator) Next() (*storagepb.HmacKeyMetadata, error) {
var item *storagepb.HmacKeyMetadata
if err := it.nextFunc(); err != nil {
return item, err
}
item = it.items[0]
it.items = it.items[1:]
return item, nil
}
func (it *HmacKeyMetadataIterator) bufLen() int {
return len(it.items)
}
func (it *HmacKeyMetadataIterator) takeBuf() interface{} {
b := it.items
it.items = nil
return b
}
// NotificationConfigIterator manages a stream of *storagepb.NotificationConfig.
type NotificationConfigIterator struct {
items []*storagepb.NotificationConfig
pageInfo *iterator.PageInfo
nextFunc func() error
// Response is the raw response for the current page.
// It must be cast to the RPC response type.
// Calling Next() or InternalFetch() updates this value.
Response interface{}
// InternalFetch is for use by the Google Cloud Libraries only.
// It is not part of the stable interface of this package.
//
// InternalFetch returns results from a single call to the underlying RPC.
// The number of results is no greater than pageSize.
// If there are no more results, nextPageToken is empty and err is nil.
InternalFetch func(pageSize int, pageToken string) (results []*storagepb.NotificationConfig, nextPageToken string, err error)
}
// PageInfo supports pagination. See the google.golang.org/api/iterator package for details.
func (it *NotificationConfigIterator) PageInfo() *iterator.PageInfo {
return it.pageInfo
}
// Next returns the next result. Its second return value is iterator.Done if there are no more
// results. Once Next returns Done, all subsequent calls will return Done.
func (it *NotificationConfigIterator) Next() (*storagepb.NotificationConfig, error) {
var item *storagepb.NotificationConfig
if err := it.nextFunc(); err != nil {
return item, err
}
item = it.items[0]
it.items = it.items[1:]
return item, nil
}
func (it *NotificationConfigIterator) bufLen() int {
return len(it.items)
}
func (it *NotificationConfigIterator) takeBuf() interface{} {
b := it.items
it.items = nil
return b
}
// ObjectIterator manages a stream of *storagepb.Object.
type ObjectIterator struct {
items []*storagepb.Object
pageInfo *iterator.PageInfo
nextFunc func() error
// Response is the raw response for the current page.
// It must be cast to the RPC response type.
// Calling Next() or InternalFetch() updates this value.
Response interface{}
// InternalFetch is for use by the Google Cloud Libraries only.
// It is not part of the stable interface of this package.
//
// InternalFetch returns results from a single call to the underlying RPC.
// The number of results is no greater than pageSize.
// If there are no more results, nextPageToken is empty and err is nil.
InternalFetch func(pageSize int, pageToken string) (results []*storagepb.Object, nextPageToken string, err error)
}
// PageInfo supports pagination. See the google.golang.org/api/iterator package for details.
func (it *ObjectIterator) PageInfo() *iterator.PageInfo {
return it.pageInfo
}
// Next returns the next result. Its second return value is iterator.Done if there are no more
// results. Once Next returns Done, all subsequent calls will return Done.
func (it *ObjectIterator) Next() (*storagepb.Object, error) {
var item *storagepb.Object
if err := it.nextFunc(); err != nil {
return item, err
}
item = it.items[0]
it.items = it.items[1:]
return item, nil
}
func (it *ObjectIterator) bufLen() int {
return len(it.items)
}
func (it *ObjectIterator) takeBuf() interface{} {
b := it.items
it.items = nil
return b
}

View File

@ -1,4 +1,4 @@
// Copyright 2023 Google LLC
// Copyright 2024 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@ -17,14 +17,31 @@
// Package storage is an auto-generated package for the
// Cloud Storage API.
//
// Lets you store and retrieve potentially-large, immutable data objects.
//
// NOTE: This package is in alpha. It is not stable, and is likely to change.
// Stop. This folder is likely not what you are looking for. This folder
// contains protocol buffer definitions for an API only accessible to select
// customers. Customers not participating should not depend on this file.
// Please contact Google Cloud sales if you are interested. Unless told
// otherwise by a Google Cloud representative, do not use or otherwise rely
// on any of the contents of this folder. If you would like to use Cloud
// Storage, please consult our official documentation (at
// https://cloud.google.com/storage/docs/apis) for details on our XML and
// JSON APIs, or else consider one of our client libraries (at
// https://cloud.google.com/storage/docs/reference/libraries). This API
// defined in this folder is unreleased and may shut off, break, or fail at
// any time for any users who are not registered as a part of a private
// preview program.
//
// # General documentation
//
// For information about setting deadlines, reusing contexts, and more
// please visit https://pkg.go.dev/cloud.google.com/go.
// For information that is relevant for all client libraries please reference
// https://pkg.go.dev/cloud.google.com/go#pkg-overview. Some information on this
// page includes:
//
// - [Authentication and Authorization]
// - [Timeouts and Cancellation]
// - [Testing against Client Libraries]
// - [Debugging Client Libraries]
// - [Inspecting errors]
//
// # Example usage
//
@ -61,15 +78,32 @@
// // TODO: Handle error.
// }
// defer c.Close()
//
// req := &storagepb.DeleteBucketRequest{
// // TODO: Fill request struct fields.
// // See https://pkg.go.dev/cloud.google.com/go/storage/internal/apiv2/stubs#DeleteBucketRequest.
// }
// err = c.DeleteBucket(ctx, req)
// stream, err := c.BidiWriteObject(ctx)
// if err != nil {
// // TODO: Handle error.
// }
// go func() {
// reqs := []*storagepb.BidiWriteObjectRequest{
// // TODO: Create requests.
// }
// for _, req := range reqs {
// if err := stream.Send(req); err != nil {
// // TODO: Handle error.
// }
// }
// stream.CloseSend()
// }()
// for {
// resp, err := stream.Recv()
// if err == io.EOF {
// break
// }
// if err != nil {
// // TODO: handle error.
// }
// // TODO: Use resp.
// _ = resp
// }
//
// # Use of Context
//
@ -78,18 +112,18 @@
// Individual methods on the client use the ctx given to them.
//
// To close the open connection, use the Close() method.
//
// [Authentication and Authorization]: https://pkg.go.dev/cloud.google.com/go#hdr-Authentication_and_Authorization
// [Timeouts and Cancellation]: https://pkg.go.dev/cloud.google.com/go#hdr-Timeouts_and_Cancellation
// [Testing against Client Libraries]: https://pkg.go.dev/cloud.google.com/go#hdr-Testing
// [Debugging Client Libraries]: https://pkg.go.dev/cloud.google.com/go#hdr-Debugging
// [Inspecting errors]: https://pkg.go.dev/cloud.google.com/go#hdr-Inspecting_errors
package storage // import "cloud.google.com/go/storage/internal/apiv2"
import (
"context"
"os"
"runtime"
"strconv"
"strings"
"unicode"
"google.golang.org/api/option"
"google.golang.org/grpc/metadata"
)
// For more information on implementing a client constructor hook, see
@ -106,27 +140,6 @@ func getVersionClient() string {
return versionClient
}
func insertMetadata(ctx context.Context, mds ...metadata.MD) context.Context {
out, _ := metadata.FromOutgoingContext(ctx)
out = out.Copy()
for _, md := range mds {
for k, v := range md {
out[k] = append(out[k], v...)
}
}
return metadata.NewOutgoingContext(ctx, out)
}
func checkDisableDeadlines() (bool, error) {
raw, ok := os.LookupEnv("GOOGLE_API_GO_EXPERIMENTAL_DISABLE_DEFAULT_DEADLINE")
if !ok {
return false, nil
}
b, err := strconv.ParseBool(raw)
return b, err
}
// DefaultAuthScopes reports the default set of authentication scopes to use with this package.
func DefaultAuthScopes() []string {
return []string{
@ -137,40 +150,3 @@ func DefaultAuthScopes() []string {
"https://www.googleapis.com/auth/devstorage.read_write",
}
}
// versionGo returns the Go runtime version. The returned string
// has no whitespace, suitable for reporting in header.
func versionGo() string {
const develPrefix = "devel +"
s := runtime.Version()
if strings.HasPrefix(s, develPrefix) {
s = s[len(develPrefix):]
if p := strings.IndexFunc(s, unicode.IsSpace); p >= 0 {
s = s[:p]
}
return s
}
notSemverRune := func(r rune) bool {
return !strings.ContainsRune("0123456789.", r)
}
if strings.HasPrefix(s, "go1") {
s = s[2:]
var prerelease string
if p := strings.IndexFunc(s, notSemverRune); p >= 0 {
s, prerelease = s[:p], s[p:]
}
if strings.HasSuffix(s, ".") {
s += "0"
} else if strings.Count(s, ".") < 2 {
s += ".0"
}
if prerelease != "" {
s += "-" + prerelease
}
return s
}
return "UNKNOWN"
}

View File

@ -10,6 +10,11 @@
"grpc": {
"libraryClient": "Client",
"rpcs": {
"BidiWriteObject": {
"methods": [
"BidiWriteObject"
]
},
"CancelResumableWrite": {
"methods": [
"CancelResumableWrite"
@ -120,6 +125,11 @@
"ReadObject"
]
},
"RestoreObject": {
"methods": [
"RestoreObject"
]
},
"RewriteObject": {
"methods": [
"RewriteObject"

View File

@ -1,26 +0,0 @@
// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package storage
import (
"context"
"google.golang.org/grpc/metadata"
)
// InsertMetadata inserts the given gRPC metadata into the outgoing context.
func InsertMetadata(ctx context.Context, mds ...metadata.MD) context.Context {
return insertMetadata(ctx, mds...)
}

File diff suppressed because it is too large Load Diff

View File

@ -15,4 +15,4 @@
package internal
// Version is the current tagged release of the library.
const Version = "1.30.1"
const Version = "1.39.1"

View File

@ -20,7 +20,6 @@ import (
"fmt"
"io"
"net"
"net/http"
"net/url"
"strings"
@ -29,6 +28,7 @@ import (
sinternal "cloud.google.com/go/storage/internal"
"github.com/google/uuid"
gax "github.com/googleapis/gax-go/v2"
"github.com/googleapis/gax-go/v2/callctx"
"google.golang.org/api/googleapi"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
@ -37,10 +37,15 @@ import (
var defaultRetry *retryConfig = &retryConfig{}
var xGoogDefaultHeader = fmt.Sprintf("gl-go/%s gccl/%s", version.Go(), sinternal.Version)
const (
xGoogHeaderKey = "x-goog-api-client"
idempotencyHeaderKey = "x-goog-gcs-idempotency-token"
)
// run determines whether a retry is necessary based on the config and
// idempotency information. It then calls the function with or without retries
// as appropriate, using the configured settings.
func run(ctx context.Context, call func() error, retry *retryConfig, isIdempotent bool, setHeader func(string, int)) error {
func run(ctx context.Context, call func(ctx context.Context) error, retry *retryConfig, isIdempotent bool) error {
attempts := 1
invocationID := uuid.New().String()
@ -48,8 +53,8 @@ func run(ctx context.Context, call func() error, retry *retryConfig, isIdempoten
retry = defaultRetry
}
if (retry.policy == RetryIdempotent && !isIdempotent) || retry.policy == RetryNever {
setHeader(invocationID, attempts)
return call()
ctxWithHeaders := setInvocationHeaders(ctx, invocationID, attempts)
return call(ctxWithHeaders)
}
bo := gax.Backoff{}
if retry.backoff != nil {
@ -63,30 +68,25 @@ func run(ctx context.Context, call func() error, retry *retryConfig, isIdempoten
}
return internal.Retry(ctx, bo, func() (stop bool, err error) {
setHeader(invocationID, attempts)
err = call()
ctxWithHeaders := setInvocationHeaders(ctx, invocationID, attempts)
err = call(ctxWithHeaders)
if retry.maxAttempts != nil && attempts >= *retry.maxAttempts {
return true, err
}
attempts++
return !errorFunc(err), err
})
}
func setRetryHeaderHTTP(req interface{ Header() http.Header }) func(string, int) {
return func(invocationID string, attempts int) {
if req == nil {
return
}
header := req.Header()
invocationHeader := fmt.Sprintf("gccl-invocation-id/%v gccl-attempt-count/%v", invocationID, attempts)
xGoogHeader := strings.Join([]string{invocationHeader, xGoogDefaultHeader}, " ")
header.Set("x-goog-api-client", xGoogHeader)
}
}
// Sets invocation ID headers on the context which will be propagated as
// headers in the call to the service (for both gRPC and HTTP).
func setInvocationHeaders(ctx context.Context, invocationID string, attempts int) context.Context {
invocationHeader := fmt.Sprintf("gccl-invocation-id/%v gccl-attempt-count/%v", invocationID, attempts)
xGoogHeader := strings.Join([]string{invocationHeader, xGoogDefaultHeader}, " ")
// TODO: Implement method setting header via context for gRPC
func setRetryHeaderGRPC(_ context.Context) func(string, int) {
return func(_ string, _ int) {
return
}
ctx = callctx.SetHeaders(ctx, xGoogHeaderKey, xGoogHeader)
ctx = callctx.SetHeaders(ctx, idempotencyHeaderKey, invocationID)
return ctx
}
// ShouldRetry returns true if an error is retryable, based on best practice
@ -131,12 +131,11 @@ func ShouldRetry(err error) bool {
return true
}
}
// HTTP 429, 502, 503, and 504 all map to gRPC UNAVAILABLE per
// https://grpc.github.io/grpc/core/md_doc_http-grpc-status-mapping.html.
//
// This is only necessary for the experimental gRPC-based media operations.
if st, ok := status.FromError(err); ok && st.Code() == codes.Unavailable {
return true
// UNAVAILABLE, RESOURCE_EXHAUSTED, and INTERNAL codes are all retryable for gRPC.
if st, ok := status.FromError(err); ok {
if code := st.Code(); code == codes.Unavailable || code == codes.ResourceExhausted || code == codes.Internal {
return true
}
}
// Unwrap is only supported in go1.13.x+
if e, ok := err.(interface{ Unwrap() error }); ok {

View File

@ -21,7 +21,7 @@ import (
"regexp"
"cloud.google.com/go/internal/trace"
storagepb "cloud.google.com/go/storage/internal/apiv2/stubs"
"cloud.google.com/go/storage/internal/apiv2/storagepb"
raw "google.golang.org/api/storage/v1"
)

View File

@ -57,7 +57,7 @@ func WithJSONReads() option.ClientOption {
}
// WithXMLReads is an option that may be passed to a Storage Client on creation.
// It sets the client to use the JSON API for object reads.
// It sets the client to use the XML API for object reads.
//
// This is the current default.
func WithXMLReads() option.ClientOption {

View File

@ -32,7 +32,7 @@ import (
// Please see https://cloud.google.com/storage/docs/xml-api/post-object
// for reference about the fields.
type PostPolicyV4Options struct {
// GoogleAccessID represents the authorizer of the signed URL generation.
// GoogleAccessID represents the authorizer of the signed post policy generation.
// It is typically the Google service account client email address from
// the Google Developers Console in the form of "xxx@developer.gserviceaccount.com".
// Required.
@ -85,7 +85,7 @@ type PostPolicyV4Options struct {
// Exactly one of PrivateKey or SignRawBytes must be non-nil.
SignRawBytes func(bytes []byte) (signature []byte, err error)
// Expires is the expiration time on the signed URL.
// Expires is the expiration time on the signed post policy.
// It must be a time in the future.
// Required.
Expires time.Time
@ -113,6 +113,12 @@ type PostPolicyV4Options struct {
// Optional.
Conditions []PostPolicyV4Condition
// Hostname sets the host of the signed post policy. This field overrides
// any endpoint set on a storage Client or through STORAGE_EMULATOR_HOST.
// Only compatible with PathStyle URLStyle.
// Optional.
Hostname string
shouldHashSignBytes bool
}
@ -128,6 +134,7 @@ func (opts *PostPolicyV4Options) clone() *PostPolicyV4Options {
Fields: opts.Fields,
Conditions: opts.Conditions,
shouldHashSignBytes: opts.shouldHashSignBytes,
Hostname: opts.Hostname,
}
}
@ -370,7 +377,7 @@ func GenerateSignedPostPolicyV4(bucket, object string, opts *PostPolicyV4Options
u := &url.URL{
Path: path,
RawPath: pathEncodeV4(path),
Host: opts.Style.host(bucket),
Host: opts.Style.host(opts.Hostname, bucket),
Scheme: scheme,
}

View File

@ -87,8 +87,9 @@ func (o *ObjectHandle) NewReader(ctx context.Context) (*Reader, error) {
// that file will be served back whole, regardless of the requested range as
// Google Cloud Storage dictates.
func (o *ObjectHandle) NewRangeReader(ctx context.Context, offset, length int64) (r *Reader, err error) {
ctx = trace.StartSpan(ctx, "cloud.google.com/go/storage.Object.NewRangeReader")
defer func() { trace.EndSpan(ctx, err) }()
// This span covers the life of the reader. It is closed via the context
// in Reader.Close.
ctx = trace.StartSpan(ctx, "cloud.google.com/go/storage.Object.Reader")
if err := o.validate(); err != nil {
return nil, err
@ -117,6 +118,14 @@ func (o *ObjectHandle) NewRangeReader(ctx context.Context, offset, length int64)
r, err = o.c.tc.NewRangeReader(ctx, params, opts...)
// Pass the context so that the span can be closed in Reader.Close, or close the
// span now if there is an error.
if err == nil {
r.ctx = ctx
} else {
trace.EndSpan(ctx, err)
}
return r, err
}
@ -178,16 +187,6 @@ func setConditionsHeaders(headers http.Header, conds *Conditions) error {
return nil
}
// Wrap a request to look similar to an apiary library request, in order to
// be used by run().
type readerRequestWrapper struct {
req *http.Request
}
func (w *readerRequestWrapper) Header() http.Header {
return w.req.Header
}
var emptyBody = ioutil.NopCloser(strings.NewReader(""))
// Reader reads a Cloud Storage object.
@ -204,11 +203,14 @@ type Reader struct {
gotCRC uint32 // running crc
reader io.ReadCloser
ctx context.Context
}
// Close closes the Reader. It must be called when done reading.
func (r *Reader) Close() error {
return r.reader.Close()
err := r.reader.Close()
trace.EndSpan(r.ctx, err)
return err
}
func (r *Reader) Read(p []byte) (int, error) {

View File

@ -41,7 +41,7 @@ import (
"cloud.google.com/go/internal/optional"
"cloud.google.com/go/internal/trace"
"cloud.google.com/go/storage/internal"
storagepb "cloud.google.com/go/storage/internal/apiv2/stubs"
"cloud.google.com/go/storage/internal/apiv2/storagepb"
"github.com/googleapis/gax-go/v2"
"golang.org/x/oauth2/google"
"google.golang.org/api/googleapi"
@ -109,8 +109,8 @@ type Client struct {
raw *raw.Service
// Scheme describes the scheme under the current host.
scheme string
// ReadHost is the default host used on the reader.
readHost string
// xmlHost is the default host used for XML requests.
xmlHost string
// May be nil.
creds *google.Credentials
retry *retryConfig
@ -123,7 +123,7 @@ type Client struct {
useGRPC bool
}
// NewClient creates a new Google Cloud Storage client.
// NewClient creates a new Google Cloud Storage client using the HTTP transport.
// The default scope is ScopeFullControl. To use a different scope, like
// ScopeReadOnly, use option.WithScopes.
//
@ -133,12 +133,6 @@ type Client struct {
// You may configure the client by passing in options from the [google.golang.org/api/option]
// package. You may also use options defined in this package, such as [WithJSONReads].
func NewClient(ctx context.Context, opts ...option.ClientOption) (*Client, error) {
// Use the experimental gRPC client if the env var is set.
// This is an experimental API and not intended for public use.
if withGRPC := os.Getenv("STORAGE_USE_GRPC"); withGRPC != "" {
return newGRPCClient(ctx, opts...)
}
var creds *google.Credentials
// In general, it is recommended to use raw.NewService instead of htransport.NewClient
@ -152,8 +146,10 @@ func NewClient(ctx context.Context, opts ...option.ClientOption) (*Client, error
// Prepend default options to avoid overriding options passed by the user.
opts = append([]option.ClientOption{option.WithScopes(ScopeFullControl, "https://www.googleapis.com/auth/cloud-platform"), option.WithUserAgent(userAgent)}, opts...)
opts = append(opts, internaloption.WithDefaultEndpoint("https://storage.googleapis.com/storage/v1/"))
opts = append(opts, internaloption.WithDefaultMTLSEndpoint("https://storage.mtls.googleapis.com/storage/v1/"))
opts = append(opts, internaloption.WithDefaultEndpointTemplate("https://storage.UNIVERSE_DOMAIN/storage/v1/"),
internaloption.WithDefaultMTLSEndpoint("https://storage.mtls.googleapis.com/storage/v1/"),
internaloption.WithDefaultUniverseDomain("googleapis.com"),
)
// Don't error out here. The user may have passed in their own HTTP
// client which does not auth with ADC or other common conventions.
@ -199,7 +195,7 @@ func NewClient(ctx context.Context, opts ...option.ClientOption) (*Client, error
if err != nil {
return nil, fmt.Errorf("storage client: %w", err)
}
// Update readHost and scheme with the chosen endpoint.
// Update xmlHost and scheme with the chosen endpoint.
u, err := url.Parse(ep)
if err != nil {
return nil, fmt.Errorf("supplied endpoint %q is not valid: %w", ep, err)
@ -211,20 +207,31 @@ func NewClient(ctx context.Context, opts ...option.ClientOption) (*Client, error
}
return &Client{
hc: hc,
raw: rawService,
scheme: u.Scheme,
readHost: u.Host,
creds: creds,
tc: tc,
hc: hc,
raw: rawService,
scheme: u.Scheme,
xmlHost: u.Host,
creds: creds,
tc: tc,
}, nil
}
// newGRPCClient creates a new Storage client that initializes a gRPC-based
// client. Calls that have not been implemented in gRPC will panic.
// NewGRPCClient creates a new Storage client using the gRPC transport and API.
// Client methods which have not been implemented in gRPC will return an error.
// In particular, methods for Cloud Pub/Sub notifications are not supported.
// Using a non-default universe domain is also not supported with the Storage
// gRPC client.
//
// This is an experimental API and not intended for public use.
func newGRPCClient(ctx context.Context, opts ...option.ClientOption) (*Client, error) {
// The storage gRPC API is still in preview and not yet publicly available.
// If you would like to use the API, please first contact your GCP account rep to
// request access. The API may be subject to breaking changes.
//
// Clients should be reused instead of created as needed. The methods of Client
// are safe for concurrent use by multiple goroutines.
//
// You may configure the client by passing in options from the [google.golang.org/api/option]
// package.
func NewGRPCClient(ctx context.Context, opts ...option.ClientOption) (*Client, error) {
opts = append(defaultGRPCOptions(), opts...)
tc, err := newGRPCStorageClient(ctx, withClientOptions(opts...))
if err != nil {
@ -262,13 +269,13 @@ const (
SigningSchemeV4
)
// URLStyle determines the style to use for the signed URL. pathStyle is the
// URLStyle determines the style to use for the signed URL. PathStyle is the
// default. All non-default options work with V4 scheme only. See
// https://cloud.google.com/storage/docs/request-endpoints for details.
type URLStyle interface {
// host should return the host portion of the signed URL, not including
// the scheme (e.g. storage.googleapis.com).
host(bucket string) string
host(hostname, bucket string) string
// path should return the path portion of the signed URL, which may include
// both the bucket and object name or only the object name depending on the
@ -284,7 +291,11 @@ type bucketBoundHostname struct {
hostname string
}
func (s pathStyle) host(bucket string) string {
func (s pathStyle) host(hostname, bucket string) string {
if hostname != "" {
return stripScheme(hostname)
}
if host := os.Getenv("STORAGE_EMULATOR_HOST"); host != "" {
return stripScheme(host)
}
@ -292,7 +303,11 @@ func (s pathStyle) host(bucket string) string {
return "storage.googleapis.com"
}
func (s virtualHostedStyle) host(bucket string) string {
func (s virtualHostedStyle) host(hostname, bucket string) string {
if hostname != "" {
return bucket + "." + stripScheme(hostname)
}
if host := os.Getenv("STORAGE_EMULATOR_HOST"); host != "" {
return bucket + "." + stripScheme(host)
}
@ -300,7 +315,7 @@ func (s virtualHostedStyle) host(bucket string) string {
return bucket + ".storage.googleapis.com"
}
func (s bucketBoundHostname) host(bucket string) string {
func (s bucketBoundHostname) host(_, bucket string) string {
return s.hostname
}
@ -321,7 +336,10 @@ func (s bucketBoundHostname) path(bucket, object string) string {
}
// PathStyle is the default style, and will generate a URL of the form
// "storage.googleapis.com/<bucket-name>/<object-name>".
// "<host-name>/<bucket-name>/<object-name>". By default, <host-name> is
// storage.googleapis.com, but setting an endpoint on the storage Client or
// through STORAGE_EMULATOR_HOST overrides this. Setting Hostname on
// SignedURLOptions or PostPolicyV4Options overrides everything else.
func PathStyle() URLStyle {
return pathStyle{}
}
@ -442,6 +460,12 @@ type SignedURLOptions struct {
// Scheme determines the version of URL signing to use. Default is
// SigningSchemeV2.
Scheme SigningScheme
// Hostname sets the host of the signed URL. This field overrides any
// endpoint set on a storage Client or through STORAGE_EMULATOR_HOST.
// Only compatible with PathStyle and VirtualHostedStyle URLStyles.
// Optional.
Hostname string
}
func (opts *SignedURLOptions) clone() *SignedURLOptions {
@ -458,6 +482,7 @@ func (opts *SignedURLOptions) clone() *SignedURLOptions {
Style: opts.Style,
Insecure: opts.Insecure,
Scheme: opts.Scheme,
Hostname: opts.Hostname,
}
}
@ -716,7 +741,7 @@ func signedURLV4(bucket, name string, opts *SignedURLOptions, now time.Time) (st
fmt.Fprintf(buf, "%s\n", escapedQuery)
// Fill in the hostname based on the desired URL style.
u.Host = opts.Style.host(bucket)
u.Host = opts.Style.host(opts.Hostname, bucket)
// Fill in the URL scheme.
if opts.Insecure {
@ -726,7 +751,7 @@ func signedURLV4(bucket, name string, opts *SignedURLOptions, now time.Time) (st
}
var headersWithValue []string
headersWithValue = append(headersWithValue, "host:"+u.Host)
headersWithValue = append(headersWithValue, "host:"+u.Hostname())
headersWithValue = append(headersWithValue, opts.Headers...)
if opts.ContentType != "" {
headersWithValue = append(headersWithValue, "content-type:"+opts.ContentType)
@ -850,7 +875,7 @@ func signedURLV2(bucket, name string, opts *SignedURLOptions) (string, error) {
}
encoded := base64.StdEncoding.EncodeToString(b)
u.Scheme = "https"
u.Host = PathStyle().host(bucket)
u.Host = PathStyle().host(opts.Hostname, bucket)
q := u.Query()
q.Set("GoogleAccessId", opts.GoogleAccessID)
q.Set("Expires", fmt.Sprintf("%d", opts.Expires.Unix()))
@ -862,16 +887,17 @@ func signedURLV2(bucket, name string, opts *SignedURLOptions) (string, error) {
// ObjectHandle provides operations on an object in a Google Cloud Storage bucket.
// Use BucketHandle.Object to get a handle.
type ObjectHandle struct {
c *Client
bucket string
object string
acl ACLHandle
gen int64 // a negative value indicates latest
conds *Conditions
encryptionKey []byte // AES-256 key
userProject string // for requester-pays buckets
readCompressed bool // Accept-Encoding: gzip
retry *retryConfig
c *Client
bucket string
object string
acl ACLHandle
gen int64 // a negative value indicates latest
conds *Conditions
encryptionKey []byte // AES-256 key
userProject string // for requester-pays buckets
readCompressed bool // Accept-Encoding: gzip
retry *retryConfig
overrideRetention *bool
}
// ACL provides access to the object's access control list.
@ -893,7 +919,9 @@ func (o *ObjectHandle) Generation(gen int64) *ObjectHandle {
}
// If returns a new ObjectHandle that applies a set of preconditions.
// Preconditions already set on the ObjectHandle are ignored.
// Preconditions already set on the ObjectHandle are ignored. The supplied
// Conditions must have at least one field set to a non-default value;
// otherwise an error will be returned from any operation on the ObjectHandle.
// Operations on the new handle will return an error if the preconditions are not
// satisfied. See https://cloud.google.com/storage/docs/generations-preconditions
// for more details.
@ -939,7 +967,15 @@ func (o *ObjectHandle) Update(ctx context.Context, uattrs ObjectAttrsToUpdate) (
}
isIdempotent := o.conds != nil && o.conds.MetagenerationMatch != 0
opts := makeStorageOpts(isIdempotent, o.retry, o.userProject)
return o.c.tc.UpdateObject(ctx, o.bucket, o.object, &uattrs, o.gen, o.encryptionKey, o.conds, opts...)
return o.c.tc.UpdateObject(ctx,
&updateObjectParams{
bucket: o.bucket,
object: o.object,
uattrs: &uattrs,
gen: o.gen,
encryptionKey: o.encryptionKey,
conds: o.conds,
overrideRetention: o.overrideRetention}, opts...)
}
// BucketName returns the name of the bucket.
@ -954,16 +990,19 @@ func (o *ObjectHandle) ObjectName() string {
// ObjectAttrsToUpdate is used to update the attributes of an object.
// Only fields set to non-nil values will be updated.
// For all fields except CustomTime, set the field to its zero value to delete
// it. CustomTime cannot be deleted or changed to an earlier time once set.
// For all fields except CustomTime and Retention, set the field to its zero
// value to delete it. CustomTime cannot be deleted or changed to an earlier
// time once set. Retention can be deleted (only if the Mode is Unlocked) by
// setting it to an empty value (not nil).
//
// For example, to change ContentType and delete ContentEncoding and
// Metadata, use
// For example, to change ContentType and delete ContentEncoding, Metadata and
// Retention, use:
//
// ObjectAttrsToUpdate{
// ContentType: "text/html",
// ContentEncoding: "",
// Metadata: map[string]string{},
// Retention: &ObjectRetention{},
// }
type ObjectAttrsToUpdate struct {
EventBasedHold optional.Bool
@ -980,6 +1019,12 @@ type ObjectAttrsToUpdate struct {
// If not empty, applies a predefined set of access controls. ACL must be nil.
// See https://cloud.google.com/storage/docs/json_api/v1/objects/patch.
PredefinedACL string
// Retention contains the retention configuration for this object.
// Operations other than setting the retention for the first time or
// extending the RetainUntil time on the object retention must be done
// on an ObjectHandle with OverrideUnlockedRetention set to true.
Retention *ObjectRetention
}
// Delete deletes the single specified object.
@ -1001,6 +1046,17 @@ func (o *ObjectHandle) ReadCompressed(compressed bool) *ObjectHandle {
return &o2
}
// OverrideUnlockedRetention provides an option for overriding an Unlocked
// Retention policy. This must be set to true in order to change a policy
// from Unlocked to Locked, to set it to null, or to reduce its
// RetainUntil attribute. It is not required for setting the ObjectRetention for
// the first time nor for extending the RetainUntil time.
func (o *ObjectHandle) OverrideUnlockedRetention(override bool) *ObjectHandle {
o2 := *o
o2.overrideRetention = &override
return &o2
}
// NewWriter returns a storage Writer that writes to the GCS object
// associated with this ObjectHandle.
//
@ -1020,6 +1076,7 @@ func (o *ObjectHandle) ReadCompressed(compressed bool) *ObjectHandle {
// It is the caller's responsibility to call Close when writing is done. To
// stop writing without saving the data, cancel the context.
func (o *ObjectHandle) NewWriter(ctx context.Context) *Writer {
ctx = trace.StartSpan(ctx, "cloud.google.com/go/storage.Object.Writer")
return &Writer{
ctx: ctx,
o: o,
@ -1039,6 +1096,10 @@ func (o *ObjectHandle) validate() error {
if !utf8.ValidString(o.object) {
return fmt.Errorf("storage: object name %q is not valid UTF-8", o.object)
}
// Names . and .. are not valid; see https://cloud.google.com/storage/docs/objects#naming
if o.object == "." || o.object == ".." {
return fmt.Errorf("storage: object name %q is not valid", o.object)
}
return nil
}
@ -1089,6 +1150,7 @@ func (o *ObjectAttrs) toRawObject(bucket string) *raw.Object {
Acl: toRawObjectACL(o.ACL),
Metadata: o.Metadata,
CustomTime: ct,
Retention: o.Retention.toRawObjectRetention(),
}
}
@ -1163,7 +1225,7 @@ func (uattrs *ObjectAttrsToUpdate) toProtoObject(bucket, object string) *storage
o.Acl = toProtoObjectACL(uattrs.ACL)
}
// TODO(cathyo): Handle metadata. Pending b/230510191.
o.Metadata = uattrs.Metadata
return o
}
@ -1324,6 +1386,42 @@ type ObjectAttrs struct {
// For non-composite objects, the value will be zero.
// This field is read-only.
ComponentCount int64
// Retention contains the retention configuration for this object.
// ObjectRetention cannot be configured or reported through the gRPC API.
Retention *ObjectRetention
}
// ObjectRetention contains the retention configuration for this object.
type ObjectRetention struct {
// Mode is the retention policy's mode on this object. Valid values are
// "Locked" and "Unlocked".
// Locked retention policies cannot be changed. Unlocked policies require an
// override to change.
Mode string
// RetainUntil is the time this object will be retained until.
RetainUntil time.Time
}
func (r *ObjectRetention) toRawObjectRetention() *raw.ObjectRetention {
if r == nil {
return nil
}
return &raw.ObjectRetention{
Mode: r.Mode,
RetainUntilTime: r.RetainUntil.Format(time.RFC3339),
}
}
func toObjectRetention(r *raw.ObjectRetention) *ObjectRetention {
if r == nil {
return nil
}
return &ObjectRetention{
Mode: r.Mode,
RetainUntil: convertTime(r.RetainUntilTime),
}
}
// convertTime converts a time in RFC3339 format to time.Time.
@ -1395,6 +1493,7 @@ func newObject(o *raw.Object) *ObjectAttrs {
Etag: o.Etag,
CustomTime: convertTime(o.CustomTime),
ComponentCount: o.ComponentCount,
Retention: toObjectRetention(o.Retention),
}
}
@ -1484,6 +1583,8 @@ type Query struct {
// aside from the prefix, contain delimiter will have their name,
// truncated after the delimiter, returned in prefixes.
// Duplicate prefixes are omitted.
// Must be set to / when used with the MatchGlob parameter to filter results
// in a directory-like mode.
// Optional.
Delimiter string
@ -1497,9 +1598,9 @@ type Query struct {
Versions bool
// attrSelection is used to select only specific fields to be returned by
// the query. It is set by the user calling calling SetAttrSelection. These
// the query. It is set by the user calling SetAttrSelection. These
// are used by toFieldMask and toFieldSelection for gRPC and HTTP/JSON
// clients repsectively.
// clients respectively.
attrSelection []string
// StartOffset is used to filter results to objects whose names are
@ -1525,6 +1626,17 @@ type Query struct {
// true, they will also be included as objects and their metadata will be
// populated in the returned ObjectAttrs.
IncludeTrailingDelimiter bool
// MatchGlob is a glob pattern used to filter results (for example, foo*bar). See
// https://cloud.google.com/storage/docs/json_api/v1/objects/list#list-object-glob
// for syntax details. When Delimiter is set in conjunction with MatchGlob,
// it must be set to /.
MatchGlob string
// IncludeFoldersAsPrefixes includes Folders and Managed Folders in the set of
// prefixes returned by the query. Only applicable if Delimiter is set to /.
// IncludeFoldersAsPrefixes is not yet implemented in the gRPC API.
IncludeFoldersAsPrefixes bool
}
// attrToFieldMap maps the field names of ObjectAttrs to the underlying field
@ -1559,6 +1671,7 @@ var attrToFieldMap = map[string]string{
"Etag": "etag",
"CustomTime": "customTime",
"ComponentCount": "componentCount",
"Retention": "retention",
}
// attrToProtoFieldMap maps the field names of ObjectAttrs to the underlying field
@ -1593,6 +1706,7 @@ var attrToProtoFieldMap = map[string]string{
"ComponentCount": "component_count",
// MediaLink was explicitly excluded from the proto as it is an HTTP-ism.
// "MediaLink": "mediaLink",
// TODO: add object retention - b/308194853
}
// SetAttrSelection makes the query populate only specific attributes of
@ -1778,7 +1892,7 @@ func (c *Conditions) isMetagenerationValid() bool {
func applyConds(method string, gen int64, conds *Conditions, call interface{}) error {
cval := reflect.ValueOf(call)
if gen >= 0 {
if !setConditionField(cval, "Generation", gen) {
if !setGeneration(cval, gen) {
return fmt.Errorf("storage: %s: generation not supported", method)
}
}
@ -1790,25 +1904,25 @@ func applyConds(method string, gen int64, conds *Conditions, call interface{}) e
}
switch {
case conds.GenerationMatch != 0:
if !setConditionField(cval, "IfGenerationMatch", conds.GenerationMatch) {
if !setIfGenerationMatch(cval, conds.GenerationMatch) {
return fmt.Errorf("storage: %s: ifGenerationMatch not supported", method)
}
case conds.GenerationNotMatch != 0:
if !setConditionField(cval, "IfGenerationNotMatch", conds.GenerationNotMatch) {
if !setIfGenerationNotMatch(cval, conds.GenerationNotMatch) {
return fmt.Errorf("storage: %s: ifGenerationNotMatch not supported", method)
}
case conds.DoesNotExist:
if !setConditionField(cval, "IfGenerationMatch", int64(0)) {
if !setIfGenerationMatch(cval, int64(0)) {
return fmt.Errorf("storage: %s: DoesNotExist not supported", method)
}
}
switch {
case conds.MetagenerationMatch != 0:
if !setConditionField(cval, "IfMetagenerationMatch", conds.MetagenerationMatch) {
if !setIfMetagenerationMatch(cval, conds.MetagenerationMatch) {
return fmt.Errorf("storage: %s: ifMetagenerationMatch not supported", method)
}
case conds.MetagenerationNotMatch != 0:
if !setConditionField(cval, "IfMetagenerationNotMatch", conds.MetagenerationNotMatch) {
if !setIfMetagenerationNotMatch(cval, conds.MetagenerationNotMatch) {
return fmt.Errorf("storage: %s: ifMetagenerationNotMatch not supported", method)
}
}
@ -1869,16 +1983,45 @@ func applySourceCondsProto(gen int64, conds *Conditions, call *storagepb.Rewrite
return nil
}
// setConditionField sets a field on a *raw.WhateverCall.
// setGeneration sets Generation on a *raw.WhateverCall.
// We can't use anonymous interfaces because the return type is
// different, since the field setters are builders.
func setConditionField(call reflect.Value, name string, value interface{}) bool {
m := call.MethodByName(name)
if !m.IsValid() {
return false
// We also make sure to supply a compile-time constant to MethodByName;
// otherwise, the Go Linker will disable dead code elimination, leading
// to larger binaries for all packages that import storage.
func setGeneration(cval reflect.Value, value interface{}) bool {
return setCondition(cval.MethodByName("Generation"), value)
}
// setIfGenerationMatch sets IfGenerationMatch on a *raw.WhateverCall.
// See also setGeneration.
func setIfGenerationMatch(cval reflect.Value, value interface{}) bool {
return setCondition(cval.MethodByName("IfGenerationMatch"), value)
}
// setIfGenerationNotMatch sets IfGenerationNotMatch on a *raw.WhateverCall.
// See also setGeneration.
func setIfGenerationNotMatch(cval reflect.Value, value interface{}) bool {
return setCondition(cval.MethodByName("IfGenerationNotMatch"), value)
}
// setIfMetagenerationMatch sets IfMetagenerationMatch on a *raw.WhateverCall.
// See also setGeneration.
func setIfMetagenerationMatch(cval reflect.Value, value interface{}) bool {
return setCondition(cval.MethodByName("IfMetagenerationMatch"), value)
}
// setIfMetagenerationNotMatch sets IfMetagenerationNotMatch on a *raw.WhateverCall.
// See also setGeneration.
func setIfMetagenerationNotMatch(cval reflect.Value, value interface{}) bool {
return setCondition(cval.MethodByName("IfMetagenerationNotMatch"), value)
}
func setCondition(setter reflect.Value, value interface{}) bool {
if setter.IsValid() {
setter.Call([]reflect.Value{reflect.ValueOf(value)})
}
m.Call([]reflect.Value{reflect.ValueOf(value)})
return true
return setter.IsValid()
}
// Retryer returns an object handle that is configured with custom retry
@ -1950,6 +2093,26 @@ func (wb *withBackoff) apply(config *retryConfig) {
config.backoff = &wb.backoff
}
// WithMaxAttempts configures the maximum number of times an API call can be made
// in the case of retryable errors.
// For example, if you set WithMaxAttempts(5), the operation will be attempted up to 5
// times total (initial call plus 4 retries).
// Without this setting, operations will continue retrying indefinitely
// until either the context is canceled or a deadline is reached.
func WithMaxAttempts(maxAttempts int) RetryOption {
return &withMaxAttempts{
maxAttempts: maxAttempts,
}
}
type withMaxAttempts struct {
maxAttempts int
}
func (wb *withMaxAttempts) apply(config *retryConfig) {
config.maxAttempts = &wb.maxAttempts
}
// RetryPolicy describes the available policies for which operations should be
// retried. The default is `RetryIdempotent`.
type RetryPolicy int
@ -2022,6 +2185,7 @@ type retryConfig struct {
backoff *gax.Backoff
policy RetryPolicy
shouldRetry func(err error) bool
maxAttempts *int
}
func (r *retryConfig) clone() *retryConfig {
@ -2042,6 +2206,7 @@ func (r *retryConfig) clone() *retryConfig {
backoff: bo,
policy: r.policy,
shouldRetry: r.shouldRetry,
maxAttempts: r.maxAttempts,
}
}
@ -2162,8 +2327,6 @@ func toProjectResource(project string) string {
// setConditionProtoField uses protobuf reflection to set named condition field
// to the given condition value if supported on the protobuf message.
//
// This is an experimental API and not intended for public use.
func setConditionProtoField(m protoreflect.Message, f string, v int64) bool {
fields := m.Descriptor().Fields()
if rf := fields.ByName(protoreflect.Name(f)); rf != nil {
@ -2176,8 +2339,6 @@ func setConditionProtoField(m protoreflect.Message, f string, v int64) bool {
// applyCondsProto validates and attempts to set the conditions on a protobuf
// message using protobuf reflection.
//
// This is an experimental API and not intended for public use.
func applyCondsProto(method string, gen int64, conds *Conditions, msg proto.Message) error {
rmsg := msg.ProtoReflect()

View File

@ -22,6 +22,8 @@ import (
"sync"
"time"
"unicode/utf8"
"cloud.google.com/go/internal/trace"
)
// A Writer writes a Cloud Storage object.
@ -86,7 +88,12 @@ type Writer struct {
// cancellation.
ChunkRetryDeadline time.Duration
// ProgressFunc can be used to monitor the progress of a large write.
// ForceEmptyContentType is an optional parameter that is used to disable
// auto-detection of Content-Type. By default, if a blank Content-Type
// is provided, then gax.DetermineContentType is called to sniff the type.
ForceEmptyContentType bool
// ProgressFunc can be used to monitor the progress of a large write
// operation. If ProgressFunc is not nil and writing requires multiple
// calls to the underlying service (see
// https://cloud.google.com/storage/docs/json_api/v1/how-tos/resumable-upload),
@ -163,6 +170,7 @@ func (w *Writer) Close() error {
<-w.donec
w.mu.Lock()
defer w.mu.Unlock()
trace.EndSpan(w.ctx, w.err)
return w.err
}
@ -177,18 +185,19 @@ func (w *Writer) openWriter() (err error) {
isIdempotent := w.o.conds != nil && (w.o.conds.GenerationMatch >= 0 || w.o.conds.DoesNotExist == true)
opts := makeStorageOpts(isIdempotent, w.o.retry, w.o.userProject)
params := &openWriterParams{
ctx: w.ctx,
chunkSize: w.ChunkSize,
chunkRetryDeadline: w.ChunkRetryDeadline,
bucket: w.o.bucket,
attrs: &w.ObjectAttrs,
conds: w.o.conds,
encryptionKey: w.o.encryptionKey,
sendCRC32C: w.SendCRC32C,
donec: w.donec,
setError: w.error,
progress: w.progress,
setObj: func(o *ObjectAttrs) { w.obj = o },
ctx: w.ctx,
chunkSize: w.ChunkSize,
chunkRetryDeadline: w.ChunkRetryDeadline,
bucket: w.o.bucket,
attrs: &w.ObjectAttrs,
conds: w.o.conds,
encryptionKey: w.o.encryptionKey,
sendCRC32C: w.SendCRC32C,
donec: w.donec,
setError: w.error,
progress: w.progress,
setObj: func(o *ObjectAttrs) { w.obj = o },
forceEmptyContentType: w.ForceEmptyContentType,
}
if err := w.ctx.Err(); err != nil {
return err // short-circuit

File diff suppressed because it is too large Load Diff

View File

@ -1,3 +1,17 @@
// Copyright 2023 The go-fuzz-headers Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package gofuzzheaders
import (
@ -35,23 +49,14 @@ func (f *ConsumeFuzzer) AddFuncs(fuzzFuncs []interface{}) {
}
func (f *ConsumeFuzzer) GenerateWithCustom(targetStruct interface{}) error {
v := reflect.ValueOf(targetStruct)
e := v.Elem()
e := reflect.ValueOf(targetStruct).Elem()
return f.fuzzStruct(e, true)
}
func (c Continue) GenerateStruct(targetStruct interface{}) error {
err := c.F.GenerateStruct(targetStruct)
if err != nil {
return err
}
return nil
return c.F.GenerateStruct(targetStruct)
}
func (c Continue) GenerateStructWithCustom(targetStruct interface{}) error {
err := c.F.GenerateWithCustom(targetStruct)
if err != nil {
return err
}
return nil
return c.F.GenerateWithCustom(targetStruct)
}

View File

@ -1,3 +1,17 @@
// Copyright 2023 The go-fuzz-headers Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package gofuzzheaders
import (
@ -16,7 +30,7 @@ func getKeyword(f *ConsumeFuzzer) (string, error) {
return k, nil
}
}
return keywords[0], fmt.Errorf("Could not get a kw")
return keywords[0], fmt.Errorf("could not get a kw")
}
// Simple utility function to check if a string
@ -101,7 +115,8 @@ var keywords = []string{
"vitess_migration", "vitess_migrations", "vitess_replication_status",
"vitess_shards", "vitess_tablets", "vschema", "warnings", "when",
"where", "while", "window", "with", "without", "work", "write", "xor",
"year", "year_month", "zerofill"}
"year", "year_month", "zerofill",
}
// Keywords that could get an additional keyword
var needCustomString = []string{
@ -131,11 +146,15 @@ var alterTableTokens = [][]string{
}
var alterTokens = [][]string{
{"DATABASE", "SCHEMA", "DEFINER = ", "EVENT", "FUNCTION", "INSTANCE",
"LOGFILE GROUP", "PROCEDURE", "SERVER"},
{
"DATABASE", "SCHEMA", "DEFINER = ", "EVENT", "FUNCTION", "INSTANCE",
"LOGFILE GROUP", "PROCEDURE", "SERVER",
},
{"CUSTOM_FUZZ_STRING"},
{"ON SCHEDULE", "ON COMPLETION PRESERVE", "ON COMPLETION NOT PRESERVE",
"ADD UNDOFILE", "OPTIONS"},
{
"ON SCHEDULE", "ON COMPLETION PRESERVE", "ON COMPLETION NOT PRESERVE",
"ADD UNDOFILE", "OPTIONS",
},
{"RENAME TO", "INITIAL_SIZE = "},
{"ENABLE", "DISABLE", "DISABLE ON SLAVE", "ENGINE"},
{"COMMENT"},
@ -150,9 +169,11 @@ var setTokens = [][]string{
var dropTokens = [][]string{
{"TEMPORARY", "UNDO"},
{"DATABASE", "SCHEMA", "EVENT", "INDEX", "LOGFILE GROUP",
{
"DATABASE", "SCHEMA", "EVENT", "INDEX", "LOGFILE GROUP",
"PROCEDURE", "FUNCTION", "SERVER", "SPATIAL REFERENCE SYSTEM",
"TABLE", "TABLESPACE", "TRIGGER", "VIEW"},
"TABLE", "TABLESPACE", "TRIGGER", "VIEW",
},
{"IF EXISTS"},
{"CUSTOM_FUZZ_STRING"},
{"ON", "ENGINE = ", "RESTRICT", "CASCADE"},
@ -172,15 +193,21 @@ var truncateTokens = [][]string{
var createTokens = [][]string{
{"OR REPLACE", "TEMPORARY", "UNDO"}, // For create spatial reference system
{"UNIQUE", "FULLTEXT", "SPATIAL", "ALGORITHM = UNDEFINED", "ALGORITHM = MERGE",
"ALGORITHM = TEMPTABLE"},
{"DATABASE", "SCHEMA", "EVENT", "FUNCTION", "INDEX", "LOGFILE GROUP",
{
"UNIQUE", "FULLTEXT", "SPATIAL", "ALGORITHM = UNDEFINED", "ALGORITHM = MERGE",
"ALGORITHM = TEMPTABLE",
},
{
"DATABASE", "SCHEMA", "EVENT", "FUNCTION", "INDEX", "LOGFILE GROUP",
"PROCEDURE", "SERVER", "SPATIAL REFERENCE SYSTEM", "TABLE", "TABLESPACE",
"TRIGGER", "VIEW"},
"TRIGGER", "VIEW",
},
{"IF NOT EXISTS"},
{"CUSTOM_FUZZ_STRING"},
}
/*
// For future use.
var updateTokens = [][]string{
{"LOW_PRIORITY"},
{"IGNORE"},
@ -189,6 +216,8 @@ var updateTokens = [][]string{
{"ORDER BY"},
{"LIMIT"},
}
*/
var replaceTokens = [][]string{
{"LOW_PRIORITY", "DELAYED"},
{"INTO"},
@ -196,6 +225,7 @@ var replaceTokens = [][]string{
{"CUSTOM_FUZZ_STRING"},
{"VALUES", "VALUE"},
}
var loadTokens = [][]string{
{"DATA"},
{"LOW_PRIORITY", "CONCURRENT", "LOCAL"},
@ -271,22 +301,24 @@ var alter_table_options = []string{
"COMPACT", "SECONDARY_ENGINE_ATTRIBUTE", "STATS_AUTO_RECALC", "STATS_PERSISTENT",
"STATS_SAMPLE_PAGES", "ZLIB", "LZ4", "ENGINE_ATTRIBUTE", "KEY_BLOCK_SIZE", "MAX_ROWS",
"MIN_ROWS", "PACK_KEYS", "PASSWORD", "COMPRESSION", "CONNECTION", "DIRECTORY",
"DELAY_KEY_WRITE", "ENCRYPTION", "STORAGE", "DISK", "MEMORY", "UNION"}
"DELAY_KEY_WRITE", "ENCRYPTION", "STORAGE", "DISK", "MEMORY", "UNION",
}
// Creates an 'alter table' statement. 'alter table' is an exception
// in that it has its own function. The majority of statements
// are created by 'createStmt()'.
func createAlterTableStmt(f *ConsumeFuzzer) (string, error) {
var stmt strings.Builder
stmt.WriteString("ALTER TABLE ")
maxArgs, err := f.GetInt()
if err != nil {
return "", err
}
maxArgs = maxArgs % 30
if maxArgs == 0 {
return "", fmt.Errorf("Could not create alter table stmt")
return "", fmt.Errorf("could not create alter table stmt")
}
var stmt strings.Builder
stmt.WriteString("ALTER TABLE ")
for i := 0; i < maxArgs; i++ {
// Calculate if we get existing token or custom string
tokenType, err := f.GetInt()
@ -298,13 +330,13 @@ func createAlterTableStmt(f *ConsumeFuzzer) (string, error) {
if err != nil {
return "", err
}
stmt.WriteString(fmt.Sprintf(" %s", customString))
stmt.WriteString(" " + customString)
} else {
tokenIndex, err := f.GetInt()
if err != nil {
return "", err
}
stmt.WriteString(fmt.Sprintf(" %s", alter_table_options[tokenIndex%len(alter_table_options)]))
stmt.WriteString(" " + alter_table_options[tokenIndex%len(alter_table_options)])
}
}
return stmt.String(), nil
@ -316,7 +348,7 @@ func chooseToken(tokens []string, f *ConsumeFuzzer) (string, error) {
return "", err
}
var token strings.Builder
token.WriteString(fmt.Sprintf(" %s", tokens[index%len(tokens)]))
token.WriteString(tokens[index%len(tokens)])
if token.String() == "CUSTOM_FUZZ_STRING" {
customFuzzString, err := f.GetString()
if err != nil {
@ -331,7 +363,7 @@ func chooseToken(tokens []string, f *ConsumeFuzzer) (string, error) {
if err != nil {
return "", err
}
token.WriteString(fmt.Sprintf(" %s", customFuzzString))
token.WriteString(" " + customFuzzString)
}
return token.String(), nil
}
@ -398,21 +430,19 @@ func createStmt(f *ConsumeFuzzer) (string, error) {
if err != nil {
return "", err
}
query.WriteString(fmt.Sprintf(" %s", queryArgs))
query.WriteString(" " + queryArgs)
return query.String(), nil
}
// Creates the arguments of a statements. In a select statement
// that would be everything after "select".
func createStmtArgs(tokenslice [][]string, f *ConsumeFuzzer) (string, error) {
var query strings.Builder
var token strings.Builder
var query, token strings.Builder
// We go through the tokens in the tokenslice,
// create the respective token and add it to
// "query"
for _, tokens := range tokenslice {
// For extra randomization, the fuzzer can
// choose to not include this token.
includeThisToken, err := f.GetBool()
@ -429,7 +459,7 @@ func createStmtArgs(tokenslice [][]string, f *ConsumeFuzzer) (string, error) {
if err != nil {
return "", err
}
query.WriteString(fmt.Sprintf(" %s", chosenToken))
query.WriteString(" " + chosenToken)
} else {
token.WriteString(tokens[0])
@ -440,7 +470,7 @@ func createStmtArgs(tokenslice [][]string, f *ConsumeFuzzer) (string, error) {
if err != nil {
return "", err
}
query.WriteString(fmt.Sprintf(" %s", customFuzzString))
query.WriteString(" " + customFuzzString)
continue
}
@ -470,7 +500,7 @@ func createQuery(f *ConsumeFuzzer) (string, error) {
}
maxLen := queryLen % 60
if maxLen == 0 {
return "", fmt.Errorf("Could not create a query")
return "", fmt.Errorf("could not create a query")
}
var query strings.Builder
for i := 0; i < maxLen; i++ {
@ -484,25 +514,27 @@ func createQuery(f *ConsumeFuzzer) (string, error) {
if err != nil {
return "", err
}
query.WriteString(fmt.Sprintf(" %s", keyword))
query.WriteString(" " + keyword)
} else {
customString, err := f.GetString()
if err != nil {
return "", err
}
query.WriteString(fmt.Sprintf(" %s", customString))
query.WriteString(" " + customString)
}
}
if query.String() == "" {
return "", fmt.Errorf("Could not create a query")
return "", fmt.Errorf("could not create a query")
}
return query.String(), nil
}
// This is the API that users will interact with.
// GetSQLString is the API that users interact with.
//
// Usage:
// f := NewConsumer(data)
// sqlString, err := f.GetSQLString()
//
// f := NewConsumer(data)
// sqlString, err := f.GetSQLString()
func (f *ConsumeFuzzer) GetSQLString() (string, error) {
var query string
veryStructured, err := f.GetBool()

View File

@ -1,28 +0,0 @@
Copyright (C) 2014-2015 Docker Inc & Go Authors. All rights reserved.
Copyright (C) 2017 SUSE LLC. All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

View File

@ -1,79 +0,0 @@
## `filepath-securejoin` ##
[![Build Status](https://github.com/cyphar/filepath-securejoin/actions/workflows/ci.yml/badge.svg)](https://github.com/cyphar/filepath-securejoin/actions/workflows/ci.yml)
An implementation of `SecureJoin`, a [candidate for inclusion in the Go
standard library][go#20126]. The purpose of this function is to be a "secure"
alternative to `filepath.Join`, and in particular it provides certain
guarantees that are not provided by `filepath.Join`.
> **NOTE**: This code is *only* safe if you are not at risk of other processes
> modifying path components after you've used `SecureJoin`. If it is possible
> for a malicious process to modify path components of the resolved path, then
> you will be vulnerable to some fairly trivial TOCTOU race conditions. [There
> are some Linux kernel patches I'm working on which might allow for a better
> solution.][lwn-obeneath]
>
> In addition, with a slightly modified API it might be possible to use
> `O_PATH` and verify that the opened path is actually the resolved one -- but
> I have not done that yet. I might add it in the future as a helper function
> to help users verify the path (we can't just return `/proc/self/fd/<foo>`
> because that doesn't always work transparently for all users).
This is the function prototype:
```go
func SecureJoin(root, unsafePath string) (string, error)
```
This library **guarantees** the following:
* If no error is set, the resulting string **must** be a child path of
`root` and will not contain any symlink path components (they will all be
expanded).
* When expanding symlinks, all symlink path components **must** be resolved
relative to the provided root. In particular, this can be considered a
userspace implementation of how `chroot(2)` operates on file paths. Note that
these symlinks will **not** be expanded lexically (`filepath.Clean` is not
called on the input before processing).
* Non-existent path components are unaffected by `SecureJoin` (similar to
`filepath.EvalSymlinks`'s semantics).
* The returned path will always be `filepath.Clean`ed and thus not contain any
`..` components.
A (trivial) implementation of this function on GNU/Linux systems could be done
with the following (note that this requires root privileges and is far more
opaque than the implementation in this library, and also requires that
`readlink` is inside the `root` path):
```go
package securejoin
import (
"os/exec"
"path/filepath"
)
func SecureJoin(root, unsafePath string) (string, error) {
unsafePath = string(filepath.Separator) + unsafePath
cmd := exec.Command("chroot", root,
"readlink", "--canonicalize-missing", "--no-newline", unsafePath)
output, err := cmd.CombinedOutput()
if err != nil {
return "", err
}
expanded := string(output)
return filepath.Join(root, expanded), nil
}
```
[lwn-obeneath]: https://lwn.net/Articles/767547/
[go#20126]: https://github.com/golang/go/issues/20126
### License ###
The license of this project is the same as Go, which is a BSD 3-clause license
available in the `LICENSE` file.

View File

@ -1 +0,0 @@
0.2.4

View File

@ -1,125 +0,0 @@
// Copyright (C) 2014-2015 Docker Inc & Go Authors. All rights reserved.
// Copyright (C) 2017 SUSE LLC. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package securejoin is an implementation of the hopefully-soon-to-be-included
// SecureJoin helper that is meant to be part of the "path/filepath" package.
// The purpose of this project is to provide a PoC implementation to make the
// SecureJoin proposal (https://github.com/golang/go/issues/20126) more
// tangible.
package securejoin
import (
"bytes"
"errors"
"os"
"path/filepath"
"strings"
"syscall"
)
// IsNotExist tells you if err is an error that implies that either the path
// accessed does not exist (or path components don't exist). This is
// effectively a more broad version of os.IsNotExist.
func IsNotExist(err error) bool {
// Check that it's not actually an ENOTDIR, which in some cases is a more
// convoluted case of ENOENT (usually involving weird paths).
return errors.Is(err, os.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) || errors.Is(err, syscall.ENOENT)
}
// SecureJoinVFS joins the two given path components (similar to Join) except
// that the returned path is guaranteed to be scoped inside the provided root
// path (when evaluated). Any symbolic links in the path are evaluated with the
// given root treated as the root of the filesystem, similar to a chroot. The
// filesystem state is evaluated through the given VFS interface (if nil, the
// standard os.* family of functions are used).
//
// Note that the guarantees provided by this function only apply if the path
// components in the returned string are not modified (in other words are not
// replaced with symlinks on the filesystem) after this function has returned.
// Such a symlink race is necessarily out-of-scope of SecureJoin.
//
// Volume names in unsafePath are always discarded, regardless if they are
// provided via direct input or when evaluating symlinks. Therefore:
//
// "C:\Temp" + "D:\path\to\file.txt" results in "C:\Temp\path\to\file.txt"
func SecureJoinVFS(root, unsafePath string, vfs VFS) (string, error) {
// Use the os.* VFS implementation if none was specified.
if vfs == nil {
vfs = osVFS{}
}
unsafePath = filepath.FromSlash(unsafePath)
var path bytes.Buffer
n := 0
for unsafePath != "" {
if n > 255 {
return "", &os.PathError{Op: "SecureJoin", Path: root + string(filepath.Separator) + unsafePath, Err: syscall.ELOOP}
}
if v := filepath.VolumeName(unsafePath); v != "" {
unsafePath = unsafePath[len(v):]
}
// Next path component, p.
i := strings.IndexRune(unsafePath, filepath.Separator)
var p string
if i == -1 {
p, unsafePath = unsafePath, ""
} else {
p, unsafePath = unsafePath[:i], unsafePath[i+1:]
}
// Create a cleaned path, using the lexical semantics of /../a, to
// create a "scoped" path component which can safely be joined to fullP
// for evaluation. At this point, path.String() doesn't contain any
// symlink components.
cleanP := filepath.Clean(string(filepath.Separator) + path.String() + p)
if cleanP == string(filepath.Separator) {
path.Reset()
continue
}
fullP := filepath.Clean(root + cleanP)
// Figure out whether the path is a symlink.
fi, err := vfs.Lstat(fullP)
if err != nil && !IsNotExist(err) {
return "", err
}
// Treat non-existent path components the same as non-symlinks (we
// can't do any better here).
if IsNotExist(err) || fi.Mode()&os.ModeSymlink == 0 {
path.WriteString(p)
path.WriteRune(filepath.Separator)
continue
}
// Only increment when we actually dereference a link.
n++
// It's a symlink, expand it by prepending it to the yet-unparsed path.
dest, err := vfs.Readlink(fullP)
if err != nil {
return "", err
}
// Absolute symlinks reset any work we've already done.
if filepath.IsAbs(dest) {
path.Reset()
}
unsafePath = dest + string(filepath.Separator) + unsafePath
}
// We have to clean path.String() here because it may contain '..'
// components that are entirely lexical, but would be misleading otherwise.
// And finally do a final clean to ensure that root is also lexically
// clean.
fullP := filepath.Clean(string(filepath.Separator) + path.String())
return filepath.Clean(root + fullP), nil
}
// SecureJoin is a wrapper around SecureJoinVFS that just uses the os.* library
// of functions as the VFS. If in doubt, use this function over SecureJoinVFS.
func SecureJoin(root, unsafePath string) (string, error) {
return SecureJoinVFS(root, unsafePath, nil)
}

View File

@ -1,41 +0,0 @@
// Copyright (C) 2017 SUSE LLC. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package securejoin
import "os"
// In future this should be moved into a separate package, because now there
// are several projects (umoci and go-mtree) that are using this sort of
// interface.
// VFS is the minimal interface necessary to use SecureJoinVFS. A nil VFS is
// equivalent to using the standard os.* family of functions. This is mainly
// used for the purposes of mock testing, but also can be used to otherwise use
// SecureJoin with VFS-like system.
type VFS interface {
// Lstat returns a FileInfo describing the named file. If the file is a
// symbolic link, the returned FileInfo describes the symbolic link. Lstat
// makes no attempt to follow the link. These semantics are identical to
// os.Lstat.
Lstat(name string) (os.FileInfo, error)
// Readlink returns the destination of the named symbolic link. These
// semantics are identical to os.Readlink.
Readlink(name string) (string, error)
}
// osVFS is the "nil" VFS, in that it just passes everything through to the os
// module.
type osVFS struct{}
// Lstat returns a FileInfo describing the named file. If the file is a
// symbolic link, the returned FileInfo describes the symbolic link. Lstat
// makes no attempt to follow the link. These semantics are identical to
// os.Lstat.
func (o osVFS) Lstat(name string) (os.FileInfo, error) { return os.Lstat(name) }
// Readlink returns the destination of the named symbolic link. These
// semantics are identical to os.Readlink.
func (o osVFS) Readlink(name string) (string, error) { return os.Readlink(name) }

201
vendor/github.com/giobart/2dfs-builder/LICENSE generated vendored Normal file
View File

@ -0,0 +1,201 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

View File

@ -0,0 +1,84 @@
package filesystem
import (
"encoding/json"
"sync"
)
func GetField() Field {
return &TwoDFilesystem{
Rows: make([]Cols, 0),
TotRows: 0,
mtx: sync.Mutex{},
}
}
func (f *TwoDFilesystem) AddAllotment(allotment Allotment) Field {
f.mtx.Lock()
defer f.mtx.Unlock()
f.genAllotments(allotment.Row, allotment.Col)
f.Rows[allotment.Row].Allotments[allotment.Col].Digest = allotment.Digest
f.Rows[allotment.Row].Allotments[allotment.Col].DiffID = allotment.DiffID
f.Rows[allotment.Row].Allotments[allotment.Col].FileName = allotment.FileName
return f
}
func (f *TwoDFilesystem) Marshal() string {
result, _ := json.Marshal(f)
return string(result)
}
func (f *TwoDFilesystem) Unmarshal(data string) (Field, error) {
fs := TwoDFilesystem{}
err := json.Unmarshal([]byte(data), &fs)
if err != nil {
return nil, err
}
return &fs, nil
}
// genRows generates (if necessary) the rows from 0 to n. If some rows exist already, they are not generated.
// E.g., f.TotRows=0, genRows(0) -> generates row[0]
// E.g., f.TotRows=4, genRows(7) -> generates row[4],row[5],row[6],row[7]
// E.g., f.TotRows=4, genRows(3) -> no new rows generated
func (f *TwoDFilesystem) genRows(n int) {
if n > (f.TotRows - 1) {
for i := f.TotRows; i <= n; i++ {
f.Rows = append(f.Rows, Cols{
Allotments: make([]Allotment, 0),
TotAllotments: 0,
})
}
f.TotRows = n + 1
}
}
// genAllotment generates (if necessary) the allotments from 0 to n on "row". If some allotments exist already, they are not generated.
func (f *TwoDFilesystem) genAllotments(row int, n int) {
f.genRows(row)
if n > (f.Rows[row].TotAllotments - 1) {
for i := f.Rows[row].TotAllotments; i <= n; i++ {
f.Rows[row].Allotments = append(f.Rows[row].Allotments, Allotment{
Row: row,
Col: i,
Digest: "",
DiffID: "",
})
}
f.Rows[row].TotAllotments = n + 1
}
}
// IterateAllotments iterates over all allotments in the filesystem
func (f *TwoDFilesystem) IterateAllotments() chan Allotment {
c := make(chan Allotment)
go func() {
for _, row := range f.Rows {
for _, allotment := range row.Allotments {
c <- allotment
}
}
close(c)
}()
return c
}

View File

@ -0,0 +1,45 @@
package filesystem
import "sync"
type Allotment struct {
Row int `json:"row"`
Col int `json:"col"`
Digest string `json:"digest"`
DiffID string `json:"diffid"`
FileName string `json:"filename"`
}
type Cols struct {
Allotments []Allotment `json:"allotments"`
TotAllotments int `json:"allotments_size"`
}
type TwoDFilesystem struct {
Rows []Cols `json:"rows"`
TotRows int `json:"rows_size"`
Owner string `json:"owner"`
mtx sync.Mutex
}
type AllotmentManifest struct {
Src string `json:"src"`
Dst string `json:"dst"`
Row int `json:"row"`
Col int `json:"col"`
}
type TwoDFsManifest struct {
Allotments []AllotmentManifest `json:"allotments"`
}
type Field interface {
// AddAllotment creates the given allotment to the 2d FileSystem
AddAllotment(allotment Allotment) Field
// Marshal gives a marshalled filesystem as string
Marshal() string
// Unmarshal Given a string marshaled from TwoDFilesystem returns a Field object
Unmarshal(string) (Field, error)
// IterateAllotments iterates over all allotments in the filesystem
IterateAllotments() chan Allotment
}

View File

@ -91,11 +91,12 @@ logr design but also left out some parts and changed others:
| Adding a name to a logger | `WithName` | no API |
| Modify verbosity of log entries in a call chain | `V` | no API |
| Grouping of key/value pairs | not supported | `WithGroup`, `GroupValue` |
| Pass context for extracting additional values | no API | API variants like `InfoCtx` |
The high-level slog API is explicitly meant to be one of many different APIs
that can be layered on top of a shared `slog.Handler`. logr is one such
alternative API, with [interoperability](#slog-interoperability) provided by the [`slogr`](slogr)
package.
alternative API, with [interoperability](#slog-interoperability) provided by
some conversion functions.
### Inspiration
@ -145,24 +146,24 @@ There are implementations for the following logging libraries:
## slog interoperability
Interoperability goes both ways, using the `logr.Logger` API with a `slog.Handler`
and using the `slog.Logger` API with a `logr.LogSink`. [slogr](./slogr) provides `NewLogr` and
`NewSlogHandler` API calls to convert between a `logr.Logger` and a `slog.Handler`.
and using the `slog.Logger` API with a `logr.LogSink`. `FromSlogHandler` and
`ToSlogHandler` convert between a `logr.Logger` and a `slog.Handler`.
As usual, `slog.New` can be used to wrap such a `slog.Handler` in the high-level
slog API. `slogr` itself leaves that to the caller.
slog API.
## Using a `logr.Sink` as backend for slog
### Using a `logr.LogSink` as backend for slog
Ideally, a logr sink implementation should support both logr and slog by
implementing both the normal logr interface(s) and `slogr.SlogSink`. Because
implementing both the normal logr interface(s) and `SlogSink`. Because
of a conflict in the parameters of the common `Enabled` method, it is [not
possible to implement both slog.Handler and logr.Sink in the same
type](https://github.com/golang/go/issues/59110).
If both are supported, log calls can go from the high-level APIs to the backend
without the need to convert parameters. `NewLogr` and `NewSlogHandler` can
without the need to convert parameters. `FromSlogHandler` and `ToSlogHandler` can
convert back and forth without adding additional wrappers, with one exception:
when `Logger.V` was used to adjust the verbosity for a `slog.Handler`, then
`NewSlogHandler` has to use a wrapper which adjusts the verbosity for future
`ToSlogHandler` has to use a wrapper which adjusts the verbosity for future
log calls.
Such an implementation should also support values that implement specific
@ -187,13 +188,13 @@ Not supporting slog has several drawbacks:
These drawbacks are severe enough that applications using a mixture of slog and
logr should switch to a different backend.
## Using a `slog.Handler` as backend for logr
### Using a `slog.Handler` as backend for logr
Using a plain `slog.Handler` without support for logr works better than the
other direction:
- All logr verbosity levels can be mapped 1:1 to their corresponding slog level
by negating them.
- Stack unwinding is done by the `slogr.SlogSink` and the resulting program
- Stack unwinding is done by the `SlogSink` and the resulting program
counter is passed to the `slog.Handler`.
- Names added via `Logger.WithName` are gathered and recorded in an additional
attribute with `logger` as key and the names separated by slash as value.
@ -205,27 +206,39 @@ ideally support both `logr.Marshaler` and `slog.Valuer`. If compatibility
with logr implementations without slog support is not important, then
`slog.Valuer` is sufficient.
## Context support for slog
### Context support for slog
Storing a logger in a `context.Context` is not supported by
slog. `logr.NewContext` and `logr.FromContext` can be used with slog like this
to fill this gap:
slog. `NewContextWithSlogLogger` and `FromContextAsSlogLogger` can be
used to fill this gap. They store and retrieve a `slog.Logger` pointer
under the same context key that is also used by `NewContext` and
`FromContext` for `logr.Logger` value.
func HandlerFromContext(ctx context.Context) slog.Handler {
logger, err := logr.FromContext(ctx)
if err == nil {
return slogr.NewSlogHandler(logger)
}
return slog.Default().Handler()
}
When `NewContextWithSlogLogger` is followed by `FromContext`, the latter will
automatically convert the `slog.Logger` to a
`logr.Logger`. `FromContextAsSlogLogger` does the same for the other direction.
func ContextWithHandler(ctx context.Context, handler slog.Handler) context.Context {
return logr.NewContext(ctx, slogr.NewLogr(handler))
}
With this approach, binaries which use either slog or logr are as efficient as
possible with no unnecessary allocations. This is also why the API stores a
`slog.Logger` pointer: when storing a `slog.Handler`, creating a `slog.Logger`
on retrieval would need to allocate one.
The downside is that storing and retrieving a `slog.Handler` needs more
allocations compared to using a `logr.Logger`. Therefore the recommendation is
to use the `logr.Logger` API in code which uses contextual logging.
The downside is that switching back and forth needs more allocations. Because
logr is the API that is already in use by different packages, in particular
Kubernetes, the recommendation is to use the `logr.Logger` API in code which
uses contextual logging.
An alternative to adding values to a logger and storing that logger in the
context is to store the values in the context and to configure a logging
backend to extract those values when emitting log entries. This only works when
log calls are passed the context, which is not supported by the logr API.
With the slog API, it is possible, but not
required. https://github.com/veqryn/slog-context is a package for slog which
provides additional support code for this approach. It also contains wrappers
for the context functions in logr, so developers who prefer to not use the logr
APIs directly can use those instead and the resulting code will still be
interoperable with logr.
## FAQ

33
vendor/github.com/go-logr/logr/context.go generated vendored Normal file
View File

@ -0,0 +1,33 @@
/*
Copyright 2023 The logr Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package logr
// contextKey is how we find Loggers in a context.Context. With Go < 1.21,
// the value is always a Logger value. With Go >= 1.21, the value can be a
// Logger value or a slog.Logger pointer.
type contextKey struct{}
// notFoundError exists to carry an IsNotFound method.
type notFoundError struct{}
func (notFoundError) Error() string {
return "no logr.Logger was present"
}
func (notFoundError) IsNotFound() bool {
return true
}

49
vendor/github.com/go-logr/logr/context_noslog.go generated vendored Normal file
View File

@ -0,0 +1,49 @@
//go:build !go1.21
// +build !go1.21
/*
Copyright 2019 The logr Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package logr
import (
"context"
)
// FromContext returns a Logger from ctx or an error if no Logger is found.
func FromContext(ctx context.Context) (Logger, error) {
if v, ok := ctx.Value(contextKey{}).(Logger); ok {
return v, nil
}
return Logger{}, notFoundError{}
}
// FromContextOrDiscard returns a Logger from ctx. If no Logger is found, this
// returns a Logger that discards all log messages.
func FromContextOrDiscard(ctx context.Context) Logger {
if v, ok := ctx.Value(contextKey{}).(Logger); ok {
return v
}
return Discard()
}
// NewContext returns a new Context, derived from ctx, which carries the
// provided Logger.
func NewContext(ctx context.Context, logger Logger) context.Context {
return context.WithValue(ctx, contextKey{}, logger)
}

83
vendor/github.com/go-logr/logr/context_slog.go generated vendored Normal file
View File

@ -0,0 +1,83 @@
//go:build go1.21
// +build go1.21
/*
Copyright 2019 The logr Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package logr
import (
"context"
"fmt"
"log/slog"
)
// FromContext returns a Logger from ctx or an error if no Logger is found.
func FromContext(ctx context.Context) (Logger, error) {
v := ctx.Value(contextKey{})
if v == nil {
return Logger{}, notFoundError{}
}
switch v := v.(type) {
case Logger:
return v, nil
case *slog.Logger:
return FromSlogHandler(v.Handler()), nil
default:
// Not reached.
panic(fmt.Sprintf("unexpected value type for logr context key: %T", v))
}
}
// FromContextAsSlogLogger returns a slog.Logger from ctx or nil if no such Logger is found.
func FromContextAsSlogLogger(ctx context.Context) *slog.Logger {
v := ctx.Value(contextKey{})
if v == nil {
return nil
}
switch v := v.(type) {
case Logger:
return slog.New(ToSlogHandler(v))
case *slog.Logger:
return v
default:
// Not reached.
panic(fmt.Sprintf("unexpected value type for logr context key: %T", v))
}
}
// FromContextOrDiscard returns a Logger from ctx. If no Logger is found, this
// returns a Logger that discards all log messages.
func FromContextOrDiscard(ctx context.Context) Logger {
if logger, err := FromContext(ctx); err == nil {
return logger
}
return Discard()
}
// NewContext returns a new Context, derived from ctx, which carries the
// provided Logger.
func NewContext(ctx context.Context, logger Logger) context.Context {
return context.WithValue(ctx, contextKey{}, logger)
}
// NewContextWithSlogLogger returns a new Context, derived from ctx, which carries the
// provided slog.Logger.
func NewContextWithSlogLogger(ctx context.Context, logger *slog.Logger) context.Context {
return context.WithValue(ctx, contextKey{}, logger)
}

View File

@ -100,6 +100,11 @@ type Options struct {
// details, see docs for Go's time.Layout.
TimestampFormat string
// LogInfoLevel tells funcr what key to use to log the info level.
// If not specified, the info level will be logged as "level".
// If this is set to "", the info level will not be logged at all.
LogInfoLevel *string
// Verbosity tells funcr which V logs to produce. Higher values enable
// more logs. Info logs at or below this level will be written, while logs
// above this level will be discarded.
@ -213,6 +218,10 @@ func newFormatter(opts Options, outfmt outputFormat) Formatter {
if opts.MaxLogDepth == 0 {
opts.MaxLogDepth = defaultMaxLogDepth
}
if opts.LogInfoLevel == nil {
opts.LogInfoLevel = new(string)
*opts.LogInfoLevel = "level"
}
f := Formatter{
outputFormat: outfmt,
prefix: "",
@ -227,12 +236,15 @@ func newFormatter(opts Options, outfmt outputFormat) Formatter {
// implementation. It should be constructed with NewFormatter. Some of
// its methods directly implement logr.LogSink.
type Formatter struct {
outputFormat outputFormat
prefix string
values []any
valuesStr string
depth int
opts *Options
outputFormat outputFormat
prefix string
values []any
valuesStr string
parentValuesStr string
depth int
opts *Options
group string // for slog groups
groupDepth int
}
// outputFormat indicates which outputFormat to use.
@ -253,33 +265,62 @@ func (f Formatter) render(builtins, args []any) string {
// Empirically bytes.Buffer is faster than strings.Builder for this.
buf := bytes.NewBuffer(make([]byte, 0, 1024))
if f.outputFormat == outputJSON {
buf.WriteByte('{')
buf.WriteByte('{') // for the whole line
}
vals := builtins
if hook := f.opts.RenderBuiltinsHook; hook != nil {
vals = hook(f.sanitize(vals))
}
f.flatten(buf, vals, false, false) // keys are ours, no need to escape
continuing := len(builtins) > 0
if len(f.valuesStr) > 0 {
if f.parentValuesStr != "" {
if continuing {
if f.outputFormat == outputJSON {
buf.WriteByte(',')
} else {
buf.WriteByte(' ')
}
buf.WriteByte(f.comma())
}
buf.WriteString(f.parentValuesStr)
continuing = true
buf.WriteString(f.valuesStr)
}
groupDepth := f.groupDepth
if f.group != "" {
if f.valuesStr != "" || len(args) != 0 {
if continuing {
buf.WriteByte(f.comma())
}
buf.WriteString(f.quoted(f.group, true)) // escape user-provided keys
buf.WriteByte(f.colon())
buf.WriteByte('{') // for the group
continuing = false
} else {
// The group was empty
groupDepth--
}
}
if f.valuesStr != "" {
if continuing {
buf.WriteByte(f.comma())
}
buf.WriteString(f.valuesStr)
continuing = true
}
vals = args
if hook := f.opts.RenderArgsHook; hook != nil {
vals = hook(f.sanitize(vals))
}
f.flatten(buf, vals, continuing, true) // escape user-provided keys
if f.outputFormat == outputJSON {
buf.WriteByte('}')
for i := 0; i < groupDepth; i++ {
buf.WriteByte('}') // for the groups
}
if f.outputFormat == outputJSON {
buf.WriteByte('}') // for the whole line
}
return buf.String()
}
@ -298,9 +339,16 @@ func (f Formatter) flatten(buf *bytes.Buffer, kvList []any, continuing bool, esc
if len(kvList)%2 != 0 {
kvList = append(kvList, noValue)
}
copied := false
for i := 0; i < len(kvList); i += 2 {
k, ok := kvList[i].(string)
if !ok {
if !copied {
newList := make([]any, len(kvList))
copy(newList, kvList)
kvList = newList
copied = true
}
k = f.nonStringKey(kvList[i])
kvList[i] = k
}
@ -308,7 +356,7 @@ func (f Formatter) flatten(buf *bytes.Buffer, kvList []any, continuing bool, esc
if i > 0 || continuing {
if f.outputFormat == outputJSON {
buf.WriteByte(',')
buf.WriteByte(f.comma())
} else {
// In theory the format could be something we don't understand. In
// practice, we control it, so it won't be.
@ -316,24 +364,35 @@ func (f Formatter) flatten(buf *bytes.Buffer, kvList []any, continuing bool, esc
}
}
if escapeKeys {
buf.WriteString(prettyString(k))
} else {
// this is faster
buf.WriteByte('"')
buf.WriteString(k)
buf.WriteByte('"')
}
if f.outputFormat == outputJSON {
buf.WriteByte(':')
} else {
buf.WriteByte('=')
}
buf.WriteString(f.quoted(k, escapeKeys))
buf.WriteByte(f.colon())
buf.WriteString(f.pretty(v))
}
return kvList
}
func (f Formatter) quoted(str string, escape bool) string {
if escape {
return prettyString(str)
}
// this is faster
return `"` + str + `"`
}
func (f Formatter) comma() byte {
if f.outputFormat == outputJSON {
return ','
}
return ' '
}
func (f Formatter) colon() byte {
if f.outputFormat == outputJSON {
return ':'
}
return '='
}
func (f Formatter) pretty(value any) string {
return f.prettyWithFlags(value, 0, 0)
}
@ -407,12 +466,12 @@ func (f Formatter) prettyWithFlags(value any, flags uint32, depth int) string {
}
for i := 0; i < len(v); i += 2 {
if i > 0 {
buf.WriteByte(',')
buf.WriteByte(f.comma())
}
k, _ := v[i].(string) // sanitize() above means no need to check success
// arbitrary keys might need escaping
buf.WriteString(prettyString(k))
buf.WriteByte(':')
buf.WriteByte(f.colon())
buf.WriteString(f.prettyWithFlags(v[i+1], 0, depth+1))
}
if flags&flagRawStruct == 0 {
@ -481,7 +540,7 @@ func (f Formatter) prettyWithFlags(value any, flags uint32, depth int) string {
continue
}
if printComma {
buf.WriteByte(',')
buf.WriteByte(f.comma())
}
printComma = true // if we got here, we are rendering a field
if fld.Anonymous && fld.Type.Kind() == reflect.Struct && name == "" {
@ -492,10 +551,8 @@ func (f Formatter) prettyWithFlags(value any, flags uint32, depth int) string {
name = fld.Name
}
// field names can't contain characters which need escaping
buf.WriteByte('"')
buf.WriteString(name)
buf.WriteByte('"')
buf.WriteByte(':')
buf.WriteString(f.quoted(name, false))
buf.WriteByte(f.colon())
buf.WriteString(f.prettyWithFlags(v.Field(i).Interface(), 0, depth+1))
}
if flags&flagRawStruct == 0 {
@ -520,7 +577,7 @@ func (f Formatter) prettyWithFlags(value any, flags uint32, depth int) string {
buf.WriteByte('[')
for i := 0; i < v.Len(); i++ {
if i > 0 {
buf.WriteByte(',')
buf.WriteByte(f.comma())
}
e := v.Index(i)
buf.WriteString(f.prettyWithFlags(e.Interface(), 0, depth+1))
@ -534,7 +591,7 @@ func (f Formatter) prettyWithFlags(value any, flags uint32, depth int) string {
i := 0
for it.Next() {
if i > 0 {
buf.WriteByte(',')
buf.WriteByte(f.comma())
}
// If a map key supports TextMarshaler, use it.
keystr := ""
@ -556,7 +613,7 @@ func (f Formatter) prettyWithFlags(value any, flags uint32, depth int) string {
}
}
buf.WriteString(keystr)
buf.WriteByte(':')
buf.WriteByte(f.colon())
buf.WriteString(f.prettyWithFlags(it.Value().Interface(), 0, depth+1))
i++
}
@ -706,6 +763,53 @@ func (f Formatter) sanitize(kvList []any) []any {
return kvList
}
// startGroup opens a new group scope (basically a sub-struct), which locks all
// the current saved values and starts them anew. This is needed to satisfy
// slog.
func (f *Formatter) startGroup(group string) {
// Unnamed groups are just inlined.
if group == "" {
return
}
// Any saved values can no longer be changed.
buf := bytes.NewBuffer(make([]byte, 0, 1024))
continuing := false
if f.parentValuesStr != "" {
buf.WriteString(f.parentValuesStr)
continuing = true
}
if f.group != "" && f.valuesStr != "" {
if continuing {
buf.WriteByte(f.comma())
}
buf.WriteString(f.quoted(f.group, true)) // escape user-provided keys
buf.WriteByte(f.colon())
buf.WriteByte('{') // for the group
continuing = false
}
if f.valuesStr != "" {
if continuing {
buf.WriteByte(f.comma())
}
buf.WriteString(f.valuesStr)
}
// NOTE: We don't close the scope here - that's done later, when a log line
// is actually rendered (because we have N scopes to close).
f.parentValuesStr = buf.String()
// Start collecting new values.
f.group = group
f.groupDepth++
f.valuesStr = ""
f.values = nil
}
// Init configures this Formatter from runtime info, such as the call depth
// imposed by logr itself.
// Note that this receiver is a pointer, so depth can be saved.
@ -740,7 +844,10 @@ func (f Formatter) FormatInfo(level int, msg string, kvList []any) (prefix, args
if policy := f.opts.LogCaller; policy == All || policy == Info {
args = append(args, "caller", f.caller())
}
args = append(args, "level", level, "msg", msg)
if key := *f.opts.LogInfoLevel; key != "" {
args = append(args, key, level)
}
args = append(args, "msg", msg)
return prefix, f.render(args, kvList)
}

105
vendor/github.com/go-logr/logr/funcr/slogsink.go generated vendored Normal file
View File

@ -0,0 +1,105 @@
//go:build go1.21
// +build go1.21
/*
Copyright 2023 The logr Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package funcr
import (
"context"
"log/slog"
"github.com/go-logr/logr"
)
var _ logr.SlogSink = &fnlogger{}
const extraSlogSinkDepth = 3 // 2 for slog, 1 for SlogSink
func (l fnlogger) Handle(_ context.Context, record slog.Record) error {
kvList := make([]any, 0, 2*record.NumAttrs())
record.Attrs(func(attr slog.Attr) bool {
kvList = attrToKVs(attr, kvList)
return true
})
if record.Level >= slog.LevelError {
l.WithCallDepth(extraSlogSinkDepth).Error(nil, record.Message, kvList...)
} else {
level := l.levelFromSlog(record.Level)
l.WithCallDepth(extraSlogSinkDepth).Info(level, record.Message, kvList...)
}
return nil
}
func (l fnlogger) WithAttrs(attrs []slog.Attr) logr.SlogSink {
kvList := make([]any, 0, 2*len(attrs))
for _, attr := range attrs {
kvList = attrToKVs(attr, kvList)
}
l.AddValues(kvList)
return &l
}
func (l fnlogger) WithGroup(name string) logr.SlogSink {
l.startGroup(name)
return &l
}
// attrToKVs appends a slog.Attr to a logr-style kvList. It handle slog Groups
// and other details of slog.
func attrToKVs(attr slog.Attr, kvList []any) []any {
attrVal := attr.Value.Resolve()
if attrVal.Kind() == slog.KindGroup {
groupVal := attrVal.Group()
grpKVs := make([]any, 0, 2*len(groupVal))
for _, attr := range groupVal {
grpKVs = attrToKVs(attr, grpKVs)
}
if attr.Key == "" {
// slog says we have to inline these
kvList = append(kvList, grpKVs...)
} else {
kvList = append(kvList, attr.Key, PseudoStruct(grpKVs))
}
} else if attr.Key != "" {
kvList = append(kvList, attr.Key, attrVal.Any())
}
return kvList
}
// levelFromSlog adjusts the level by the logger's verbosity and negates it.
// It ensures that the result is >= 0. This is necessary because the result is
// passed to a LogSink and that API did not historically document whether
// levels could be negative or what that meant.
//
// Some example usage:
//
// logrV0 := getMyLogger()
// logrV2 := logrV0.V(2)
// slogV2 := slog.New(logr.ToSlogHandler(logrV2))
// slogV2.Debug("msg") // =~ logrV2.V(4) =~ logrV0.V(6)
// slogV2.Info("msg") // =~ logrV2.V(0) =~ logrV0.V(2)
// slogv2.Warn("msg") // =~ logrV2.V(-4) =~ logrV0.V(0)
func (l fnlogger) levelFromSlog(level slog.Level) int {
result := -level
if result < 0 {
result = 0 // because LogSink doesn't expect negative V levels
}
return int(result)
}

View File

@ -207,10 +207,6 @@ limitations under the License.
// those.
package logr
import (
"context"
)
// New returns a new Logger instance. This is primarily used by libraries
// implementing LogSink, rather than end users. Passing a nil sink will create
// a Logger which discards all log lines.
@ -410,45 +406,6 @@ func (l Logger) IsZero() bool {
return l.sink == nil
}
// contextKey is how we find Loggers in a context.Context.
type contextKey struct{}
// FromContext returns a Logger from ctx or an error if no Logger is found.
func FromContext(ctx context.Context) (Logger, error) {
if v, ok := ctx.Value(contextKey{}).(Logger); ok {
return v, nil
}
return Logger{}, notFoundError{}
}
// notFoundError exists to carry an IsNotFound method.
type notFoundError struct{}
func (notFoundError) Error() string {
return "no logr.Logger was present"
}
func (notFoundError) IsNotFound() bool {
return true
}
// FromContextOrDiscard returns a Logger from ctx. If no Logger is found, this
// returns a Logger that discards all log messages.
func FromContextOrDiscard(ctx context.Context) Logger {
if v, ok := ctx.Value(contextKey{}).(Logger); ok {
return v
}
return Discard()
}
// NewContext returns a new Context, derived from ctx, which carries the
// provided Logger.
func NewContext(ctx context.Context, logger Logger) context.Context {
return context.WithValue(ctx, contextKey{}, logger)
}
// RuntimeInfo holds information that the logr "core" library knows which
// LogSinks might want to know.
type RuntimeInfo struct {

192
vendor/github.com/go-logr/logr/sloghandler.go generated vendored Normal file
View File

@ -0,0 +1,192 @@
//go:build go1.21
// +build go1.21
/*
Copyright 2023 The logr Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package logr
import (
"context"
"log/slog"
)
type slogHandler struct {
// May be nil, in which case all logs get discarded.
sink LogSink
// Non-nil if sink is non-nil and implements SlogSink.
slogSink SlogSink
// groupPrefix collects values from WithGroup calls. It gets added as
// prefix to value keys when handling a log record.
groupPrefix string
// levelBias can be set when constructing the handler to influence the
// slog.Level of log records. A positive levelBias reduces the
// slog.Level value. slog has no API to influence this value after the
// handler got created, so it can only be set indirectly through
// Logger.V.
levelBias slog.Level
}
var _ slog.Handler = &slogHandler{}
// groupSeparator is used to concatenate WithGroup names and attribute keys.
const groupSeparator = "."
// GetLevel is used for black box unit testing.
func (l *slogHandler) GetLevel() slog.Level {
return l.levelBias
}
func (l *slogHandler) Enabled(_ context.Context, level slog.Level) bool {
return l.sink != nil && (level >= slog.LevelError || l.sink.Enabled(l.levelFromSlog(level)))
}
func (l *slogHandler) Handle(ctx context.Context, record slog.Record) error {
if l.slogSink != nil {
// Only adjust verbosity level of log entries < slog.LevelError.
if record.Level < slog.LevelError {
record.Level -= l.levelBias
}
return l.slogSink.Handle(ctx, record)
}
// No need to check for nil sink here because Handle will only be called
// when Enabled returned true.
kvList := make([]any, 0, 2*record.NumAttrs())
record.Attrs(func(attr slog.Attr) bool {
kvList = attrToKVs(attr, l.groupPrefix, kvList)
return true
})
if record.Level >= slog.LevelError {
l.sinkWithCallDepth().Error(nil, record.Message, kvList...)
} else {
level := l.levelFromSlog(record.Level)
l.sinkWithCallDepth().Info(level, record.Message, kvList...)
}
return nil
}
// sinkWithCallDepth adjusts the stack unwinding so that when Error or Info
// are called by Handle, code in slog gets skipped.
//
// This offset currently (Go 1.21.0) works for calls through
// slog.New(ToSlogHandler(...)). There's no guarantee that the call
// chain won't change. Wrapping the handler will also break unwinding. It's
// still better than not adjusting at all....
//
// This cannot be done when constructing the handler because FromSlogHandler needs
// access to the original sink without this adjustment. A second copy would
// work, but then WithAttrs would have to be called for both of them.
func (l *slogHandler) sinkWithCallDepth() LogSink {
if sink, ok := l.sink.(CallDepthLogSink); ok {
return sink.WithCallDepth(2)
}
return l.sink
}
func (l *slogHandler) WithAttrs(attrs []slog.Attr) slog.Handler {
if l.sink == nil || len(attrs) == 0 {
return l
}
clone := *l
if l.slogSink != nil {
clone.slogSink = l.slogSink.WithAttrs(attrs)
clone.sink = clone.slogSink
} else {
kvList := make([]any, 0, 2*len(attrs))
for _, attr := range attrs {
kvList = attrToKVs(attr, l.groupPrefix, kvList)
}
clone.sink = l.sink.WithValues(kvList...)
}
return &clone
}
func (l *slogHandler) WithGroup(name string) slog.Handler {
if l.sink == nil {
return l
}
if name == "" {
// slog says to inline empty groups
return l
}
clone := *l
if l.slogSink != nil {
clone.slogSink = l.slogSink.WithGroup(name)
clone.sink = clone.slogSink
} else {
clone.groupPrefix = addPrefix(clone.groupPrefix, name)
}
return &clone
}
// attrToKVs appends a slog.Attr to a logr-style kvList. It handle slog Groups
// and other details of slog.
func attrToKVs(attr slog.Attr, groupPrefix string, kvList []any) []any {
attrVal := attr.Value.Resolve()
if attrVal.Kind() == slog.KindGroup {
groupVal := attrVal.Group()
grpKVs := make([]any, 0, 2*len(groupVal))
prefix := groupPrefix
if attr.Key != "" {
prefix = addPrefix(groupPrefix, attr.Key)
}
for _, attr := range groupVal {
grpKVs = attrToKVs(attr, prefix, grpKVs)
}
kvList = append(kvList, grpKVs...)
} else if attr.Key != "" {
kvList = append(kvList, addPrefix(groupPrefix, attr.Key), attrVal.Any())
}
return kvList
}
func addPrefix(prefix, name string) string {
if prefix == "" {
return name
}
if name == "" {
return prefix
}
return prefix + groupSeparator + name
}
// levelFromSlog adjusts the level by the logger's verbosity and negates it.
// It ensures that the result is >= 0. This is necessary because the result is
// passed to a LogSink and that API did not historically document whether
// levels could be negative or what that meant.
//
// Some example usage:
//
// logrV0 := getMyLogger()
// logrV2 := logrV0.V(2)
// slogV2 := slog.New(logr.ToSlogHandler(logrV2))
// slogV2.Debug("msg") // =~ logrV2.V(4) =~ logrV0.V(6)
// slogV2.Info("msg") // =~ logrV2.V(0) =~ logrV0.V(2)
// slogv2.Warn("msg") // =~ logrV2.V(-4) =~ logrV0.V(0)
func (l *slogHandler) levelFromSlog(level slog.Level) int {
result := -level
result += l.levelBias // in case the original Logger had a V level
if result < 0 {
result = 0 // because LogSink doesn't expect negative V levels
}
return int(result)
}

100
vendor/github.com/go-logr/logr/slogr.go generated vendored Normal file
View File

@ -0,0 +1,100 @@
//go:build go1.21
// +build go1.21
/*
Copyright 2023 The logr Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package logr
import (
"context"
"log/slog"
)
// FromSlogHandler returns a Logger which writes to the slog.Handler.
//
// The logr verbosity level is mapped to slog levels such that V(0) becomes
// slog.LevelInfo and V(4) becomes slog.LevelDebug.
func FromSlogHandler(handler slog.Handler) Logger {
if handler, ok := handler.(*slogHandler); ok {
if handler.sink == nil {
return Discard()
}
return New(handler.sink).V(int(handler.levelBias))
}
return New(&slogSink{handler: handler})
}
// ToSlogHandler returns a slog.Handler which writes to the same sink as the Logger.
//
// The returned logger writes all records with level >= slog.LevelError as
// error log entries with LogSink.Error, regardless of the verbosity level of
// the Logger:
//
// logger := <some Logger with 0 as verbosity level>
// slog.New(ToSlogHandler(logger.V(10))).Error(...) -> logSink.Error(...)
//
// The level of all other records gets reduced by the verbosity
// level of the Logger and the result is negated. If it happens
// to be negative, then it gets replaced by zero because a LogSink
// is not expected to handled negative levels:
//
// slog.New(ToSlogHandler(logger)).Debug(...) -> logger.GetSink().Info(level=4, ...)
// slog.New(ToSlogHandler(logger)).Warning(...) -> logger.GetSink().Info(level=0, ...)
// slog.New(ToSlogHandler(logger)).Info(...) -> logger.GetSink().Info(level=0, ...)
// slog.New(ToSlogHandler(logger.V(4))).Info(...) -> logger.GetSink().Info(level=4, ...)
func ToSlogHandler(logger Logger) slog.Handler {
if sink, ok := logger.GetSink().(*slogSink); ok && logger.GetV() == 0 {
return sink.handler
}
handler := &slogHandler{sink: logger.GetSink(), levelBias: slog.Level(logger.GetV())}
if slogSink, ok := handler.sink.(SlogSink); ok {
handler.slogSink = slogSink
}
return handler
}
// SlogSink is an optional interface that a LogSink can implement to support
// logging through the slog.Logger or slog.Handler APIs better. It then should
// also support special slog values like slog.Group. When used as a
// slog.Handler, the advantages are:
//
// - stack unwinding gets avoided in favor of logging the pre-recorded PC,
// as intended by slog
// - proper grouping of key/value pairs via WithGroup
// - verbosity levels > slog.LevelInfo can be recorded
// - less overhead
//
// Both APIs (Logger and slog.Logger/Handler) then are supported equally
// well. Developers can pick whatever API suits them better and/or mix
// packages which use either API in the same binary with a common logging
// implementation.
//
// This interface is necessary because the type implementing the LogSink
// interface cannot also implement the slog.Handler interface due to the
// different prototype of the common Enabled method.
//
// An implementation could support both interfaces in two different types, but then
// additional interfaces would be needed to convert between those types in FromSlogHandler
// and ToSlogHandler.
type SlogSink interface {
LogSink
Handle(ctx context.Context, record slog.Record) error
WithAttrs(attrs []slog.Attr) SlogSink
WithGroup(name string) SlogSink
}

120
vendor/github.com/go-logr/logr/slogsink.go generated vendored Normal file
View File

@ -0,0 +1,120 @@
//go:build go1.21
// +build go1.21
/*
Copyright 2023 The logr Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package logr
import (
"context"
"log/slog"
"runtime"
"time"
)
var (
_ LogSink = &slogSink{}
_ CallDepthLogSink = &slogSink{}
_ Underlier = &slogSink{}
)
// Underlier is implemented by the LogSink returned by NewFromLogHandler.
type Underlier interface {
// GetUnderlying returns the Handler used by the LogSink.
GetUnderlying() slog.Handler
}
const (
// nameKey is used to log the `WithName` values as an additional attribute.
nameKey = "logger"
// errKey is used to log the error parameter of Error as an additional attribute.
errKey = "err"
)
type slogSink struct {
callDepth int
name string
handler slog.Handler
}
func (l *slogSink) Init(info RuntimeInfo) {
l.callDepth = info.CallDepth
}
func (l *slogSink) GetUnderlying() slog.Handler {
return l.handler
}
func (l *slogSink) WithCallDepth(depth int) LogSink {
newLogger := *l
newLogger.callDepth += depth
return &newLogger
}
func (l *slogSink) Enabled(level int) bool {
return l.handler.Enabled(context.Background(), slog.Level(-level))
}
func (l *slogSink) Info(level int, msg string, kvList ...interface{}) {
l.log(nil, msg, slog.Level(-level), kvList...)
}
func (l *slogSink) Error(err error, msg string, kvList ...interface{}) {
l.log(err, msg, slog.LevelError, kvList...)
}
func (l *slogSink) log(err error, msg string, level slog.Level, kvList ...interface{}) {
var pcs [1]uintptr
// skip runtime.Callers, this function, Info/Error, and all helper functions above that.
runtime.Callers(3+l.callDepth, pcs[:])
record := slog.NewRecord(time.Now(), level, msg, pcs[0])
if l.name != "" {
record.AddAttrs(slog.String(nameKey, l.name))
}
if err != nil {
record.AddAttrs(slog.Any(errKey, err))
}
record.Add(kvList...)
_ = l.handler.Handle(context.Background(), record)
}
func (l slogSink) WithName(name string) LogSink {
if l.name != "" {
l.name += "/"
}
l.name += name
return &l
}
func (l slogSink) WithValues(kvList ...interface{}) LogSink {
l.handler = l.handler.WithAttrs(kvListToAttrs(kvList...))
return &l
}
func kvListToAttrs(kvList ...interface{}) []slog.Attr {
// We don't need the record itself, only its Add method.
record := slog.NewRecord(time.Time{}, 0, "", 0)
record.Add(kvList...)
attrs := make([]slog.Attr, 0, record.NumAttrs())
record.Attrs(func(attr slog.Attr) bool {
attrs = append(attrs, attr)
return true
})
return attrs
}

View File

@ -1,530 +0,0 @@
// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package jsonpb
import (
"encoding/json"
"errors"
"fmt"
"io"
"math"
"reflect"
"strconv"
"strings"
"time"
"github.com/golang/protobuf/proto"
"google.golang.org/protobuf/encoding/protojson"
protoV2 "google.golang.org/protobuf/proto"
"google.golang.org/protobuf/reflect/protoreflect"
"google.golang.org/protobuf/reflect/protoregistry"
)
const wrapJSONUnmarshalV2 = false
// UnmarshalNext unmarshals the next JSON object from d into m.
func UnmarshalNext(d *json.Decoder, m proto.Message) error {
return new(Unmarshaler).UnmarshalNext(d, m)
}
// Unmarshal unmarshals a JSON object from r into m.
func Unmarshal(r io.Reader, m proto.Message) error {
return new(Unmarshaler).Unmarshal(r, m)
}
// UnmarshalString unmarshals a JSON object from s into m.
func UnmarshalString(s string, m proto.Message) error {
return new(Unmarshaler).Unmarshal(strings.NewReader(s), m)
}
// Unmarshaler is a configurable object for converting from a JSON
// representation to a protocol buffer object.
type Unmarshaler struct {
// AllowUnknownFields specifies whether to allow messages to contain
// unknown JSON fields, as opposed to failing to unmarshal.
AllowUnknownFields bool
// AnyResolver is used to resolve the google.protobuf.Any well-known type.
// If unset, the global registry is used by default.
AnyResolver AnyResolver
}
// JSONPBUnmarshaler is implemented by protobuf messages that customize the way
// they are unmarshaled from JSON. Messages that implement this should also
// implement JSONPBMarshaler so that the custom format can be produced.
//
// The JSON unmarshaling must follow the JSON to proto specification:
// https://developers.google.com/protocol-buffers/docs/proto3#json
//
// Deprecated: Custom types should implement protobuf reflection instead.
type JSONPBUnmarshaler interface {
UnmarshalJSONPB(*Unmarshaler, []byte) error
}
// Unmarshal unmarshals a JSON object from r into m.
func (u *Unmarshaler) Unmarshal(r io.Reader, m proto.Message) error {
return u.UnmarshalNext(json.NewDecoder(r), m)
}
// UnmarshalNext unmarshals the next JSON object from d into m.
func (u *Unmarshaler) UnmarshalNext(d *json.Decoder, m proto.Message) error {
if m == nil {
return errors.New("invalid nil message")
}
// Parse the next JSON object from the stream.
raw := json.RawMessage{}
if err := d.Decode(&raw); err != nil {
return err
}
// Check for custom unmarshalers first since they may not properly
// implement protobuf reflection that the logic below relies on.
if jsu, ok := m.(JSONPBUnmarshaler); ok {
return jsu.UnmarshalJSONPB(u, raw)
}
mr := proto.MessageReflect(m)
// NOTE: For historical reasons, a top-level null is treated as a noop.
// This is incorrect, but kept for compatibility.
if string(raw) == "null" && mr.Descriptor().FullName() != "google.protobuf.Value" {
return nil
}
if wrapJSONUnmarshalV2 {
// NOTE: If input message is non-empty, we need to preserve merge semantics
// of the old jsonpb implementation. These semantics are not supported by
// the protobuf JSON specification.
isEmpty := true
mr.Range(func(protoreflect.FieldDescriptor, protoreflect.Value) bool {
isEmpty = false // at least one iteration implies non-empty
return false
})
if !isEmpty {
// Perform unmarshaling into a newly allocated, empty message.
mr = mr.New()
// Use a defer to copy all unmarshaled fields into the original message.
dst := proto.MessageReflect(m)
defer mr.Range(func(fd protoreflect.FieldDescriptor, v protoreflect.Value) bool {
dst.Set(fd, v)
return true
})
}
// Unmarshal using the v2 JSON unmarshaler.
opts := protojson.UnmarshalOptions{
DiscardUnknown: u.AllowUnknownFields,
}
if u.AnyResolver != nil {
opts.Resolver = anyResolver{u.AnyResolver}
}
return opts.Unmarshal(raw, mr.Interface())
} else {
if err := u.unmarshalMessage(mr, raw); err != nil {
return err
}
return protoV2.CheckInitialized(mr.Interface())
}
}
func (u *Unmarshaler) unmarshalMessage(m protoreflect.Message, in []byte) error {
md := m.Descriptor()
fds := md.Fields()
if jsu, ok := proto.MessageV1(m.Interface()).(JSONPBUnmarshaler); ok {
return jsu.UnmarshalJSONPB(u, in)
}
if string(in) == "null" && md.FullName() != "google.protobuf.Value" {
return nil
}
switch wellKnownType(md.FullName()) {
case "Any":
var jsonObject map[string]json.RawMessage
if err := json.Unmarshal(in, &jsonObject); err != nil {
return err
}
rawTypeURL, ok := jsonObject["@type"]
if !ok {
return errors.New("Any JSON doesn't have '@type'")
}
typeURL, err := unquoteString(string(rawTypeURL))
if err != nil {
return fmt.Errorf("can't unmarshal Any's '@type': %q", rawTypeURL)
}
m.Set(fds.ByNumber(1), protoreflect.ValueOfString(typeURL))
var m2 protoreflect.Message
if u.AnyResolver != nil {
mi, err := u.AnyResolver.Resolve(typeURL)
if err != nil {
return err
}
m2 = proto.MessageReflect(mi)
} else {
mt, err := protoregistry.GlobalTypes.FindMessageByURL(typeURL)
if err != nil {
if err == protoregistry.NotFound {
return fmt.Errorf("could not resolve Any message type: %v", typeURL)
}
return err
}
m2 = mt.New()
}
if wellKnownType(m2.Descriptor().FullName()) != "" {
rawValue, ok := jsonObject["value"]
if !ok {
return errors.New("Any JSON doesn't have 'value'")
}
if err := u.unmarshalMessage(m2, rawValue); err != nil {
return fmt.Errorf("can't unmarshal Any nested proto %v: %v", typeURL, err)
}
} else {
delete(jsonObject, "@type")
rawJSON, err := json.Marshal(jsonObject)
if err != nil {
return fmt.Errorf("can't generate JSON for Any's nested proto to be unmarshaled: %v", err)
}
if err = u.unmarshalMessage(m2, rawJSON); err != nil {
return fmt.Errorf("can't unmarshal Any nested proto %v: %v", typeURL, err)
}
}
rawWire, err := protoV2.Marshal(m2.Interface())
if err != nil {
return fmt.Errorf("can't marshal proto %v into Any.Value: %v", typeURL, err)
}
m.Set(fds.ByNumber(2), protoreflect.ValueOfBytes(rawWire))
return nil
case "BoolValue", "BytesValue", "StringValue",
"Int32Value", "UInt32Value", "FloatValue",
"Int64Value", "UInt64Value", "DoubleValue":
fd := fds.ByNumber(1)
v, err := u.unmarshalValue(m.NewField(fd), in, fd)
if err != nil {
return err
}
m.Set(fd, v)
return nil
case "Duration":
v, err := unquoteString(string(in))
if err != nil {
return err
}
d, err := time.ParseDuration(v)
if err != nil {
return fmt.Errorf("bad Duration: %v", err)
}
sec := d.Nanoseconds() / 1e9
nsec := d.Nanoseconds() % 1e9
m.Set(fds.ByNumber(1), protoreflect.ValueOfInt64(int64(sec)))
m.Set(fds.ByNumber(2), protoreflect.ValueOfInt32(int32(nsec)))
return nil
case "Timestamp":
v, err := unquoteString(string(in))
if err != nil {
return err
}
t, err := time.Parse(time.RFC3339Nano, v)
if err != nil {
return fmt.Errorf("bad Timestamp: %v", err)
}
sec := t.Unix()
nsec := t.Nanosecond()
m.Set(fds.ByNumber(1), protoreflect.ValueOfInt64(int64(sec)))
m.Set(fds.ByNumber(2), protoreflect.ValueOfInt32(int32(nsec)))
return nil
case "Value":
switch {
case string(in) == "null":
m.Set(fds.ByNumber(1), protoreflect.ValueOfEnum(0))
case string(in) == "true":
m.Set(fds.ByNumber(4), protoreflect.ValueOfBool(true))
case string(in) == "false":
m.Set(fds.ByNumber(4), protoreflect.ValueOfBool(false))
case hasPrefixAndSuffix('"', in, '"'):
s, err := unquoteString(string(in))
if err != nil {
return fmt.Errorf("unrecognized type for Value %q", in)
}
m.Set(fds.ByNumber(3), protoreflect.ValueOfString(s))
case hasPrefixAndSuffix('[', in, ']'):
v := m.Mutable(fds.ByNumber(6))
return u.unmarshalMessage(v.Message(), in)
case hasPrefixAndSuffix('{', in, '}'):
v := m.Mutable(fds.ByNumber(5))
return u.unmarshalMessage(v.Message(), in)
default:
f, err := strconv.ParseFloat(string(in), 0)
if err != nil {
return fmt.Errorf("unrecognized type for Value %q", in)
}
m.Set(fds.ByNumber(2), protoreflect.ValueOfFloat64(f))
}
return nil
case "ListValue":
var jsonArray []json.RawMessage
if err := json.Unmarshal(in, &jsonArray); err != nil {
return fmt.Errorf("bad ListValue: %v", err)
}
lv := m.Mutable(fds.ByNumber(1)).List()
for _, raw := range jsonArray {
ve := lv.NewElement()
if err := u.unmarshalMessage(ve.Message(), raw); err != nil {
return err
}
lv.Append(ve)
}
return nil
case "Struct":
var jsonObject map[string]json.RawMessage
if err := json.Unmarshal(in, &jsonObject); err != nil {
return fmt.Errorf("bad StructValue: %v", err)
}
mv := m.Mutable(fds.ByNumber(1)).Map()
for key, raw := range jsonObject {
kv := protoreflect.ValueOf(key).MapKey()
vv := mv.NewValue()
if err := u.unmarshalMessage(vv.Message(), raw); err != nil {
return fmt.Errorf("bad value in StructValue for key %q: %v", key, err)
}
mv.Set(kv, vv)
}
return nil
}
var jsonObject map[string]json.RawMessage
if err := json.Unmarshal(in, &jsonObject); err != nil {
return err
}
// Handle known fields.
for i := 0; i < fds.Len(); i++ {
fd := fds.Get(i)
if fd.IsWeak() && fd.Message().IsPlaceholder() {
continue // weak reference is not linked in
}
// Search for any raw JSON value associated with this field.
var raw json.RawMessage
name := string(fd.Name())
if fd.Kind() == protoreflect.GroupKind {
name = string(fd.Message().Name())
}
if v, ok := jsonObject[name]; ok {
delete(jsonObject, name)
raw = v
}
name = string(fd.JSONName())
if v, ok := jsonObject[name]; ok {
delete(jsonObject, name)
raw = v
}
field := m.NewField(fd)
// Unmarshal the field value.
if raw == nil || (string(raw) == "null" && !isSingularWellKnownValue(fd) && !isSingularJSONPBUnmarshaler(field, fd)) {
continue
}
v, err := u.unmarshalValue(field, raw, fd)
if err != nil {
return err
}
m.Set(fd, v)
}
// Handle extension fields.
for name, raw := range jsonObject {
if !strings.HasPrefix(name, "[") || !strings.HasSuffix(name, "]") {
continue
}
// Resolve the extension field by name.
xname := protoreflect.FullName(name[len("[") : len(name)-len("]")])
xt, _ := protoregistry.GlobalTypes.FindExtensionByName(xname)
if xt == nil && isMessageSet(md) {
xt, _ = protoregistry.GlobalTypes.FindExtensionByName(xname.Append("message_set_extension"))
}
if xt == nil {
continue
}
delete(jsonObject, name)
fd := xt.TypeDescriptor()
if fd.ContainingMessage().FullName() != m.Descriptor().FullName() {
return fmt.Errorf("extension field %q does not extend message %q", xname, m.Descriptor().FullName())
}
field := m.NewField(fd)
// Unmarshal the field value.
if raw == nil || (string(raw) == "null" && !isSingularWellKnownValue(fd) && !isSingularJSONPBUnmarshaler(field, fd)) {
continue
}
v, err := u.unmarshalValue(field, raw, fd)
if err != nil {
return err
}
m.Set(fd, v)
}
if !u.AllowUnknownFields && len(jsonObject) > 0 {
for name := range jsonObject {
return fmt.Errorf("unknown field %q in %v", name, md.FullName())
}
}
return nil
}
func isSingularWellKnownValue(fd protoreflect.FieldDescriptor) bool {
if fd.Cardinality() == protoreflect.Repeated {
return false
}
if md := fd.Message(); md != nil {
return md.FullName() == "google.protobuf.Value"
}
if ed := fd.Enum(); ed != nil {
return ed.FullName() == "google.protobuf.NullValue"
}
return false
}
func isSingularJSONPBUnmarshaler(v protoreflect.Value, fd protoreflect.FieldDescriptor) bool {
if fd.Message() != nil && fd.Cardinality() != protoreflect.Repeated {
_, ok := proto.MessageV1(v.Interface()).(JSONPBUnmarshaler)
return ok
}
return false
}
func (u *Unmarshaler) unmarshalValue(v protoreflect.Value, in []byte, fd protoreflect.FieldDescriptor) (protoreflect.Value, error) {
switch {
case fd.IsList():
var jsonArray []json.RawMessage
if err := json.Unmarshal(in, &jsonArray); err != nil {
return v, err
}
lv := v.List()
for _, raw := range jsonArray {
ve, err := u.unmarshalSingularValue(lv.NewElement(), raw, fd)
if err != nil {
return v, err
}
lv.Append(ve)
}
return v, nil
case fd.IsMap():
var jsonObject map[string]json.RawMessage
if err := json.Unmarshal(in, &jsonObject); err != nil {
return v, err
}
kfd := fd.MapKey()
vfd := fd.MapValue()
mv := v.Map()
for key, raw := range jsonObject {
var kv protoreflect.MapKey
if kfd.Kind() == protoreflect.StringKind {
kv = protoreflect.ValueOf(key).MapKey()
} else {
v, err := u.unmarshalSingularValue(kfd.Default(), []byte(key), kfd)
if err != nil {
return v, err
}
kv = v.MapKey()
}
vv, err := u.unmarshalSingularValue(mv.NewValue(), raw, vfd)
if err != nil {
return v, err
}
mv.Set(kv, vv)
}
return v, nil
default:
return u.unmarshalSingularValue(v, in, fd)
}
}
var nonFinite = map[string]float64{
`"NaN"`: math.NaN(),
`"Infinity"`: math.Inf(+1),
`"-Infinity"`: math.Inf(-1),
}
func (u *Unmarshaler) unmarshalSingularValue(v protoreflect.Value, in []byte, fd protoreflect.FieldDescriptor) (protoreflect.Value, error) {
switch fd.Kind() {
case protoreflect.BoolKind:
return unmarshalValue(in, new(bool))
case protoreflect.Int32Kind, protoreflect.Sint32Kind, protoreflect.Sfixed32Kind:
return unmarshalValue(trimQuote(in), new(int32))
case protoreflect.Int64Kind, protoreflect.Sint64Kind, protoreflect.Sfixed64Kind:
return unmarshalValue(trimQuote(in), new(int64))
case protoreflect.Uint32Kind, protoreflect.Fixed32Kind:
return unmarshalValue(trimQuote(in), new(uint32))
case protoreflect.Uint64Kind, protoreflect.Fixed64Kind:
return unmarshalValue(trimQuote(in), new(uint64))
case protoreflect.FloatKind:
if f, ok := nonFinite[string(in)]; ok {
return protoreflect.ValueOfFloat32(float32(f)), nil
}
return unmarshalValue(trimQuote(in), new(float32))
case protoreflect.DoubleKind:
if f, ok := nonFinite[string(in)]; ok {
return protoreflect.ValueOfFloat64(float64(f)), nil
}
return unmarshalValue(trimQuote(in), new(float64))
case protoreflect.StringKind:
return unmarshalValue(in, new(string))
case protoreflect.BytesKind:
return unmarshalValue(in, new([]byte))
case protoreflect.EnumKind:
if hasPrefixAndSuffix('"', in, '"') {
vd := fd.Enum().Values().ByName(protoreflect.Name(trimQuote(in)))
if vd == nil {
return v, fmt.Errorf("unknown value %q for enum %s", in, fd.Enum().FullName())
}
return protoreflect.ValueOfEnum(vd.Number()), nil
}
return unmarshalValue(in, new(protoreflect.EnumNumber))
case protoreflect.MessageKind, protoreflect.GroupKind:
err := u.unmarshalMessage(v.Message(), in)
return v, err
default:
panic(fmt.Sprintf("invalid kind %v", fd.Kind()))
}
}
func unmarshalValue(in []byte, v interface{}) (protoreflect.Value, error) {
err := json.Unmarshal(in, v)
return protoreflect.ValueOf(reflect.ValueOf(v).Elem().Interface()), err
}
func unquoteString(in string) (out string, err error) {
err = json.Unmarshal([]byte(in), &out)
return out, err
}
func hasPrefixAndSuffix(prefix byte, in []byte, suffix byte) bool {
if len(in) >= 2 && in[0] == prefix && in[len(in)-1] == suffix {
return true
}
return false
}
// trimQuote is like unquoteString but simply strips surrounding quotes.
// This is incorrect, but is behavior done by the legacy implementation.
func trimQuote(in []byte) []byte {
if len(in) >= 2 && in[0] == '"' && in[len(in)-1] == '"' {
in = in[1 : len(in)-1]
}
return in
}

View File

@ -1,559 +0,0 @@
// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package jsonpb
import (
"encoding/json"
"errors"
"fmt"
"io"
"math"
"reflect"
"sort"
"strconv"
"strings"
"time"
"github.com/golang/protobuf/proto"
"google.golang.org/protobuf/encoding/protojson"
protoV2 "google.golang.org/protobuf/proto"
"google.golang.org/protobuf/reflect/protoreflect"
"google.golang.org/protobuf/reflect/protoregistry"
)
const wrapJSONMarshalV2 = false
// Marshaler is a configurable object for marshaling protocol buffer messages
// to the specified JSON representation.
type Marshaler struct {
// OrigName specifies whether to use the original protobuf name for fields.
OrigName bool
// EnumsAsInts specifies whether to render enum values as integers,
// as opposed to string values.
EnumsAsInts bool
// EmitDefaults specifies whether to render fields with zero values.
EmitDefaults bool
// Indent controls whether the output is compact or not.
// If empty, the output is compact JSON. Otherwise, every JSON object
// entry and JSON array value will be on its own line.
// Each line will be preceded by repeated copies of Indent, where the
// number of copies is the current indentation depth.
Indent string
// AnyResolver is used to resolve the google.protobuf.Any well-known type.
// If unset, the global registry is used by default.
AnyResolver AnyResolver
}
// JSONPBMarshaler is implemented by protobuf messages that customize the
// way they are marshaled to JSON. Messages that implement this should also
// implement JSONPBUnmarshaler so that the custom format can be parsed.
//
// The JSON marshaling must follow the proto to JSON specification:
// https://developers.google.com/protocol-buffers/docs/proto3#json
//
// Deprecated: Custom types should implement protobuf reflection instead.
type JSONPBMarshaler interface {
MarshalJSONPB(*Marshaler) ([]byte, error)
}
// Marshal serializes a protobuf message as JSON into w.
func (jm *Marshaler) Marshal(w io.Writer, m proto.Message) error {
b, err := jm.marshal(m)
if len(b) > 0 {
if _, err := w.Write(b); err != nil {
return err
}
}
return err
}
// MarshalToString serializes a protobuf message as JSON in string form.
func (jm *Marshaler) MarshalToString(m proto.Message) (string, error) {
b, err := jm.marshal(m)
if err != nil {
return "", err
}
return string(b), nil
}
func (jm *Marshaler) marshal(m proto.Message) ([]byte, error) {
v := reflect.ValueOf(m)
if m == nil || (v.Kind() == reflect.Ptr && v.IsNil()) {
return nil, errors.New("Marshal called with nil")
}
// Check for custom marshalers first since they may not properly
// implement protobuf reflection that the logic below relies on.
if jsm, ok := m.(JSONPBMarshaler); ok {
return jsm.MarshalJSONPB(jm)
}
if wrapJSONMarshalV2 {
opts := protojson.MarshalOptions{
UseProtoNames: jm.OrigName,
UseEnumNumbers: jm.EnumsAsInts,
EmitUnpopulated: jm.EmitDefaults,
Indent: jm.Indent,
}
if jm.AnyResolver != nil {
opts.Resolver = anyResolver{jm.AnyResolver}
}
return opts.Marshal(proto.MessageReflect(m).Interface())
} else {
// Check for unpopulated required fields first.
m2 := proto.MessageReflect(m)
if err := protoV2.CheckInitialized(m2.Interface()); err != nil {
return nil, err
}
w := jsonWriter{Marshaler: jm}
err := w.marshalMessage(m2, "", "")
return w.buf, err
}
}
type jsonWriter struct {
*Marshaler
buf []byte
}
func (w *jsonWriter) write(s string) {
w.buf = append(w.buf, s...)
}
func (w *jsonWriter) marshalMessage(m protoreflect.Message, indent, typeURL string) error {
if jsm, ok := proto.MessageV1(m.Interface()).(JSONPBMarshaler); ok {
b, err := jsm.MarshalJSONPB(w.Marshaler)
if err != nil {
return err
}
if typeURL != "" {
// we are marshaling this object to an Any type
var js map[string]*json.RawMessage
if err = json.Unmarshal(b, &js); err != nil {
return fmt.Errorf("type %T produced invalid JSON: %v", m.Interface(), err)
}
turl, err := json.Marshal(typeURL)
if err != nil {
return fmt.Errorf("failed to marshal type URL %q to JSON: %v", typeURL, err)
}
js["@type"] = (*json.RawMessage)(&turl)
if b, err = json.Marshal(js); err != nil {
return err
}
}
w.write(string(b))
return nil
}
md := m.Descriptor()
fds := md.Fields()
// Handle well-known types.
const secondInNanos = int64(time.Second / time.Nanosecond)
switch wellKnownType(md.FullName()) {
case "Any":
return w.marshalAny(m, indent)
case "BoolValue", "BytesValue", "StringValue",
"Int32Value", "UInt32Value", "FloatValue",
"Int64Value", "UInt64Value", "DoubleValue":
fd := fds.ByNumber(1)
return w.marshalValue(fd, m.Get(fd), indent)
case "Duration":
const maxSecondsInDuration = 315576000000
// "Generated output always contains 0, 3, 6, or 9 fractional digits,
// depending on required precision."
s := m.Get(fds.ByNumber(1)).Int()
ns := m.Get(fds.ByNumber(2)).Int()
if s < -maxSecondsInDuration || s > maxSecondsInDuration {
return fmt.Errorf("seconds out of range %v", s)
}
if ns <= -secondInNanos || ns >= secondInNanos {
return fmt.Errorf("ns out of range (%v, %v)", -secondInNanos, secondInNanos)
}
if (s > 0 && ns < 0) || (s < 0 && ns > 0) {
return errors.New("signs of seconds and nanos do not match")
}
var sign string
if s < 0 || ns < 0 {
sign, s, ns = "-", -1*s, -1*ns
}
x := fmt.Sprintf("%s%d.%09d", sign, s, ns)
x = strings.TrimSuffix(x, "000")
x = strings.TrimSuffix(x, "000")
x = strings.TrimSuffix(x, ".000")
w.write(fmt.Sprintf(`"%vs"`, x))
return nil
case "Timestamp":
// "RFC 3339, where generated output will always be Z-normalized
// and uses 0, 3, 6 or 9 fractional digits."
s := m.Get(fds.ByNumber(1)).Int()
ns := m.Get(fds.ByNumber(2)).Int()
if ns < 0 || ns >= secondInNanos {
return fmt.Errorf("ns out of range [0, %v)", secondInNanos)
}
t := time.Unix(s, ns).UTC()
// time.RFC3339Nano isn't exactly right (we need to get 3/6/9 fractional digits).
x := t.Format("2006-01-02T15:04:05.000000000")
x = strings.TrimSuffix(x, "000")
x = strings.TrimSuffix(x, "000")
x = strings.TrimSuffix(x, ".000")
w.write(fmt.Sprintf(`"%vZ"`, x))
return nil
case "Value":
// JSON value; which is a null, number, string, bool, object, or array.
od := md.Oneofs().Get(0)
fd := m.WhichOneof(od)
if fd == nil {
return errors.New("nil Value")
}
return w.marshalValue(fd, m.Get(fd), indent)
case "Struct", "ListValue":
// JSON object or array.
fd := fds.ByNumber(1)
return w.marshalValue(fd, m.Get(fd), indent)
}
w.write("{")
if w.Indent != "" {
w.write("\n")
}
firstField := true
if typeURL != "" {
if err := w.marshalTypeURL(indent, typeURL); err != nil {
return err
}
firstField = false
}
for i := 0; i < fds.Len(); {
fd := fds.Get(i)
if od := fd.ContainingOneof(); od != nil {
fd = m.WhichOneof(od)
i += od.Fields().Len()
if fd == nil {
continue
}
} else {
i++
}
v := m.Get(fd)
if !m.Has(fd) {
if !w.EmitDefaults || fd.ContainingOneof() != nil {
continue
}
if fd.Cardinality() != protoreflect.Repeated && (fd.Message() != nil || fd.Syntax() == protoreflect.Proto2) {
v = protoreflect.Value{} // use "null" for singular messages or proto2 scalars
}
}
if !firstField {
w.writeComma()
}
if err := w.marshalField(fd, v, indent); err != nil {
return err
}
firstField = false
}
// Handle proto2 extensions.
if md.ExtensionRanges().Len() > 0 {
// Collect a sorted list of all extension descriptor and values.
type ext struct {
desc protoreflect.FieldDescriptor
val protoreflect.Value
}
var exts []ext
m.Range(func(fd protoreflect.FieldDescriptor, v protoreflect.Value) bool {
if fd.IsExtension() {
exts = append(exts, ext{fd, v})
}
return true
})
sort.Slice(exts, func(i, j int) bool {
return exts[i].desc.Number() < exts[j].desc.Number()
})
for _, ext := range exts {
if !firstField {
w.writeComma()
}
if err := w.marshalField(ext.desc, ext.val, indent); err != nil {
return err
}
firstField = false
}
}
if w.Indent != "" {
w.write("\n")
w.write(indent)
}
w.write("}")
return nil
}
func (w *jsonWriter) writeComma() {
if w.Indent != "" {
w.write(",\n")
} else {
w.write(",")
}
}
func (w *jsonWriter) marshalAny(m protoreflect.Message, indent string) error {
// "If the Any contains a value that has a special JSON mapping,
// it will be converted as follows: {"@type": xxx, "value": yyy}.
// Otherwise, the value will be converted into a JSON object,
// and the "@type" field will be inserted to indicate the actual data type."
md := m.Descriptor()
typeURL := m.Get(md.Fields().ByNumber(1)).String()
rawVal := m.Get(md.Fields().ByNumber(2)).Bytes()
var m2 protoreflect.Message
if w.AnyResolver != nil {
mi, err := w.AnyResolver.Resolve(typeURL)
if err != nil {
return err
}
m2 = proto.MessageReflect(mi)
} else {
mt, err := protoregistry.GlobalTypes.FindMessageByURL(typeURL)
if err != nil {
return err
}
m2 = mt.New()
}
if err := protoV2.Unmarshal(rawVal, m2.Interface()); err != nil {
return err
}
if wellKnownType(m2.Descriptor().FullName()) == "" {
return w.marshalMessage(m2, indent, typeURL)
}
w.write("{")
if w.Indent != "" {
w.write("\n")
}
if err := w.marshalTypeURL(indent, typeURL); err != nil {
return err
}
w.writeComma()
if w.Indent != "" {
w.write(indent)
w.write(w.Indent)
w.write(`"value": `)
} else {
w.write(`"value":`)
}
if err := w.marshalMessage(m2, indent+w.Indent, ""); err != nil {
return err
}
if w.Indent != "" {
w.write("\n")
w.write(indent)
}
w.write("}")
return nil
}
func (w *jsonWriter) marshalTypeURL(indent, typeURL string) error {
if w.Indent != "" {
w.write(indent)
w.write(w.Indent)
}
w.write(`"@type":`)
if w.Indent != "" {
w.write(" ")
}
b, err := json.Marshal(typeURL)
if err != nil {
return err
}
w.write(string(b))
return nil
}
// marshalField writes field description and value to the Writer.
func (w *jsonWriter) marshalField(fd protoreflect.FieldDescriptor, v protoreflect.Value, indent string) error {
if w.Indent != "" {
w.write(indent)
w.write(w.Indent)
}
w.write(`"`)
switch {
case fd.IsExtension():
// For message set, use the fname of the message as the extension name.
name := string(fd.FullName())
if isMessageSet(fd.ContainingMessage()) {
name = strings.TrimSuffix(name, ".message_set_extension")
}
w.write("[" + name + "]")
case w.OrigName:
name := string(fd.Name())
if fd.Kind() == protoreflect.GroupKind {
name = string(fd.Message().Name())
}
w.write(name)
default:
w.write(string(fd.JSONName()))
}
w.write(`":`)
if w.Indent != "" {
w.write(" ")
}
return w.marshalValue(fd, v, indent)
}
func (w *jsonWriter) marshalValue(fd protoreflect.FieldDescriptor, v protoreflect.Value, indent string) error {
switch {
case fd.IsList():
w.write("[")
comma := ""
lv := v.List()
for i := 0; i < lv.Len(); i++ {
w.write(comma)
if w.Indent != "" {
w.write("\n")
w.write(indent)
w.write(w.Indent)
w.write(w.Indent)
}
if err := w.marshalSingularValue(fd, lv.Get(i), indent+w.Indent); err != nil {
return err
}
comma = ","
}
if w.Indent != "" {
w.write("\n")
w.write(indent)
w.write(w.Indent)
}
w.write("]")
return nil
case fd.IsMap():
kfd := fd.MapKey()
vfd := fd.MapValue()
mv := v.Map()
// Collect a sorted list of all map keys and values.
type entry struct{ key, val protoreflect.Value }
var entries []entry
mv.Range(func(k protoreflect.MapKey, v protoreflect.Value) bool {
entries = append(entries, entry{k.Value(), v})
return true
})
sort.Slice(entries, func(i, j int) bool {
switch kfd.Kind() {
case protoreflect.BoolKind:
return !entries[i].key.Bool() && entries[j].key.Bool()
case protoreflect.Int32Kind, protoreflect.Sint32Kind, protoreflect.Sfixed32Kind, protoreflect.Int64Kind, protoreflect.Sint64Kind, protoreflect.Sfixed64Kind:
return entries[i].key.Int() < entries[j].key.Int()
case protoreflect.Uint32Kind, protoreflect.Fixed32Kind, protoreflect.Uint64Kind, protoreflect.Fixed64Kind:
return entries[i].key.Uint() < entries[j].key.Uint()
case protoreflect.StringKind:
return entries[i].key.String() < entries[j].key.String()
default:
panic("invalid kind")
}
})
w.write(`{`)
comma := ""
for _, entry := range entries {
w.write(comma)
if w.Indent != "" {
w.write("\n")
w.write(indent)
w.write(w.Indent)
w.write(w.Indent)
}
s := fmt.Sprint(entry.key.Interface())
b, err := json.Marshal(s)
if err != nil {
return err
}
w.write(string(b))
w.write(`:`)
if w.Indent != "" {
w.write(` `)
}
if err := w.marshalSingularValue(vfd, entry.val, indent+w.Indent); err != nil {
return err
}
comma = ","
}
if w.Indent != "" {
w.write("\n")
w.write(indent)
w.write(w.Indent)
}
w.write(`}`)
return nil
default:
return w.marshalSingularValue(fd, v, indent)
}
}
func (w *jsonWriter) marshalSingularValue(fd protoreflect.FieldDescriptor, v protoreflect.Value, indent string) error {
switch {
case !v.IsValid():
w.write("null")
return nil
case fd.Message() != nil:
return w.marshalMessage(v.Message(), indent+w.Indent, "")
case fd.Enum() != nil:
if fd.Enum().FullName() == "google.protobuf.NullValue" {
w.write("null")
return nil
}
vd := fd.Enum().Values().ByNumber(v.Enum())
if vd == nil || w.EnumsAsInts {
w.write(strconv.Itoa(int(v.Enum())))
} else {
w.write(`"` + string(vd.Name()) + `"`)
}
return nil
default:
switch v.Interface().(type) {
case float32, float64:
switch {
case math.IsInf(v.Float(), +1):
w.write(`"Infinity"`)
return nil
case math.IsInf(v.Float(), -1):
w.write(`"-Infinity"`)
return nil
case math.IsNaN(v.Float()):
w.write(`"NaN"`)
return nil
}
case int64, uint64:
w.write(fmt.Sprintf(`"%d"`, v.Interface()))
return nil
}
b, err := json.Marshal(v.Interface())
if err != nil {
return err
}
w.write(string(b))
return nil
}
}

View File

@ -1,69 +0,0 @@
// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package jsonpb provides functionality to marshal and unmarshal between a
// protocol buffer message and JSON. It follows the specification at
// https://developers.google.com/protocol-buffers/docs/proto3#json.
//
// Do not rely on the default behavior of the standard encoding/json package
// when called on generated message types as it does not operate correctly.
//
// Deprecated: Use the "google.golang.org/protobuf/encoding/protojson"
// package instead.
package jsonpb
import (
"github.com/golang/protobuf/proto"
"google.golang.org/protobuf/reflect/protoreflect"
"google.golang.org/protobuf/reflect/protoregistry"
"google.golang.org/protobuf/runtime/protoimpl"
)
// AnyResolver takes a type URL, present in an Any message,
// and resolves it into an instance of the associated message.
type AnyResolver interface {
Resolve(typeURL string) (proto.Message, error)
}
type anyResolver struct{ AnyResolver }
func (r anyResolver) FindMessageByName(message protoreflect.FullName) (protoreflect.MessageType, error) {
return r.FindMessageByURL(string(message))
}
func (r anyResolver) FindMessageByURL(url string) (protoreflect.MessageType, error) {
m, err := r.Resolve(url)
if err != nil {
return nil, err
}
return protoimpl.X.MessageTypeOf(m), nil
}
func (r anyResolver) FindExtensionByName(field protoreflect.FullName) (protoreflect.ExtensionType, error) {
return protoregistry.GlobalTypes.FindExtensionByName(field)
}
func (r anyResolver) FindExtensionByNumber(message protoreflect.FullName, field protoreflect.FieldNumber) (protoreflect.ExtensionType, error) {
return protoregistry.GlobalTypes.FindExtensionByNumber(message, field)
}
func wellKnownType(s protoreflect.FullName) string {
if s.Parent() == "google.protobuf" {
switch s.Name() {
case "Empty", "Any",
"BoolValue", "BytesValue", "StringValue",
"Int32Value", "UInt32Value", "FloatValue",
"Int64Value", "UInt64Value", "DoubleValue",
"Duration", "Timestamp",
"NullValue", "Struct", "Value", "ListValue":
return string(s.Name())
}
}
return ""
}
func isMessageSet(md protoreflect.MessageDescriptor) bool {
ms, ok := md.(interface{ IsMessageSet() bool })
return ok && ms.IsMessageSet()
}

View File

@ -1,179 +0,0 @@
// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package ptypes
import (
"fmt"
"strings"
"github.com/golang/protobuf/proto"
"google.golang.org/protobuf/reflect/protoreflect"
"google.golang.org/protobuf/reflect/protoregistry"
anypb "github.com/golang/protobuf/ptypes/any"
)
const urlPrefix = "type.googleapis.com/"
// AnyMessageName returns the message name contained in an anypb.Any message.
// Most type assertions should use the Is function instead.
//
// Deprecated: Call the any.MessageName method instead.
func AnyMessageName(any *anypb.Any) (string, error) {
name, err := anyMessageName(any)
return string(name), err
}
func anyMessageName(any *anypb.Any) (protoreflect.FullName, error) {
if any == nil {
return "", fmt.Errorf("message is nil")
}
name := protoreflect.FullName(any.TypeUrl)
if i := strings.LastIndex(any.TypeUrl, "/"); i >= 0 {
name = name[i+len("/"):]
}
if !name.IsValid() {
return "", fmt.Errorf("message type url %q is invalid", any.TypeUrl)
}
return name, nil
}
// MarshalAny marshals the given message m into an anypb.Any message.
//
// Deprecated: Call the anypb.New function instead.
func MarshalAny(m proto.Message) (*anypb.Any, error) {
switch dm := m.(type) {
case DynamicAny:
m = dm.Message
case *DynamicAny:
if dm == nil {
return nil, proto.ErrNil
}
m = dm.Message
}
b, err := proto.Marshal(m)
if err != nil {
return nil, err
}
return &anypb.Any{TypeUrl: urlPrefix + proto.MessageName(m), Value: b}, nil
}
// Empty returns a new message of the type specified in an anypb.Any message.
// It returns protoregistry.NotFound if the corresponding message type could not
// be resolved in the global registry.
//
// Deprecated: Use protoregistry.GlobalTypes.FindMessageByName instead
// to resolve the message name and create a new instance of it.
func Empty(any *anypb.Any) (proto.Message, error) {
name, err := anyMessageName(any)
if err != nil {
return nil, err
}
mt, err := protoregistry.GlobalTypes.FindMessageByName(name)
if err != nil {
return nil, err
}
return proto.MessageV1(mt.New().Interface()), nil
}
// UnmarshalAny unmarshals the encoded value contained in the anypb.Any message
// into the provided message m. It returns an error if the target message
// does not match the type in the Any message or if an unmarshal error occurs.
//
// The target message m may be a *DynamicAny message. If the underlying message
// type could not be resolved, then this returns protoregistry.NotFound.
//
// Deprecated: Call the any.UnmarshalTo method instead.
func UnmarshalAny(any *anypb.Any, m proto.Message) error {
if dm, ok := m.(*DynamicAny); ok {
if dm.Message == nil {
var err error
dm.Message, err = Empty(any)
if err != nil {
return err
}
}
m = dm.Message
}
anyName, err := AnyMessageName(any)
if err != nil {
return err
}
msgName := proto.MessageName(m)
if anyName != msgName {
return fmt.Errorf("mismatched message type: got %q want %q", anyName, msgName)
}
return proto.Unmarshal(any.Value, m)
}
// Is reports whether the Any message contains a message of the specified type.
//
// Deprecated: Call the any.MessageIs method instead.
func Is(any *anypb.Any, m proto.Message) bool {
if any == nil || m == nil {
return false
}
name := proto.MessageName(m)
if !strings.HasSuffix(any.TypeUrl, name) {
return false
}
return len(any.TypeUrl) == len(name) || any.TypeUrl[len(any.TypeUrl)-len(name)-1] == '/'
}
// DynamicAny is a value that can be passed to UnmarshalAny to automatically
// allocate a proto.Message for the type specified in an anypb.Any message.
// The allocated message is stored in the embedded proto.Message.
//
// Example:
// var x ptypes.DynamicAny
// if err := ptypes.UnmarshalAny(a, &x); err != nil { ... }
// fmt.Printf("unmarshaled message: %v", x.Message)
//
// Deprecated: Use the any.UnmarshalNew method instead to unmarshal
// the any message contents into a new instance of the underlying message.
type DynamicAny struct{ proto.Message }
func (m DynamicAny) String() string {
if m.Message == nil {
return "<nil>"
}
return m.Message.String()
}
func (m DynamicAny) Reset() {
if m.Message == nil {
return
}
m.Message.Reset()
}
func (m DynamicAny) ProtoMessage() {
return
}
func (m DynamicAny) ProtoReflect() protoreflect.Message {
if m.Message == nil {
return nil
}
return dynamicAny{proto.MessageReflect(m.Message)}
}
type dynamicAny struct{ protoreflect.Message }
func (m dynamicAny) Type() protoreflect.MessageType {
return dynamicAnyType{m.Message.Type()}
}
func (m dynamicAny) New() protoreflect.Message {
return dynamicAnyType{m.Message.Type()}.New()
}
func (m dynamicAny) Interface() protoreflect.ProtoMessage {
return DynamicAny{proto.MessageV1(m.Message.Interface())}
}
type dynamicAnyType struct{ protoreflect.MessageType }
func (t dynamicAnyType) New() protoreflect.Message {
return dynamicAny{t.MessageType.New()}
}
func (t dynamicAnyType) Zero() protoreflect.Message {
return dynamicAny{t.MessageType.Zero()}
}

View File

@ -1,62 +0,0 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// source: github.com/golang/protobuf/ptypes/any/any.proto
package any
import (
protoreflect "google.golang.org/protobuf/reflect/protoreflect"
protoimpl "google.golang.org/protobuf/runtime/protoimpl"
anypb "google.golang.org/protobuf/types/known/anypb"
reflect "reflect"
)
// Symbols defined in public import of google/protobuf/any.proto.
type Any = anypb.Any
var File_github_com_golang_protobuf_ptypes_any_any_proto protoreflect.FileDescriptor
var file_github_com_golang_protobuf_ptypes_any_any_proto_rawDesc = []byte{
0x0a, 0x2f, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x6f, 0x6c,
0x61, 0x6e, 0x67, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x70, 0x74, 0x79,
0x70, 0x65, 0x73, 0x2f, 0x61, 0x6e, 0x79, 0x2f, 0x61, 0x6e, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74,
0x6f, 0x1a, 0x19, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62,
0x75, 0x66, 0x2f, 0x61, 0x6e, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x42, 0x2b, 0x5a, 0x29,
0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x6f, 0x6c, 0x61, 0x6e,
0x67, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x70, 0x74, 0x79, 0x70, 0x65,
0x73, 0x2f, 0x61, 0x6e, 0x79, 0x3b, 0x61, 0x6e, 0x79, 0x50, 0x00, 0x62, 0x06, 0x70, 0x72, 0x6f,
0x74, 0x6f, 0x33,
}
var file_github_com_golang_protobuf_ptypes_any_any_proto_goTypes = []interface{}{}
var file_github_com_golang_protobuf_ptypes_any_any_proto_depIdxs = []int32{
0, // [0:0] is the sub-list for method output_type
0, // [0:0] is the sub-list for method input_type
0, // [0:0] is the sub-list for extension type_name
0, // [0:0] is the sub-list for extension extendee
0, // [0:0] is the sub-list for field type_name
}
func init() { file_github_com_golang_protobuf_ptypes_any_any_proto_init() }
func file_github_com_golang_protobuf_ptypes_any_any_proto_init() {
if File_github_com_golang_protobuf_ptypes_any_any_proto != nil {
return
}
type x struct{}
out := protoimpl.TypeBuilder{
File: protoimpl.DescBuilder{
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: file_github_com_golang_protobuf_ptypes_any_any_proto_rawDesc,
NumEnums: 0,
NumMessages: 0,
NumExtensions: 0,
NumServices: 0,
},
GoTypes: file_github_com_golang_protobuf_ptypes_any_any_proto_goTypes,
DependencyIndexes: file_github_com_golang_protobuf_ptypes_any_any_proto_depIdxs,
}.Build()
File_github_com_golang_protobuf_ptypes_any_any_proto = out.File
file_github_com_golang_protobuf_ptypes_any_any_proto_rawDesc = nil
file_github_com_golang_protobuf_ptypes_any_any_proto_goTypes = nil
file_github_com_golang_protobuf_ptypes_any_any_proto_depIdxs = nil
}

View File

@ -1,10 +0,0 @@
// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package ptypes provides functionality for interacting with well-known types.
//
// Deprecated: Well-known types have specialized functionality directly
// injected into the generated packages for each message type.
// See the deprecation notice for each function for the suggested alternative.
package ptypes

View File

@ -1,76 +0,0 @@
// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package ptypes
import (
"errors"
"fmt"
"time"
durationpb "github.com/golang/protobuf/ptypes/duration"
)
// Range of google.protobuf.Duration as specified in duration.proto.
// This is about 10,000 years in seconds.
const (
maxSeconds = int64(10000 * 365.25 * 24 * 60 * 60)
minSeconds = -maxSeconds
)
// Duration converts a durationpb.Duration to a time.Duration.
// Duration returns an error if dur is invalid or overflows a time.Duration.
//
// Deprecated: Call the dur.AsDuration and dur.CheckValid methods instead.
func Duration(dur *durationpb.Duration) (time.Duration, error) {
if err := validateDuration(dur); err != nil {
return 0, err
}
d := time.Duration(dur.Seconds) * time.Second
if int64(d/time.Second) != dur.Seconds {
return 0, fmt.Errorf("duration: %v is out of range for time.Duration", dur)
}
if dur.Nanos != 0 {
d += time.Duration(dur.Nanos) * time.Nanosecond
if (d < 0) != (dur.Nanos < 0) {
return 0, fmt.Errorf("duration: %v is out of range for time.Duration", dur)
}
}
return d, nil
}
// DurationProto converts a time.Duration to a durationpb.Duration.
//
// Deprecated: Call the durationpb.New function instead.
func DurationProto(d time.Duration) *durationpb.Duration {
nanos := d.Nanoseconds()
secs := nanos / 1e9
nanos -= secs * 1e9
return &durationpb.Duration{
Seconds: int64(secs),
Nanos: int32(nanos),
}
}
// validateDuration determines whether the durationpb.Duration is valid
// according to the definition in google/protobuf/duration.proto.
// A valid durpb.Duration may still be too large to fit into a time.Duration
// Note that the range of durationpb.Duration is about 10,000 years,
// while the range of time.Duration is about 290 years.
func validateDuration(dur *durationpb.Duration) error {
if dur == nil {
return errors.New("duration: nil Duration")
}
if dur.Seconds < minSeconds || dur.Seconds > maxSeconds {
return fmt.Errorf("duration: %v: seconds out of range", dur)
}
if dur.Nanos <= -1e9 || dur.Nanos >= 1e9 {
return fmt.Errorf("duration: %v: nanos out of range", dur)
}
// Seconds and Nanos must have the same sign, unless d.Nanos is zero.
if (dur.Seconds < 0 && dur.Nanos > 0) || (dur.Seconds > 0 && dur.Nanos < 0) {
return fmt.Errorf("duration: %v: seconds and nanos have different signs", dur)
}
return nil
}

View File

@ -1,63 +0,0 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// source: github.com/golang/protobuf/ptypes/duration/duration.proto
package duration
import (
protoreflect "google.golang.org/protobuf/reflect/protoreflect"
protoimpl "google.golang.org/protobuf/runtime/protoimpl"
durationpb "google.golang.org/protobuf/types/known/durationpb"
reflect "reflect"
)
// Symbols defined in public import of google/protobuf/duration.proto.
type Duration = durationpb.Duration
var File_github_com_golang_protobuf_ptypes_duration_duration_proto protoreflect.FileDescriptor
var file_github_com_golang_protobuf_ptypes_duration_duration_proto_rawDesc = []byte{
0x0a, 0x39, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x6f, 0x6c,
0x61, 0x6e, 0x67, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x70, 0x74, 0x79,
0x70, 0x65, 0x73, 0x2f, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x64, 0x75, 0x72,
0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1e, 0x67, 0x6f, 0x6f,
0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x64, 0x75, 0x72,
0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x42, 0x35, 0x5a, 0x33, 0x67,
0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x6f, 0x6c, 0x61, 0x6e, 0x67,
0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x70, 0x74, 0x79, 0x70, 0x65, 0x73,
0x2f, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x3b, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69,
0x6f, 0x6e, 0x50, 0x00, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
}
var file_github_com_golang_protobuf_ptypes_duration_duration_proto_goTypes = []interface{}{}
var file_github_com_golang_protobuf_ptypes_duration_duration_proto_depIdxs = []int32{
0, // [0:0] is the sub-list for method output_type
0, // [0:0] is the sub-list for method input_type
0, // [0:0] is the sub-list for extension type_name
0, // [0:0] is the sub-list for extension extendee
0, // [0:0] is the sub-list for field type_name
}
func init() { file_github_com_golang_protobuf_ptypes_duration_duration_proto_init() }
func file_github_com_golang_protobuf_ptypes_duration_duration_proto_init() {
if File_github_com_golang_protobuf_ptypes_duration_duration_proto != nil {
return
}
type x struct{}
out := protoimpl.TypeBuilder{
File: protoimpl.DescBuilder{
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: file_github_com_golang_protobuf_ptypes_duration_duration_proto_rawDesc,
NumEnums: 0,
NumMessages: 0,
NumExtensions: 0,
NumServices: 0,
},
GoTypes: file_github_com_golang_protobuf_ptypes_duration_duration_proto_goTypes,
DependencyIndexes: file_github_com_golang_protobuf_ptypes_duration_duration_proto_depIdxs,
}.Build()
File_github_com_golang_protobuf_ptypes_duration_duration_proto = out.File
file_github_com_golang_protobuf_ptypes_duration_duration_proto_rawDesc = nil
file_github_com_golang_protobuf_ptypes_duration_duration_proto_goTypes = nil
file_github_com_golang_protobuf_ptypes_duration_duration_proto_depIdxs = nil
}

View File

@ -1,112 +0,0 @@
// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package ptypes
import (
"errors"
"fmt"
"time"
timestamppb "github.com/golang/protobuf/ptypes/timestamp"
)
// Range of google.protobuf.Duration as specified in timestamp.proto.
const (
// Seconds field of the earliest valid Timestamp.
// This is time.Date(1, 1, 1, 0, 0, 0, 0, time.UTC).Unix().
minValidSeconds = -62135596800
// Seconds field just after the latest valid Timestamp.
// This is time.Date(10000, 1, 1, 0, 0, 0, 0, time.UTC).Unix().
maxValidSeconds = 253402300800
)
// Timestamp converts a timestamppb.Timestamp to a time.Time.
// It returns an error if the argument is invalid.
//
// Unlike most Go functions, if Timestamp returns an error, the first return
// value is not the zero time.Time. Instead, it is the value obtained from the
// time.Unix function when passed the contents of the Timestamp, in the UTC
// locale. This may or may not be a meaningful time; many invalid Timestamps
// do map to valid time.Times.
//
// A nil Timestamp returns an error. The first return value in that case is
// undefined.
//
// Deprecated: Call the ts.AsTime and ts.CheckValid methods instead.
func Timestamp(ts *timestamppb.Timestamp) (time.Time, error) {
// Don't return the zero value on error, because corresponds to a valid
// timestamp. Instead return whatever time.Unix gives us.
var t time.Time
if ts == nil {
t = time.Unix(0, 0).UTC() // treat nil like the empty Timestamp
} else {
t = time.Unix(ts.Seconds, int64(ts.Nanos)).UTC()
}
return t, validateTimestamp(ts)
}
// TimestampNow returns a google.protobuf.Timestamp for the current time.
//
// Deprecated: Call the timestamppb.Now function instead.
func TimestampNow() *timestamppb.Timestamp {
ts, err := TimestampProto(time.Now())
if err != nil {
panic("ptypes: time.Now() out of Timestamp range")
}
return ts
}
// TimestampProto converts the time.Time to a google.protobuf.Timestamp proto.
// It returns an error if the resulting Timestamp is invalid.
//
// Deprecated: Call the timestamppb.New function instead.
func TimestampProto(t time.Time) (*timestamppb.Timestamp, error) {
ts := &timestamppb.Timestamp{
Seconds: t.Unix(),
Nanos: int32(t.Nanosecond()),
}
if err := validateTimestamp(ts); err != nil {
return nil, err
}
return ts, nil
}
// TimestampString returns the RFC 3339 string for valid Timestamps.
// For invalid Timestamps, it returns an error message in parentheses.
//
// Deprecated: Call the ts.AsTime method instead,
// followed by a call to the Format method on the time.Time value.
func TimestampString(ts *timestamppb.Timestamp) string {
t, err := Timestamp(ts)
if err != nil {
return fmt.Sprintf("(%v)", err)
}
return t.Format(time.RFC3339Nano)
}
// validateTimestamp determines whether a Timestamp is valid.
// A valid timestamp represents a time in the range [0001-01-01, 10000-01-01)
// and has a Nanos field in the range [0, 1e9).
//
// If the Timestamp is valid, validateTimestamp returns nil.
// Otherwise, it returns an error that describes the problem.
//
// Every valid Timestamp can be represented by a time.Time,
// but the converse is not true.
func validateTimestamp(ts *timestamppb.Timestamp) error {
if ts == nil {
return errors.New("timestamp: nil Timestamp")
}
if ts.Seconds < minValidSeconds {
return fmt.Errorf("timestamp: %v before 0001-01-01", ts)
}
if ts.Seconds >= maxValidSeconds {
return fmt.Errorf("timestamp: %v after 10000-01-01", ts)
}
if ts.Nanos < 0 || ts.Nanos >= 1e9 {
return fmt.Errorf("timestamp: %v: nanos not in range [0, 1e9)", ts)
}
return nil
}

View File

@ -1,64 +0,0 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// source: github.com/golang/protobuf/ptypes/timestamp/timestamp.proto
package timestamp
import (
protoreflect "google.golang.org/protobuf/reflect/protoreflect"
protoimpl "google.golang.org/protobuf/runtime/protoimpl"
timestamppb "google.golang.org/protobuf/types/known/timestamppb"
reflect "reflect"
)
// Symbols defined in public import of google/protobuf/timestamp.proto.
type Timestamp = timestamppb.Timestamp
var File_github_com_golang_protobuf_ptypes_timestamp_timestamp_proto protoreflect.FileDescriptor
var file_github_com_golang_protobuf_ptypes_timestamp_timestamp_proto_rawDesc = []byte{
0x0a, 0x3b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x6f, 0x6c,
0x61, 0x6e, 0x67, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x70, 0x74, 0x79,
0x70, 0x65, 0x73, 0x2f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x2f, 0x74, 0x69,
0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1f, 0x67,
0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x74,
0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x42, 0x37,
0x5a, 0x35, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x6f, 0x6c,
0x61, 0x6e, 0x67, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x70, 0x74, 0x79,
0x70, 0x65, 0x73, 0x2f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x3b, 0x74, 0x69,
0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x50, 0x00, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f,
0x33,
}
var file_github_com_golang_protobuf_ptypes_timestamp_timestamp_proto_goTypes = []interface{}{}
var file_github_com_golang_protobuf_ptypes_timestamp_timestamp_proto_depIdxs = []int32{
0, // [0:0] is the sub-list for method output_type
0, // [0:0] is the sub-list for method input_type
0, // [0:0] is the sub-list for extension type_name
0, // [0:0] is the sub-list for extension extendee
0, // [0:0] is the sub-list for field type_name
}
func init() { file_github_com_golang_protobuf_ptypes_timestamp_timestamp_proto_init() }
func file_github_com_golang_protobuf_ptypes_timestamp_timestamp_proto_init() {
if File_github_com_golang_protobuf_ptypes_timestamp_timestamp_proto != nil {
return
}
type x struct{}
out := protoimpl.TypeBuilder{
File: protoimpl.DescBuilder{
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: file_github_com_golang_protobuf_ptypes_timestamp_timestamp_proto_rawDesc,
NumEnums: 0,
NumMessages: 0,
NumExtensions: 0,
NumServices: 0,
},
GoTypes: file_github_com_golang_protobuf_ptypes_timestamp_timestamp_proto_goTypes,
DependencyIndexes: file_github_com_golang_protobuf_ptypes_timestamp_timestamp_proto_depIdxs,
}.Build()
File_github_com_golang_protobuf_ptypes_timestamp_timestamp_proto = out.File
file_github_com_golang_protobuf_ptypes_timestamp_timestamp_proto_rawDesc = nil
file_github_com_golang_protobuf_ptypes_timestamp_timestamp_proto_goTypes = nil
file_github_com_golang_protobuf_ptypes_timestamp_timestamp_proto_depIdxs = nil
}

View File

@ -10,8 +10,5 @@ Session Agent during the TLS handshake, and to encrypt traffic to the peer
after the TLS handshake is complete.
This repository contains the source code for the Secure Session Agent's Go
client libraries, which allow gRPC-Go applications to use the Secure Session
Agent. This repository supports the Bazel and Golang build systems.
All code in this repository is experimental and subject to change. We do not
guarantee API stability at this time.
client libraries, which allow gRPC and HTTP Go applications to use the Secure Session
Agent.

View File

@ -21,50 +21,27 @@ package service
import (
"context"
"net"
"os"
"strings"
"sync"
"time"
"google.golang.org/appengine"
"google.golang.org/appengine/socket"
grpc "google.golang.org/grpc"
"google.golang.org/grpc/grpclog"
"google.golang.org/grpc/credentials"
"google.golang.org/grpc/credentials/insecure"
)
// An environment variable, if true, opportunistically use AppEngine-specific dialer to call S2A.
const enableAppEngineDialerEnv = "S2A_ENABLE_APP_ENGINE_DIALER"
var (
// appEngineDialerHook is an AppEngine-specific dial option that is set
// during init time. If nil, then the application is not running on Google
// AppEngine.
appEngineDialerHook func(context.Context) grpc.DialOption
// mu guards hsConnMap and hsDialer.
mu sync.Mutex
// hsConnMap represents a mapping from an S2A handshaker service address
// to a corresponding connection to an S2A handshaker service instance.
hsConnMap = make(map[string]*grpc.ClientConn)
// hsDialer will be reassigned in tests.
hsDialer = grpc.Dial
hsDialer = grpc.DialContext
)
func init() {
if !appengine.IsAppEngine() && !appengine.IsDevAppServer() {
return
}
appEngineDialerHook = func(ctx context.Context) grpc.DialOption {
return grpc.WithDialer(func(addr string, timeout time.Duration) (net.Conn, error) {
return socket.DialTimeout(ctx, "tcp", addr, timeout)
})
}
}
// Dial dials the S2A handshaker service. If a connection has already been
// established, this function returns it. Otherwise, a new connection is
// created.
func Dial(handshakerServiceAddress string) (*grpc.ClientConn, error) {
func Dial(ctx context.Context, handshakerServiceAddress string, transportCreds credentials.TransportCredentials) (*grpc.ClientConn, error) {
mu.Lock()
defer mu.Unlock()
@ -72,17 +49,14 @@ func Dial(handshakerServiceAddress string) (*grpc.ClientConn, error) {
if !ok {
// Create a new connection to the S2A handshaker service. Note that
// this connection stays open until the application is closed.
grpcOpts := []grpc.DialOption{
grpc.WithInsecure(),
}
if enableAppEngineDialer() && appEngineDialerHook != nil {
if grpclog.V(1) {
grpclog.Info("Using AppEngine-specific dialer to talk to S2A.")
}
grpcOpts = append(grpcOpts, appEngineDialerHook(context.Background()))
var grpcOpts []grpc.DialOption
if transportCreds != nil {
grpcOpts = append(grpcOpts, grpc.WithTransportCredentials(transportCreds))
} else {
grpcOpts = append(grpcOpts, grpc.WithTransportCredentials(insecure.NewCredentials()))
}
var err error
hsConn, err = hsDialer(handshakerServiceAddress, grpcOpts...)
hsConn, err = hsDialer(ctx, handshakerServiceAddress, grpcOpts...)
if err != nil {
return nil, err
}
@ -90,10 +64,3 @@ func Dial(handshakerServiceAddress string) (*grpc.ClientConn, error) {
}
return hsConn, nil
}
func enableAppEngineDialer() bool {
if strings.ToLower(os.Getenv(enableAppEngineDialerEnv)) == "true" {
return true
}
return false
}

View File

@ -83,13 +83,15 @@ func (t *ticketSender) sendTicketsToS2A(sessionTickets [][]byte, callComplete ch
t.ensureProcessSessionTickets.Done()
}
}()
hsConn, err := service.Dial(t.hsAddr)
ctx, cancel := context.WithTimeout(context.Background(), sessionTimeout)
defer cancel()
// The transportCreds only needs to be set when talking to S2AV2 and also
// if mTLS is required.
hsConn, err := service.Dial(ctx, t.hsAddr, nil)
if err != nil {
return err
}
client := s2apb.NewS2AServiceClient(hsConn)
ctx, cancel := context.WithTimeout(context.Background(), sessionTimeout)
defer cancel()
session, err := client.SetUpSession(ctx)
if err != nil {
return err

View File

@ -33,6 +33,7 @@ import (
"github.com/google/s2a-go/internal/handshaker/service"
"github.com/google/s2a-go/internal/tokenmanager"
"github.com/google/s2a-go/internal/v2/tlsconfigstore"
"github.com/google/s2a-go/retry"
"github.com/google/s2a-go/stream"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials"
@ -44,18 +45,19 @@ import (
const (
s2aSecurityProtocol = "tls"
defaultS2ATimeout = 3 * time.Second
defaultS2ATimeout = 6 * time.Second
)
// An environment variable, which sets the timeout enforced on the connection to the S2A service for handshake.
const s2aTimeoutEnv = "S2A_TIMEOUT"
type s2av2TransportCreds struct {
info *credentials.ProtocolInfo
isClient bool
serverName string
s2av2Address string
tokenManager *tokenmanager.AccessTokenManager
info *credentials.ProtocolInfo
isClient bool
serverName string
s2av2Address string
transportCreds credentials.TransportCredentials
tokenManager *tokenmanager.AccessTokenManager
// localIdentity should only be used by the client.
localIdentity *commonpbv1.Identity
// localIdentities should only be used by the server.
@ -68,7 +70,7 @@ type s2av2TransportCreds struct {
// NewClientCreds returns a client-side transport credentials object that uses
// the S2Av2 to establish a secure connection with a server.
func NewClientCreds(s2av2Address string, localIdentity *commonpbv1.Identity, verificationMode s2av2pb.ValidatePeerCertificateChainReq_VerificationMode, fallbackClientHandshakeFunc fallback.ClientHandshake, getS2AStream func(ctx context.Context, s2av2Address string) (stream.S2AStream, error), serverAuthorizationPolicy []byte) (credentials.TransportCredentials, error) {
func NewClientCreds(s2av2Address string, transportCreds credentials.TransportCredentials, localIdentity *commonpbv1.Identity, verificationMode s2av2pb.ValidatePeerCertificateChainReq_VerificationMode, fallbackClientHandshakeFunc fallback.ClientHandshake, getS2AStream func(ctx context.Context, s2av2Address string) (stream.S2AStream, error), serverAuthorizationPolicy []byte) (credentials.TransportCredentials, error) {
// Create an AccessTokenManager instance to use to authenticate to S2Av2.
accessTokenManager, err := tokenmanager.NewSingleTokenAccessTokenManager()
@ -79,6 +81,7 @@ func NewClientCreds(s2av2Address string, localIdentity *commonpbv1.Identity, ver
isClient: true,
serverName: "",
s2av2Address: s2av2Address,
transportCreds: transportCreds,
localIdentity: localIdentity,
verificationMode: verificationMode,
fallbackClientHandshake: fallbackClientHandshakeFunc,
@ -98,7 +101,7 @@ func NewClientCreds(s2av2Address string, localIdentity *commonpbv1.Identity, ver
// NewServerCreds returns a server-side transport credentials object that uses
// the S2Av2 to establish a secure connection with a client.
func NewServerCreds(s2av2Address string, localIdentities []*commonpbv1.Identity, verificationMode s2av2pb.ValidatePeerCertificateChainReq_VerificationMode, getS2AStream func(ctx context.Context, s2av2Address string) (stream.S2AStream, error)) (credentials.TransportCredentials, error) {
func NewServerCreds(s2av2Address string, transportCreds credentials.TransportCredentials, localIdentities []*commonpbv1.Identity, verificationMode s2av2pb.ValidatePeerCertificateChainReq_VerificationMode, getS2AStream func(ctx context.Context, s2av2Address string) (stream.S2AStream, error)) (credentials.TransportCredentials, error) {
// Create an AccessTokenManager instance to use to authenticate to S2Av2.
accessTokenManager, err := tokenmanager.NewSingleTokenAccessTokenManager()
creds := &s2av2TransportCreds{
@ -107,6 +110,7 @@ func NewServerCreds(s2av2Address string, localIdentities []*commonpbv1.Identity,
},
isClient: false,
s2av2Address: s2av2Address,
transportCreds: transportCreds,
localIdentities: localIdentities,
verificationMode: verificationMode,
getS2AStream: getS2AStream,
@ -131,7 +135,13 @@ func (c *s2av2TransportCreds) ClientHandshake(ctx context.Context, serverAuthori
serverName := removeServerNamePort(serverAuthority)
timeoutCtx, cancel := context.WithTimeout(ctx, GetS2ATimeout())
defer cancel()
s2AStream, err := createStream(timeoutCtx, c.s2av2Address, c.getS2AStream)
var s2AStream stream.S2AStream
var err error
retry.Run(timeoutCtx,
func() error {
s2AStream, err = createStream(timeoutCtx, c.s2av2Address, c.transportCreds, c.getS2AStream)
return err
})
if err != nil {
grpclog.Infof("Failed to connect to S2Av2: %v", err)
if c.fallbackClientHandshake != nil {
@ -152,31 +162,34 @@ func (c *s2av2TransportCreds) ClientHandshake(ctx context.Context, serverAuthori
tokenManager = *c.tokenManager
}
if c.serverName == "" {
config, err = tlsconfigstore.GetTLSConfigurationForClient(serverName, s2AStream, tokenManager, c.localIdentity, c.verificationMode, c.serverAuthorizationPolicy)
if err != nil {
grpclog.Info("Failed to get client TLS config from S2Av2: %v", err)
if c.fallbackClientHandshake != nil {
return c.fallbackClientHandshake(ctx, serverAuthority, rawConn, err)
}
return nil, nil, err
}
} else {
config, err = tlsconfigstore.GetTLSConfigurationForClient(c.serverName, s2AStream, tokenManager, c.localIdentity, c.verificationMode, c.serverAuthorizationPolicy)
if err != nil {
grpclog.Info("Failed to get client TLS config from S2Av2: %v", err)
if c.fallbackClientHandshake != nil {
return c.fallbackClientHandshake(ctx, serverAuthority, rawConn, err)
}
return nil, nil, err
sn := serverName
if c.serverName != "" {
sn = c.serverName
}
retry.Run(timeoutCtx,
func() error {
config, err = tlsconfigstore.GetTLSConfigurationForClient(sn, s2AStream, tokenManager, c.localIdentity, c.verificationMode, c.serverAuthorizationPolicy)
return err
})
if err != nil {
grpclog.Info("Failed to get client TLS config from S2Av2: %v", err)
if c.fallbackClientHandshake != nil {
return c.fallbackClientHandshake(ctx, serverAuthority, rawConn, err)
}
return nil, nil, err
}
if grpclog.V(1) {
grpclog.Infof("Got client TLS config from S2Av2.")
}
creds := credentials.NewTLS(config)
conn, authInfo, err := creds.ClientHandshake(ctx, serverName, rawConn)
creds := credentials.NewTLS(config)
var conn net.Conn
var authInfo credentials.AuthInfo
retry.Run(timeoutCtx,
func() error {
conn, authInfo, err = creds.ClientHandshake(timeoutCtx, serverName, rawConn)
return err
})
if err != nil {
grpclog.Infof("Failed to do client handshake using S2Av2: %v", err)
if c.fallbackClientHandshake != nil {
@ -196,7 +209,13 @@ func (c *s2av2TransportCreds) ServerHandshake(rawConn net.Conn) (net.Conn, crede
}
ctx, cancel := context.WithTimeout(context.Background(), GetS2ATimeout())
defer cancel()
s2AStream, err := createStream(ctx, c.s2av2Address, c.getS2AStream)
var s2AStream stream.S2AStream
var err error
retry.Run(ctx,
func() error {
s2AStream, err = createStream(ctx, c.s2av2Address, c.transportCreds, c.getS2AStream)
return err
})
if err != nil {
grpclog.Infof("Failed to connect to S2Av2: %v", err)
return nil, nil, err
@ -213,7 +232,12 @@ func (c *s2av2TransportCreds) ServerHandshake(rawConn net.Conn) (net.Conn, crede
tokenManager = *c.tokenManager
}
config, err := tlsconfigstore.GetTLSConfigurationForServer(s2AStream, tokenManager, c.localIdentities, c.verificationMode)
var config *tls.Config
retry.Run(ctx,
func() error {
config, err = tlsconfigstore.GetTLSConfigurationForServer(s2AStream, tokenManager, c.localIdentities, c.verificationMode)
return err
})
if err != nil {
grpclog.Infof("Failed to get server TLS config from S2Av2: %v", err)
return nil, nil, err
@ -221,8 +245,20 @@ func (c *s2av2TransportCreds) ServerHandshake(rawConn net.Conn) (net.Conn, crede
if grpclog.V(1) {
grpclog.Infof("Got server TLS config from S2Av2.")
}
creds := credentials.NewTLS(config)
return creds.ServerHandshake(rawConn)
var conn net.Conn
var authInfo credentials.AuthInfo
retry.Run(ctx,
func() error {
conn, authInfo, err = creds.ServerHandshake(rawConn)
return err
})
if err != nil {
grpclog.Infof("Failed to do server handshake using S2Av2: %v", err)
return nil, nil, err
}
return conn, authInfo, err
}
// Info returns protocol info of s2av2TransportCreds.
@ -278,11 +314,12 @@ func (c *s2av2TransportCreds) Clone() credentials.TransportCredentials {
func NewClientTLSConfig(
ctx context.Context,
s2av2Address string,
transportCreds credentials.TransportCredentials,
tokenManager tokenmanager.AccessTokenManager,
verificationMode s2av2pb.ValidatePeerCertificateChainReq_VerificationMode,
serverName string,
serverAuthorizationPolicy []byte) (*tls.Config, error) {
s2AStream, err := createStream(ctx, s2av2Address, nil)
s2AStream, err := createStream(ctx, s2av2Address, transportCreds, nil)
if err != nil {
grpclog.Infof("Failed to connect to S2Av2: %v", err)
return nil, err
@ -325,12 +362,12 @@ func (x s2AGrpcStream) CloseSend() error {
return x.stream.CloseSend()
}
func createStream(ctx context.Context, s2av2Address string, getS2AStream func(ctx context.Context, s2av2Address string) (stream.S2AStream, error)) (stream.S2AStream, error) {
func createStream(ctx context.Context, s2av2Address string, transportCreds credentials.TransportCredentials, getS2AStream func(ctx context.Context, s2av2Address string) (stream.S2AStream, error)) (stream.S2AStream, error) {
if getS2AStream != nil {
return getS2AStream(ctx, s2av2Address)
}
// TODO(rmehta19): Consider whether to close the connection to S2Av2.
conn, err := service.Dial(s2av2Address)
conn, err := service.Dial(ctx, s2av2Address, transportCreds)
if err != nil {
return nil, err
}

144
vendor/github.com/google/s2a-go/retry/retry.go generated vendored Normal file
View File

@ -0,0 +1,144 @@
/*
*
* Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
// Package retry provides a retry helper for talking to S2A gRPC server.
// The implementation is modeled after
// https://github.com/googleapis/google-cloud-go/blob/main/compute/metadata/retry.go
package retry
import (
"context"
"math/rand"
"time"
"google.golang.org/grpc/grpclog"
)
const (
maxRetryAttempts = 5
maxRetryForLoops = 10
)
type defaultBackoff struct {
max time.Duration
mul float64
cur time.Duration
}
// Pause returns a duration, which is used as the backoff wait time
// before the next retry.
func (b *defaultBackoff) Pause() time.Duration {
d := time.Duration(1 + rand.Int63n(int64(b.cur)))
b.cur = time.Duration(float64(b.cur) * b.mul)
if b.cur > b.max {
b.cur = b.max
}
return d
}
// Sleep will wait for the specified duration or return on context
// expiration.
func Sleep(ctx context.Context, d time.Duration) error {
t := time.NewTimer(d)
select {
case <-ctx.Done():
t.Stop()
return ctx.Err()
case <-t.C:
return nil
}
}
// NewRetryer creates an instance of S2ARetryer using the defaultBackoff
// implementation.
var NewRetryer = func() *S2ARetryer {
return &S2ARetryer{bo: &defaultBackoff{
cur: 100 * time.Millisecond,
max: 30 * time.Second,
mul: 2,
}}
}
type backoff interface {
Pause() time.Duration
}
// S2ARetryer implements a retry helper for talking to S2A gRPC server.
type S2ARetryer struct {
bo backoff
attempts int
}
// Attempts return the number of retries attempted.
func (r *S2ARetryer) Attempts() int {
return r.attempts
}
// Retry returns a boolean indicating whether retry should be performed
// and the backoff duration.
func (r *S2ARetryer) Retry(err error) (time.Duration, bool) {
if err == nil {
return 0, false
}
if r.attempts >= maxRetryAttempts {
return 0, false
}
r.attempts++
return r.bo.Pause(), true
}
// Run uses S2ARetryer to execute the function passed in, until success or reaching
// max number of retry attempts.
func Run(ctx context.Context, f func() error) {
retryer := NewRetryer()
forLoopCnt := 0
var err error
for {
err = f()
if bo, shouldRetry := retryer.Retry(err); shouldRetry {
if grpclog.V(1) {
grpclog.Infof("will attempt retry: %v", err)
}
if ctx.Err() != nil {
if grpclog.V(1) {
grpclog.Infof("exit retry loop due to context error: %v", ctx.Err())
}
break
}
if errSleep := Sleep(ctx, bo); errSleep != nil {
if grpclog.V(1) {
grpclog.Infof("exit retry loop due to sleep error: %v", errSleep)
}
break
}
// This shouldn't happen, just make sure we are not stuck in the for loops.
forLoopCnt++
if forLoopCnt > maxRetryForLoops {
if grpclog.V(1) {
grpclog.Infof("exit the for loop after too many retries")
}
break
}
continue
}
if grpclog.V(1) {
grpclog.Infof("retry conditions not met, exit the loop")
}
break
}
}

View File

@ -35,6 +35,7 @@ import (
"github.com/google/s2a-go/internal/handshaker/service"
"github.com/google/s2a-go/internal/tokenmanager"
"github.com/google/s2a-go/internal/v2"
"github.com/google/s2a-go/retry"
"google.golang.org/grpc/credentials"
"google.golang.org/grpc/grpclog"
@ -111,7 +112,7 @@ func NewClientCreds(opts *ClientOptions) (credentials.TransportCredentials, erro
if opts.FallbackOpts != nil && opts.FallbackOpts.FallbackClientHandshakeFunc != nil {
fallbackFunc = opts.FallbackOpts.FallbackClientHandshakeFunc
}
return v2.NewClientCreds(opts.S2AAddress, localIdentity, verificationMode, fallbackFunc, opts.getS2AStream, opts.serverAuthorizationPolicy)
return v2.NewClientCreds(opts.S2AAddress, opts.TransportCreds, localIdentity, verificationMode, fallbackFunc, opts.getS2AStream, opts.serverAuthorizationPolicy)
}
// NewServerCreds returns a server-side transport credentials object that uses
@ -146,7 +147,7 @@ func NewServerCreds(opts *ServerOptions) (credentials.TransportCredentials, erro
}, nil
}
verificationMode := getVerificationMode(opts.VerificationMode)
return v2.NewServerCreds(opts.S2AAddress, localIdentities, verificationMode, opts.getS2AStream)
return v2.NewServerCreds(opts.S2AAddress, opts.TransportCreds, localIdentities, verificationMode, opts.getS2AStream)
}
// ClientHandshake initiates a client-side TLS handshake using the S2A.
@ -155,17 +156,17 @@ func (c *s2aTransportCreds) ClientHandshake(ctx context.Context, serverAuthority
return nil, nil, errors.New("client handshake called using server transport credentials")
}
var cancel context.CancelFunc
ctx, cancel = context.WithCancel(ctx)
defer cancel()
// Connect to the S2A.
hsConn, err := service.Dial(c.s2aAddr)
hsConn, err := service.Dial(ctx, c.s2aAddr, nil)
if err != nil {
grpclog.Infof("Failed to connect to S2A: %v", err)
return nil, nil, err
}
var cancel context.CancelFunc
ctx, cancel = context.WithCancel(ctx)
defer cancel()
opts := &handshaker.ClientHandshakerOptions{
MinTLSVersion: c.minTLSVersion,
MaxTLSVersion: c.maxTLSVersion,
@ -203,16 +204,16 @@ func (c *s2aTransportCreds) ServerHandshake(rawConn net.Conn) (net.Conn, credent
return nil, nil, errors.New("server handshake called using client transport credentials")
}
ctx, cancel := context.WithTimeout(context.Background(), defaultTimeout)
defer cancel()
// Connect to the S2A.
hsConn, err := service.Dial(c.s2aAddr)
hsConn, err := service.Dial(ctx, c.s2aAddr, nil)
if err != nil {
grpclog.Infof("Failed to connect to S2A: %v", err)
return nil, nil, err
}
ctx, cancel := context.WithTimeout(context.Background(), defaultTimeout)
defer cancel()
opts := &handshaker.ServerHandshakerOptions{
MinTLSVersion: c.minTLSVersion,
MaxTLSVersion: c.maxTLSVersion,
@ -312,6 +313,7 @@ func NewTLSClientConfigFactory(opts *ClientOptions) (TLSClientConfigFactory, err
grpclog.Infof("Access token manager not initialized: %v", err)
return &s2aTLSClientConfigFactory{
s2av2Address: opts.S2AAddress,
transportCreds: opts.TransportCreds,
tokenManager: nil,
verificationMode: getVerificationMode(opts.VerificationMode),
serverAuthorizationPolicy: opts.serverAuthorizationPolicy,
@ -319,6 +321,7 @@ func NewTLSClientConfigFactory(opts *ClientOptions) (TLSClientConfigFactory, err
}
return &s2aTLSClientConfigFactory{
s2av2Address: opts.S2AAddress,
transportCreds: opts.TransportCreds,
tokenManager: tokenManager,
verificationMode: getVerificationMode(opts.VerificationMode),
serverAuthorizationPolicy: opts.serverAuthorizationPolicy,
@ -327,6 +330,7 @@ func NewTLSClientConfigFactory(opts *ClientOptions) (TLSClientConfigFactory, err
type s2aTLSClientConfigFactory struct {
s2av2Address string
transportCreds credentials.TransportCredentials
tokenManager tokenmanager.AccessTokenManager
verificationMode s2av2pb.ValidatePeerCertificateChainReq_VerificationMode
serverAuthorizationPolicy []byte
@ -338,7 +342,7 @@ func (f *s2aTLSClientConfigFactory) Build(
if opts != nil && opts.ServerName != "" {
serverName = opts.ServerName
}
return v2.NewClientTLSConfig(ctx, f.s2av2Address, f.tokenManager, f.verificationMode, serverName, f.serverAuthorizationPolicy)
return v2.NewClientTLSConfig(ctx, f.s2av2Address, f.transportCreds, f.tokenManager, f.verificationMode, serverName, f.serverAuthorizationPolicy)
}
func getVerificationMode(verificationMode VerificationModeType) s2av2pb.ValidatePeerCertificateChainReq_VerificationMode {
@ -390,9 +394,15 @@ func NewS2ADialTLSContextFunc(opts *ClientOptions) func(ctx context.Context, net
}
timeoutCtx, cancel := context.WithTimeout(ctx, v2.GetS2ATimeout())
defer cancel()
s2aTLSConfig, err := factory.Build(timeoutCtx, &TLSClientConfigOptions{
ServerName: serverName,
})
var s2aTLSConfig *tls.Config
retry.Run(timeoutCtx,
func() error {
s2aTLSConfig, err = factory.Build(timeoutCtx, &TLSClientConfigOptions{
ServerName: serverName,
})
return err
})
if err != nil {
grpclog.Infof("error building S2A TLS config: %v", err)
return fallback(err)
@ -401,7 +411,12 @@ func NewS2ADialTLSContextFunc(opts *ClientOptions) func(ctx context.Context, net
s2aDialer := &tls.Dialer{
Config: s2aTLSConfig,
}
c, err := s2aDialer.DialContext(ctx, network, addr)
var c net.Conn
retry.Run(timeoutCtx,
func() error {
c, err = s2aDialer.DialContext(timeoutCtx, network, addr)
return err
})
if err != nil {
grpclog.Infof("error dialing with S2A to %s: %v", addr, err)
return fallback(err)

View File

@ -26,6 +26,7 @@ import (
"github.com/google/s2a-go/fallback"
"github.com/google/s2a-go/stream"
"google.golang.org/grpc/credentials"
s2apb "github.com/google/s2a-go/internal/proto/common_go_proto"
)
@ -92,6 +93,9 @@ type ClientOptions struct {
LocalIdentity Identity
// S2AAddress is the address of the S2A.
S2AAddress string
// Optional transport credentials.
// If set, this will be used for the gRPC connection to the S2A server.
TransportCreds credentials.TransportCredentials
// EnsureProcessSessionTickets waits for all session tickets to be sent to
// S2A before a process completes.
//
@ -173,6 +177,9 @@ type ServerOptions struct {
LocalIdentities []Identity
// S2AAddress is the address of the S2A.
S2AAddress string
// Optional transport credentials.
// If set, this will be used for the gRPC connection to the S2A server.
TransportCreds credentials.TransportCredentials
// If true, enables the use of legacy S2Av1.
EnableLegacyMode bool
// VerificationMode specifies the mode that S2A must use to verify the

View File

@ -35,6 +35,8 @@ import (
const signAPI = "EnterpriseCertSigner.Sign"
const certificateChainAPI = "EnterpriseCertSigner.CertificateChain"
const publicKeyAPI = "EnterpriseCertSigner.Public"
const encryptAPI = "EnterpriseCertSigner.Encrypt"
const decryptAPI = "EnterpriseCertSigner.Decrypt"
// A Connection wraps a pair of unidirectional streams as an io.ReadWriteCloser.
type Connection struct {
@ -54,13 +56,28 @@ func (c *Connection) Close() error {
func init() {
gob.Register(crypto.SHA256)
gob.Register(crypto.SHA384)
gob.Register(crypto.SHA512)
gob.Register(&rsa.PSSOptions{})
gob.Register(&rsa.OAEPOptions{})
}
// SignArgs contains arguments to a crypto Signer.Sign method.
// SignArgs contains arguments for a Sign API call.
type SignArgs struct {
Digest []byte // The content to sign.
Opts crypto.SignerOpts // Options for signing, such as Hash identifier.
Opts crypto.SignerOpts // Options for signing. Must implement HashFunc().
}
// EncryptArgs contains arguments for an Encrypt API call.
type EncryptArgs struct {
Plaintext []byte // The plaintext to encrypt.
Opts any // Options for encryption. Ex: an instance of crypto.Hash.
}
// DecryptArgs contains arguments to for a Decrypt API call.
type DecryptArgs struct {
Ciphertext []byte // The ciphertext to decrypt.
Opts crypto.DecrypterOpts // Options for decryption. Ex: an instance of *rsa.OAEPOptions.
}
// Key implements credential.Credential by holding the executed signer subprocess.
@ -98,7 +115,7 @@ func (k *Key) Public() crypto.PublicKey {
return k.publicKey
}
// Sign signs a message digest, using the specified signer options.
// Sign signs a message digest, using the specified signer opts. Implements crypto.Signer interface.
func (k *Key) Sign(_ io.Reader, digest []byte, opts crypto.SignerOpts) (signed []byte, err error) {
if opts != nil && opts.HashFunc() != 0 && len(digest) != opts.HashFunc().Size() {
return nil, fmt.Errorf("Digest length of %v bytes does not match Hash function size of %v bytes", len(digest), opts.HashFunc().Size())
@ -107,6 +124,18 @@ func (k *Key) Sign(_ io.Reader, digest []byte, opts crypto.SignerOpts) (signed [
return
}
// Encrypt encrypts a plaintext msg into ciphertext, using the specified encrypt opts.
func (k *Key) Encrypt(_ io.Reader, msg []byte, opts any) (ciphertext []byte, err error) {
err = k.client.Call(encryptAPI, EncryptArgs{Plaintext: msg, Opts: opts}, &ciphertext)
return
}
// Decrypt decrypts a ciphertext msg into plaintext, using the specified decrypter opts. Implements crypto.Decrypter interface.
func (k *Key) Decrypt(_ io.Reader, msg []byte, opts crypto.DecrypterOpts) (plaintext []byte, err error) {
err = k.client.Call(decryptAPI, DecryptArgs{Ciphertext: msg, Opts: opts}, &plaintext)
return
}
// ErrCredUnavailable is a sentinel error that indicates ECP Cred is unavailable,
// possibly due to missing config or missing binary path.
var ErrCredUnavailable = errors.New("Cred is unavailable")
@ -120,7 +149,12 @@ var ErrCredUnavailable = errors.New("Cred is unavailable")
// The config file also specifies which certificate the signer should use.
func Cred(configFilePath string) (*Key, error) {
if configFilePath == "" {
configFilePath = util.GetDefaultConfigFilePath()
envFilePath := util.GetConfigFilePathFromEnv()
if envFilePath != "" {
configFilePath = envFilePath
} else {
configFilePath = util.GetDefaultConfigFilePath()
}
}
enterpriseCertSignerPath, err := util.LoadSignerBinaryPath(configFilePath)
if err != nil {

View File

@ -22,6 +22,7 @@ import (
"os/user"
"path/filepath"
"runtime"
"strings"
)
const configFileName = "certificate_config.json"
@ -63,6 +64,9 @@ func LoadSignerBinaryPath(configFilePath string) (path string, err error) {
if signerBinaryPath == "" {
return "", ErrConfigUnavailable
}
signerBinaryPath = strings.ReplaceAll(signerBinaryPath, "~", guessHomeDir())
signerBinaryPath = strings.ReplaceAll(signerBinaryPath, "$HOME", guessHomeDir())
return signerBinaryPath, nil
}
@ -89,3 +93,8 @@ func getDefaultConfigFileDirectory() (directory string) {
func GetDefaultConfigFilePath() (path string) {
return filepath.Join(getDefaultConfigFileDirectory(), configFileName)
}
// GetConfigFilePathFromEnv returns the path associated with environment variable GOOGLE_API_CERTIFICATE_CONFIG
func GetConfigFilePathFromEnv() (path string) {
return os.Getenv("GOOGLE_API_CERTIFICATE_CONFIG")
}

View File

@ -1,3 +1,3 @@
{
"v2": "2.11.0"
"v2": "2.12.3"
}

View File

@ -1,5 +1,34 @@
# Changelog
## [2.12.3](https://github.com/googleapis/gax-go/compare/v2.12.2...v2.12.3) (2024-03-14)
### Bug Fixes
* bump protobuf dep to v1.33 ([#333](https://github.com/googleapis/gax-go/issues/333)) ([2892b22](https://github.com/googleapis/gax-go/commit/2892b22c1ae8a70dec3448d82e634643fe6c1be2))
## [2.12.2](https://github.com/googleapis/gax-go/compare/v2.12.1...v2.12.2) (2024-02-23)
### Bug Fixes
* **v2/callctx:** fix SetHeader race by cloning header map ([#326](https://github.com/googleapis/gax-go/issues/326)) ([534311f](https://github.com/googleapis/gax-go/commit/534311f0f163d101f30657736c0e6f860e9c39dc))
## [2.12.1](https://github.com/googleapis/gax-go/compare/v2.12.0...v2.12.1) (2024-02-13)
### Bug Fixes
* add XGoogFieldMaskHeader constant ([#321](https://github.com/googleapis/gax-go/issues/321)) ([666ee08](https://github.com/googleapis/gax-go/commit/666ee08931041b7fed56bed7132649785b2d3dfe))
## [2.12.0](https://github.com/googleapis/gax-go/compare/v2.11.0...v2.12.0) (2023-06-26)
### Features
* **v2/callctx:** add new callctx package ([#291](https://github.com/googleapis/gax-go/issues/291)) ([11503ed](https://github.com/googleapis/gax-go/commit/11503ed98df4ae1bbdedf91ff64d47e63f187d68))
* **v2:** add BuildHeaders and InsertMetadataIntoOutgoingContext to header ([#290](https://github.com/googleapis/gax-go/issues/290)) ([6a4b89f](https://github.com/googleapis/gax-go/commit/6a4b89f5551a40262e7c3caf2e1bdc7321b76ea1))
## [2.11.0](https://github.com/googleapis/gax-go/compare/v2.10.0...v2.11.0) (2023-06-13)

View File

@ -0,0 +1,100 @@
// Copyright 2023, Google Inc.
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
// Package callctx provides helpers for storing and retrieving values out of
// [context.Context]. These values are used by our client libraries in various
// ways across the stack.
package callctx
import (
"context"
"fmt"
)
const (
// XGoogFieldMaskHeader is the canonical header key for the [System Parameter]
// that specifies the response read mask. The value(s) for this header
// must adhere to format described in [fieldmaskpb].
//
// [System Parameter]: https://cloud.google.com/apis/docs/system-parameters
// [fieldmaskpb]: https://google.golang.org/protobuf/types/known/fieldmaskpb
XGoogFieldMaskHeader = "x-goog-fieldmask"
headerKey = contextKey("header")
)
// contextKey is a private type used to store/retrieve context values.
type contextKey string
// HeadersFromContext retrieves headers set from [SetHeaders]. These headers
// can then be cast to http.Header or metadata.MD to send along on requests.
func HeadersFromContext(ctx context.Context) map[string][]string {
m, ok := ctx.Value(headerKey).(map[string][]string)
if !ok {
return nil
}
return m
}
// SetHeaders stores key value pairs in the returned context that can later
// be retrieved by [HeadersFromContext]. Values stored in this manner will
// automatically be retrieved by client libraries and sent as outgoing headers
// on all requests. keyvals should have a corresponding value for every key
// provided. If there is an odd number of keyvals this method will panic.
func SetHeaders(ctx context.Context, keyvals ...string) context.Context {
if len(keyvals)%2 != 0 {
panic(fmt.Sprintf("callctx: an even number of key value pairs must be provided, got %d", len(keyvals)))
}
h, ok := ctx.Value(headerKey).(map[string][]string)
if !ok {
h = make(map[string][]string)
} else {
h = cloneHeaders(h)
}
for i := 0; i < len(keyvals); i = i + 2 {
h[keyvals[i]] = append(h[keyvals[i]], keyvals[i+1])
}
return context.WithValue(ctx, headerKey, h)
}
// cloneHeaders makes a new key-value map while reusing the value slices.
// As such, new values should be appended to the value slice, and modifying
// indexed values is not thread safe.
//
// TODO: Replace this with maps.Clone when Go 1.21 is the minimum version.
func cloneHeaders(h map[string][]string) map[string][]string {
c := make(map[string][]string, len(h))
for k, v := range h {
vc := make([]string, len(v))
copy(vc, v)
c[k] = vc
}
return c
}

View File

@ -31,9 +31,15 @@ package gax
import (
"bytes"
"context"
"fmt"
"net/http"
"runtime"
"strings"
"unicode"
"github.com/googleapis/gax-go/v2/callctx"
"google.golang.org/grpc/metadata"
)
var (
@ -97,7 +103,9 @@ func goVersion() string {
return "UNKNOWN"
}
// XGoogHeader is for use by the Google Cloud Libraries only.
// XGoogHeader is for use by the Google Cloud Libraries only. See package
// [github.com/googleapis/gax-go/v2/callctx] for help setting/retrieving
// request/response headers.
//
// XGoogHeader formats key-value pairs.
// The resulting string is suitable for x-goog-api-client header.
@ -117,3 +125,49 @@ func XGoogHeader(keyval ...string) string {
}
return buf.String()[1:]
}
// InsertMetadataIntoOutgoingContext is for use by the Google Cloud Libraries
// only. See package [github.com/googleapis/gax-go/v2/callctx] for help
// setting/retrieving request/response headers.
//
// InsertMetadataIntoOutgoingContext returns a new context that merges the
// provided keyvals metadata pairs with any existing metadata/headers in the
// provided context. keyvals should have a corresponding value for every key
// provided. If there is an odd number of keyvals this method will panic.
// Existing values for keys will not be overwritten, instead provided values
// will be appended to the list of existing values.
func InsertMetadataIntoOutgoingContext(ctx context.Context, keyvals ...string) context.Context {
return metadata.NewOutgoingContext(ctx, insertMetadata(ctx, keyvals...))
}
// BuildHeaders is for use by the Google Cloud Libraries only. See package
// [github.com/googleapis/gax-go/v2/callctx] for help setting/retrieving
// request/response headers.
//
// BuildHeaders returns a new http.Header that merges the provided
// keyvals header pairs with any existing metadata/headers in the provided
// context. keyvals should have a corresponding value for every key provided.
// If there is an odd number of keyvals this method will panic.
// Existing values for keys will not be overwritten, instead provided values
// will be appended to the list of existing values.
func BuildHeaders(ctx context.Context, keyvals ...string) http.Header {
return http.Header(insertMetadata(ctx, keyvals...))
}
func insertMetadata(ctx context.Context, keyvals ...string) metadata.MD {
if len(keyvals)%2 != 0 {
panic(fmt.Sprintf("gax: an even number of key value pairs must be provided, got %d", len(keyvals)))
}
out, ok := metadata.FromOutgoingContext(ctx)
if !ok {
out = metadata.MD(make(map[string][]string))
}
headers := callctx.HeadersFromContext(ctx)
for k, v := range headers {
out[k] = append(out[k], v...)
}
for i := 0; i < len(keyvals); i = i + 2 {
out[keyvals[i]] = append(out[keyvals[i]], keyvals[i+1])
}
return out
}

View File

@ -30,4 +30,4 @@
package internal
// Version is the current tagged release of the library.
const Version = "2.11.0"
const Version = "2.12.3"

View File

@ -24,7 +24,7 @@ go_test(
embed = [":httprule"],
deps = [
"//utilities",
"@com_github_golang_glog//:glog",
"@org_golang_google_grpc//grpclog",
],
)

View File

@ -26,7 +26,7 @@ go_library(
deps = [
"//internal/httprule",
"//utilities",
"@go_googleapis//google/api:httpbody_go_proto",
"@org_golang_google_genproto_googleapis_api//httpbody",
"@org_golang_google_grpc//codes",
"@org_golang_google_grpc//grpclog",
"@org_golang_google_grpc//health/grpc_health_v1",
@ -70,9 +70,9 @@ go_test(
"//utilities",
"@com_github_google_go_cmp//cmp",
"@com_github_google_go_cmp//cmp/cmpopts",
"@go_googleapis//google/api:httpbody_go_proto",
"@go_googleapis//google/rpc:errdetails_go_proto",
"@go_googleapis//google/rpc:status_go_proto",
"@org_golang_google_genproto_googleapis_api//httpbody",
"@org_golang_google_genproto_googleapis_rpc//errdetails",
"@org_golang_google_genproto_googleapis_rpc//status",
"@org_golang_google_grpc//:go_default_library",
"@org_golang_google_grpc//codes",
"@org_golang_google_grpc//health/grpc_health_v1",

View File

@ -137,7 +137,7 @@ func DefaultHTTPErrorHandler(ctx context.Context, mux *ServeMux, marshaler Marsh
doForwardTrailers := requestAcceptsTrailers(r)
if doForwardTrailers {
handleForwardResponseTrailerHeader(w, md)
handleForwardResponseTrailerHeader(w, mux, md)
w.Header().Set("Transfer-Encoding", "chunked")
}
@ -152,7 +152,7 @@ func DefaultHTTPErrorHandler(ctx context.Context, mux *ServeMux, marshaler Marsh
}
if doForwardTrailers {
handleForwardResponseTrailer(w, md)
handleForwardResponseTrailer(w, mux, md)
}
}

View File

@ -27,7 +27,7 @@ func FieldMaskFromRequestBody(r io.Reader, msg proto.Message) (*field_mask.Field
var root interface{}
if err := json.NewDecoder(r).Decode(&root); err != nil {
if err == io.EOF {
if errors.Is(err, io.EOF) {
return fm, nil
}
return nil, err
@ -41,7 +41,7 @@ func FieldMaskFromRequestBody(r io.Reader, msg proto.Message) (*field_mask.Field
m, ok := item.node.(map[string]interface{})
switch {
case ok:
case ok && len(m) > 0:
// if the item is an object, then enqueue all of its children
for k, v := range m {
if item.msg == nil {
@ -96,6 +96,8 @@ func FieldMaskFromRequestBody(r io.Reader, msg proto.Message) (*field_mask.Field
queue = append(queue, child)
}
}
case ok && len(m) == 0:
fallthrough
case len(item.path) > 0:
// otherwise, it's a leaf node so print its path
fm.Paths = append(fm.Paths, item.path)

View File

@ -2,7 +2,7 @@ package runtime
import (
"context"
"fmt"
"errors"
"io"
"net/http"
"net/textproto"
@ -48,7 +48,7 @@ func ForwardResponseStream(ctx context.Context, mux *ServeMux, marshaler Marshal
var wroteHeader bool
for {
resp, err := recv()
if err == io.EOF {
if errors.Is(err, io.EOF) {
return
}
if err != nil {
@ -108,18 +108,20 @@ func handleForwardResponseServerMetadata(w http.ResponseWriter, mux *ServeMux, m
}
}
func handleForwardResponseTrailerHeader(w http.ResponseWriter, md ServerMetadata) {
func handleForwardResponseTrailerHeader(w http.ResponseWriter, mux *ServeMux, md ServerMetadata) {
for k := range md.TrailerMD {
tKey := textproto.CanonicalMIMEHeaderKey(fmt.Sprintf("%s%s", MetadataTrailerPrefix, k))
w.Header().Add("Trailer", tKey)
if h, ok := mux.outgoingTrailerMatcher(k); ok {
w.Header().Add("Trailer", textproto.CanonicalMIMEHeaderKey(h))
}
}
}
func handleForwardResponseTrailer(w http.ResponseWriter, md ServerMetadata) {
func handleForwardResponseTrailer(w http.ResponseWriter, mux *ServeMux, md ServerMetadata) {
for k, vs := range md.TrailerMD {
tKey := fmt.Sprintf("%s%s", MetadataTrailerPrefix, k)
for _, v := range vs {
w.Header().Add(tKey, v)
if h, ok := mux.outgoingTrailerMatcher(k); ok {
for _, v := range vs {
w.Header().Add(h, v)
}
}
}
}
@ -147,12 +149,10 @@ func ForwardResponseMessage(ctx context.Context, mux *ServeMux, marshaler Marsha
doForwardTrailers := requestAcceptsTrailers(req)
if doForwardTrailers {
handleForwardResponseTrailerHeader(w, md)
handleForwardResponseTrailerHeader(w, mux, md)
w.Header().Set("Transfer-Encoding", "chunked")
}
handleForwardResponseTrailerHeader(w, md)
contentType := marshaler.ContentType(resp)
w.Header().Set("Content-Type", contentType)
@ -178,7 +178,7 @@ func ForwardResponseMessage(ctx context.Context, mux *ServeMux, marshaler Marsha
}
if doForwardTrailers {
handleForwardResponseTrailer(w, md)
handleForwardResponseTrailer(w, mux, md)
}
}

View File

@ -26,7 +26,7 @@ func (h *HTTPBodyMarshaler) ContentType(v interface{}) string {
// google.api.HttpBody message, otherwise it falls back to the default Marshaler.
func (h *HTTPBodyMarshaler) Marshal(v interface{}) ([]byte, error) {
if httpBody, ok := v.(*httpbody.HttpBody); ok {
return httpBody.Data, nil
return httpBody.GetData(), nil
}
return h.Marshaler.Marshal(v)
}

View File

@ -57,6 +57,7 @@ type ServeMux struct {
marshalers marshalerRegistry
incomingHeaderMatcher HeaderMatcherFunc
outgoingHeaderMatcher HeaderMatcherFunc
outgoingTrailerMatcher HeaderMatcherFunc
metadataAnnotators []func(context.Context, *http.Request) metadata.MD
errorHandler ErrorHandlerFunc
streamErrorHandler StreamErrorHandlerFunc
@ -114,10 +115,18 @@ func DefaultHeaderMatcher(key string) (string, bool) {
return "", false
}
func defaultOutgoingHeaderMatcher(key string) (string, bool) {
return fmt.Sprintf("%s%s", MetadataHeaderPrefix, key), true
}
func defaultOutgoingTrailerMatcher(key string) (string, bool) {
return fmt.Sprintf("%s%s", MetadataTrailerPrefix, key), true
}
// WithIncomingHeaderMatcher returns a ServeMuxOption representing a headerMatcher for incoming request to gateway.
//
// This matcher will be called with each header in http.Request. If matcher returns true, that header will be
// passed to gRPC context. To transform the header before passing to gRPC context, matcher should return modified header.
// passed to gRPC context. To transform the header before passing to gRPC context, matcher should return the modified header.
func WithIncomingHeaderMatcher(fn HeaderMatcherFunc) ServeMuxOption {
for _, header := range fn.matchedMalformedHeaders() {
grpclog.Warningf("The configured forwarding filter would allow %q to be sent to the gRPC server, which will likely cause errors. See https://github.com/grpc/grpc-go/pull/4803#issuecomment-986093310 for more information.", header)
@ -147,13 +156,24 @@ func (fn HeaderMatcherFunc) matchedMalformedHeaders() []string {
//
// This matcher will be called with each header in response header metadata. If matcher returns true, that header will be
// passed to http response returned from gateway. To transform the header before passing to response,
// matcher should return modified header.
// matcher should return the modified header.
func WithOutgoingHeaderMatcher(fn HeaderMatcherFunc) ServeMuxOption {
return func(mux *ServeMux) {
mux.outgoingHeaderMatcher = fn
}
}
// WithOutgoingTrailerMatcher returns a ServeMuxOption representing a headerMatcher for outgoing response from gateway.
//
// This matcher will be called with each header in response trailer metadata. If matcher returns true, that header will be
// passed to http response returned from gateway. To transform the header before passing to response,
// matcher should return the modified header.
func WithOutgoingTrailerMatcher(fn HeaderMatcherFunc) ServeMuxOption {
return func(mux *ServeMux) {
mux.outgoingTrailerMatcher = fn
}
}
// WithMetadata returns a ServeMuxOption for passing metadata to a gRPC context.
//
// This can be used by services that need to read from http.Request and modify gRPC context. A common use case
@ -273,11 +293,11 @@ func NewServeMux(opts ...ServeMuxOption) *ServeMux {
if serveMux.incomingHeaderMatcher == nil {
serveMux.incomingHeaderMatcher = DefaultHeaderMatcher
}
if serveMux.outgoingHeaderMatcher == nil {
serveMux.outgoingHeaderMatcher = func(key string) (string, bool) {
return fmt.Sprintf("%s%s", MetadataHeaderPrefix, key), true
}
serveMux.outgoingHeaderMatcher = defaultOutgoingHeaderMatcher
}
if serveMux.outgoingTrailerMatcher == nil {
serveMux.outgoingTrailerMatcher = defaultOutgoingTrailerMatcher
}
return serveMux
@ -321,13 +341,13 @@ func (s *ServeMux) ServeHTTP(w http.ResponseWriter, r *http.Request) {
}
if override := r.Header.Get("X-HTTP-Method-Override"); override != "" && s.isPathLengthFallback(r) {
r.Method = strings.ToUpper(override)
if err := r.ParseForm(); err != nil {
_, outboundMarshaler := MarshalerForRequest(s, r)
sterr := status.Error(codes.InvalidArgument, err.Error())
s.errorHandler(ctx, s, outboundMarshaler, w, r, sterr)
return
}
r.Method = strings.ToUpper(override)
}
var pathComponents []string

View File

@ -51,11 +51,13 @@ func (*DefaultQueryParser) Parse(msg proto.Message, values url.Values, filter *u
key = match[1]
values = append([]string{match[2]}, values...)
}
fieldPath := strings.Split(key, ".")
msgValue := msg.ProtoReflect()
fieldPath := normalizeFieldPath(msgValue, strings.Split(key, "."))
if filter.HasCommonPrefix(fieldPath) {
continue
}
if err := populateFieldValueFromPath(msg.ProtoReflect(), fieldPath, values); err != nil {
if err := populateFieldValueFromPath(msgValue, fieldPath, values); err != nil {
return err
}
}
@ -68,6 +70,38 @@ func PopulateFieldFromPath(msg proto.Message, fieldPathString string, value stri
return populateFieldValueFromPath(msg.ProtoReflect(), fieldPath, []string{value})
}
func normalizeFieldPath(msgValue protoreflect.Message, fieldPath []string) []string {
newFieldPath := make([]string, 0, len(fieldPath))
for i, fieldName := range fieldPath {
fields := msgValue.Descriptor().Fields()
fieldDesc := fields.ByTextName(fieldName)
if fieldDesc == nil {
fieldDesc = fields.ByJSONName(fieldName)
}
if fieldDesc == nil {
// return initial field path values if no matching message field was found
return fieldPath
}
newFieldPath = append(newFieldPath, string(fieldDesc.Name()))
// If this is the last element, we're done
if i == len(fieldPath)-1 {
break
}
// Only singular message fields are allowed
if fieldDesc.Message() == nil || fieldDesc.Cardinality() == protoreflect.Repeated {
return fieldPath
}
// Get the nested message
msgValue = msgValue.Get(fieldDesc).Message()
}
return newFieldPath
}
func populateFieldValueFromPath(msgValue protoreflect.Message, fieldPath []string, values []string) error {
if len(fieldPath) < 1 {
return errors.New("no field path")

View File

@ -3,7 +3,6 @@
before:
hooks:
- ./gen.sh
- go install mvdan.cc/garble@v0.10.1
builds:
-
@ -32,7 +31,6 @@ builds:
- mips64le
goarm:
- 7
gobinary: garble
-
id: "s2d"
binary: s2d
@ -59,7 +57,6 @@ builds:
- mips64le
goarm:
- 7
gobinary: garble
-
id: "s2sx"
binary: s2sx
@ -87,7 +84,6 @@ builds:
- mips64le
goarm:
- 7
gobinary: garble
archives:
-

View File

@ -16,6 +16,30 @@ This package provides various compression algorithms.
# changelog
* Feb 5th, 2024 - [1.17.6](https://github.com/klauspost/compress/releases/tag/v1.17.6)
* zstd: Fix incorrect repeat coding in best mode https://github.com/klauspost/compress/pull/923
* s2: Fix DecodeConcurrent deadlock on errors https://github.com/klauspost/compress/pull/925
* Jan 26th, 2024 - [v1.17.5](https://github.com/klauspost/compress/releases/tag/v1.17.5)
* flate: Fix reset with dictionary on custom window encodes https://github.com/klauspost/compress/pull/912
* zstd: Add Frame header encoding and stripping https://github.com/klauspost/compress/pull/908
* zstd: Limit better/best default window to 8MB https://github.com/klauspost/compress/pull/913
* zstd: Speed improvements by @greatroar in https://github.com/klauspost/compress/pull/896 https://github.com/klauspost/compress/pull/910
* s2: Fix callbacks for skippable blocks and disallow 0xfe (Padding) by @Jille in https://github.com/klauspost/compress/pull/916 https://github.com/klauspost/compress/pull/917
https://github.com/klauspost/compress/pull/919 https://github.com/klauspost/compress/pull/918
* Dec 1st, 2023 - [v1.17.4](https://github.com/klauspost/compress/releases/tag/v1.17.4)
* huff0: Speed up symbol counting by @greatroar in https://github.com/klauspost/compress/pull/887
* huff0: Remove byteReader by @greatroar in https://github.com/klauspost/compress/pull/886
* gzhttp: Allow overriding decompression on transport https://github.com/klauspost/compress/pull/892
* gzhttp: Clamp compression level https://github.com/klauspost/compress/pull/890
* gzip: Error out if reserved bits are set https://github.com/klauspost/compress/pull/891
* Nov 15th, 2023 - [v1.17.3](https://github.com/klauspost/compress/releases/tag/v1.17.3)
* fse: Fix max header size https://github.com/klauspost/compress/pull/881
* zstd: Improve better/best compression https://github.com/klauspost/compress/pull/877
* gzhttp: Fix missing content type on Close https://github.com/klauspost/compress/pull/883
* Oct 22nd, 2023 - [v1.17.2](https://github.com/klauspost/compress/releases/tag/v1.17.2)
* zstd: Fix rare *CORRUPTION* output in "best" mode. See https://github.com/klauspost/compress/pull/876
@ -31,6 +55,10 @@ This package provides various compression algorithms.
* s2: Do 2 overlapping match checks https://github.com/klauspost/compress/pull/839
* flate: Add amd64 assembly matchlen https://github.com/klauspost/compress/pull/837
* gzip: Copy bufio.Reader on Reset by @thatguystone in https://github.com/klauspost/compress/pull/860
<details>
<summary>See changes to v1.16.x</summary>
* July 1st, 2023 - [v1.16.7](https://github.com/klauspost/compress/releases/tag/v1.16.7)
* zstd: Fix default level first dictionary encode https://github.com/klauspost/compress/pull/829
@ -69,6 +97,7 @@ This package provides various compression algorithms.
* s2: Add LZ4 block converter. https://github.com/klauspost/compress/pull/748
* s2: Support io.ReaderAt in ReadSeeker. https://github.com/klauspost/compress/pull/747
* s2c/s2sx: Use concurrent decoding. https://github.com/klauspost/compress/pull/746
</details>
<details>
<summary>See changes to v1.15.x</summary>
@ -536,6 +565,8 @@ the stateless compress described below.
For compression performance, see: [this spreadsheet](https://docs.google.com/spreadsheets/d/1nuNE2nPfuINCZJRMt6wFWhKpToF95I47XjSsc-1rbPQ/edit?usp=sharing).
To disable all assembly add `-tags=noasm`. This works across all packages.
# Stateless compression
This package offers stateless compression as a special option for gzip/deflate.
@ -554,7 +585,7 @@ For direct deflate use, NewStatelessWriter and StatelessDeflate are available. S
A `bufio.Writer` can of course be used to control write sizes. For example, to use a 4KB buffer:
```
```go
// replace 'ioutil.Discard' with your output.
gzw, err := gzip.NewWriterLevel(ioutil.Discard, gzip.StatelessCompression)
if err != nil {

View File

@ -51,7 +51,7 @@ func emitCopy(dst []byte, offset, length int) int {
i := 0
// The maximum length for a single tagCopy1 or tagCopy2 op is 64 bytes. The
// threshold for this loop is a little higher (at 68 = 64 + 4), and the
// length emitted down below is is a little lower (at 60 = 64 - 4), because
// length emitted down below is a little lower (at 60 = 64 - 4), because
// it's shorter to encode a length 67 copy as a length 60 tagCopy2 followed
// by a length 7 tagCopy1 (which encodes as 3+2 bytes) than to encode it as
// a length 64 tagCopy2 followed by a length 3 tagCopy2 (which encodes as

View File

@ -1,4 +1,4 @@
module github.com/klauspost/compress
go 1.16
go 1.19

View File

@ -554,6 +554,9 @@ func (b *blockDec) prepareSequences(in []byte, hist *history) (err error) {
if debugDecoder {
printf("Compression modes: 0b%b", compMode)
}
if compMode&3 != 0 {
return errors.New("corrupt block: reserved bits not zero")
}
for i := uint(0); i < 3; i++ {
mode := seqCompMode((compMode >> (6 - i*2)) & 3)
if debugDecoder {

View File

@ -427,6 +427,16 @@ func (b *blockEnc) encodeLits(lits []byte, raw bool) error {
return nil
}
// encodeRLE will encode an RLE block.
func (b *blockEnc) encodeRLE(val byte, length uint32) {
var bh blockHeader
bh.setLast(b.last)
bh.setSize(length)
bh.setType(blockTypeRLE)
b.output = bh.appendTo(b.output)
b.output = append(b.output, val)
}
// fuzzFseEncoder can be used to fuzz the FSE encoder.
func fuzzFseEncoder(data []byte) int {
if len(data) > maxSequences || len(data) < 2 {
@ -479,6 +489,16 @@ func (b *blockEnc) encode(org []byte, raw, rawAllLits bool) error {
if len(b.sequences) == 0 {
return b.encodeLits(b.literals, rawAllLits)
}
if len(b.sequences) == 1 && len(org) > 0 && len(b.literals) <= 1 {
// Check common RLE cases.
seq := b.sequences[0]
if seq.litLen == uint32(len(b.literals)) && seq.offset-3 == 1 {
// Offset == 1 and 0 or 1 literals.
b.encodeRLE(org[0], b.sequences[0].matchLen+zstdMinMatch+seq.litLen)
return nil
}
}
// We want some difference to at least account for the headers.
saved := b.size - len(b.literals) - (b.size >> 6)
if saved < 16 {

View File

@ -95,42 +95,54 @@ type Header struct {
// If there isn't enough input, io.ErrUnexpectedEOF is returned.
// The FirstBlock.OK will indicate if enough information was available to decode the first block header.
func (h *Header) Decode(in []byte) error {
_, err := h.DecodeAndStrip(in)
return err
}
// DecodeAndStrip will decode the header from the beginning of the stream
// and on success return the remaining bytes.
// This will decode the frame header and the first block header if enough bytes are provided.
// It is recommended to provide at least HeaderMaxSize bytes.
// If the frame header cannot be read an error will be returned.
// If there isn't enough input, io.ErrUnexpectedEOF is returned.
// The FirstBlock.OK will indicate if enough information was available to decode the first block header.
func (h *Header) DecodeAndStrip(in []byte) (remain []byte, err error) {
*h = Header{}
if len(in) < 4 {
return io.ErrUnexpectedEOF
return nil, io.ErrUnexpectedEOF
}
h.HeaderSize += 4
b, in := in[:4], in[4:]
if string(b) != frameMagic {
if string(b[1:4]) != skippableFrameMagic || b[0]&0xf0 != 0x50 {
return ErrMagicMismatch
return nil, ErrMagicMismatch
}
if len(in) < 4 {
return io.ErrUnexpectedEOF
return nil, io.ErrUnexpectedEOF
}
h.HeaderSize += 4
h.Skippable = true
h.SkippableID = int(b[0] & 0xf)
h.SkippableSize = binary.LittleEndian.Uint32(in)
return nil
return in[4:], nil
}
// Read Window_Descriptor
// https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#window_descriptor
if len(in) < 1 {
return io.ErrUnexpectedEOF
return nil, io.ErrUnexpectedEOF
}
fhd, in := in[0], in[1:]
h.HeaderSize++
h.SingleSegment = fhd&(1<<5) != 0
h.HasCheckSum = fhd&(1<<2) != 0
if fhd&(1<<3) != 0 {
return errors.New("reserved bit set on frame header")
return nil, errors.New("reserved bit set on frame header")
}
if !h.SingleSegment {
if len(in) < 1 {
return io.ErrUnexpectedEOF
return nil, io.ErrUnexpectedEOF
}
var wd byte
wd, in = in[0], in[1:]
@ -148,7 +160,7 @@ func (h *Header) Decode(in []byte) error {
size = 4
}
if len(in) < int(size) {
return io.ErrUnexpectedEOF
return nil, io.ErrUnexpectedEOF
}
b, in = in[:size], in[size:]
h.HeaderSize += int(size)
@ -178,7 +190,7 @@ func (h *Header) Decode(in []byte) error {
if fcsSize > 0 {
h.HasFCS = true
if len(in) < fcsSize {
return io.ErrUnexpectedEOF
return nil, io.ErrUnexpectedEOF
}
b, in = in[:fcsSize], in[fcsSize:]
h.HeaderSize += int(fcsSize)
@ -199,7 +211,7 @@ func (h *Header) Decode(in []byte) error {
// Frame Header done, we will not fail from now on.
if len(in) < 3 {
return nil
return in, nil
}
tmp := in[:3]
bh := uint32(tmp[0]) | (uint32(tmp[1]) << 8) | (uint32(tmp[2]) << 16)
@ -209,7 +221,7 @@ func (h *Header) Decode(in []byte) error {
cSize := int(bh >> 3)
switch blockType {
case blockTypeReserved:
return nil
return in, nil
case blockTypeRLE:
h.FirstBlock.Compressed = true
h.FirstBlock.DecompressedSize = cSize
@ -225,5 +237,25 @@ func (h *Header) Decode(in []byte) error {
}
h.FirstBlock.OK = true
return nil
return in, nil
}
// AppendTo will append the encoded header to the dst slice.
// There is no error checking performed on the header values.
func (h *Header) AppendTo(dst []byte) ([]byte, error) {
if h.Skippable {
magic := [4]byte{0x50, 0x2a, 0x4d, 0x18}
magic[0] |= byte(h.SkippableID & 0xf)
dst = append(dst, magic[:]...)
f := h.SkippableSize
return append(dst, uint8(f), uint8(f>>8), uint8(f>>16), uint8(f>>24)), nil
}
f := frameHeader{
ContentSize: h.FrameContentSize,
WindowSize: uint32(h.WindowSize),
SingleSegment: h.SingleSegment,
Checksum: h.HasCheckSum,
DictID: h.DictionaryID,
}
return f.appendTo(dst), nil
}

View File

@ -82,7 +82,7 @@ var (
// can run multiple concurrent stateless decodes. It is even possible to
// use stateless decodes while a stream is being decoded.
//
// The Reset function can be used to initiate a new stream, which is will considerably
// The Reset function can be used to initiate a new stream, which will considerably
// reduce the allocations normally caused by NewReader.
func NewReader(r io.Reader, opts ...DOption) (*Decoder, error) {
initPredefined()

View File

@ -135,8 +135,20 @@ func (e *bestFastEncoder) Encode(blk *blockEnc, src []byte) {
break
}
// Add block to history
s := e.addBlock(src)
blk.size = len(src)
// Check RLE first
if len(src) > zstdMinMatch {
ml := matchLen(src[1:], src)
if ml == len(src)-1 {
blk.literals = append(blk.literals, src[0])
blk.sequences = append(blk.sequences, seq{litLen: 1, matchLen: uint32(len(src)-1) - zstdMinMatch, offset: 1 + 3})
return
}
}
if len(src) < minNonLiteralBlockSize {
blk.extraLits = len(src)
blk.literals = blk.literals[:len(src)]
@ -201,14 +213,6 @@ encodeLoop:
if delta >= e.maxMatchOff || delta <= 0 || load3232(src, offset) != first {
return
}
if debugAsserts {
if offset >= s {
panic(fmt.Sprintf("offset: %d - s:%d - rep: %d - cur :%d - max: %d", offset, s, rep, e.cur, e.maxMatchOff))
}
if !bytes.Equal(src[s:s+4], src[offset:offset+4]) {
panic(fmt.Sprintf("first match mismatch: %v != %v, first: %08x", src[s:s+4], src[offset:offset+4], first))
}
}
// Try to quick reject if we already have a long match.
if m.length > 16 {
left := len(src) - int(m.s+m.length)
@ -227,8 +231,10 @@ encodeLoop:
}
}
l := 4 + e.matchlen(s+4, offset+4, src)
if true {
if m.rep <= 0 {
// Extend candidate match backwards as far as possible.
// Do not extend repeats as we can assume they are optimal
// and offsets change if s == nextEmit.
tMin := s - e.maxMatchOff
if tMin < 0 {
tMin = 0
@ -239,7 +245,14 @@ encodeLoop:
l++
}
}
if debugAsserts {
if offset >= s {
panic(fmt.Sprintf("offset: %d - s:%d - rep: %d - cur :%d - max: %d", offset, s, rep, e.cur, e.maxMatchOff))
}
if !bytes.Equal(src[s:s+l], src[offset:offset+l]) {
panic(fmt.Sprintf("second match mismatch: %v != %v, first: %08x", src[s:s+4], src[offset:offset+4], first))
}
}
cand := match{offset: offset, s: s, length: l, rep: rep}
cand.estBits(bitsPerByte)
if m.est >= highScore || cand.est-m.est+(cand.s-m.s)*bitsPerByte>>10 < 0 {
@ -336,24 +349,31 @@ encodeLoop:
}
if debugAsserts {
if best.offset >= best.s {
panic(fmt.Sprintf("best.offset > s: %d >= %d", best.offset, best.s))
}
if best.s < nextEmit {
panic(fmt.Sprintf("s %d < nextEmit %d", best.s, nextEmit))
}
if best.offset < s-e.maxMatchOff {
panic(fmt.Sprintf("best.offset < s-e.maxMatchOff: %d < %d", best.offset, s-e.maxMatchOff))
}
if !bytes.Equal(src[best.s:best.s+best.length], src[best.offset:best.offset+best.length]) {
panic(fmt.Sprintf("match mismatch: %v != %v", src[best.s:best.s+best.length], src[best.offset:best.offset+best.length]))
}
}
// We have a match, we can store the forward value
s = best.s
if best.rep > 0 {
var seq seq
seq.matchLen = uint32(best.length - zstdMinMatch)
if debugAsserts && s < nextEmit {
panic("s < nextEmit")
}
addLiterals(&seq, best.s)
// Repeat. If bit 4 is set, this is a non-lit repeat.
seq.offset = uint32(best.rep & 3)
if debugSequences {
println("repeat sequence", seq, "next s:", s)
println("repeat sequence", seq, "next s:", best.s, "off:", best.s-best.offset)
}
blk.sequences = append(blk.sequences, seq)
@ -396,7 +416,6 @@ encodeLoop:
// A 4-byte match has been found. Update recent offsets.
// We'll later see if more than 4 bytes.
s = best.s
t := best.offset
offset1, offset2, offset3 = s-t, offset1, offset2

Some files were not shown because too many files have changed in this diff Show More