diff --git a/go.mod b/go.mod index 1af389d5170..08c78f4713b 100644 --- a/go.mod +++ b/go.mod @@ -29,10 +29,10 @@ require ( github.com/go-xorm/core v0.6.2 github.com/go-xorm/xorm v0.7.1 github.com/gobwas/glob v0.2.3 - github.com/google/go-cmp v0.2.0 // indirect github.com/gorilla/websocket v1.4.0 github.com/gosimple/slug v1.4.2 github.com/grafana/grafana-plugin-model v0.0.0-20190930120109-1fc953a61fb4 + github.com/grafana/grafana-plugin-sdk-go v0.0.0-20191024130641-6756418f682c github.com/hashicorp/go-hclog v0.8.0 github.com/hashicorp/go-plugin v1.0.1 github.com/hashicorp/go-version v1.1.0 diff --git a/go.sum b/go.sum index f03ffdd538b..fce9bf43acf 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,4 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0 h1:eOI3/cP2VTU6uZLDYAoic+eyzzB9YyGmJ7eIjl8rOPg= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= @@ -6,6 +7,8 @@ github.com/VividCortex/mysqlerr v0.0.0-20170204212430-6c6b55f8796f h1:HR5nRmUQgX github.com/VividCortex/mysqlerr v0.0.0-20170204212430-6c6b55f8796f/go.mod h1:f3HiCrHjHBdcm6E83vGaXh1KomZMA2P6aeo3hKx/wg0= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/apache/arrow/go/arrow v0.0.0-20190716210558-5f564424c71c h1:iHUHzx3S1TU5xt+D7vLb0PAk3e+RfayF9IhR6+hyO/k= +github.com/apache/arrow/go/arrow v0.0.0-20190716210558-5f564424c71c/go.mod h1:VTxUBvSJ3s3eHAg65PNgrsn5BtqCRPdmyXh6rAfdxN0= github.com/aws/aws-sdk-go v1.25.6 h1:Rmg2pgKXoCfNe0KQb4LNSNmHqMdcgBjpMeXK9IjHWq8= github.com/aws/aws-sdk-go v1.25.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/beevik/etree v1.1.0 h1:T0xke/WvNtMoCqgzPhkX2r4rjY3GDZFi+FjpRZY2Jbs= @@ -17,6 +20,7 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24 github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= github.com/bradfitz/gomemcache v0.0.0-20190329173943-551aad21a668 h1:U/lr3Dgy4WK+hNk4tyD+nuGjpVLPEHuJSFXMw11/HPA= github.com/bradfitz/gomemcache v0.0.0-20190329173943-551aad21a668/go.mod h1:H0wQNHz2YrLsuXOZozoeDmnHXkNCRmMW0gwFWDfEZDA= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I= github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w= @@ -81,6 +85,7 @@ github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJA github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -88,6 +93,8 @@ github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/flatbuffers v1.11.0 h1:O7CEyB8Cb3/DmtxODGtLHcEvpr81Jm5qLg/hsHnxA2A= +github.com/google/flatbuffers v1.11.0/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= @@ -101,6 +108,8 @@ github.com/gosimple/slug v1.4.2 h1:jDmprx3q/9Lfk4FkGZtvzDQ9Cj9eAmsjzeQGp24PeiQ= github.com/gosimple/slug v1.4.2/go.mod h1:ER78kgg1Mv0NQGlXiDe57DpCyfbNywXXZ9mIorhxAf0= github.com/grafana/grafana-plugin-model v0.0.0-20190930120109-1fc953a61fb4 h1:SPdxCL9BChFTlyi0Khv64vdCW4TMna8+sxL7+Chx+Ag= github.com/grafana/grafana-plugin-model v0.0.0-20190930120109-1fc953a61fb4/go.mod h1:nc0XxBzjeGcrMltCDw269LoWF9S8ibhgxolCdA1R8To= +github.com/grafana/grafana-plugin-sdk-go v0.0.0-20191024130641-6756418f682c h1:VqEGTH0daEXSpqoLNv87ohtflLi0nuvZsSsrmw3uNnY= +github.com/grafana/grafana-plugin-sdk-go v0.0.0-20191024130641-6756418f682c/go.mod h1:nH8fL+JRTcD8u34ZZw8mYq5mVNtJlVl7AfvFUZ4fzUc= github.com/hashicorp/go-hclog v0.0.0-20180709165350-ff2cf002a8dd/go.mod h1:9bjs9uLqI8l75knNv3lV1kA55veR+WUPSiKIWcQHudI= github.com/hashicorp/go-hclog v0.8.0 h1:z3ollgGRg8RjfJH6UVBaG54R70GFd++QOkvnJH3VSBY= github.com/hashicorp/go-hclog v0.8.0/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= @@ -150,6 +159,8 @@ github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lunny/log v0.0.0-20160921050905-7887c61bf0de/go.mod h1:3q8WtuPQsoRbatJuy3nvq/hRSvuBJrHHr+ybPPiNvHQ= github.com/lunny/nodb v0.0.0-20160621015157-fc1ef06ad4af/go.mod h1:Cqz6pqow14VObJ7peltM+2n3PWOz7yTrfUuGbVFkzN0= +github.com/mattetti/filebuffer v1.0.0 h1:ixTvQ0JjBTwWbdpDZ98lLrydo7KRi8xNRIi5RFszsbY= +github.com/mattetti/filebuffer v1.0.0/go.mod h1:X6nyAIge2JGVmuJt2MFCqmHrb/5IHiphfHtot0s5cnI= github.com/mattn/go-colorable v0.1.1 h1:G1f5SKeVxmagw/IyvzvtZE4Gybcc4Tr1tf7I8z0XgOg= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= @@ -228,6 +239,7 @@ github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337/go.mod h1:s github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.0/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -265,6 +277,7 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 h1:HuIa8hRrWRSrqYzx1qI49NNxhdi2PrY7gxVSq1JjLDc= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/image v0.0.0-20190507092727-e4e5bf290fec/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -277,6 +290,7 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80 h1:Ao/3l156eZf2AW5wK8a7/smtodRU+gha3+BeqJ69lRk= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190319182350-c85d3e98c914 h1:jIOcLT9BZzyJ9ce+IwwZ+aF9yeCqzrR+NrD68a/SHKw= golang.org/x/oauth2 v0.0.0-20190319182350-c85d3e98c914/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -300,12 +314,15 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190802220118-1d1727260058/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI= golang.org/x/tools v0.0.0-20190805222050-c5a2fd39b72a/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 h1:9zdDQZ7Thm29KFXgAX/+yaf3eVbP7djjWp/dXAppNCc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1 h1:QzqyMA1tlu6CgqCDUtU9V+ZKhLFT2dkJuANu5QaxI3I= @@ -314,6 +331,8 @@ google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8 h1:Nw54tB0rB7hY/N0 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/grpc v1.14.0 h1:ArxJuB1NWfPY6r9Gp9gqwplT0Ge7nqv9msgu03lHLmo= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= +google.golang.org/grpc v1.23.1 h1:q4XQuHFC6I28BKZpo6IYyb3mNO+l7lSOxRuYTCiDfXk= +google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc h1:2gGKlE2+asNV9m7xrywl36YYNnBG5ZQ0r/BOOxqPpmk= gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc/go.mod h1:m7x9LTH6d71AHyAX77c9yqWCCa3UKHcVEj9y7hAtKDk= @@ -344,3 +363,4 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWD gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/pkg/plugins/datasource/wrapper/datasource_plugin_wrapper_v2.go b/pkg/plugins/datasource/wrapper/datasource_plugin_wrapper_v2.go new file mode 100644 index 00000000000..740df9fc551 --- /dev/null +++ b/pkg/plugins/datasource/wrapper/datasource_plugin_wrapper_v2.go @@ -0,0 +1,176 @@ +package wrapper + +import ( + "context" + "errors" + "fmt" + + sdk "github.com/grafana/grafana-plugin-sdk-go" + "github.com/grafana/grafana-plugin-sdk-go/dataframe" + "github.com/grafana/grafana-plugin-sdk-go/genproto/datasource" + "github.com/grafana/grafana/pkg/bus" + "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/infra/log" + "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/tsdb" +) + +type grafanaAPI struct { + logger log.Logger +} + +func (s *grafanaAPI) QueryDatasource(ctx context.Context, req *datasource.QueryDatasourceRequest) (*datasource.QueryDatasourceResponse, error) { + if len(req.Queries) == 0 { + return nil, fmt.Errorf("zero queries found in datasource request") + } + getDsInfo := &models.GetDataSourceByIdQuery{ + Id: req.DatasourceId, + OrgId: req.OrgId, + } + + if err := bus.Dispatch(getDsInfo); err != nil { + return nil, fmt.Errorf("Could not find datasource %v", err) + } + + // Convert plugin-model (datasource) queries to tsdb queries + queries := make([]*tsdb.Query, len(req.Queries)) + for i, query := range req.Queries { + sj, err := simplejson.NewJson([]byte(query.ModelJson)) + if err != nil { + return nil, err + } + queries[i] = &tsdb.Query{ + RefId: query.RefId, + IntervalMs: query.IntervalMs, + MaxDataPoints: query.MaxDataPoints, + DataSource: getDsInfo.Result, + Model: sj, + } + } + + timeRange := tsdb.NewTimeRange(req.TimeRange.FromRaw, req.TimeRange.ToRaw) + tQ := &tsdb.TsdbQuery{ + TimeRange: timeRange, + Queries: queries, + } + + // Execute the converted queries + tsdbRes, err := tsdb.HandleRequest(ctx, getDsInfo.Result, tQ) + if err != nil { + return nil, err + } + // Convert tsdb results (map) to plugin-model/datasource (slice) results + // Only error and Series responses mapped. + results := make([]*datasource.QueryResult, len(tsdbRes.Results)) + resIdx := 0 + for refID, res := range tsdbRes.Results { + qr := &datasource.QueryResult{ + RefId: refID, + } + if res.Error != nil { + qr.Error = res.ErrorString + results[resIdx] = qr + resIdx++ + continue + } + + encodedFrames := make([][]byte, len(res.Series)) + for sIdx, series := range res.Series { + frame, err := tsdb.SeriesToFrame(series) + if err != nil { + return nil, err + } + encodedFrames[sIdx], err = dataframe.MarshalArrow(frame) + if err != nil { + return nil, err + } + } + qr.Dataframes = encodedFrames + results[resIdx] = qr + + resIdx++ + } + return &datasource.QueryDatasourceResponse{Results: results}, nil +} + +func NewDatasourcePluginWrapperV2(log log.Logger, plugin sdk.DatasourcePlugin) *DatasourcePluginWrapperV2 { + return &DatasourcePluginWrapperV2{DatasourcePlugin: plugin, logger: log} +} + +type DatasourcePluginWrapperV2 struct { + sdk.DatasourcePlugin + logger log.Logger +} + +func (tw *DatasourcePluginWrapperV2) Query(ctx context.Context, ds *models.DataSource, query *tsdb.TsdbQuery) (*tsdb.Response, error) { + jsonData, err := ds.JsonData.MarshalJSON() + if err != nil { + return nil, err + } + + pbQuery := &datasource.DatasourceRequest{ + Datasource: &datasource.DatasourceInfo{ + Name: ds.Name, + Type: ds.Type, + Url: ds.Url, + Id: ds.Id, + OrgId: ds.OrgId, + JsonData: string(jsonData), + DecryptedSecureJsonData: ds.SecureJsonData.Decrypt(), + }, + TimeRange: &datasource.TimeRange{ + FromRaw: query.TimeRange.From, + ToRaw: query.TimeRange.To, + ToEpochMs: query.TimeRange.GetToAsMsEpoch(), + FromEpochMs: query.TimeRange.GetFromAsMsEpoch(), + }, + Queries: []*datasource.Query{}, + } + + for _, q := range query.Queries { + modelJSON, err := q.Model.MarshalJSON() + if err != nil { + return nil, err + } + pbQuery.Queries = append(pbQuery.Queries, &datasource.Query{ + ModelJson: string(modelJSON), + IntervalMs: q.IntervalMs, + RefId: q.RefId, + MaxDataPoints: q.MaxDataPoints, + }) + } + + pbres, err := tw.DatasourcePlugin.Query(ctx, pbQuery, &grafanaAPI{logger: tw.logger}) + + if err != nil { + return nil, err + } + + res := &tsdb.Response{ + Results: map[string]*tsdb.QueryResult{}, + } + + for _, r := range pbres.Results { + qr := &tsdb.QueryResult{ + RefId: r.RefId, + } + + if r.Error != "" { + qr.Error = errors.New(r.Error) + qr.ErrorString = r.Error + } + + if r.MetaJson != "" { + metaJSON, err := simplejson.NewJson([]byte(r.MetaJson)) + if err != nil { + tw.logger.Error("Error parsing JSON Meta field: " + err.Error()) + } + qr.Meta = metaJSON + } + qr.Dataframes = r.Dataframes + + res.Results[r.RefId] = qr + } + + return res, nil +} diff --git a/pkg/plugins/datasource_plugin.go b/pkg/plugins/datasource_plugin.go index 1da105860ad..6d4a6a8aaf7 100644 --- a/pkg/plugins/datasource_plugin.go +++ b/pkg/plugins/datasource_plugin.go @@ -3,11 +3,16 @@ package plugins import ( "context" "encoding/json" + "errors" + "fmt" "os/exec" "path" "time" - "github.com/grafana/grafana-plugin-model/go/datasource" + "github.com/grafana/grafana/pkg/setting" + + datasourceV1 "github.com/grafana/grafana-plugin-model/go/datasource" + sdk "github.com/grafana/grafana-plugin-sdk-go" "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/plugins/datasource/wrapper" @@ -34,16 +39,29 @@ type DataSourcePlugin struct { Backend bool `json:"backend,omitempty"` Executable string `json:"executable,omitempty"` + SDK bool `json:"sdk,omitempty"` log log.Logger client *plugin.Client } +func isExpressionsEnabled() bool { + v, ok := setting.FeatureToggles["expressions"] + if !ok { + return false + } + return v +} + func (p *DataSourcePlugin) Load(decoder *json.Decoder, pluginDir string) error { if err := decoder.Decode(&p); err != nil { return err } + if !p.isVersionOne() && !isExpressionsEnabled() { + return errors.New("A plugin version 2 was found but expressions feature toggle are not enabled") + } + if err := p.registerPlugin(pluginDir); err != nil { return err } @@ -73,18 +91,35 @@ func (p *DataSourcePlugin) startBackendPlugin(ctx context.Context, log log.Logge return nil } +func (p *DataSourcePlugin) isVersionOne() bool { + return !p.SDK +} func (p *DataSourcePlugin) spawnSubProcess() error { cmd := ComposePluginStartCommmand(p.Executable) fullpath := path.Join(p.PluginDir, cmd) - p.client = plugin.NewClient(&plugin.ClientConfig{ - HandshakeConfig: handshakeConfig, - Plugins: map[string]plugin.Plugin{p.Id: &datasource.DatasourcePluginImpl{}}, - Cmd: exec.Command(fullpath), - AllowedProtocols: []plugin.Protocol{plugin.ProtocolGRPC}, - Logger: LogWrapper{Logger: p.log}, - }) + var newClient *plugin.Client + if p.isVersionOne() { + newClient = plugin.NewClient(&plugin.ClientConfig{ + HandshakeConfig: handshakeConfig, + Plugins: map[string]plugin.Plugin{p.Id: &datasourceV1.DatasourcePluginImpl{}}, + Cmd: exec.Command(fullpath), + AllowedProtocols: []plugin.Protocol{plugin.ProtocolGRPC}, + Logger: LogWrapper{Logger: p.log}, + }) + + } else { + newClient = plugin.NewClient(&plugin.ClientConfig{ + HandshakeConfig: handshakeConfig, + Plugins: map[string]plugin.Plugin{p.Id: &sdk.DatasourcePluginImpl{}}, + Cmd: exec.Command(fullpath), + AllowedProtocols: []plugin.Protocol{plugin.ProtocolGRPC}, + Logger: LogWrapper{Logger: p.log}, + }) + } + + p.client = newClient rpcClient, err := p.client.Client() if err != nil { @@ -96,10 +131,22 @@ func (p *DataSourcePlugin) spawnSubProcess() error { return err } - plugin := raw.(datasource.DatasourcePlugin) + if p.isVersionOne() { + plugin := raw.(datasourceV1.DatasourcePlugin) + + tsdb.RegisterTsdbQueryEndpoint(p.Id, func(dsInfo *models.DataSource) (tsdb.TsdbQueryEndpoint, error) { + return wrapper.NewDatasourcePluginWrapper(p.log, plugin), nil + }) + return nil + } + + plugin, ok := raw.(sdk.DatasourcePlugin) + if !ok { + return fmt.Errorf("unxpected type %T, expeced sdk.DatasourcePlugin", raw) + } tsdb.RegisterTsdbQueryEndpoint(p.Id, func(dsInfo *models.DataSource) (tsdb.TsdbQueryEndpoint, error) { - return wrapper.NewDatasourcePluginWrapper(p.log, plugin), nil + return wrapper.NewDatasourcePluginWrapperV2(p.log, plugin), nil }) return nil diff --git a/pkg/plugins/datasource_plugin_test.go b/pkg/plugins/datasource_plugin_test.go new file mode 100644 index 00000000000..a29961902ec --- /dev/null +++ b/pkg/plugins/datasource_plugin_test.go @@ -0,0 +1,34 @@ +package plugins + +import ( + "bytes" + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestLoadDatasourceVersion(t *testing.T) { + t.Run("If plugin version is not set, it should be treated as plugin version one", func(t *testing.T) { + pluginJSON, _ := json.Marshal(DataSourcePlugin{}) + datasourcePlugin := DataSourcePlugin{} + (&datasourcePlugin).Load(json.NewDecoder(bytes.NewReader(pluginJSON)), "/tmp") + assert.True(t, datasourcePlugin.isVersionOne()) + }) + + t.Run("If plugin version is set to one, it should be treated as plugin version one", func(t *testing.T) { + pluginJSON, _ := json.Marshal(DataSourcePlugin{SDK: false}) + datasourcePlugin := DataSourcePlugin{} + (&datasourcePlugin).Load(json.NewDecoder(bytes.NewReader(pluginJSON)), "/tmp") + assert.True(t, datasourcePlugin.isVersionOne()) + assert.False(t, datasourcePlugin.SDK) + }) + + t.Run("If plugin version is set to two, it should not be treated as plugin version one", func(t *testing.T) { + pluginJSON, _ := json.Marshal(DataSourcePlugin{SDK: true}) + datasourcePlugin := DataSourcePlugin{} + (&datasourcePlugin).Load(json.NewDecoder(bytes.NewReader(pluginJSON)), "/tmp") + assert.False(t, datasourcePlugin.isVersionOne()) + assert.True(t, datasourcePlugin.SDK) + }) +} diff --git a/pkg/setting/setting.go b/pkg/setting/setting.go index 9729daf8bb0..7d1208662c9 100644 --- a/pkg/setting/setting.go +++ b/pkg/setting/setting.go @@ -206,6 +206,7 @@ var ( S3TempImageStoreSecretKey string ImageUploadProvider string + FeatureToggles map[string]bool ) // TODO move all global vars to this struct @@ -955,6 +956,7 @@ func (cfg *Cfg) Load(args *CommandLineArgs) error { for _, feature := range util.SplitString(featuresTogglesStr) { cfg.FeatureToggles[feature] = true } + FeatureToggles = cfg.FeatureToggles // check old location for this option if panelsSection.Key("enable_alpha").MustBool(false) { diff --git a/pkg/tsdb/frame_util.go b/pkg/tsdb/frame_util.go new file mode 100644 index 00000000000..eb8de55f1a8 --- /dev/null +++ b/pkg/tsdb/frame_util.go @@ -0,0 +1,37 @@ +package tsdb + +import ( + "time" + + "github.com/grafana/grafana-plugin-sdk-go/dataframe" +) + +// SeriesToFrame converts a TimeSeries to a sdk Frame +func SeriesToFrame(series *TimeSeries) (*dataframe.Frame, error) { + timeVec := make([]*time.Time, len(series.Points)) + floatVec := make([]*float64, len(series.Points)) + for idx, point := range series.Points { + timeVec[idx], floatVec[idx] = convertTSDBTimePoint(point) + } + frame := dataframe.New(series.Name, dataframe.Labels(series.Tags), + dataframe.NewField("time", dataframe.FieldTypeTime, timeVec), + dataframe.NewField("value", dataframe.FieldTypeNumber, floatVec), + ) + + return frame, nil +} + +// convertTSDBTimePoint coverts a tsdb.TimePoint into two values appropriate +// for Series values. +func convertTSDBTimePoint(point TimePoint) (t *time.Time, f *float64) { + timeIdx, valueIdx := 1, 0 + if point[timeIdx].Valid { // Assuming valid is null? + tI := int64(point[timeIdx].Float64) + uT := time.Unix(tI/int64(1e+3), (tI%int64(1e+3))*int64(1e+6)) // time.Time from millisecond unix ts + t = &uT + } + if point[valueIdx].Valid { + f = &point[valueIdx].Float64 + } + return +} diff --git a/pkg/tsdb/models.go b/pkg/tsdb/models.go index 8a81d6fb237..27111d22e24 100644 --- a/pkg/tsdb/models.go +++ b/pkg/tsdb/models.go @@ -32,6 +32,7 @@ type QueryResult struct { Meta *simplejson.Json `json:"meta,omitempty"` Series TimeSeriesSlice `json:"series"` Tables []*Table `json:"tables"` + Dataframes [][]byte `json:"dataframes"` } type TimeSeries struct { diff --git a/vendor/github.com/apache/arrow/go/arrow/.gitignore b/vendor/github.com/apache/arrow/go/arrow/.gitignore new file mode 100644 index 00000000000..d4b831ae811 --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/.gitignore @@ -0,0 +1,35 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +### Go template +# Binaries for programs and plugins +*.exe +*.dll +*.so +*.dylib +*.o + +# Test binary, build with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736 +.glide/ + +bin/ +vendor/ \ No newline at end of file diff --git a/vendor/github.com/apache/arrow/go/arrow/Gopkg.lock b/vendor/github.com/apache/arrow/go/arrow/Gopkg.lock new file mode 100644 index 00000000000..143e4f93b5e --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/Gopkg.lock @@ -0,0 +1,44 @@ +# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. + + +[[projects]] + digest = "1:56c130d885a4aacae1dd9c7b71cfe39912c7ebc1ff7d2b46083c8812996dc43b" + name = "github.com/davecgh/go-spew" + packages = ["spew"] + pruneopts = "" + revision = "346938d642f2ec3594ed81d874461961cd0faa76" + version = "v1.1.0" + +[[projects]] + digest = "1:1d7e1867c49a6dd9856598ef7c3123604ea3daabf5b83f303ff457bcbc410b1d" + name = "github.com/pkg/errors" + packages = ["."] + pruneopts = "" + revision = "ba968bfe8b2f7e042a574c888954fccecfa385b4" + version = "v0.8.1" + +[[projects]] + digest = "1:256484dbbcd271f9ecebc6795b2df8cad4c458dd0f5fd82a8c2fa0c29f233411" + name = "github.com/pmezard/go-difflib" + packages = ["difflib"] + pruneopts = "" + revision = "792786c7400a136282c1664665ae0a8db921c6c2" + version = "v1.0.0" + +[[projects]] + digest = "1:2d0dc026c4aef5e2f3a0e06a4dabe268b840d8f63190cf6894e02134a03f52c5" + name = "github.com/stretchr/testify" + packages = ["assert"] + pruneopts = "" + revision = "b91bfb9ebec76498946beb6af7c0230c7cc7ba6c" + version = "v1.2.0" + +[solve-meta] + analyzer-name = "dep" + analyzer-version = 1 + input-imports = [ + "github.com/pkg/errors", + "github.com/stretchr/testify/assert", + ] + solver-name = "gps-cdcl" + solver-version = 1 diff --git a/vendor/github.com/apache/arrow/go/arrow/Gopkg.toml b/vendor/github.com/apache/arrow/go/arrow/Gopkg.toml new file mode 100644 index 00000000000..b27807d69f9 --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/Gopkg.toml @@ -0,0 +1,23 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[[constraint]] + name = "github.com/stretchr/testify" + version = "1.2.0" + +[[constraint]] + name = "github.com/pkg/errors" + version = "0.8.1" \ No newline at end of file diff --git a/vendor/github.com/apache/arrow/go/arrow/Makefile b/vendor/github.com/apache/arrow/go/arrow/Makefile new file mode 100644 index 00000000000..9c4a23262d0 --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/Makefile @@ -0,0 +1,54 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +GO_BUILD=go build +GO_GEN=go generate +GO_TEST?=go test +GOPATH=$(realpath ../../../../../..) + +GO_SOURCES := $(shell find . -path ./_lib -prune -o -name '*.go' -not -name '*_test.go') +ALL_SOURCES := $(shell find . -path ./_lib -prune -o -name '*.go' -name '*.s' -not -name '*_test.go') +SOURCES_NO_VENDOR := $(shell find . -path ./vendor -prune -o -name "*.go" -not -name '*_test.go' -print) + +.PHONEY: test bench assembly generate + +assembly: + @$(MAKE) -C memory assembly + @$(MAKE) -C math assembly + +generate: bin/tmpl + bin/tmpl -i -data=numeric.tmpldata type_traits_numeric.gen.go.tmpl type_traits_numeric.gen_test.go.tmpl array/numeric.gen.go.tmpl array/numericbuilder.gen_test.go.tmpl array/numericbuilder.gen.go.tmpl array/bufferbuilder_numeric.gen.go.tmpl + bin/tmpl -i -data=datatype_numeric.gen.go.tmpldata datatype_numeric.gen.go.tmpl + @$(MAKE) -C math generate + +fmt: $(SOURCES_NO_VENDOR) + goimports -w $^ + +bench: $(GO_SOURCES) | assembly + $(GO_TEST) $(GO_TEST_ARGS) -bench=. -run=- ./... + +bench-noasm: $(GO_SOURCES) + $(GO_TEST) $(GO_TEST_ARGS) -tags='noasm' -bench=. -run=- ./... + +test: $(GO_SOURCES) | assembly + $(GO_TEST) $(GO_TEST_ARGS) ./... + +test-noasm: $(GO_SOURCES) + $(GO_TEST) $(GO_TEST_ARGS) -tags='noasm' ./... + +bin/tmpl: _tools/tmpl/main.go + $(GO_BUILD) -o $@ ./_tools/tmpl + diff --git a/vendor/github.com/apache/arrow/go/arrow/array/array.go b/vendor/github.com/apache/arrow/go/arrow/array/array.go new file mode 100644 index 00000000000..c3577337a7b --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/array/array.go @@ -0,0 +1,206 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package array // import "github.com/apache/arrow/go/arrow/array" + +import ( + "sync/atomic" + + "github.com/apache/arrow/go/arrow" + "github.com/apache/arrow/go/arrow/internal/bitutil" + "github.com/apache/arrow/go/arrow/internal/debug" +) + +// A type which satisfies array.Interface represents an immutable sequence of values. +type Interface interface { + // DataType returns the type metadata for this instance. + DataType() arrow.DataType + + // NullN returns the number of null values in the array. + NullN() int + + // NullBitmapBytes returns a byte slice of the validity bitmap. + NullBitmapBytes() []byte + + // IsNull returns true if value at index is null. + // NOTE: IsNull will panic if NullBitmapBytes is not empty and 0 > i ≥ Len. + IsNull(i int) bool + + // IsValid returns true if value at index is not null. + // NOTE: IsValid will panic if NullBitmapBytes is not empty and 0 > i ≥ Len. + IsValid(i int) bool + + Data() *Data + + // Len returns the number of elements in the array. + Len() int + + // Retain increases the reference count by 1. + // Retain may be called simultaneously from multiple goroutines. + Retain() + + // Release decreases the reference count by 1. + // Release may be called simultaneously from multiple goroutines. + // When the reference count goes to zero, the memory is freed. + Release() +} + +const ( + // UnknownNullCount specifies the NullN should be calculated from the null bitmap buffer. + UnknownNullCount = -1 +) + +type array struct { + refCount int64 + data *Data + nullBitmapBytes []byte +} + +// Retain increases the reference count by 1. +// Retain may be called simultaneously from multiple goroutines. +func (a *array) Retain() { + atomic.AddInt64(&a.refCount, 1) +} + +// Release decreases the reference count by 1. +// Release may be called simultaneously from multiple goroutines. +// When the reference count goes to zero, the memory is freed. +func (a *array) Release() { + debug.Assert(atomic.LoadInt64(&a.refCount) > 0, "too many releases") + + if atomic.AddInt64(&a.refCount, -1) == 0 { + a.data.Release() + a.data, a.nullBitmapBytes = nil, nil + } +} + +// DataType returns the type metadata for this instance. +func (a *array) DataType() arrow.DataType { return a.data.dtype } + +// NullN returns the number of null values in the array. +func (a *array) NullN() int { + if a.data.nulls < 0 { + a.data.nulls = a.data.length - bitutil.CountSetBits(a.nullBitmapBytes, a.data.offset, a.data.length) + } + return a.data.nulls +} + +// NullBitmapBytes returns a byte slice of the validity bitmap. +func (a *array) NullBitmapBytes() []byte { return a.nullBitmapBytes } + +func (a *array) Data() *Data { return a.data } + +// Len returns the number of elements in the array. +func (a *array) Len() int { return a.data.length } + +// IsNull returns true if value at index is null. +// NOTE: IsNull will panic if NullBitmapBytes is not empty and 0 > i ≥ Len. +func (a *array) IsNull(i int) bool { + return len(a.nullBitmapBytes) != 0 && bitutil.BitIsNotSet(a.nullBitmapBytes, a.data.offset+i) +} + +// IsValid returns true if value at index is not null. +// NOTE: IsValid will panic if NullBitmapBytes is not empty and 0 > i ≥ Len. +func (a *array) IsValid(i int) bool { + return len(a.nullBitmapBytes) == 0 || bitutil.BitIsSet(a.nullBitmapBytes, a.data.offset+i) +} + +func (a *array) setData(data *Data) { + if a.data != nil { + a.data.Release() + } + + data.Retain() + if len(data.buffers) > 0 && data.buffers[0] != nil { + a.nullBitmapBytes = data.buffers[0].Bytes() + } + a.data = data +} + +func (a *array) Offset() int { + return a.data.Offset() +} + +type arrayConstructorFn func(*Data) Interface + +var ( + makeArrayFn [32]arrayConstructorFn +) + +func unsupportedArrayType(data *Data) Interface { + panic("unsupported data type: " + data.dtype.ID().String()) +} + +func invalidDataType(data *Data) Interface { + panic("invalid data type: " + data.dtype.ID().String()) +} + +// MakeFromData constructs a strongly-typed array instance from generic Data. +func MakeFromData(data *Data) Interface { + return makeArrayFn[byte(data.dtype.ID()&0x1f)](data) +} + +// NewSlice constructs a zero-copy slice of the array with the indicated +// indices i and j, corresponding to array[i:j]. +// The returned array must be Release()'d after use. +// +// NewSlice panics if the slice is outside the valid range of the input array. +// NewSlice panics if j < i. +func NewSlice(arr Interface, i, j int64) Interface { + data := NewSliceData(arr.Data(), i, j) + slice := MakeFromData(data) + data.Release() + return slice +} + +func init() { + makeArrayFn = [...]arrayConstructorFn{ + arrow.NULL: func(data *Data) Interface { return NewNullData(data) }, + arrow.BOOL: func(data *Data) Interface { return NewBooleanData(data) }, + arrow.UINT8: func(data *Data) Interface { return NewUint8Data(data) }, + arrow.INT8: func(data *Data) Interface { return NewInt8Data(data) }, + arrow.UINT16: func(data *Data) Interface { return NewUint16Data(data) }, + arrow.INT16: func(data *Data) Interface { return NewInt16Data(data) }, + arrow.UINT32: func(data *Data) Interface { return NewUint32Data(data) }, + arrow.INT32: func(data *Data) Interface { return NewInt32Data(data) }, + arrow.UINT64: func(data *Data) Interface { return NewUint64Data(data) }, + arrow.INT64: func(data *Data) Interface { return NewInt64Data(data) }, + arrow.FLOAT16: func(data *Data) Interface { return NewFloat16Data(data) }, + arrow.FLOAT32: func(data *Data) Interface { return NewFloat32Data(data) }, + arrow.FLOAT64: func(data *Data) Interface { return NewFloat64Data(data) }, + arrow.STRING: func(data *Data) Interface { return NewStringData(data) }, + arrow.BINARY: func(data *Data) Interface { return NewBinaryData(data) }, + arrow.FIXED_SIZE_BINARY: func(data *Data) Interface { return NewFixedSizeBinaryData(data) }, + arrow.DATE32: func(data *Data) Interface { return NewDate32Data(data) }, + arrow.DATE64: func(data *Data) Interface { return NewDate64Data(data) }, + arrow.TIMESTAMP: func(data *Data) Interface { return NewTimestampData(data) }, + arrow.TIME32: func(data *Data) Interface { return NewTime32Data(data) }, + arrow.TIME64: func(data *Data) Interface { return NewTime64Data(data) }, + arrow.INTERVAL: func(data *Data) Interface { return NewIntervalData(data) }, + arrow.DECIMAL: func(data *Data) Interface { return NewDecimal128Data(data) }, + arrow.LIST: func(data *Data) Interface { return NewListData(data) }, + arrow.STRUCT: func(data *Data) Interface { return NewStructData(data) }, + arrow.UNION: unsupportedArrayType, + arrow.DICTIONARY: unsupportedArrayType, + arrow.MAP: unsupportedArrayType, + arrow.EXTENSION: unsupportedArrayType, + arrow.FIXED_SIZE_LIST: func(data *Data) Interface { return NewFixedSizeListData(data) }, + arrow.DURATION: func(data *Data) Interface { return NewDurationData(data) }, + + // invalid data types to fill out array size 2⁵-1 + 31: invalidDataType, + } +} diff --git a/vendor/github.com/apache/arrow/go/arrow/array/binary.go b/vendor/github.com/apache/arrow/go/arrow/array/binary.go new file mode 100644 index 00000000000..ed58910919a --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/array/binary.go @@ -0,0 +1,134 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package array + +import ( + "bytes" + "fmt" + "strings" + "unsafe" + + "github.com/apache/arrow/go/arrow" +) + +// A type which represents an immutable sequence of variable-length binary strings. +type Binary struct { + array + valueOffsets []int32 + valueBytes []byte +} + +// NewBinaryData constructs a new Binary array from data. +func NewBinaryData(data *Data) *Binary { + a := &Binary{} + a.refCount = 1 + a.setData(data) + return a +} + +// Value returns the slice at index i. This value should not be mutated. +func (a *Binary) Value(i int) []byte { + if i < 0 || i >= a.array.data.length { + panic("arrow/array: index out of range") + } + idx := a.array.data.offset + i + return a.valueBytes[a.valueOffsets[idx]:a.valueOffsets[idx+1]] +} + +// ValueString returns the string at index i without performing additional allocations. +// The string is only valid for the lifetime of the Binary array. +func (a *Binary) ValueString(i int) string { + b := a.Value(i) + return *(*string)(unsafe.Pointer(&b)) +} + +func (a *Binary) ValueOffset(i int) int { + if i < 0 || i >= a.array.data.length { + panic("arrow/array: index out of range") + } + return int(a.valueOffsets[a.array.data.offset+i]) +} + +func (a *Binary) ValueLen(i int) int { + if i < 0 || i >= a.array.data.length { + panic("arrow/array: index out of range") + } + beg := a.array.data.offset + i + return int(a.valueOffsets[beg+1] - a.valueOffsets[beg]) +} + +func (a *Binary) ValueOffsets() []int32 { + beg := a.array.data.offset + end := beg + a.array.data.length + 1 + return a.valueOffsets[beg:end] +} + +func (a *Binary) ValueBytes() []byte { + beg := a.array.data.offset + end := beg + a.array.data.length + return a.valueBytes[a.valueOffsets[beg]:a.valueOffsets[end]] +} + +func (a *Binary) String() string { + o := new(strings.Builder) + o.WriteString("[") + for i := 0; i < a.Len(); i++ { + if i > 0 { + o.WriteString(" ") + } + switch { + case a.IsNull(i): + o.WriteString("(null)") + default: + fmt.Fprintf(o, "%q", a.ValueString(i)) + } + } + o.WriteString("]") + return o.String() +} + +func (a *Binary) setData(data *Data) { + if len(data.buffers) != 3 { + panic("len(data.buffers) != 3") + } + + a.array.setData(data) + + if valueData := data.buffers[2]; valueData != nil { + a.valueBytes = valueData.Bytes() + } + + if valueOffsets := data.buffers[1]; valueOffsets != nil { + a.valueOffsets = arrow.Int32Traits.CastFromBytes(valueOffsets.Bytes()) + } +} + +func arrayEqualBinary(left, right *Binary) bool { + for i := 0; i < left.Len(); i++ { + if left.IsNull(i) { + continue + } + if bytes.Compare(left.Value(i), right.Value(i)) != 0 { + return false + } + } + return true +} + +var ( + _ Interface = (*Binary)(nil) +) diff --git a/vendor/github.com/apache/arrow/go/arrow/array/binarybuilder.go b/vendor/github.com/apache/arrow/go/arrow/array/binarybuilder.go new file mode 100644 index 00000000000..17562fcc765 --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/array/binarybuilder.go @@ -0,0 +1,217 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package array + +import ( + "math" + "sync/atomic" + + "github.com/apache/arrow/go/arrow" + "github.com/apache/arrow/go/arrow/internal/debug" + "github.com/apache/arrow/go/arrow/memory" +) + +const ( + binaryArrayMaximumCapacity = math.MaxInt32 +) + +// A BinaryBuilder is used to build a Binary array using the Append methods. +type BinaryBuilder struct { + builder + + dtype arrow.BinaryDataType + offsets *int32BufferBuilder + values *byteBufferBuilder +} + +func NewBinaryBuilder(mem memory.Allocator, dtype arrow.BinaryDataType) *BinaryBuilder { + b := &BinaryBuilder{ + builder: builder{refCount: 1, mem: mem}, + dtype: dtype, + offsets: newInt32BufferBuilder(mem), + values: newByteBufferBuilder(mem), + } + return b +} + +// Release decreases the reference count by 1. +// When the reference count goes to zero, the memory is freed. +// Release may be called simultaneously from multiple goroutines. +func (b *BinaryBuilder) Release() { + debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases") + + if atomic.AddInt64(&b.refCount, -1) == 0 { + if b.nullBitmap != nil { + b.nullBitmap.Release() + b.nullBitmap = nil + } + if b.offsets != nil { + b.offsets.Release() + b.offsets = nil + } + if b.values != nil { + b.values.Release() + b.values = nil + } + } +} + +func (b *BinaryBuilder) Append(v []byte) { + b.Reserve(1) + b.appendNextOffset() + b.values.Append(v) + b.UnsafeAppendBoolToBitmap(true) +} + +func (b *BinaryBuilder) AppendString(v string) { + b.Append([]byte(v)) +} + +func (b *BinaryBuilder) AppendNull() { + b.Reserve(1) + b.appendNextOffset() + b.UnsafeAppendBoolToBitmap(false) +} + +// AppendValues will append the values in the v slice. The valid slice determines which values +// in v are valid (not null). The valid slice must either be empty or be equal in length to v. If empty, +// all values in v are appended and considered valid. +func (b *BinaryBuilder) AppendValues(v [][]byte, valid []bool) { + if len(v) != len(valid) && len(valid) != 0 { + panic("len(v) != len(valid) && len(valid) != 0") + } + + if len(v) == 0 { + return + } + + b.Reserve(len(v)) + for _, vv := range v { + b.appendNextOffset() + b.values.Append(vv) + } + + b.builder.unsafeAppendBoolsToBitmap(valid, len(v)) +} + +// AppendStringValues will append the values in the v slice. The valid slice determines which values +// in v are valid (not null). The valid slice must either be empty or be equal in length to v. If empty, +// all values in v are appended and considered valid. +func (b *BinaryBuilder) AppendStringValues(v []string, valid []bool) { + if len(v) != len(valid) && len(valid) != 0 { + panic("len(v) != len(valid) && len(valid) != 0") + } + + if len(v) == 0 { + return + } + + b.Reserve(len(v)) + for _, vv := range v { + b.appendNextOffset() + b.values.Append([]byte(vv)) + } + + b.builder.unsafeAppendBoolsToBitmap(valid, len(v)) +} + +func (b *BinaryBuilder) Value(i int) []byte { + offsets := b.offsets.Values() + start := int(offsets[i]) + var end int + if i == (b.length - 1) { + end = b.values.Len() + } else { + end = int(offsets[i+1]) + } + return b.values.Bytes()[start:end] +} + +func (b *BinaryBuilder) init(capacity int) { + b.builder.init(capacity) + b.offsets.resize((capacity + 1) * arrow.Int32SizeBytes) +} + +// DataLen returns the number of bytes in the data array. +func (b *BinaryBuilder) DataLen() int { return b.values.length } + +// DataCap returns the total number of bytes that can be stored +// without allocating additional memory. +func (b *BinaryBuilder) DataCap() int { return b.values.capacity } + +// Reserve ensures there is enough space for appending n elements +// by checking the capacity and calling Resize if necessary. +func (b *BinaryBuilder) Reserve(n int) { + b.builder.reserve(n, b.Resize) +} + +// ReserveData ensures there is enough space for appending n bytes +// by checking the capacity and resizing the data buffer if necessary. +func (b *BinaryBuilder) ReserveData(n int) { + if b.values.capacity < b.values.length+n { + b.values.resize(b.values.Len() + n) + } +} + +// Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(), +// additional memory will be allocated. If n is smaller, the allocated memory may be reduced. +func (b *BinaryBuilder) Resize(n int) { + b.offsets.resize((n + 1) * arrow.Int32SizeBytes) + b.builder.resize(n, b.init) +} + +// NewArray creates a Binary array from the memory buffers used by the builder and resets the BinaryBuilder +// so it can be used to build a new array. +func (b *BinaryBuilder) NewArray() Interface { + return b.NewBinaryArray() +} + +// NewBinaryArray creates a Binary array from the memory buffers used by the builder and resets the BinaryBuilder +// so it can be used to build a new array. +func (b *BinaryBuilder) NewBinaryArray() (a *Binary) { + data := b.newData() + a = NewBinaryData(data) + data.Release() + return +} + +func (b *BinaryBuilder) newData() (data *Data) { + b.appendNextOffset() + offsets, values := b.offsets.Finish(), b.values.Finish() + data = NewData(b.dtype, b.length, []*memory.Buffer{b.nullBitmap, offsets, values}, nil, b.nulls, 0) + if offsets != nil { + offsets.Release() + } + + if values != nil { + values.Release() + } + + b.builder.reset() + + return +} + +func (b *BinaryBuilder) appendNextOffset() { + numBytes := b.values.Len() + // TODO(sgc): check binaryArrayMaximumCapacity? + b.offsets.AppendValue(int32(numBytes)) +} + +var ( + _ Builder = (*BinaryBuilder)(nil) +) diff --git a/vendor/github.com/apache/arrow/go/arrow/array/boolean.go b/vendor/github.com/apache/arrow/go/arrow/array/boolean.go new file mode 100644 index 00000000000..5095b17318a --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/array/boolean.go @@ -0,0 +1,95 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package array + +import ( + "fmt" + "strings" + + "github.com/apache/arrow/go/arrow" + "github.com/apache/arrow/go/arrow/internal/bitutil" + "github.com/apache/arrow/go/arrow/memory" +) + +// A type which represents an immutable sequence of boolean values. +type Boolean struct { + array + values []byte +} + +// NewBoolean creates a boolean array from the data memory.Buffer and contains length elements. +// The nullBitmap buffer can be nil of there are no null values. +// If nulls is not known, use UnknownNullCount to calculate the value of NullN at runtime from the nullBitmap buffer. +func NewBoolean(length int, data *memory.Buffer, nullBitmap *memory.Buffer, nulls int) *Boolean { + return NewBooleanData(NewData(arrow.FixedWidthTypes.Boolean, length, []*memory.Buffer{nullBitmap, data}, nil, nulls, 0)) +} + +func NewBooleanData(data *Data) *Boolean { + a := &Boolean{} + a.refCount = 1 + a.setData(data) + return a +} + +func (a *Boolean) Value(i int) bool { + if i < 0 || i >= a.array.data.length { + panic("arrow/array: index out of range") + } + return bitutil.BitIsSet(a.values, a.array.data.offset+i) +} + +func (a *Boolean) String() string { + o := new(strings.Builder) + o.WriteString("[") + for i := 0; i < a.Len(); i++ { + if i > 0 { + fmt.Fprintf(o, " ") + } + switch { + case a.IsNull(i): + o.WriteString("(null)") + default: + fmt.Fprintf(o, "%v", a.Value(i)) + } + } + o.WriteString("]") + return o.String() +} + +func (a *Boolean) setData(data *Data) { + a.array.setData(data) + vals := data.buffers[1] + if vals != nil { + a.values = vals.Bytes() + } +} + +func arrayEqualBoolean(left, right *Boolean) bool { + for i := 0; i < left.Len(); i++ { + if left.IsNull(i) { + continue + } + if left.Value(i) != right.Value(i) { + return false + } + } + return true +} + +var ( + _ Interface = (*Boolean)(nil) +) diff --git a/vendor/github.com/apache/arrow/go/arrow/array/booleanbuilder.go b/vendor/github.com/apache/arrow/go/arrow/array/booleanbuilder.go new file mode 100644 index 00000000000..943df1f1200 --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/array/booleanbuilder.go @@ -0,0 +1,165 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package array + +import ( + "sync/atomic" + + "github.com/apache/arrow/go/arrow" + "github.com/apache/arrow/go/arrow/internal/bitutil" + "github.com/apache/arrow/go/arrow/internal/debug" + "github.com/apache/arrow/go/arrow/memory" +) + +type BooleanBuilder struct { + builder + + data *memory.Buffer + rawData []byte +} + +func NewBooleanBuilder(mem memory.Allocator) *BooleanBuilder { + return &BooleanBuilder{builder: builder{refCount: 1, mem: mem}} +} + +// Release decreases the reference count by 1. +// When the reference count goes to zero, the memory is freed. +// Release may be called simultaneously from multiple goroutines. +func (b *BooleanBuilder) Release() { + debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases") + + if atomic.AddInt64(&b.refCount, -1) == 0 { + if b.nullBitmap != nil { + b.nullBitmap.Release() + b.nullBitmap = nil + } + if b.data != nil { + b.data.Release() + b.data = nil + b.rawData = nil + } + } +} + +func (b *BooleanBuilder) Append(v bool) { + b.Reserve(1) + b.UnsafeAppend(v) +} + +func (b *BooleanBuilder) AppendByte(v byte) { + b.Reserve(1) + b.UnsafeAppend(v != 0) +} + +func (b *BooleanBuilder) AppendNull() { + b.Reserve(1) + b.UnsafeAppendBoolToBitmap(false) +} + +func (b *BooleanBuilder) UnsafeAppend(v bool) { + bitutil.SetBit(b.nullBitmap.Bytes(), b.length) + if v { + bitutil.SetBit(b.rawData, b.length) + } else { + bitutil.ClearBit(b.rawData, b.length) + } + b.length++ +} + +func (b *BooleanBuilder) AppendValues(v []bool, valid []bool) { + if len(v) != len(valid) && len(valid) != 0 { + panic("len(v) != len(valid) && len(valid) != 0") + } + + if len(v) == 0 { + return + } + + b.Reserve(len(v)) + for i, vv := range v { + bitutil.SetBitTo(b.rawData, b.length+i, vv) + } + b.builder.unsafeAppendBoolsToBitmap(valid, len(v)) +} + +func (b *BooleanBuilder) init(capacity int) { + b.builder.init(capacity) + + b.data = memory.NewResizableBuffer(b.mem) + bytesN := arrow.BooleanTraits.BytesRequired(capacity) + b.data.Resize(bytesN) + b.rawData = b.data.Bytes() +} + +// Reserve ensures there is enough space for appending n elements +// by checking the capacity and calling Resize if necessary. +func (b *BooleanBuilder) Reserve(n int) { + b.builder.reserve(n, b.Resize) +} + +// Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(), +// additional memory will be allocated. If n is smaller, the allocated memory may reduced. +func (b *BooleanBuilder) Resize(n int) { + if n < minBuilderCapacity { + n = minBuilderCapacity + } + + if b.capacity == 0 { + b.init(n) + } else { + b.builder.resize(n, b.init) + b.data.Resize(arrow.BooleanTraits.BytesRequired(n)) + b.rawData = b.data.Bytes() + } +} + +// NewArray creates a Boolean array from the memory buffers used by the builder and resets the BooleanBuilder +// so it can be used to build a new array. +func (b *BooleanBuilder) NewArray() Interface { + return b.NewBooleanArray() +} + +// NewBooleanArray creates a Boolean array from the memory buffers used by the builder and resets the BooleanBuilder +// so it can be used to build a new array. +func (b *BooleanBuilder) NewBooleanArray() (a *Boolean) { + data := b.newData() + a = NewBooleanData(data) + data.Release() + return +} + +func (b *BooleanBuilder) newData() *Data { + bytesRequired := arrow.BooleanTraits.BytesRequired(b.length) + if bytesRequired > 0 && bytesRequired < b.data.Len() { + // trim buffers + b.data.Resize(bytesRequired) + } + res := NewData(arrow.FixedWidthTypes.Boolean, b.length, []*memory.Buffer{b.nullBitmap, b.data}, nil, b.nulls, 0) + b.reset() + + if b.data != nil { + b.data.Release() + b.data = nil + b.rawData = nil + } + + return res +} + +var ( + _ Builder = (*BooleanBuilder)(nil) +) diff --git a/vendor/github.com/apache/arrow/go/arrow/array/bufferbuilder.go b/vendor/github.com/apache/arrow/go/arrow/array/bufferbuilder.go new file mode 100644 index 00000000000..29d97fc8d41 --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/array/bufferbuilder.go @@ -0,0 +1,127 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package array + +import ( + "sync/atomic" + + "github.com/apache/arrow/go/arrow/internal/bitutil" + "github.com/apache/arrow/go/arrow/internal/debug" + "github.com/apache/arrow/go/arrow/memory" +) + +// A bufferBuilder provides common functionality for populating memory with a sequence of type-specific values. +// Specialized implementations provide type-safe APIs for appending and accessing the memory. +type bufferBuilder struct { + refCount int64 + mem memory.Allocator + buffer *memory.Buffer + length int + capacity int + + bytes []byte +} + +// Retain increases the reference count by 1. +// Retain may be called simultaneously from multiple goroutines. +func (b *bufferBuilder) Retain() { + atomic.AddInt64(&b.refCount, 1) +} + +// Release decreases the reference count by 1. +// When the reference count goes to zero, the memory is freed. +// Release may be called simultaneously from multiple goroutines. +func (b *bufferBuilder) Release() { + debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases") + + if atomic.AddInt64(&b.refCount, -1) == 0 { + if b.buffer != nil { + b.buffer.Release() + b.buffer, b.bytes = nil, nil + } + } +} + +// Len returns the length of the memory buffer in bytes. +func (b *bufferBuilder) Len() int { return b.length } + +// Cap returns the total number of bytes that can be stored without allocating additional memory. +func (b *bufferBuilder) Cap() int { return b.capacity } + +// Bytes returns a slice of length b.Len(). +// The slice is only valid for use until the next buffer modification. That is, until the next call +// to Advance, Reset, Finish or any Append function. The slice aliases the buffer content at least until the next +// buffer modification. +func (b *bufferBuilder) Bytes() []byte { return b.bytes[:b.length] } + +func (b *bufferBuilder) resize(elements int) { + if b.buffer == nil { + b.buffer = memory.NewResizableBuffer(b.mem) + } + + b.buffer.Resize(elements) + oldCapacity := b.capacity + b.capacity = b.buffer.Cap() + b.bytes = b.buffer.Buf() + + if b.capacity > oldCapacity { + memory.Set(b.bytes[oldCapacity:], 0) + } +} + +// Advance increases the buffer by length and initializes the skipped bytes to zero. +func (b *bufferBuilder) Advance(length int) { + if b.capacity < b.length+length { + newCapacity := bitutil.NextPowerOf2(b.length + length) + b.resize(newCapacity) + } + b.length += length +} + +// Append appends the contents of v to the buffer, resizing it if necessary. +func (b *bufferBuilder) Append(v []byte) { + if b.capacity < b.length+len(v) { + newCapacity := bitutil.NextPowerOf2(b.length + len(v)) + b.resize(newCapacity) + } + b.unsafeAppend(v) +} + +// Reset returns the buffer to an empty state. Reset releases the memory and sets the length and capacity to zero. +func (b *bufferBuilder) Reset() { + if b.buffer != nil { + b.buffer.Release() + } + b.buffer, b.bytes = nil, nil + b.capacity, b.length = 0, 0 +} + +// Finish TODO(sgc) +func (b *bufferBuilder) Finish() (buffer *memory.Buffer) { + if b.length > 0 { + b.buffer.ResizeNoShrink(b.length) + } + buffer = b.buffer + b.buffer = nil + b.Reset() + return +} + +func (b *bufferBuilder) unsafeAppend(data []byte) { + copy(b.bytes[b.length:], data) + b.length += len(data) +} diff --git a/vendor/github.com/apache/arrow/go/arrow/array/bufferbuilder_byte.go b/vendor/github.com/apache/arrow/go/arrow/array/bufferbuilder_byte.go new file mode 100644 index 00000000000..f5f5445b7ac --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/array/bufferbuilder_byte.go @@ -0,0 +1,30 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package array + +import "github.com/apache/arrow/go/arrow/memory" + +type byteBufferBuilder struct { + bufferBuilder +} + +func newByteBufferBuilder(mem memory.Allocator) *byteBufferBuilder { + return &byteBufferBuilder{bufferBuilder: bufferBuilder{refCount: 1, mem: mem}} +} + +func (b *byteBufferBuilder) Values() []byte { return b.Bytes() } +func (b *byteBufferBuilder) Value(i int) byte { return b.bytes[i] } diff --git a/vendor/github.com/apache/arrow/go/arrow/array/bufferbuilder_numeric.gen.go b/vendor/github.com/apache/arrow/go/arrow/array/bufferbuilder_numeric.gen.go new file mode 100644 index 00000000000..e39b2528a9e --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/array/bufferbuilder_numeric.gen.go @@ -0,0 +1,58 @@ +// Code generated by array/bufferbuilder_numeric.gen.go.tmpl. DO NOT EDIT. + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package array + +import ( + "github.com/apache/arrow/go/arrow" + "github.com/apache/arrow/go/arrow/internal/bitutil" + "github.com/apache/arrow/go/arrow/memory" +) + +type int32BufferBuilder struct { + bufferBuilder +} + +func newInt32BufferBuilder(mem memory.Allocator) *int32BufferBuilder { + return &int32BufferBuilder{bufferBuilder: bufferBuilder{refCount: 1, mem: mem}} +} + +// AppendValues appends the contents of v to the buffer, growing the buffer as needed. +func (b *int32BufferBuilder) AppendValues(v []int32) { b.Append(arrow.Int32Traits.CastToBytes(v)) } + +// Values returns a slice of length b.Len(). +// The slice is only valid for use until the next buffer modification. That is, until the next call +// to Advance, Reset, Finish or any Append function. The slice aliases the buffer content at least until the next +// buffer modification. +func (b *int32BufferBuilder) Values() []int32 { return arrow.Int32Traits.CastFromBytes(b.Bytes()) } + +// Value returns the int32 element at the index i. Value will panic if i is negative or ≥ Len. +func (b *int32BufferBuilder) Value(i int) int32 { return b.Values()[i] } + +// Len returns the number of int32 elements in the buffer. +func (b *int32BufferBuilder) Len() int { return b.length / arrow.Int32SizeBytes } + +// AppendValue appends v to the buffer, growing the buffer as needed. +func (b *int32BufferBuilder) AppendValue(v int32) { + if b.capacity < b.length+arrow.Int32SizeBytes { + newCapacity := bitutil.NextPowerOf2(b.length + arrow.Int32SizeBytes) + b.resize(newCapacity) + } + arrow.Int32Traits.PutValue(b.bytes[b.length:], v) + b.length += arrow.Int32SizeBytes +} diff --git a/vendor/github.com/apache/arrow/go/arrow/array/bufferbuilder_numeric.gen.go.tmpl b/vendor/github.com/apache/arrow/go/arrow/array/bufferbuilder_numeric.gen.go.tmpl new file mode 100644 index 00000000000..d8b03b06ce2 --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/array/bufferbuilder_numeric.gen.go.tmpl @@ -0,0 +1,61 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package array + +import ( + "github.com/apache/arrow/go/arrow" + "github.com/apache/arrow/go/arrow/internal/bitutil" + "github.com/apache/arrow/go/arrow/memory" +) + +{{range .In}} +{{$TypeNamePrefix := .name}} +{{if .Opt.BufferBuilder}} +type {{$TypeNamePrefix}}BufferBuilder struct { + bufferBuilder +} + +func new{{.Name}}BufferBuilder(mem memory.Allocator) *{{$TypeNamePrefix}}BufferBuilder { + return &{{$TypeNamePrefix}}BufferBuilder{bufferBuilder:bufferBuilder{refCount: 1, mem:mem}} +} + +// AppendValues appends the contents of v to the buffer, growing the buffer as needed. +func (b *{{$TypeNamePrefix}}BufferBuilder) AppendValues(v []{{.Type}}) { b.Append(arrow.{{.Name}}Traits.CastToBytes(v)) } + +// Values returns a slice of length b.Len(). +// The slice is only valid for use until the next buffer modification. That is, until the next call +// to Advance, Reset, Finish or any Append function. The slice aliases the buffer content at least until the next +// buffer modification. +func (b *{{$TypeNamePrefix}}BufferBuilder) Values() []{{.Type}} { return arrow.{{.Name}}Traits.CastFromBytes(b.Bytes()) } + +// Value returns the {{.Type}} element at the index i. Value will panic if i is negative or ≥ Len. +func (b *{{$TypeNamePrefix}}BufferBuilder) Value(i int) {{.Type}} { return b.Values()[i] } + +// Len returns the number of {{.Type}} elements in the buffer. +func (b *{{$TypeNamePrefix}}BufferBuilder) Len() int { return b.length/arrow.{{.Name}}SizeBytes } + +// AppendValue appends v to the buffer, growing the buffer as needed. +func (b *{{$TypeNamePrefix}}BufferBuilder) AppendValue(v {{.Type}}) { + if b.capacity < b.length+arrow.{{.Name}}SizeBytes { + newCapacity := bitutil.NextPowerOf2(b.length + arrow.{{.Name}}SizeBytes) + b.resize(newCapacity) + } + arrow.{{.Name}}Traits.PutValue(b.bytes[b.length:], v) + b.length+=arrow.{{.Name}}SizeBytes +} +{{end}} +{{end}} \ No newline at end of file diff --git a/vendor/github.com/apache/arrow/go/arrow/array/builder.go b/vendor/github.com/apache/arrow/go/arrow/array/builder.go new file mode 100644 index 00000000000..6f5beb355d3 --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/array/builder.go @@ -0,0 +1,273 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package array + +import ( + "fmt" + "sync/atomic" + + "github.com/apache/arrow/go/arrow" + "github.com/apache/arrow/go/arrow/internal/bitutil" + "github.com/apache/arrow/go/arrow/memory" +) + +const ( + minBuilderCapacity = 1 << 5 +) + +// Builder provides an interface to build arrow arrays. +type Builder interface { + // Retain increases the reference count by 1. + // Retain may be called simultaneously from multiple goroutines. + Retain() + + // Release decreases the reference count by 1. + Release() + + // Len returns the number of elements in the array builder. + Len() int + + // Cap returns the total number of elements that can be stored + // without allocating additional memory. + Cap() int + + // NullN returns the number of null values in the array builder. + NullN() int + + // AppendNull adds a new null value to the array being built. + AppendNull() + + // Reserve ensures there is enough space for appending n elements + // by checking the capacity and calling Resize if necessary. + Reserve(n int) + + // Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(), + // additional memory will be allocated. If n is smaller, the allocated memory may reduced. + Resize(n int) + + // NewArray creates a new array from the memory buffers used + // by the builder and resets the Builder so it can be used to build + // a new array. + NewArray() Interface + + init(capacity int) + resize(newBits int, init func(int)) +} + +// builder provides common functionality for managing the validity bitmap (nulls) when building arrays. +type builder struct { + refCount int64 + mem memory.Allocator + nullBitmap *memory.Buffer + nulls int + length int + capacity int +} + +// Retain increases the reference count by 1. +// Retain may be called simultaneously from multiple goroutines. +func (b *builder) Retain() { + atomic.AddInt64(&b.refCount, 1) +} + +// Len returns the number of elements in the array builder. +func (b *builder) Len() int { return b.length } + +// Cap returns the total number of elements that can be stored without allocating additional memory. +func (b *builder) Cap() int { return b.capacity } + +// NullN returns the number of null values in the array builder. +func (b *builder) NullN() int { return b.nulls } + +func (b *builder) init(capacity int) { + toAlloc := bitutil.CeilByte(capacity) / 8 + b.nullBitmap = memory.NewResizableBuffer(b.mem) + b.nullBitmap.Resize(toAlloc) + b.capacity = capacity + memory.Set(b.nullBitmap.Buf(), 0) +} + +func (b *builder) reset() { + if b.nullBitmap != nil { + b.nullBitmap.Release() + b.nullBitmap = nil + } + + b.nulls = 0 + b.length = 0 + b.capacity = 0 +} + +func (b *builder) resize(newBits int, init func(int)) { + if b.nullBitmap == nil { + init(newBits) + return + } + + newBytesN := bitutil.CeilByte(newBits) / 8 + oldBytesN := b.nullBitmap.Len() + b.nullBitmap.Resize(newBytesN) + b.capacity = newBits + if oldBytesN < newBytesN { + // TODO(sgc): necessary? + memory.Set(b.nullBitmap.Buf()[oldBytesN:], 0) + } + if newBits < b.length { + b.length = newBits + b.nulls = newBits - bitutil.CountSetBits(b.nullBitmap.Buf(), 0, newBits) + } +} + +func (b *builder) reserve(elements int, resize func(int)) { + if b.length+elements > b.capacity { + newCap := bitutil.NextPowerOf2(b.length + elements) + resize(newCap) + } +} + +// unsafeAppendBoolsToBitmap appends the contents of valid to the validity bitmap. +// As an optimization, if the valid slice is empty, the next length bits will be set to valid (not null). +func (b *builder) unsafeAppendBoolsToBitmap(valid []bool, length int) { + if len(valid) == 0 { + b.unsafeSetValid(length) + return + } + + byteOffset := b.length / 8 + bitOffset := byte(b.length % 8) + nullBitmap := b.nullBitmap.Bytes() + bitSet := nullBitmap[byteOffset] + + for _, v := range valid { + if bitOffset == 8 { + bitOffset = 0 + nullBitmap[byteOffset] = bitSet + byteOffset++ + bitSet = nullBitmap[byteOffset] + } + + if v { + bitSet |= bitutil.BitMask[bitOffset] + } else { + bitSet &= bitutil.FlippedBitMask[bitOffset] + b.nulls++ + } + bitOffset++ + } + + if bitOffset != 0 { + nullBitmap[byteOffset] = bitSet + } + b.length += len(valid) +} + +// unsafeSetValid sets the next length bits to valid in the validity bitmap. +func (b *builder) unsafeSetValid(length int) { + padToByte := min(8-(b.length%8), length) + if padToByte == 8 { + padToByte = 0 + } + bits := b.nullBitmap.Bytes() + for i := b.length; i < b.length+padToByte; i++ { + bitutil.SetBit(bits, i) + } + + start := (b.length + padToByte) / 8 + fastLength := (length - padToByte) / 8 + memory.Set(bits[start:start+fastLength], 0xff) + + newLength := b.length + length + // trailing bytes + for i := b.length + padToByte + (fastLength * 8); i < newLength; i++ { + bitutil.SetBit(bits, i) + } + + b.length = newLength +} + +func (b *builder) UnsafeAppendBoolToBitmap(isValid bool) { + if isValid { + bitutil.SetBit(b.nullBitmap.Bytes(), b.length) + } else { + b.nulls++ + } + b.length++ +} + +func newBuilder(mem memory.Allocator, dtype arrow.DataType) Builder { + // FIXME(sbinet): use a type switch on dtype instead? + switch dtype.ID() { + case arrow.NULL: + case arrow.BOOL: + return NewBooleanBuilder(mem) + case arrow.UINT8: + return NewUint8Builder(mem) + case arrow.INT8: + return NewInt8Builder(mem) + case arrow.UINT16: + return NewUint16Builder(mem) + case arrow.INT16: + return NewInt16Builder(mem) + case arrow.UINT32: + return NewUint32Builder(mem) + case arrow.INT32: + return NewInt32Builder(mem) + case arrow.UINT64: + return NewUint64Builder(mem) + case arrow.INT64: + return NewInt64Builder(mem) + case arrow.FLOAT16: + return NewFloat16Builder(mem) + case arrow.FLOAT32: + return NewFloat32Builder(mem) + case arrow.FLOAT64: + return NewFloat64Builder(mem) + case arrow.STRING: + return NewStringBuilder(mem) + case arrow.BINARY: + return NewBinaryBuilder(mem, arrow.BinaryTypes.Binary) + case arrow.FIXED_SIZE_BINARY: + typ := dtype.(*arrow.FixedSizeBinaryType) + return NewFixedSizeBinaryBuilder(mem, typ) + case arrow.DATE32: + case arrow.DATE64: + case arrow.TIMESTAMP: + case arrow.TIME32: + typ := dtype.(*arrow.Time32Type) + return NewTime32Builder(mem, typ) + case arrow.TIME64: + typ := dtype.(*arrow.Time64Type) + return NewTime64Builder(mem, typ) + case arrow.INTERVAL: + case arrow.DECIMAL: + case arrow.LIST: + typ := dtype.(*arrow.ListType) + return NewListBuilder(mem, typ.Elem()) + case arrow.STRUCT: + typ := dtype.(*arrow.StructType) + return NewStructBuilder(mem, typ) + case arrow.UNION: + case arrow.DICTIONARY: + case arrow.MAP: + case arrow.EXTENSION: + case arrow.FIXED_SIZE_LIST: + typ := dtype.(*arrow.FixedSizeListType) + return NewFixedSizeListBuilder(mem, typ.Len(), typ.Elem()) + case arrow.DURATION: + } + panic(fmt.Errorf("arrow/array: unsupported builder for %T", dtype)) +} diff --git a/vendor/github.com/apache/arrow/go/arrow/array/compare.go b/vendor/github.com/apache/arrow/go/arrow/array/compare.go new file mode 100644 index 00000000000..17839c65600 --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/array/compare.go @@ -0,0 +1,474 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package array + +import ( + "math" + + "github.com/apache/arrow/go/arrow" + "github.com/apache/arrow/go/arrow/float16" + "github.com/pkg/errors" +) + +// RecordEqual reports whether the two provided records are equal. +func RecordEqual(left, right Record) bool { + switch { + case left.NumCols() != right.NumCols(): + return false + case left.NumRows() != right.NumRows(): + return false + } + + for i := range left.Columns() { + lc := left.Column(i) + rc := right.Column(i) + if !ArrayEqual(lc, rc) { + return false + } + } + return true +} + +// RecordApproxEqual reports whether the two provided records are approximately equal. +// For non-floating point columns, it is equivalent to RecordEqual. +func RecordApproxEqual(left, right Record, opts ...EqualOption) bool { + switch { + case left.NumCols() != right.NumCols(): + return false + case left.NumRows() != right.NumRows(): + return false + } + + opt := newEqualOption(opts...) + + for i := range left.Columns() { + lc := left.Column(i) + rc := right.Column(i) + if !arrayApproxEqual(lc, rc, opt) { + return false + } + } + return true +} + +// ArrayEqual reports whether the two provided arrays are equal. +func ArrayEqual(left, right Interface) bool { + switch { + case !baseArrayEqual(left, right): + return false + case left.Len() == 0: + return true + case left.NullN() == left.Len(): + return true + } + + // at this point, we know both arrays have same type, same length, same number of nulls + // and nulls at the same place. + // compare the values. + + switch l := left.(type) { + case *Null: + return true + case *Boolean: + r := right.(*Boolean) + return arrayEqualBoolean(l, r) + case *FixedSizeBinary: + r := right.(*FixedSizeBinary) + return arrayEqualFixedSizeBinary(l, r) + case *Binary: + r := right.(*Binary) + return arrayEqualBinary(l, r) + case *String: + r := right.(*String) + return arrayEqualString(l, r) + case *Int8: + r := right.(*Int8) + return arrayEqualInt8(l, r) + case *Int16: + r := right.(*Int16) + return arrayEqualInt16(l, r) + case *Int32: + r := right.(*Int32) + return arrayEqualInt32(l, r) + case *Int64: + r := right.(*Int64) + return arrayEqualInt64(l, r) + case *Uint8: + r := right.(*Uint8) + return arrayEqualUint8(l, r) + case *Uint16: + r := right.(*Uint16) + return arrayEqualUint16(l, r) + case *Uint32: + r := right.(*Uint32) + return arrayEqualUint32(l, r) + case *Uint64: + r := right.(*Uint64) + return arrayEqualUint64(l, r) + case *Float16: + r := right.(*Float16) + return arrayEqualFloat16(l, r) + case *Float32: + r := right.(*Float32) + return arrayEqualFloat32(l, r) + case *Float64: + r := right.(*Float64) + return arrayEqualFloat64(l, r) + case *Decimal128: + r := right.(*Decimal128) + return arrayEqualDecimal128(l, r) + case *Date32: + r := right.(*Date32) + return arrayEqualDate32(l, r) + case *Date64: + r := right.(*Date64) + return arrayEqualDate64(l, r) + case *Time32: + r := right.(*Time32) + return arrayEqualTime32(l, r) + case *Time64: + r := right.(*Time64) + return arrayEqualTime64(l, r) + case *Timestamp: + r := right.(*Timestamp) + return arrayEqualTimestamp(l, r) + case *List: + r := right.(*List) + return arrayEqualList(l, r) + case *FixedSizeList: + r := right.(*FixedSizeList) + return arrayEqualFixedSizeList(l, r) + case *Struct: + r := right.(*Struct) + return arrayEqualStruct(l, r) + case *MonthInterval: + r := right.(*MonthInterval) + return arrayEqualMonthInterval(l, r) + case *DayTimeInterval: + r := right.(*DayTimeInterval) + return arrayEqualDayTimeInterval(l, r) + case *Duration: + r := right.(*Duration) + return arrayEqualDuration(l, r) + + default: + panic(errors.Errorf("arrow/array: unknown array type %T", l)) + } +} + +// ArraySliceEqual reports whether slices left[lbeg:lend] and right[rbeg:rend] are equal. +func ArraySliceEqual(left Interface, lbeg, lend int64, right Interface, rbeg, rend int64) bool { + l := NewSlice(left, lbeg, lend) + defer l.Release() + r := NewSlice(right, rbeg, rend) + defer r.Release() + + return ArrayEqual(l, r) +} + +const defaultAbsoluteTolerance = 1e-5 + +type equalOption struct { + atol float64 // absolute tolerance + nansEq bool // whether NaNs are considered equal. +} + +func (eq equalOption) f16(f1, f2 float16.Num) bool { + v1 := float64(f1.Float32()) + v2 := float64(f2.Float32()) + switch { + case eq.nansEq: + return math.Abs(v1-v2) <= eq.atol || (math.IsNaN(v1) && math.IsNaN(v2)) + default: + return math.Abs(v1-v2) <= eq.atol + } +} + +func (eq equalOption) f32(f1, f2 float32) bool { + v1 := float64(f1) + v2 := float64(f2) + switch { + case eq.nansEq: + return math.Abs(v1-v2) <= eq.atol || (math.IsNaN(v1) && math.IsNaN(v2)) + default: + return math.Abs(v1-v2) <= eq.atol + } +} + +func (eq equalOption) f64(v1, v2 float64) bool { + switch { + case eq.nansEq: + return math.Abs(v1-v2) <= eq.atol || (math.IsNaN(v1) && math.IsNaN(v2)) + default: + return math.Abs(v1-v2) <= eq.atol + } +} + +func newEqualOption(opts ...EqualOption) equalOption { + eq := equalOption{ + atol: defaultAbsoluteTolerance, + nansEq: false, + } + for _, opt := range opts { + opt(&eq) + } + + return eq +} + +// EqualOption is a functional option type used to configure how Records and Arrays are compared. +type EqualOption func(*equalOption) + +// WithNaNsEqual configures the comparison functions so that NaNs are considered equal. +func WithNaNsEqual(v bool) EqualOption { + return func(o *equalOption) { + o.nansEq = v + } +} + +// WithAbsTolerance configures the comparison functions so that 2 floating point values +// v1 and v2 are considered equal if |v1-v2| <= atol. +func WithAbsTolerance(atol float64) EqualOption { + return func(o *equalOption) { + o.atol = atol + } +} + +// ArrayApproxEqual reports whether the two provided arrays are approximately equal. +// For non-floating point arrays, it is equivalent to ArrayEqual. +func ArrayApproxEqual(left, right Interface, opts ...EqualOption) bool { + opt := newEqualOption(opts...) + return arrayApproxEqual(left, right, opt) +} + +func arrayApproxEqual(left, right Interface, opt equalOption) bool { + switch { + case !baseArrayEqual(left, right): + return false + case left.Len() == 0: + return true + case left.NullN() == left.Len(): + return true + } + + // at this point, we know both arrays have same type, same length, same number of nulls + // and nulls at the same place. + // compare the values. + + switch l := left.(type) { + case *Null: + return true + case *Boolean: + r := right.(*Boolean) + return arrayEqualBoolean(l, r) + case *FixedSizeBinary: + r := right.(*FixedSizeBinary) + return arrayEqualFixedSizeBinary(l, r) + case *Binary: + r := right.(*Binary) + return arrayEqualBinary(l, r) + case *String: + r := right.(*String) + return arrayEqualString(l, r) + case *Int8: + r := right.(*Int8) + return arrayEqualInt8(l, r) + case *Int16: + r := right.(*Int16) + return arrayEqualInt16(l, r) + case *Int32: + r := right.(*Int32) + return arrayEqualInt32(l, r) + case *Int64: + r := right.(*Int64) + return arrayEqualInt64(l, r) + case *Uint8: + r := right.(*Uint8) + return arrayEqualUint8(l, r) + case *Uint16: + r := right.(*Uint16) + return arrayEqualUint16(l, r) + case *Uint32: + r := right.(*Uint32) + return arrayEqualUint32(l, r) + case *Uint64: + r := right.(*Uint64) + return arrayEqualUint64(l, r) + case *Float16: + r := right.(*Float16) + return arrayApproxEqualFloat16(l, r, opt) + case *Float32: + r := right.(*Float32) + return arrayApproxEqualFloat32(l, r, opt) + case *Float64: + r := right.(*Float64) + return arrayApproxEqualFloat64(l, r, opt) + case *Decimal128: + r := right.(*Decimal128) + return arrayEqualDecimal128(l, r) + case *Date32: + r := right.(*Date32) + return arrayEqualDate32(l, r) + case *Date64: + r := right.(*Date64) + return arrayEqualDate64(l, r) + case *Time32: + r := right.(*Time32) + return arrayEqualTime32(l, r) + case *Time64: + r := right.(*Time64) + return arrayEqualTime64(l, r) + case *Timestamp: + r := right.(*Timestamp) + return arrayEqualTimestamp(l, r) + case *List: + r := right.(*List) + return arrayApproxEqualList(l, r, opt) + case *FixedSizeList: + r := right.(*FixedSizeList) + return arrayApproxEqualFixedSizeList(l, r, opt) + case *Struct: + r := right.(*Struct) + return arrayApproxEqualStruct(l, r, opt) + case *MonthInterval: + r := right.(*MonthInterval) + return arrayEqualMonthInterval(l, r) + case *DayTimeInterval: + r := right.(*DayTimeInterval) + return arrayEqualDayTimeInterval(l, r) + case *Duration: + r := right.(*Duration) + return arrayEqualDuration(l, r) + + default: + panic(errors.Errorf("arrow/array: unknown array type %T", l)) + } + + return false +} + +func baseArrayEqual(left, right Interface) bool { + switch { + case left.Len() != right.Len(): + return false + case left.NullN() != right.NullN(): + return false + case !arrow.TypeEquals(left.DataType(), right.DataType()): // We do not check for metadata as in the C++ implementation. + return false + case !validityBitmapEqual(left, right): + return false + } + return true +} + +func validityBitmapEqual(left, right Interface) bool { + // TODO(alexandreyc): make it faster by comparing byte slices of the validity bitmap? + n := left.Len() + if n != right.Len() { + return false + } + for i := 0; i < n; i++ { + if left.IsNull(i) != right.IsNull(i) { + return false + } + } + return true +} + +func arrayApproxEqualFloat16(left, right *Float16, opt equalOption) bool { + for i := 0; i < left.Len(); i++ { + if left.IsNull(i) { + continue + } + if !opt.f16(left.Value(i), right.Value(i)) { + return false + } + } + return true +} + +func arrayApproxEqualFloat32(left, right *Float32, opt equalOption) bool { + for i := 0; i < left.Len(); i++ { + if left.IsNull(i) { + continue + } + if !opt.f32(left.Value(i), right.Value(i)) { + return false + } + } + return true +} + +func arrayApproxEqualFloat64(left, right *Float64, opt equalOption) bool { + for i := 0; i < left.Len(); i++ { + if left.IsNull(i) { + continue + } + if !opt.f64(left.Value(i), right.Value(i)) { + return false + } + } + return true +} + +func arrayApproxEqualList(left, right *List, opt equalOption) bool { + for i := 0; i < left.Len(); i++ { + if left.IsNull(i) { + continue + } + o := func() bool { + l := left.newListValue(i) + defer l.Release() + r := right.newListValue(i) + defer r.Release() + return arrayApproxEqual(l, r, opt) + }() + if !o { + return false + } + } + return true +} + +func arrayApproxEqualFixedSizeList(left, right *FixedSizeList, opt equalOption) bool { + for i := 0; i < left.Len(); i++ { + if left.IsNull(i) { + continue + } + o := func() bool { + l := left.newListValue(i) + defer l.Release() + r := right.newListValue(i) + defer r.Release() + return arrayApproxEqual(l, r, opt) + }() + if !o { + return false + } + } + return true +} + +func arrayApproxEqualStruct(left, right *Struct, opt equalOption) bool { + for i, lf := range left.fields { + rf := right.fields[i] + if !arrayApproxEqual(lf, rf, opt) { + return false + } + } + return true +} diff --git a/vendor/github.com/apache/arrow/go/arrow/array/data.go b/vendor/github.com/apache/arrow/go/arrow/array/data.go new file mode 100644 index 00000000000..58b16483bb8 --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/array/data.go @@ -0,0 +1,133 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package array + +import ( + "sync/atomic" + + "github.com/apache/arrow/go/arrow" + "github.com/apache/arrow/go/arrow/internal/debug" + "github.com/apache/arrow/go/arrow/memory" +) + +// A type which represents the memory and metadata for an Arrow array. +type Data struct { + refCount int64 + dtype arrow.DataType + nulls int + offset int + length int + buffers []*memory.Buffer // TODO(sgc): should this be an interface? + childData []*Data // TODO(sgc): managed by ListArray, StructArray and UnionArray types +} + +func NewData(dtype arrow.DataType, length int, buffers []*memory.Buffer, childData []*Data, nulls, offset int) *Data { + for _, b := range buffers { + if b != nil { + b.Retain() + } + } + + for _, child := range childData { + if child != nil { + child.Retain() + } + } + + return &Data{ + refCount: 1, + dtype: dtype, + nulls: nulls, + length: length, + offset: offset, + buffers: buffers, + childData: childData, + } +} + +// Retain increases the reference count by 1. +// Retain may be called simultaneously from multiple goroutines. +func (d *Data) Retain() { + atomic.AddInt64(&d.refCount, 1) +} + +// Release decreases the reference count by 1. +// When the reference count goes to zero, the memory is freed. +// Release may be called simultaneously from multiple goroutines. +func (d *Data) Release() { + debug.Assert(atomic.LoadInt64(&d.refCount) > 0, "too many releases") + + if atomic.AddInt64(&d.refCount, -1) == 0 { + for _, b := range d.buffers { + if b != nil { + b.Release() + } + } + + for _, b := range d.childData { + b.Release() + } + d.buffers, d.childData = nil, nil + } +} + +func (d *Data) DataType() arrow.DataType { return d.dtype } +func (d *Data) NullN() int { return d.nulls } +func (d *Data) Len() int { return d.length } +func (d *Data) Offset() int { return d.offset } +func (d *Data) Buffers() []*memory.Buffer { return d.buffers } + +// NewSliceData returns a new slice that shares backing data with the input. +// The returned Data slice starts at i and extends j-i elements, such as: +// slice := data[i:j] +// The returned value must be Release'd after use. +// +// NewSliceData panics if the slice is outside the valid range of the input Data. +// NewSliceData panics if j < i. +func NewSliceData(data *Data, i, j int64) *Data { + if j > int64(data.length) || i > j || data.offset+int(i) > data.length { + panic("arrow/array: index out of range") + } + + for _, b := range data.buffers { + if b != nil { + b.Retain() + } + } + + for _, child := range data.childData { + if child != nil { + child.Retain() + } + } + + o := &Data{ + refCount: 1, + dtype: data.dtype, + nulls: UnknownNullCount, + length: int(j - i), + offset: data.offset + int(i), + buffers: data.buffers, + childData: data.childData, + } + + if data.nulls == 0 { + o.nulls = 0 + } + + return o +} diff --git a/vendor/github.com/apache/arrow/go/arrow/array/decimal128.go b/vendor/github.com/apache/arrow/go/arrow/array/decimal128.go new file mode 100644 index 00000000000..390d87e9a49 --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/array/decimal128.go @@ -0,0 +1,235 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package array // import "github.com/apache/arrow/go/arrow/array" + +import ( + "fmt" + "strings" + "sync/atomic" + + "github.com/apache/arrow/go/arrow" + "github.com/apache/arrow/go/arrow/decimal128" + "github.com/apache/arrow/go/arrow/internal/bitutil" + "github.com/apache/arrow/go/arrow/internal/debug" + "github.com/apache/arrow/go/arrow/memory" +) + +// A type which represents an immutable sequence of 128-bit decimal values. +type Decimal128 struct { + array + + values []decimal128.Num +} + +func NewDecimal128Data(data *Data) *Decimal128 { + a := &Decimal128{} + a.refCount = 1 + a.setData(data) + return a +} + +func (a *Decimal128) Value(i int) decimal128.Num { return a.values[i] } + +func (a *Decimal128) Values() []decimal128.Num { return a.values } + +func (a *Decimal128) String() string { + o := new(strings.Builder) + o.WriteString("[") + for i := 0; i < a.Len(); i++ { + if i > 0 { + fmt.Fprintf(o, " ") + } + switch { + case a.IsNull(i): + o.WriteString("(null)") + default: + fmt.Fprintf(o, "%v", a.Value(i)) + } + } + o.WriteString("]") + return o.String() +} + +func (a *Decimal128) setData(data *Data) { + a.array.setData(data) + vals := data.buffers[1] + if vals != nil { + a.values = arrow.Decimal128Traits.CastFromBytes(vals.Bytes()) + beg := a.array.data.offset + end := beg + a.array.data.length + a.values = a.values[beg:end] + } +} + +func arrayEqualDecimal128(left, right *Decimal128) bool { + for i := 0; i < left.Len(); i++ { + if left.IsNull(i) { + continue + } + if left.Value(i) != right.Value(i) { + return false + } + } + return true +} + +type Decimal128Builder struct { + builder + + dtype *arrow.Decimal128Type + data *memory.Buffer + rawData []decimal128.Num +} + +func NewDecimal128Builder(mem memory.Allocator, dtype *arrow.Decimal128Type) *Decimal128Builder { + return &Decimal128Builder{ + builder: builder{refCount: 1, mem: mem}, + dtype: dtype, + } +} + +// Release decreases the reference count by 1. +// When the reference count goes to zero, the memory is freed. +func (b *Decimal128Builder) Release() { + debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases") + + if atomic.AddInt64(&b.refCount, -1) == 0 { + if b.nullBitmap != nil { + b.nullBitmap.Release() + b.nullBitmap = nil + } + if b.data != nil { + b.data.Release() + b.data = nil + b.rawData = nil + } + } +} + +func (b *Decimal128Builder) Append(v decimal128.Num) { + b.Reserve(1) + b.UnsafeAppend(v) +} + +func (b *Decimal128Builder) UnsafeAppend(v decimal128.Num) { + bitutil.SetBit(b.nullBitmap.Bytes(), b.length) + b.rawData[b.length] = v + b.length++ +} + +func (b *Decimal128Builder) AppendNull() { + b.Reserve(1) + b.UnsafeAppendBoolToBitmap(false) +} + +func (b *Decimal128Builder) UnsafeAppendBoolToBitmap(isValid bool) { + if isValid { + bitutil.SetBit(b.nullBitmap.Bytes(), b.length) + } else { + b.nulls++ + } + b.length++ +} + +// AppendValues will append the values in the v slice. The valid slice determines which values +// in v are valid (not null). The valid slice must either be empty or be equal in length to v. If empty, +// all values in v are appended and considered valid. +func (b *Decimal128Builder) AppendValues(v []decimal128.Num, valid []bool) { + if len(v) != len(valid) && len(valid) != 0 { + panic("len(v) != len(valid) && len(valid) != 0") + } + + if len(v) == 0 { + return + } + + b.Reserve(len(v)) + if len(v) > 0 { + arrow.Decimal128Traits.Copy(b.rawData[b.length:], v) + } + b.builder.unsafeAppendBoolsToBitmap(valid, len(v)) +} + +func (b *Decimal128Builder) init(capacity int) { + b.builder.init(capacity) + + b.data = memory.NewResizableBuffer(b.mem) + bytesN := arrow.Decimal128Traits.BytesRequired(capacity) + b.data.Resize(bytesN) + b.rawData = arrow.Decimal128Traits.CastFromBytes(b.data.Bytes()) +} + +// Reserve ensures there is enough space for appending n elements +// by checking the capacity and calling Resize if necessary. +func (b *Decimal128Builder) Reserve(n int) { + b.builder.reserve(n, b.Resize) +} + +// Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(), +// additional memory will be allocated. If n is smaller, the allocated memory may reduced. +func (b *Decimal128Builder) Resize(n int) { + nBuilder := n + if n < minBuilderCapacity { + n = minBuilderCapacity + } + + if b.capacity == 0 { + b.init(n) + } else { + b.builder.resize(nBuilder, b.init) + b.data.Resize(arrow.Decimal128Traits.BytesRequired(n)) + b.rawData = arrow.Decimal128Traits.CastFromBytes(b.data.Bytes()) + } +} + +// NewArray creates a Decimal128 array from the memory buffers used by the builder and resets the Decimal128Builder +// so it can be used to build a new array. +func (b *Decimal128Builder) NewArray() Interface { + return b.NewDecimal128Array() +} + +// NewDecimal128Array creates a Decimal128 array from the memory buffers used by the builder and resets the Decimal128Builder +// so it can be used to build a new array. +func (b *Decimal128Builder) NewDecimal128Array() (a *Decimal128) { + data := b.newData() + a = NewDecimal128Data(data) + data.Release() + return +} + +func (b *Decimal128Builder) newData() (data *Data) { + bytesRequired := arrow.Decimal128Traits.BytesRequired(b.length) + if bytesRequired > 0 && bytesRequired < b.data.Len() { + // trim buffers + b.data.Resize(bytesRequired) + } + data = NewData(b.dtype, b.length, []*memory.Buffer{b.nullBitmap, b.data}, nil, b.nulls, 0) + b.reset() + + if b.data != nil { + b.data.Release() + b.data = nil + b.rawData = nil + } + + return +} + +var ( + _ Interface = (*Decimal128)(nil) + _ Builder = (*Decimal128Builder)(nil) +) diff --git a/vendor/github.com/apache/arrow/go/arrow/array/doc.go b/vendor/github.com/apache/arrow/go/arrow/array/doc.go new file mode 100644 index 00000000000..5cf85408626 --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/array/doc.go @@ -0,0 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/* +Package array provides implementations of various Arrow array types. +*/ +package array diff --git a/vendor/github.com/apache/arrow/go/arrow/array/fixed_size_list.go b/vendor/github.com/apache/arrow/go/arrow/array/fixed_size_list.go new file mode 100644 index 00000000000..6c61513cf0e --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/array/fixed_size_list.go @@ -0,0 +1,240 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package array + +import ( + "fmt" + "strings" + "sync/atomic" + + "github.com/apache/arrow/go/arrow" + "github.com/apache/arrow/go/arrow/internal/bitutil" + "github.com/apache/arrow/go/arrow/internal/debug" + "github.com/apache/arrow/go/arrow/memory" +) + +// FixedSizeList represents an immutable sequence of N array values. +type FixedSizeList struct { + array + n int32 + values Interface +} + +// NewFixedSizeListData returns a new List array value, from data. +func NewFixedSizeListData(data *Data) *FixedSizeList { + a := &FixedSizeList{} + a.refCount = 1 + a.setData(data) + return a +} + +func (a *FixedSizeList) ListValues() Interface { return a.values } + +func (a *FixedSizeList) String() string { + o := new(strings.Builder) + o.WriteString("[") + for i := 0; i < a.Len(); i++ { + if i > 0 { + o.WriteString(" ") + } + if !a.IsValid(i) { + o.WriteString("(null)") + continue + } + sub := a.newListValue(i) + fmt.Fprintf(o, "%v", sub) + sub.Release() + } + o.WriteString("]") + return o.String() +} + +func (a *FixedSizeList) newListValue(i int) Interface { + n := int64(a.n) + off := int64(a.array.data.offset) + beg := (off + int64(i)) * n + end := (off + int64(i+1)) * n + sli := NewSlice(a.values, beg, end) + return sli +} + +func (a *FixedSizeList) setData(data *Data) { + a.array.setData(data) + a.n = a.DataType().(*arrow.FixedSizeListType).Len() + a.values = MakeFromData(data.childData[0]) +} + +func arrayEqualFixedSizeList(left, right *FixedSizeList) bool { + for i := 0; i < left.Len(); i++ { + if left.IsNull(i) { + continue + } + o := func() bool { + l := left.newListValue(i) + defer l.Release() + r := right.newListValue(i) + defer r.Release() + return ArrayEqual(l, r) + }() + if !o { + return false + } + } + return true +} + +// Len returns the number of elements in the array. +func (a *FixedSizeList) Len() int { return a.array.Len() } + +func (a *FixedSizeList) Retain() { + a.array.Retain() + a.values.Retain() +} + +func (a *FixedSizeList) Release() { + a.array.Release() + a.values.Release() +} + +type FixedSizeListBuilder struct { + builder + + etype arrow.DataType // data type of the list's elements. + n int32 // number of elements in the fixed-size list. + values Builder // value builder for the list's elements. +} + +// NewFixedSizeListBuilder returns a builder, using the provided memory allocator. +// The created list builder will create a list whose elements will be of type etype. +func NewFixedSizeListBuilder(mem memory.Allocator, n int32, etype arrow.DataType) *FixedSizeListBuilder { + return &FixedSizeListBuilder{ + builder: builder{refCount: 1, mem: mem}, + etype: etype, + n: n, + values: newBuilder(mem, etype), + } +} + +// Release decreases the reference count by 1. +// When the reference count goes to zero, the memory is freed. +func (b *FixedSizeListBuilder) Release() { + debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases") + + if atomic.AddInt64(&b.refCount, -1) == 0 { + if b.nullBitmap != nil { + b.nullBitmap.Release() + b.nullBitmap = nil + } + if b.values != nil { + b.values.Release() + b.values = nil + } + } +} + +func (b *FixedSizeListBuilder) Append(v bool) { + b.Reserve(1) + b.unsafeAppendBoolToBitmap(v) +} + +func (b *FixedSizeListBuilder) AppendNull() { + b.Reserve(1) + b.unsafeAppendBoolToBitmap(false) +} + +func (b *FixedSizeListBuilder) AppendValues(valid []bool) { + b.Reserve(len(valid)) + b.builder.unsafeAppendBoolsToBitmap(valid, len(valid)) +} + +func (b *FixedSizeListBuilder) unsafeAppend(v bool) { + bitutil.SetBit(b.nullBitmap.Bytes(), b.length) + b.length++ +} + +func (b *FixedSizeListBuilder) unsafeAppendBoolToBitmap(isValid bool) { + if isValid { + bitutil.SetBit(b.nullBitmap.Bytes(), b.length) + } else { + b.nulls++ + } + b.length++ +} + +func (b *FixedSizeListBuilder) init(capacity int) { + b.builder.init(capacity) +} + +// Reserve ensures there is enough space for appending n elements +// by checking the capacity and calling Resize if necessary. +func (b *FixedSizeListBuilder) Reserve(n int) { + b.builder.reserve(n, b.Resize) +} + +// Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(), +// additional memory will be allocated. If n is smaller, the allocated memory may reduced. +func (b *FixedSizeListBuilder) Resize(n int) { + if n < minBuilderCapacity { + n = minBuilderCapacity + } + + if b.capacity == 0 { + b.init(n) + } else { + b.builder.resize(n, b.builder.init) + } +} + +func (b *FixedSizeListBuilder) ValueBuilder() Builder { + return b.values +} + +// NewArray creates a List array from the memory buffers used by the builder and resets the FixedSizeListBuilder +// so it can be used to build a new array. +func (b *FixedSizeListBuilder) NewArray() Interface { + return b.NewListArray() +} + +// NewListArray creates a List array from the memory buffers used by the builder and resets the FixedSizeListBuilder +// so it can be used to build a new array. +func (b *FixedSizeListBuilder) NewListArray() (a *FixedSizeList) { + data := b.newData() + a = NewFixedSizeListData(data) + data.Release() + return +} + +func (b *FixedSizeListBuilder) newData() (data *Data) { + values := b.values.NewArray() + defer values.Release() + + data = NewData( + arrow.FixedSizeListOf(b.n, b.etype), b.length, + []*memory.Buffer{b.nullBitmap}, + []*Data{values.Data()}, + b.nulls, + 0, + ) + b.reset() + + return +} + +var ( + _ Interface = (*FixedSizeList)(nil) + _ Builder = (*FixedSizeListBuilder)(nil) +) diff --git a/vendor/github.com/apache/arrow/go/arrow/array/fixedsize_binary.go b/vendor/github.com/apache/arrow/go/arrow/array/fixedsize_binary.go new file mode 100644 index 00000000000..502fb99e511 --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/array/fixedsize_binary.go @@ -0,0 +1,95 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package array + +import ( + "bytes" + "fmt" + "strings" + + "github.com/apache/arrow/go/arrow" +) + +// A type which represents an immutable sequence of fixed-length binary strings. +type FixedSizeBinary struct { + array + + valueBytes []byte + bytewidth int32 +} + +// NewFixedSizeBinaryData constructs a new fixed-size binary array from data. +func NewFixedSizeBinaryData(data *Data) *FixedSizeBinary { + a := &FixedSizeBinary{bytewidth: int32(data.DataType().(arrow.FixedWidthDataType).BitWidth() / 8)} + a.refCount = 1 + a.setData(data) + return a +} + +// Value returns the fixed-size slice at index i. This value should not be mutated. +func (a *FixedSizeBinary) Value(i int) []byte { + i += a.array.data.offset + var ( + bw = int(a.bytewidth) + beg = i * bw + end = (i + 1) * bw + ) + return a.valueBytes[beg:end] +} + +func (a *FixedSizeBinary) String() string { + o := new(strings.Builder) + o.WriteString("[") + for i := 0; i < a.Len(); i++ { + if i > 0 { + o.WriteString(" ") + } + switch { + case a.IsNull(i): + o.WriteString("(null)") + default: + fmt.Fprintf(o, "%q", a.Value(i)) + } + } + o.WriteString("]") + return o.String() +} + +func (a *FixedSizeBinary) setData(data *Data) { + a.array.setData(data) + vals := data.buffers[1] + if vals != nil { + a.valueBytes = vals.Bytes() + } + +} + +func arrayEqualFixedSizeBinary(left, right *FixedSizeBinary) bool { + for i := 0; i < left.Len(); i++ { + if left.IsNull(i) { + continue + } + if bytes.Compare(left.Value(i), right.Value(i)) != 0 { + return false + } + } + return true +} + +var ( + _ Interface = (*FixedSizeBinary)(nil) +) diff --git a/vendor/github.com/apache/arrow/go/arrow/array/fixedsize_binarybuilder.go b/vendor/github.com/apache/arrow/go/arrow/array/fixedsize_binarybuilder.go new file mode 100644 index 00000000000..8a2f65f59c6 --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/array/fixedsize_binarybuilder.go @@ -0,0 +1,154 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package array + +import ( + "fmt" + "sync/atomic" + + "github.com/apache/arrow/go/arrow" + "github.com/apache/arrow/go/arrow/internal/debug" + "github.com/apache/arrow/go/arrow/memory" +) + +// A FixedSizeBinaryBuilder is used to build a FixedSizeBinary array using the Append methods. +type FixedSizeBinaryBuilder struct { + builder + + dtype *arrow.FixedSizeBinaryType + values *byteBufferBuilder +} + +func NewFixedSizeBinaryBuilder(mem memory.Allocator, dtype *arrow.FixedSizeBinaryType) *FixedSizeBinaryBuilder { + b := &FixedSizeBinaryBuilder{ + builder: builder{refCount: 1, mem: mem}, + dtype: dtype, + values: newByteBufferBuilder(mem), + } + return b +} + +// Release decreases the reference count by 1. +// When the reference count goes to zero, the memory is freed. +// Release may be called simultaneously from multiple goroutines. +func (b *FixedSizeBinaryBuilder) Release() { + debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases") + + if atomic.AddInt64(&b.refCount, -1) == 0 { + if b.nullBitmap != nil { + b.nullBitmap.Release() + b.nullBitmap = nil + } + if b.values != nil { + b.values.Release() + b.values = nil + } + } +} + +func (b *FixedSizeBinaryBuilder) Append(v []byte) { + if len(v) != b.dtype.ByteWidth { + // TODO(alexandre): should we return an error instead? + panic("len(v) != b.dtype.ByteWidth") + } + + b.Reserve(1) + b.values.Append(v) + b.UnsafeAppendBoolToBitmap(true) +} + +func (b *FixedSizeBinaryBuilder) AppendNull() { + b.Reserve(1) + b.values.Advance(b.dtype.ByteWidth) + b.UnsafeAppendBoolToBitmap(false) +} + +// AppendValues will append the values in the v slice. The valid slice determines which values +// in v are valid (not null). The valid slice must either be empty or be equal in length to v. If empty, +// all values in v are appended and considered valid. +func (b *FixedSizeBinaryBuilder) AppendValues(v [][]byte, valid []bool) { + if len(v) != len(valid) && len(valid) != 0 { + panic("len(v) != len(valid) && len(valid) != 0") + } + + if len(v) == 0 { + return + } + + b.Reserve(len(v)) + for _, vv := range v { + switch len(vv) { + case 0: + b.values.Advance(b.dtype.ByteWidth) + case b.dtype.ByteWidth: + b.values.Append(vv) + default: + panic(fmt.Errorf("array: invalid binary length (got=%d, want=%d)", len(vv), b.dtype.ByteWidth)) + } + } + + b.builder.unsafeAppendBoolsToBitmap(valid, len(v)) +} + +func (b *FixedSizeBinaryBuilder) init(capacity int) { + b.builder.init(capacity) + b.values.resize(capacity * b.dtype.ByteWidth) +} + +// Reserve ensures there is enough space for appending n elements +// by checking the capacity and calling Resize if necessary. +func (b *FixedSizeBinaryBuilder) Reserve(n int) { + b.builder.reserve(n, b.Resize) +} + +// Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(), +// additional memory will be allocated. If n is smaller, the allocated memory may reduced. +func (b *FixedSizeBinaryBuilder) Resize(n int) { + b.builder.resize(n, b.init) +} + +// NewArray creates a FixedSizeBinary array from the memory buffers used by the +// builder and resets the FixedSizeBinaryBuilder so it can be used to build a new array. +func (b *FixedSizeBinaryBuilder) NewArray() Interface { + return b.NewFixedSizeBinaryArray() +} + +// NewFixedSizeBinaryArray creates a FixedSizeBinary array from the memory buffers used by the builder and resets the FixedSizeBinaryBuilder +// so it can be used to build a new array. +func (b *FixedSizeBinaryBuilder) NewFixedSizeBinaryArray() (a *FixedSizeBinary) { + data := b.newData() + a = NewFixedSizeBinaryData(data) + data.Release() + return +} + +func (b *FixedSizeBinaryBuilder) newData() (data *Data) { + values := b.values.Finish() + data = NewData(b.dtype, b.length, []*memory.Buffer{b.nullBitmap, values}, nil, b.nulls, 0) + + if values != nil { + values.Release() + } + + b.builder.reset() + + return +} + +var ( + _ Builder = (*FixedSizeBinaryBuilder)(nil) +) diff --git a/vendor/github.com/apache/arrow/go/arrow/array/float16.go b/vendor/github.com/apache/arrow/go/arrow/array/float16.go new file mode 100644 index 00000000000..931e2d96c07 --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/array/float16.go @@ -0,0 +1,87 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package array + +import ( + "fmt" + "strings" + + "github.com/apache/arrow/go/arrow" + "github.com/apache/arrow/go/arrow/float16" +) + +// A type which represents an immutable sequence of Float16 values. +type Float16 struct { + array + values []float16.Num +} + +func NewFloat16Data(data *Data) *Float16 { + a := &Float16{} + a.refCount = 1 + a.setData(data) + return a +} + +func (a *Float16) Value(i int) float16.Num { return a.values[i] } + +func (a *Float16) Values() []float16.Num { return a.values } + +func (a *Float16) String() string { + o := new(strings.Builder) + o.WriteString("[") + for i := 0; i < a.Len(); i++ { + if i > 0 { + fmt.Fprintf(o, " ") + } + switch { + case a.IsNull(i): + o.WriteString("(null)") + default: + fmt.Fprintf(o, "%v", a.values[i].Float32()) + } + } + o.WriteString("]") + return o.String() +} + +func (a *Float16) setData(data *Data) { + a.array.setData(data) + vals := data.buffers[1] + if vals != nil { + a.values = arrow.Float16Traits.CastFromBytes(vals.Bytes()) + beg := a.array.data.offset + end := beg + a.array.data.length + a.values = a.values[beg:end] + } +} + +func arrayEqualFloat16(left, right *Float16) bool { + for i := 0; i < left.Len(); i++ { + if left.IsNull(i) { + continue + } + if left.Value(i) != right.Value(i) { + return false + } + } + return true +} + +var ( + _ Interface = (*Float16)(nil) +) diff --git a/vendor/github.com/apache/arrow/go/arrow/array/float16_builder.go b/vendor/github.com/apache/arrow/go/arrow/array/float16_builder.go new file mode 100644 index 00000000000..9c36dd172a1 --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/array/float16_builder.go @@ -0,0 +1,165 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package array + +import ( + "sync/atomic" + + "github.com/apache/arrow/go/arrow" + "github.com/apache/arrow/go/arrow/float16" + "github.com/apache/arrow/go/arrow/internal/bitutil" + "github.com/apache/arrow/go/arrow/internal/debug" + "github.com/apache/arrow/go/arrow/memory" +) + +type Float16Builder struct { + builder + + data *memory.Buffer + rawData []float16.Num +} + +func NewFloat16Builder(mem memory.Allocator) *Float16Builder { + return &Float16Builder{builder: builder{refCount: 1, mem: mem}} +} + +// Release decreases the reference count by 1. +// When the reference count goes to zero, the memory is freed. +func (b *Float16Builder) Release() { + debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases") + + if atomic.AddInt64(&b.refCount, -1) == 0 { + if b.nullBitmap != nil { + b.nullBitmap.Release() + b.nullBitmap = nil + } + if b.data != nil { + b.data.Release() + b.data = nil + b.rawData = nil + } + } +} + +func (b *Float16Builder) Append(v float16.Num) { + b.Reserve(1) + b.UnsafeAppend(v) +} + +func (b *Float16Builder) UnsafeAppend(v float16.Num) { + bitutil.SetBit(b.nullBitmap.Bytes(), b.length) + b.rawData[b.length] = v + b.length++ +} + +func (b *Float16Builder) AppendNull() { + b.Reserve(1) + b.UnsafeAppendBoolToBitmap(false) +} + +func (b *Float16Builder) UnsafeAppendBoolToBitmap(isValid bool) { + if isValid { + bitutil.SetBit(b.nullBitmap.Bytes(), b.length) + } else { + b.nulls++ + } + b.length++ +} + +// AppendValues will append the values in the v slice. The valid slice determines which values +// in v are valid (not null). The valid slice must either be empty or be equal in length to v. If empty, +// all values in v are appended and considered valid. +func (b *Float16Builder) AppendValues(v []float16.Num, valid []bool) { + if len(v) != len(valid) && len(valid) != 0 { + panic("len(v) != len(valid) && len(valid) != 0") + } + + if len(v) == 0 { + return + } + + b.Reserve(len(v)) + if len(v) > 0 { + arrow.Float16Traits.Copy(b.rawData[b.length:], v) + } + b.builder.unsafeAppendBoolsToBitmap(valid, len(v)) +} + +func (b *Float16Builder) init(capacity int) { + b.builder.init(capacity) + + b.data = memory.NewResizableBuffer(b.mem) + bytesN := arrow.Uint16Traits.BytesRequired(capacity) + b.data.Resize(bytesN) + b.rawData = arrow.Float16Traits.CastFromBytes(b.data.Bytes()) +} + +// Reserve ensures there is enough space for appending n elements +// by checking the capacity and calling Resize if necessary. +func (b *Float16Builder) Reserve(n int) { + b.builder.reserve(n, b.Resize) +} + +// Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(), +// additional memory will be allocated. If n is smaller, the allocated memory may reduced. +func (b *Float16Builder) Resize(n int) { + nBuilder := n + if n < minBuilderCapacity { + n = minBuilderCapacity + } + + if b.capacity == 0 { + b.init(n) + } else { + b.builder.resize(nBuilder, b.init) + b.data.Resize(arrow.Float16Traits.BytesRequired(n)) + b.rawData = arrow.Float16Traits.CastFromBytes(b.data.Bytes()) + } +} + +// NewArray creates a Float16 array from the memory buffers used by the builder and resets the Float16Builder +// so it can be used to build a new array. +func (b *Float16Builder) NewArray() Interface { + return b.NewFloat16Array() +} + +// NewFloat16Array creates a Float16 array from the memory buffers used by the builder and resets the Float16Builder +// so it can be used to build a new array. +func (b *Float16Builder) NewFloat16Array() (a *Float16) { + data := b.newData() + a = NewFloat16Data(data) + data.Release() + return +} + +func (b *Float16Builder) newData() (data *Data) { + bytesRequired := arrow.Float16Traits.BytesRequired(b.length) + if bytesRequired > 0 && bytesRequired < b.data.Len() { + // trim buffers + b.data.Resize(bytesRequired) + } + data = NewData(arrow.FixedWidthTypes.Float16, b.length, []*memory.Buffer{b.nullBitmap, b.data}, nil, b.nulls, 0) + b.reset() + + if b.data != nil { + b.data.Release() + b.data = nil + b.rawData = nil + } + + return +} diff --git a/vendor/github.com/apache/arrow/go/arrow/array/interval.go b/vendor/github.com/apache/arrow/go/arrow/array/interval.go new file mode 100644 index 00000000000..21efd6e4078 --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/array/interval.go @@ -0,0 +1,434 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package array // import "github.com/apache/arrow/go/arrow/array" + +import ( + "fmt" + "strings" + "sync/atomic" + + "github.com/apache/arrow/go/arrow" + "github.com/apache/arrow/go/arrow/internal/bitutil" + "github.com/apache/arrow/go/arrow/internal/debug" + "github.com/apache/arrow/go/arrow/memory" + "github.com/pkg/errors" +) + +func NewIntervalData(data *Data) Interface { + switch data.dtype.(type) { + case *arrow.MonthIntervalType: + return NewMonthIntervalData(data) + case *arrow.DayTimeIntervalType: + return NewDayTimeIntervalData(data) + default: + panic(errors.Errorf("arrow/array: unknown interval data type %T", data.dtype)) + } +} + +// A type which represents an immutable sequence of arrow.MonthInterval values. +type MonthInterval struct { + array + values []arrow.MonthInterval +} + +func NewMonthIntervalData(data *Data) *MonthInterval { + a := &MonthInterval{} + a.refCount = 1 + a.setData(data) + return a +} + +func (a *MonthInterval) Value(i int) arrow.MonthInterval { return a.values[i] } +func (a *MonthInterval) MonthIntervalValues() []arrow.MonthInterval { return a.values } + +func (a *MonthInterval) String() string { + o := new(strings.Builder) + o.WriteString("[") + for i, v := range a.values { + if i > 0 { + fmt.Fprintf(o, " ") + } + switch { + case a.IsNull(i): + o.WriteString("(null)") + default: + fmt.Fprintf(o, "%v", v) + } + } + o.WriteString("]") + return o.String() +} + +func (a *MonthInterval) setData(data *Data) { + a.array.setData(data) + vals := data.buffers[1] + if vals != nil { + a.values = arrow.MonthIntervalTraits.CastFromBytes(vals.Bytes()) + beg := a.array.data.offset + end := beg + a.array.data.length + a.values = a.values[beg:end] + } +} + +func arrayEqualMonthInterval(left, right *MonthInterval) bool { + for i := 0; i < left.Len(); i++ { + if left.IsNull(i) { + continue + } + if left.Value(i) != right.Value(i) { + return false + } + } + return true +} + +type MonthIntervalBuilder struct { + builder + + data *memory.Buffer + rawData []arrow.MonthInterval +} + +func NewMonthIntervalBuilder(mem memory.Allocator) *MonthIntervalBuilder { + return &MonthIntervalBuilder{builder: builder{refCount: 1, mem: mem}} +} + +// Release decreases the reference count by 1. +// When the reference count goes to zero, the memory is freed. +func (b *MonthIntervalBuilder) Release() { + debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases") + + if atomic.AddInt64(&b.refCount, -1) == 0 { + if b.nullBitmap != nil { + b.nullBitmap.Release() + b.nullBitmap = nil + } + if b.data != nil { + b.data.Release() + b.data = nil + b.rawData = nil + } + } +} + +func (b *MonthIntervalBuilder) Append(v arrow.MonthInterval) { + b.Reserve(1) + b.UnsafeAppend(v) +} + +func (b *MonthIntervalBuilder) AppendNull() { + b.Reserve(1) + b.UnsafeAppendBoolToBitmap(false) +} + +func (b *MonthIntervalBuilder) UnsafeAppend(v arrow.MonthInterval) { + bitutil.SetBit(b.nullBitmap.Bytes(), b.length) + b.rawData[b.length] = v + b.length++ +} + +func (b *MonthIntervalBuilder) UnsafeAppendBoolToBitmap(isValid bool) { + if isValid { + bitutil.SetBit(b.nullBitmap.Bytes(), b.length) + } else { + b.nulls++ + } + b.length++ +} + +// AppendValues will append the values in the v slice. The valid slice determines which values +// in v are valid (not null). The valid slice must either be empty or be equal in length to v. If empty, +// all values in v are appended and considered valid. +func (b *MonthIntervalBuilder) AppendValues(v []arrow.MonthInterval, valid []bool) { + if len(v) != len(valid) && len(valid) != 0 { + panic("len(v) != len(valid) && len(valid) != 0") + } + + if len(v) == 0 { + return + } + + b.Reserve(len(v)) + arrow.MonthIntervalTraits.Copy(b.rawData[b.length:], v) + b.builder.unsafeAppendBoolsToBitmap(valid, len(v)) +} + +func (b *MonthIntervalBuilder) init(capacity int) { + b.builder.init(capacity) + + b.data = memory.NewResizableBuffer(b.mem) + bytesN := arrow.MonthIntervalTraits.BytesRequired(capacity) + b.data.Resize(bytesN) + b.rawData = arrow.MonthIntervalTraits.CastFromBytes(b.data.Bytes()) +} + +// Reserve ensures there is enough space for appending n elements +// by checking the capacity and calling Resize if necessary. +func (b *MonthIntervalBuilder) Reserve(n int) { + b.builder.reserve(n, b.Resize) +} + +// Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(), +// additional memory will be allocated. If n is smaller, the allocated memory may reduced. +func (b *MonthIntervalBuilder) Resize(n int) { + nBuilder := n + if n < minBuilderCapacity { + n = minBuilderCapacity + } + + if b.capacity == 0 { + b.init(n) + } else { + b.builder.resize(nBuilder, b.init) + b.data.Resize(arrow.MonthIntervalTraits.BytesRequired(n)) + b.rawData = arrow.MonthIntervalTraits.CastFromBytes(b.data.Bytes()) + } +} + +// NewArray creates a MonthInterval array from the memory buffers used by the builder and resets the MonthIntervalBuilder +// so it can be used to build a new array. +func (b *MonthIntervalBuilder) NewArray() Interface { + return b.NewMonthIntervalArray() +} + +// NewMonthIntervalArray creates a MonthInterval array from the memory buffers used by the builder and resets the MonthIntervalBuilder +// so it can be used to build a new array. +func (b *MonthIntervalBuilder) NewMonthIntervalArray() (a *MonthInterval) { + data := b.newData() + a = NewMonthIntervalData(data) + data.Release() + return +} + +func (b *MonthIntervalBuilder) newData() (data *Data) { + bytesRequired := arrow.MonthIntervalTraits.BytesRequired(b.length) + if bytesRequired > 0 && bytesRequired < b.data.Len() { + // trim buffers + b.data.Resize(bytesRequired) + } + data = NewData(arrow.FixedWidthTypes.MonthInterval, b.length, []*memory.Buffer{b.nullBitmap, b.data}, nil, b.nulls, 0) + b.reset() + + if b.data != nil { + b.data.Release() + b.data = nil + b.rawData = nil + } + + return +} + +// A type which represents an immutable sequence of arrow.DayTimeInterval values. +type DayTimeInterval struct { + array + values []arrow.DayTimeInterval +} + +func NewDayTimeIntervalData(data *Data) *DayTimeInterval { + a := &DayTimeInterval{} + a.refCount = 1 + a.setData(data) + return a +} + +func (a *DayTimeInterval) Value(i int) arrow.DayTimeInterval { return a.values[i] } +func (a *DayTimeInterval) DayTimeIntervalValues() []arrow.DayTimeInterval { return a.values } + +func (a *DayTimeInterval) String() string { + o := new(strings.Builder) + o.WriteString("[") + for i, v := range a.values { + if i > 0 { + fmt.Fprintf(o, " ") + } + switch { + case a.IsNull(i): + o.WriteString("(null)") + default: + fmt.Fprintf(o, "%v", v) + } + } + o.WriteString("]") + return o.String() +} + +func (a *DayTimeInterval) setData(data *Data) { + a.array.setData(data) + vals := data.buffers[1] + if vals != nil { + a.values = arrow.DayTimeIntervalTraits.CastFromBytes(vals.Bytes()) + beg := a.array.data.offset + end := beg + a.array.data.length + a.values = a.values[beg:end] + } +} + +func arrayEqualDayTimeInterval(left, right *DayTimeInterval) bool { + for i := 0; i < left.Len(); i++ { + if left.IsNull(i) { + continue + } + if left.Value(i) != right.Value(i) { + return false + } + } + return true +} + +type DayTimeIntervalBuilder struct { + builder + + data *memory.Buffer + rawData []arrow.DayTimeInterval +} + +func NewDayTimeIntervalBuilder(mem memory.Allocator) *DayTimeIntervalBuilder { + return &DayTimeIntervalBuilder{builder: builder{refCount: 1, mem: mem}} +} + +// Release decreases the reference count by 1. +// When the reference count goes to zero, the memory is freed. +func (b *DayTimeIntervalBuilder) Release() { + debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases") + + if atomic.AddInt64(&b.refCount, -1) == 0 { + if b.nullBitmap != nil { + b.nullBitmap.Release() + b.nullBitmap = nil + } + if b.data != nil { + b.data.Release() + b.data = nil + b.rawData = nil + } + } +} + +func (b *DayTimeIntervalBuilder) Append(v arrow.DayTimeInterval) { + b.Reserve(1) + b.UnsafeAppend(v) +} + +func (b *DayTimeIntervalBuilder) AppendNull() { + b.Reserve(1) + b.UnsafeAppendBoolToBitmap(false) +} + +func (b *DayTimeIntervalBuilder) UnsafeAppend(v arrow.DayTimeInterval) { + bitutil.SetBit(b.nullBitmap.Bytes(), b.length) + b.rawData[b.length] = v + b.length++ +} + +func (b *DayTimeIntervalBuilder) UnsafeAppendBoolToBitmap(isValid bool) { + if isValid { + bitutil.SetBit(b.nullBitmap.Bytes(), b.length) + } else { + b.nulls++ + } + b.length++ +} + +// AppendValues will append the values in the v slice. The valid slice determines which values +// in v are valid (not null). The valid slice must either be empty or be equal in length to v. If empty, +// all values in v are appended and considered valid. +func (b *DayTimeIntervalBuilder) AppendValues(v []arrow.DayTimeInterval, valid []bool) { + if len(v) != len(valid) && len(valid) != 0 { + panic("len(v) != len(valid) && len(valid) != 0") + } + + if len(v) == 0 { + return + } + + b.Reserve(len(v)) + arrow.DayTimeIntervalTraits.Copy(b.rawData[b.length:], v) + b.builder.unsafeAppendBoolsToBitmap(valid, len(v)) +} + +func (b *DayTimeIntervalBuilder) init(capacity int) { + b.builder.init(capacity) + + b.data = memory.NewResizableBuffer(b.mem) + bytesN := arrow.DayTimeIntervalTraits.BytesRequired(capacity) + b.data.Resize(bytesN) + b.rawData = arrow.DayTimeIntervalTraits.CastFromBytes(b.data.Bytes()) +} + +// Reserve ensures there is enough space for appending n elements +// by checking the capacity and calling Resize if necessary. +func (b *DayTimeIntervalBuilder) Reserve(n int) { + b.builder.reserve(n, b.Resize) +} + +// Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(), +// additional memory will be allocated. If n is smaller, the allocated memory may reduced. +func (b *DayTimeIntervalBuilder) Resize(n int) { + nBuilder := n + if n < minBuilderCapacity { + n = minBuilderCapacity + } + + if b.capacity == 0 { + b.init(n) + } else { + b.builder.resize(nBuilder, b.init) + b.data.Resize(arrow.DayTimeIntervalTraits.BytesRequired(n)) + b.rawData = arrow.DayTimeIntervalTraits.CastFromBytes(b.data.Bytes()) + } +} + +// NewArray creates a DayTimeInterval array from the memory buffers used by the builder and resets the DayTimeIntervalBuilder +// so it can be used to build a new array. +func (b *DayTimeIntervalBuilder) NewArray() Interface { + return b.NewDayTimeIntervalArray() +} + +// NewDayTimeIntervalArray creates a DayTimeInterval array from the memory buffers used by the builder and resets the DayTimeIntervalBuilder +// so it can be used to build a new array. +func (b *DayTimeIntervalBuilder) NewDayTimeIntervalArray() (a *DayTimeInterval) { + data := b.newData() + a = NewDayTimeIntervalData(data) + data.Release() + return +} + +func (b *DayTimeIntervalBuilder) newData() (data *Data) { + bytesRequired := arrow.DayTimeIntervalTraits.BytesRequired(b.length) + if bytesRequired > 0 && bytesRequired < b.data.Len() { + // trim buffers + b.data.Resize(bytesRequired) + } + data = NewData(arrow.FixedWidthTypes.DayTimeInterval, b.length, []*memory.Buffer{b.nullBitmap, b.data}, nil, b.nulls, 0) + b.reset() + + if b.data != nil { + b.data.Release() + b.data = nil + b.rawData = nil + } + + return +} + +var ( + _ Interface = (*MonthInterval)(nil) + _ Interface = (*DayTimeInterval)(nil) + + _ Builder = (*MonthIntervalBuilder)(nil) + _ Builder = (*DayTimeIntervalBuilder)(nil) +) diff --git a/vendor/github.com/apache/arrow/go/arrow/array/list.go b/vendor/github.com/apache/arrow/go/arrow/array/list.go new file mode 100644 index 00000000000..2fccdeadb94 --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/array/list.go @@ -0,0 +1,264 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package array + +import ( + "fmt" + "strings" + "sync/atomic" + + "github.com/apache/arrow/go/arrow" + "github.com/apache/arrow/go/arrow/internal/bitutil" + "github.com/apache/arrow/go/arrow/internal/debug" + "github.com/apache/arrow/go/arrow/memory" +) + +// List represents an immutable sequence of array values. +type List struct { + array + values Interface + offsets []int32 +} + +// NewListData returns a new List array value, from data. +func NewListData(data *Data) *List { + a := &List{} + a.refCount = 1 + a.setData(data) + return a +} + +func (a *List) ListValues() Interface { return a.values } + +func (a *List) String() string { + o := new(strings.Builder) + o.WriteString("[") + for i := 0; i < a.Len(); i++ { + if i > 0 { + o.WriteString(" ") + } + if !a.IsValid(i) { + o.WriteString("(null)") + continue + } + sub := a.newListValue(i) + fmt.Fprintf(o, "%v", sub) + sub.Release() + } + o.WriteString("]") + return o.String() +} + +func (a *List) newListValue(i int) Interface { + j := i + a.array.data.offset + beg := int64(a.offsets[j]) + end := int64(a.offsets[j+1]) + return NewSlice(a.values, beg, end) +} + +func (a *List) setData(data *Data) { + a.array.setData(data) + vals := data.buffers[1] + if vals != nil { + a.offsets = arrow.Int32Traits.CastFromBytes(vals.Bytes()) + } + a.values = MakeFromData(data.childData[0]) +} + +func arrayEqualList(left, right *List) bool { + for i := 0; i < left.Len(); i++ { + if left.IsNull(i) { + continue + } + o := func() bool { + l := left.newListValue(i) + defer l.Release() + r := right.newListValue(i) + defer r.Release() + return ArrayEqual(l, r) + }() + if !o { + return false + } + } + return true +} + +// Len returns the number of elements in the array. +func (a *List) Len() int { return a.array.Len() } + +func (a *List) Offsets() []int32 { return a.offsets } + +func (a *List) Retain() { + a.array.Retain() + a.values.Retain() +} + +func (a *List) Release() { + a.array.Release() + a.values.Release() +} + +type ListBuilder struct { + builder + + etype arrow.DataType // data type of the list's elements. + values Builder // value builder for the list's elements. + offsets *Int32Builder +} + +// NewListBuilder returns a builder, using the provided memory allocator. +// The created list builder will create a list whose elements will be of type etype. +func NewListBuilder(mem memory.Allocator, etype arrow.DataType) *ListBuilder { + return &ListBuilder{ + builder: builder{refCount: 1, mem: mem}, + etype: etype, + values: newBuilder(mem, etype), + offsets: NewInt32Builder(mem), + } +} + +// Release decreases the reference count by 1. +// When the reference count goes to zero, the memory is freed. +func (b *ListBuilder) Release() { + debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases") + + if atomic.AddInt64(&b.refCount, -1) == 0 { + if b.nullBitmap != nil { + b.nullBitmap.Release() + b.nullBitmap = nil + } + } + + b.values.Release() + b.offsets.Release() +} + +func (b *ListBuilder) appendNextOffset() { + b.offsets.Append(int32(b.values.Len())) +} + +func (b *ListBuilder) Append(v bool) { + b.Reserve(1) + b.unsafeAppendBoolToBitmap(v) + b.appendNextOffset() +} + +func (b *ListBuilder) AppendNull() { + b.Reserve(1) + b.unsafeAppendBoolToBitmap(false) + b.appendNextOffset() +} + +func (b *ListBuilder) AppendValues(offsets []int32, valid []bool) { + b.Reserve(len(valid)) + b.offsets.AppendValues(offsets, nil) + b.builder.unsafeAppendBoolsToBitmap(valid, len(valid)) +} + +func (b *ListBuilder) unsafeAppend(v bool) { + bitutil.SetBit(b.nullBitmap.Bytes(), b.length) + b.length++ +} + +func (b *ListBuilder) unsafeAppendBoolToBitmap(isValid bool) { + if isValid { + bitutil.SetBit(b.nullBitmap.Bytes(), b.length) + } else { + b.nulls++ + } + b.length++ +} + +func (b *ListBuilder) init(capacity int) { + b.builder.init(capacity) + b.offsets.init(capacity + 1) +} + +// Reserve ensures there is enough space for appending n elements +// by checking the capacity and calling Resize if necessary. +func (b *ListBuilder) Reserve(n int) { + b.builder.reserve(n, b.Resize) +} + +// Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(), +// additional memory will be allocated. If n is smaller, the allocated memory may reduced. +func (b *ListBuilder) Resize(n int) { + if n < minBuilderCapacity { + n = minBuilderCapacity + } + + if b.capacity == 0 { + b.init(n) + } else { + b.builder.resize(n, b.builder.init) + b.offsets.resize(n+1, b.offsets.init) + } +} + +func (b *ListBuilder) ValueBuilder() Builder { + return b.values +} + +// NewArray creates a List array from the memory buffers used by the builder and resets the ListBuilder +// so it can be used to build a new array. +func (b *ListBuilder) NewArray() Interface { + return b.NewListArray() +} + +// NewListArray creates a List array from the memory buffers used by the builder and resets the ListBuilder +// so it can be used to build a new array. +func (b *ListBuilder) NewListArray() (a *List) { + if b.offsets.Len() != b.length+1 { + b.appendNextOffset() + } + data := b.newData() + a = NewListData(data) + data.Release() + return +} + +func (b *ListBuilder) newData() (data *Data) { + values := b.values.NewArray() + defer values.Release() + + var offsets *memory.Buffer + if b.offsets != nil { + arr := b.offsets.NewInt32Array() + defer arr.Release() + offsets = arr.Data().buffers[1] + } + + data = NewData( + arrow.ListOf(b.etype), b.length, + []*memory.Buffer{ + b.nullBitmap, + offsets, + }, + []*Data{values.Data()}, + b.nulls, + 0, + ) + b.reset() + + return +} + +var ( + _ Interface = (*List)(nil) + _ Builder = (*ListBuilder)(nil) +) diff --git a/vendor/github.com/apache/arrow/go/arrow/array/null.go b/vendor/github.com/apache/arrow/go/arrow/array/null.go new file mode 100644 index 00000000000..346445d5bec --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/array/null.go @@ -0,0 +1,126 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package array + +import ( + "sync/atomic" + + "github.com/apache/arrow/go/arrow" + "github.com/apache/arrow/go/arrow/internal/debug" + "github.com/apache/arrow/go/arrow/memory" +) + +// Null represents an immutable, degenerate array with no physical storage. +type Null struct { + array +} + +// NewNull returns a new Null array value of size n. +func NewNull(n int) *Null { + a := &Null{} + a.refCount = 1 + data := NewData( + arrow.Null, n, + []*memory.Buffer{nil}, + nil, + n, + 0, + ) + a.setData(data) + data.Release() + return a +} + +// NewNullData returns a new Null array value, from data. +func NewNullData(data *Data) *Null { + a := &Null{} + a.refCount = 1 + a.setData(data) + return a +} + +func (a *Null) setData(data *Data) { + a.array.setData(data) + a.array.nullBitmapBytes = nil + a.array.data.nulls = a.array.data.length +} + +type NullBuilder struct { + builder +} + +// NewNullBuilder returns a builder, using the provided memory allocator. +func NewNullBuilder(mem memory.Allocator) *NullBuilder { + return &NullBuilder{builder: builder{refCount: 1, mem: mem}} +} + +// Release decreases the reference count by 1. +// When the reference count goes to zero, the memory is freed. +func (b *NullBuilder) Release() { + debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases") + + if atomic.AddInt64(&b.refCount, -1) == 0 { + if b.nullBitmap != nil { + b.nullBitmap.Release() + b.nullBitmap = nil + } + } +} + +func (b *NullBuilder) AppendNull() { + b.builder.length++ + b.builder.nulls++ +} + +func (*NullBuilder) Reserve(size int) {} +func (*NullBuilder) Resize(size int) {} + +func (*NullBuilder) init(cap int) {} +func (*NullBuilder) resize(newBits int, init func(int)) {} + +// NewArray creates a Null array from the memory buffers used by the builder and resets the NullBuilder +// so it can be used to build a new array. +func (b *NullBuilder) NewArray() Interface { + return b.NewNullArray() +} + +// NewNullArray creates a Null array from the memory buffers used by the builder and resets the NullBuilder +// so it can be used to build a new array. +func (b *NullBuilder) NewNullArray() (a *Null) { + data := b.newData() + a = NewNullData(data) + data.Release() + return +} + +func (b *NullBuilder) newData() (data *Data) { + data = NewData( + arrow.Null, b.length, + []*memory.Buffer{nil}, + nil, + b.nulls, + 0, + ) + b.reset() + + return +} + +var ( + _ Interface = (*Null)(nil) + _ Builder = (*NullBuilder)(nil) +) diff --git a/vendor/github.com/apache/arrow/go/arrow/array/numeric.gen.go b/vendor/github.com/apache/arrow/go/arrow/array/numeric.gen.go new file mode 100644 index 00000000000..21c4e4b5e81 --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/array/numeric.gen.go @@ -0,0 +1,938 @@ +// Code generated by array/numeric.gen.go.tmpl. DO NOT EDIT. + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package array + +import ( + "fmt" + "strings" + + "github.com/apache/arrow/go/arrow" +) + +// A type which represents an immutable sequence of int64 values. +type Int64 struct { + array + values []int64 +} + +func NewInt64Data(data *Data) *Int64 { + a := &Int64{} + a.refCount = 1 + a.setData(data) + return a +} + +func (a *Int64) Value(i int) int64 { return a.values[i] } +func (a *Int64) Int64Values() []int64 { return a.values } + +func (a *Int64) String() string { + o := new(strings.Builder) + o.WriteString("[") + for i, v := range a.values { + if i > 0 { + fmt.Fprintf(o, " ") + } + switch { + case a.IsNull(i): + o.WriteString("(null)") + default: + fmt.Fprintf(o, "%v", v) + } + } + o.WriteString("]") + return o.String() +} + +func (a *Int64) setData(data *Data) { + a.array.setData(data) + vals := data.buffers[1] + if vals != nil { + a.values = arrow.Int64Traits.CastFromBytes(vals.Bytes()) + beg := a.array.data.offset + end := beg + a.array.data.length + a.values = a.values[beg:end] + } +} + +func arrayEqualInt64(left, right *Int64) bool { + for i := 0; i < left.Len(); i++ { + if left.IsNull(i) { + continue + } + if left.Value(i) != right.Value(i) { + return false + } + } + return true +} + +// A type which represents an immutable sequence of uint64 values. +type Uint64 struct { + array + values []uint64 +} + +func NewUint64Data(data *Data) *Uint64 { + a := &Uint64{} + a.refCount = 1 + a.setData(data) + return a +} + +func (a *Uint64) Value(i int) uint64 { return a.values[i] } +func (a *Uint64) Uint64Values() []uint64 { return a.values } + +func (a *Uint64) String() string { + o := new(strings.Builder) + o.WriteString("[") + for i, v := range a.values { + if i > 0 { + fmt.Fprintf(o, " ") + } + switch { + case a.IsNull(i): + o.WriteString("(null)") + default: + fmt.Fprintf(o, "%v", v) + } + } + o.WriteString("]") + return o.String() +} + +func (a *Uint64) setData(data *Data) { + a.array.setData(data) + vals := data.buffers[1] + if vals != nil { + a.values = arrow.Uint64Traits.CastFromBytes(vals.Bytes()) + beg := a.array.data.offset + end := beg + a.array.data.length + a.values = a.values[beg:end] + } +} + +func arrayEqualUint64(left, right *Uint64) bool { + for i := 0; i < left.Len(); i++ { + if left.IsNull(i) { + continue + } + if left.Value(i) != right.Value(i) { + return false + } + } + return true +} + +// A type which represents an immutable sequence of float64 values. +type Float64 struct { + array + values []float64 +} + +func NewFloat64Data(data *Data) *Float64 { + a := &Float64{} + a.refCount = 1 + a.setData(data) + return a +} + +func (a *Float64) Value(i int) float64 { return a.values[i] } +func (a *Float64) Float64Values() []float64 { return a.values } + +func (a *Float64) String() string { + o := new(strings.Builder) + o.WriteString("[") + for i, v := range a.values { + if i > 0 { + fmt.Fprintf(o, " ") + } + switch { + case a.IsNull(i): + o.WriteString("(null)") + default: + fmt.Fprintf(o, "%v", v) + } + } + o.WriteString("]") + return o.String() +} + +func (a *Float64) setData(data *Data) { + a.array.setData(data) + vals := data.buffers[1] + if vals != nil { + a.values = arrow.Float64Traits.CastFromBytes(vals.Bytes()) + beg := a.array.data.offset + end := beg + a.array.data.length + a.values = a.values[beg:end] + } +} + +func arrayEqualFloat64(left, right *Float64) bool { + for i := 0; i < left.Len(); i++ { + if left.IsNull(i) { + continue + } + if left.Value(i) != right.Value(i) { + return false + } + } + return true +} + +// A type which represents an immutable sequence of int32 values. +type Int32 struct { + array + values []int32 +} + +func NewInt32Data(data *Data) *Int32 { + a := &Int32{} + a.refCount = 1 + a.setData(data) + return a +} + +func (a *Int32) Value(i int) int32 { return a.values[i] } +func (a *Int32) Int32Values() []int32 { return a.values } + +func (a *Int32) String() string { + o := new(strings.Builder) + o.WriteString("[") + for i, v := range a.values { + if i > 0 { + fmt.Fprintf(o, " ") + } + switch { + case a.IsNull(i): + o.WriteString("(null)") + default: + fmt.Fprintf(o, "%v", v) + } + } + o.WriteString("]") + return o.String() +} + +func (a *Int32) setData(data *Data) { + a.array.setData(data) + vals := data.buffers[1] + if vals != nil { + a.values = arrow.Int32Traits.CastFromBytes(vals.Bytes()) + beg := a.array.data.offset + end := beg + a.array.data.length + a.values = a.values[beg:end] + } +} + +func arrayEqualInt32(left, right *Int32) bool { + for i := 0; i < left.Len(); i++ { + if left.IsNull(i) { + continue + } + if left.Value(i) != right.Value(i) { + return false + } + } + return true +} + +// A type which represents an immutable sequence of uint32 values. +type Uint32 struct { + array + values []uint32 +} + +func NewUint32Data(data *Data) *Uint32 { + a := &Uint32{} + a.refCount = 1 + a.setData(data) + return a +} + +func (a *Uint32) Value(i int) uint32 { return a.values[i] } +func (a *Uint32) Uint32Values() []uint32 { return a.values } + +func (a *Uint32) String() string { + o := new(strings.Builder) + o.WriteString("[") + for i, v := range a.values { + if i > 0 { + fmt.Fprintf(o, " ") + } + switch { + case a.IsNull(i): + o.WriteString("(null)") + default: + fmt.Fprintf(o, "%v", v) + } + } + o.WriteString("]") + return o.String() +} + +func (a *Uint32) setData(data *Data) { + a.array.setData(data) + vals := data.buffers[1] + if vals != nil { + a.values = arrow.Uint32Traits.CastFromBytes(vals.Bytes()) + beg := a.array.data.offset + end := beg + a.array.data.length + a.values = a.values[beg:end] + } +} + +func arrayEqualUint32(left, right *Uint32) bool { + for i := 0; i < left.Len(); i++ { + if left.IsNull(i) { + continue + } + if left.Value(i) != right.Value(i) { + return false + } + } + return true +} + +// A type which represents an immutable sequence of float32 values. +type Float32 struct { + array + values []float32 +} + +func NewFloat32Data(data *Data) *Float32 { + a := &Float32{} + a.refCount = 1 + a.setData(data) + return a +} + +func (a *Float32) Value(i int) float32 { return a.values[i] } +func (a *Float32) Float32Values() []float32 { return a.values } + +func (a *Float32) String() string { + o := new(strings.Builder) + o.WriteString("[") + for i, v := range a.values { + if i > 0 { + fmt.Fprintf(o, " ") + } + switch { + case a.IsNull(i): + o.WriteString("(null)") + default: + fmt.Fprintf(o, "%v", v) + } + } + o.WriteString("]") + return o.String() +} + +func (a *Float32) setData(data *Data) { + a.array.setData(data) + vals := data.buffers[1] + if vals != nil { + a.values = arrow.Float32Traits.CastFromBytes(vals.Bytes()) + beg := a.array.data.offset + end := beg + a.array.data.length + a.values = a.values[beg:end] + } +} + +func arrayEqualFloat32(left, right *Float32) bool { + for i := 0; i < left.Len(); i++ { + if left.IsNull(i) { + continue + } + if left.Value(i) != right.Value(i) { + return false + } + } + return true +} + +// A type which represents an immutable sequence of int16 values. +type Int16 struct { + array + values []int16 +} + +func NewInt16Data(data *Data) *Int16 { + a := &Int16{} + a.refCount = 1 + a.setData(data) + return a +} + +func (a *Int16) Value(i int) int16 { return a.values[i] } +func (a *Int16) Int16Values() []int16 { return a.values } + +func (a *Int16) String() string { + o := new(strings.Builder) + o.WriteString("[") + for i, v := range a.values { + if i > 0 { + fmt.Fprintf(o, " ") + } + switch { + case a.IsNull(i): + o.WriteString("(null)") + default: + fmt.Fprintf(o, "%v", v) + } + } + o.WriteString("]") + return o.String() +} + +func (a *Int16) setData(data *Data) { + a.array.setData(data) + vals := data.buffers[1] + if vals != nil { + a.values = arrow.Int16Traits.CastFromBytes(vals.Bytes()) + beg := a.array.data.offset + end := beg + a.array.data.length + a.values = a.values[beg:end] + } +} + +func arrayEqualInt16(left, right *Int16) bool { + for i := 0; i < left.Len(); i++ { + if left.IsNull(i) { + continue + } + if left.Value(i) != right.Value(i) { + return false + } + } + return true +} + +// A type which represents an immutable sequence of uint16 values. +type Uint16 struct { + array + values []uint16 +} + +func NewUint16Data(data *Data) *Uint16 { + a := &Uint16{} + a.refCount = 1 + a.setData(data) + return a +} + +func (a *Uint16) Value(i int) uint16 { return a.values[i] } +func (a *Uint16) Uint16Values() []uint16 { return a.values } + +func (a *Uint16) String() string { + o := new(strings.Builder) + o.WriteString("[") + for i, v := range a.values { + if i > 0 { + fmt.Fprintf(o, " ") + } + switch { + case a.IsNull(i): + o.WriteString("(null)") + default: + fmt.Fprintf(o, "%v", v) + } + } + o.WriteString("]") + return o.String() +} + +func (a *Uint16) setData(data *Data) { + a.array.setData(data) + vals := data.buffers[1] + if vals != nil { + a.values = arrow.Uint16Traits.CastFromBytes(vals.Bytes()) + beg := a.array.data.offset + end := beg + a.array.data.length + a.values = a.values[beg:end] + } +} + +func arrayEqualUint16(left, right *Uint16) bool { + for i := 0; i < left.Len(); i++ { + if left.IsNull(i) { + continue + } + if left.Value(i) != right.Value(i) { + return false + } + } + return true +} + +// A type which represents an immutable sequence of int8 values. +type Int8 struct { + array + values []int8 +} + +func NewInt8Data(data *Data) *Int8 { + a := &Int8{} + a.refCount = 1 + a.setData(data) + return a +} + +func (a *Int8) Value(i int) int8 { return a.values[i] } +func (a *Int8) Int8Values() []int8 { return a.values } + +func (a *Int8) String() string { + o := new(strings.Builder) + o.WriteString("[") + for i, v := range a.values { + if i > 0 { + fmt.Fprintf(o, " ") + } + switch { + case a.IsNull(i): + o.WriteString("(null)") + default: + fmt.Fprintf(o, "%v", v) + } + } + o.WriteString("]") + return o.String() +} + +func (a *Int8) setData(data *Data) { + a.array.setData(data) + vals := data.buffers[1] + if vals != nil { + a.values = arrow.Int8Traits.CastFromBytes(vals.Bytes()) + beg := a.array.data.offset + end := beg + a.array.data.length + a.values = a.values[beg:end] + } +} + +func arrayEqualInt8(left, right *Int8) bool { + for i := 0; i < left.Len(); i++ { + if left.IsNull(i) { + continue + } + if left.Value(i) != right.Value(i) { + return false + } + } + return true +} + +// A type which represents an immutable sequence of uint8 values. +type Uint8 struct { + array + values []uint8 +} + +func NewUint8Data(data *Data) *Uint8 { + a := &Uint8{} + a.refCount = 1 + a.setData(data) + return a +} + +func (a *Uint8) Value(i int) uint8 { return a.values[i] } +func (a *Uint8) Uint8Values() []uint8 { return a.values } + +func (a *Uint8) String() string { + o := new(strings.Builder) + o.WriteString("[") + for i, v := range a.values { + if i > 0 { + fmt.Fprintf(o, " ") + } + switch { + case a.IsNull(i): + o.WriteString("(null)") + default: + fmt.Fprintf(o, "%v", v) + } + } + o.WriteString("]") + return o.String() +} + +func (a *Uint8) setData(data *Data) { + a.array.setData(data) + vals := data.buffers[1] + if vals != nil { + a.values = arrow.Uint8Traits.CastFromBytes(vals.Bytes()) + beg := a.array.data.offset + end := beg + a.array.data.length + a.values = a.values[beg:end] + } +} + +func arrayEqualUint8(left, right *Uint8) bool { + for i := 0; i < left.Len(); i++ { + if left.IsNull(i) { + continue + } + if left.Value(i) != right.Value(i) { + return false + } + } + return true +} + +// A type which represents an immutable sequence of arrow.Timestamp values. +type Timestamp struct { + array + values []arrow.Timestamp +} + +func NewTimestampData(data *Data) *Timestamp { + a := &Timestamp{} + a.refCount = 1 + a.setData(data) + return a +} + +func (a *Timestamp) Value(i int) arrow.Timestamp { return a.values[i] } +func (a *Timestamp) TimestampValues() []arrow.Timestamp { return a.values } + +func (a *Timestamp) String() string { + o := new(strings.Builder) + o.WriteString("[") + for i, v := range a.values { + if i > 0 { + fmt.Fprintf(o, " ") + } + switch { + case a.IsNull(i): + o.WriteString("(null)") + default: + fmt.Fprintf(o, "%v", v) + } + } + o.WriteString("]") + return o.String() +} + +func (a *Timestamp) setData(data *Data) { + a.array.setData(data) + vals := data.buffers[1] + if vals != nil { + a.values = arrow.TimestampTraits.CastFromBytes(vals.Bytes()) + beg := a.array.data.offset + end := beg + a.array.data.length + a.values = a.values[beg:end] + } +} + +func arrayEqualTimestamp(left, right *Timestamp) bool { + for i := 0; i < left.Len(); i++ { + if left.IsNull(i) { + continue + } + if left.Value(i) != right.Value(i) { + return false + } + } + return true +} + +// A type which represents an immutable sequence of arrow.Time32 values. +type Time32 struct { + array + values []arrow.Time32 +} + +func NewTime32Data(data *Data) *Time32 { + a := &Time32{} + a.refCount = 1 + a.setData(data) + return a +} + +func (a *Time32) Value(i int) arrow.Time32 { return a.values[i] } +func (a *Time32) Time32Values() []arrow.Time32 { return a.values } + +func (a *Time32) String() string { + o := new(strings.Builder) + o.WriteString("[") + for i, v := range a.values { + if i > 0 { + fmt.Fprintf(o, " ") + } + switch { + case a.IsNull(i): + o.WriteString("(null)") + default: + fmt.Fprintf(o, "%v", v) + } + } + o.WriteString("]") + return o.String() +} + +func (a *Time32) setData(data *Data) { + a.array.setData(data) + vals := data.buffers[1] + if vals != nil { + a.values = arrow.Time32Traits.CastFromBytes(vals.Bytes()) + beg := a.array.data.offset + end := beg + a.array.data.length + a.values = a.values[beg:end] + } +} + +func arrayEqualTime32(left, right *Time32) bool { + for i := 0; i < left.Len(); i++ { + if left.IsNull(i) { + continue + } + if left.Value(i) != right.Value(i) { + return false + } + } + return true +} + +// A type which represents an immutable sequence of arrow.Time64 values. +type Time64 struct { + array + values []arrow.Time64 +} + +func NewTime64Data(data *Data) *Time64 { + a := &Time64{} + a.refCount = 1 + a.setData(data) + return a +} + +func (a *Time64) Value(i int) arrow.Time64 { return a.values[i] } +func (a *Time64) Time64Values() []arrow.Time64 { return a.values } + +func (a *Time64) String() string { + o := new(strings.Builder) + o.WriteString("[") + for i, v := range a.values { + if i > 0 { + fmt.Fprintf(o, " ") + } + switch { + case a.IsNull(i): + o.WriteString("(null)") + default: + fmt.Fprintf(o, "%v", v) + } + } + o.WriteString("]") + return o.String() +} + +func (a *Time64) setData(data *Data) { + a.array.setData(data) + vals := data.buffers[1] + if vals != nil { + a.values = arrow.Time64Traits.CastFromBytes(vals.Bytes()) + beg := a.array.data.offset + end := beg + a.array.data.length + a.values = a.values[beg:end] + } +} + +func arrayEqualTime64(left, right *Time64) bool { + for i := 0; i < left.Len(); i++ { + if left.IsNull(i) { + continue + } + if left.Value(i) != right.Value(i) { + return false + } + } + return true +} + +// A type which represents an immutable sequence of arrow.Date32 values. +type Date32 struct { + array + values []arrow.Date32 +} + +func NewDate32Data(data *Data) *Date32 { + a := &Date32{} + a.refCount = 1 + a.setData(data) + return a +} + +func (a *Date32) Value(i int) arrow.Date32 { return a.values[i] } +func (a *Date32) Date32Values() []arrow.Date32 { return a.values } + +func (a *Date32) String() string { + o := new(strings.Builder) + o.WriteString("[") + for i, v := range a.values { + if i > 0 { + fmt.Fprintf(o, " ") + } + switch { + case a.IsNull(i): + o.WriteString("(null)") + default: + fmt.Fprintf(o, "%v", v) + } + } + o.WriteString("]") + return o.String() +} + +func (a *Date32) setData(data *Data) { + a.array.setData(data) + vals := data.buffers[1] + if vals != nil { + a.values = arrow.Date32Traits.CastFromBytes(vals.Bytes()) + beg := a.array.data.offset + end := beg + a.array.data.length + a.values = a.values[beg:end] + } +} + +func arrayEqualDate32(left, right *Date32) bool { + for i := 0; i < left.Len(); i++ { + if left.IsNull(i) { + continue + } + if left.Value(i) != right.Value(i) { + return false + } + } + return true +} + +// A type which represents an immutable sequence of arrow.Date64 values. +type Date64 struct { + array + values []arrow.Date64 +} + +func NewDate64Data(data *Data) *Date64 { + a := &Date64{} + a.refCount = 1 + a.setData(data) + return a +} + +func (a *Date64) Value(i int) arrow.Date64 { return a.values[i] } +func (a *Date64) Date64Values() []arrow.Date64 { return a.values } + +func (a *Date64) String() string { + o := new(strings.Builder) + o.WriteString("[") + for i, v := range a.values { + if i > 0 { + fmt.Fprintf(o, " ") + } + switch { + case a.IsNull(i): + o.WriteString("(null)") + default: + fmt.Fprintf(o, "%v", v) + } + } + o.WriteString("]") + return o.String() +} + +func (a *Date64) setData(data *Data) { + a.array.setData(data) + vals := data.buffers[1] + if vals != nil { + a.values = arrow.Date64Traits.CastFromBytes(vals.Bytes()) + beg := a.array.data.offset + end := beg + a.array.data.length + a.values = a.values[beg:end] + } +} + +func arrayEqualDate64(left, right *Date64) bool { + for i := 0; i < left.Len(); i++ { + if left.IsNull(i) { + continue + } + if left.Value(i) != right.Value(i) { + return false + } + } + return true +} + +// A type which represents an immutable sequence of arrow.Duration values. +type Duration struct { + array + values []arrow.Duration +} + +func NewDurationData(data *Data) *Duration { + a := &Duration{} + a.refCount = 1 + a.setData(data) + return a +} + +func (a *Duration) Value(i int) arrow.Duration { return a.values[i] } +func (a *Duration) DurationValues() []arrow.Duration { return a.values } + +func (a *Duration) String() string { + o := new(strings.Builder) + o.WriteString("[") + for i, v := range a.values { + if i > 0 { + fmt.Fprintf(o, " ") + } + switch { + case a.IsNull(i): + o.WriteString("(null)") + default: + fmt.Fprintf(o, "%v", v) + } + } + o.WriteString("]") + return o.String() +} + +func (a *Duration) setData(data *Data) { + a.array.setData(data) + vals := data.buffers[1] + if vals != nil { + a.values = arrow.DurationTraits.CastFromBytes(vals.Bytes()) + beg := a.array.data.offset + end := beg + a.array.data.length + a.values = a.values[beg:end] + } +} + +func arrayEqualDuration(left, right *Duration) bool { + for i := 0; i < left.Len(); i++ { + if left.IsNull(i) { + continue + } + if left.Value(i) != right.Value(i) { + return false + } + } + return true +} diff --git a/vendor/github.com/apache/arrow/go/arrow/array/numeric.gen.go.tmpl b/vendor/github.com/apache/arrow/go/arrow/array/numeric.gen.go.tmpl new file mode 100644 index 00000000000..1e4a2f210da --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/array/numeric.gen.go.tmpl @@ -0,0 +1,85 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package array + +import ( + "fmt" + "strings" + + "github.com/apache/arrow/go/arrow" +) + +{{range .In}} + +// A type which represents an immutable sequence of {{or .QualifiedType .Type}} values. +type {{.Name}} struct { + array + values []{{or .QualifiedType .Type}} +} + +func New{{.Name}}Data(data *Data) *{{.Name}} { + a := &{{.Name}}{} + a.refCount = 1 + a.setData(data) + return a +} + +func (a *{{.Name}}) Value(i int) {{or .QualifiedType .Type}} { return a.values[i] } +func (a *{{.Name}}) {{.Name}}Values() []{{or .QualifiedType .Type}} { return a.values } + +func (a *{{.Name}}) String() string { + o := new(strings.Builder) + o.WriteString("[") + for i, v := range a.values { + if i > 0 { + fmt.Fprintf(o, " ") + } + switch { + case a.IsNull(i): + o.WriteString("(null)") + default: + fmt.Fprintf(o, "%v", v) + } + } + o.WriteString("]") + return o.String() +} + +func (a *{{.Name}}) setData(data *Data) { + a.array.setData(data) + vals := data.buffers[1] + if vals != nil { + a.values = arrow.{{.Name}}Traits.CastFromBytes(vals.Bytes()) + beg := a.array.data.offset + end := beg + a.array.data.length + a.values = a.values[beg:end] + } +} + +func arrayEqual{{.Name}}(left, right *{{.Name}}) bool { + for i := 0; i < left.Len(); i++ { + if left.IsNull(i) { + continue + } + if left.Value(i) != right.Value(i) { + return false + } + } + return true +} + +{{end}} diff --git a/vendor/github.com/apache/arrow/go/arrow/array/numericbuilder.gen.go b/vendor/github.com/apache/arrow/go/arrow/array/numericbuilder.gen.go new file mode 100644 index 00000000000..b113908af67 --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/array/numericbuilder.gen.go @@ -0,0 +1,2227 @@ +// Code generated by array/numericbuilder.gen.go.tmpl. DO NOT EDIT. + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package array + +import ( + "sync/atomic" + + "github.com/apache/arrow/go/arrow" + "github.com/apache/arrow/go/arrow/internal/bitutil" + "github.com/apache/arrow/go/arrow/internal/debug" + "github.com/apache/arrow/go/arrow/memory" +) + +type Int64Builder struct { + builder + + data *memory.Buffer + rawData []int64 +} + +func NewInt64Builder(mem memory.Allocator) *Int64Builder { + return &Int64Builder{builder: builder{refCount: 1, mem: mem}} +} + +// Release decreases the reference count by 1. +// When the reference count goes to zero, the memory is freed. +func (b *Int64Builder) Release() { + debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases") + + if atomic.AddInt64(&b.refCount, -1) == 0 { + if b.nullBitmap != nil { + b.nullBitmap.Release() + b.nullBitmap = nil + } + if b.data != nil { + b.data.Release() + b.data = nil + b.rawData = nil + } + } +} + +func (b *Int64Builder) Append(v int64) { + b.Reserve(1) + b.UnsafeAppend(v) +} + +func (b *Int64Builder) AppendNull() { + b.Reserve(1) + b.UnsafeAppendBoolToBitmap(false) +} + +func (b *Int64Builder) UnsafeAppend(v int64) { + bitutil.SetBit(b.nullBitmap.Bytes(), b.length) + b.rawData[b.length] = v + b.length++ +} + +func (b *Int64Builder) UnsafeAppendBoolToBitmap(isValid bool) { + if isValid { + bitutil.SetBit(b.nullBitmap.Bytes(), b.length) + } else { + b.nulls++ + } + b.length++ +} + +// AppendValues will append the values in the v slice. The valid slice determines which values +// in v are valid (not null). The valid slice must either be empty or be equal in length to v. If empty, +// all values in v are appended and considered valid. +func (b *Int64Builder) AppendValues(v []int64, valid []bool) { + if len(v) != len(valid) && len(valid) != 0 { + panic("len(v) != len(valid) && len(valid) != 0") + } + + if len(v) == 0 { + return + } + + b.Reserve(len(v)) + arrow.Int64Traits.Copy(b.rawData[b.length:], v) + b.builder.unsafeAppendBoolsToBitmap(valid, len(v)) +} + +func (b *Int64Builder) init(capacity int) { + b.builder.init(capacity) + + b.data = memory.NewResizableBuffer(b.mem) + bytesN := arrow.Int64Traits.BytesRequired(capacity) + b.data.Resize(bytesN) + b.rawData = arrow.Int64Traits.CastFromBytes(b.data.Bytes()) +} + +// Reserve ensures there is enough space for appending n elements +// by checking the capacity and calling Resize if necessary. +func (b *Int64Builder) Reserve(n int) { + b.builder.reserve(n, b.Resize) +} + +// Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(), +// additional memory will be allocated. If n is smaller, the allocated memory may reduced. +func (b *Int64Builder) Resize(n int) { + nBuilder := n + if n < minBuilderCapacity { + n = minBuilderCapacity + } + + if b.capacity == 0 { + b.init(n) + } else { + b.builder.resize(nBuilder, b.init) + b.data.Resize(arrow.Int64Traits.BytesRequired(n)) + b.rawData = arrow.Int64Traits.CastFromBytes(b.data.Bytes()) + } +} + +// NewArray creates a Int64 array from the memory buffers used by the builder and resets the Int64Builder +// so it can be used to build a new array. +func (b *Int64Builder) NewArray() Interface { + return b.NewInt64Array() +} + +// NewInt64Array creates a Int64 array from the memory buffers used by the builder and resets the Int64Builder +// so it can be used to build a new array. +func (b *Int64Builder) NewInt64Array() (a *Int64) { + data := b.newData() + a = NewInt64Data(data) + data.Release() + return +} + +func (b *Int64Builder) newData() (data *Data) { + bytesRequired := arrow.Int64Traits.BytesRequired(b.length) + if bytesRequired > 0 && bytesRequired < b.data.Len() { + // trim buffers + b.data.Resize(bytesRequired) + } + data = NewData(arrow.PrimitiveTypes.Int64, b.length, []*memory.Buffer{b.nullBitmap, b.data}, nil, b.nulls, 0) + b.reset() + + if b.data != nil { + b.data.Release() + b.data = nil + b.rawData = nil + } + + return +} + +type Uint64Builder struct { + builder + + data *memory.Buffer + rawData []uint64 +} + +func NewUint64Builder(mem memory.Allocator) *Uint64Builder { + return &Uint64Builder{builder: builder{refCount: 1, mem: mem}} +} + +// Release decreases the reference count by 1. +// When the reference count goes to zero, the memory is freed. +func (b *Uint64Builder) Release() { + debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases") + + if atomic.AddInt64(&b.refCount, -1) == 0 { + if b.nullBitmap != nil { + b.nullBitmap.Release() + b.nullBitmap = nil + } + if b.data != nil { + b.data.Release() + b.data = nil + b.rawData = nil + } + } +} + +func (b *Uint64Builder) Append(v uint64) { + b.Reserve(1) + b.UnsafeAppend(v) +} + +func (b *Uint64Builder) AppendNull() { + b.Reserve(1) + b.UnsafeAppendBoolToBitmap(false) +} + +func (b *Uint64Builder) UnsafeAppend(v uint64) { + bitutil.SetBit(b.nullBitmap.Bytes(), b.length) + b.rawData[b.length] = v + b.length++ +} + +func (b *Uint64Builder) UnsafeAppendBoolToBitmap(isValid bool) { + if isValid { + bitutil.SetBit(b.nullBitmap.Bytes(), b.length) + } else { + b.nulls++ + } + b.length++ +} + +// AppendValues will append the values in the v slice. The valid slice determines which values +// in v are valid (not null). The valid slice must either be empty or be equal in length to v. If empty, +// all values in v are appended and considered valid. +func (b *Uint64Builder) AppendValues(v []uint64, valid []bool) { + if len(v) != len(valid) && len(valid) != 0 { + panic("len(v) != len(valid) && len(valid) != 0") + } + + if len(v) == 0 { + return + } + + b.Reserve(len(v)) + arrow.Uint64Traits.Copy(b.rawData[b.length:], v) + b.builder.unsafeAppendBoolsToBitmap(valid, len(v)) +} + +func (b *Uint64Builder) init(capacity int) { + b.builder.init(capacity) + + b.data = memory.NewResizableBuffer(b.mem) + bytesN := arrow.Uint64Traits.BytesRequired(capacity) + b.data.Resize(bytesN) + b.rawData = arrow.Uint64Traits.CastFromBytes(b.data.Bytes()) +} + +// Reserve ensures there is enough space for appending n elements +// by checking the capacity and calling Resize if necessary. +func (b *Uint64Builder) Reserve(n int) { + b.builder.reserve(n, b.Resize) +} + +// Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(), +// additional memory will be allocated. If n is smaller, the allocated memory may reduced. +func (b *Uint64Builder) Resize(n int) { + nBuilder := n + if n < minBuilderCapacity { + n = minBuilderCapacity + } + + if b.capacity == 0 { + b.init(n) + } else { + b.builder.resize(nBuilder, b.init) + b.data.Resize(arrow.Uint64Traits.BytesRequired(n)) + b.rawData = arrow.Uint64Traits.CastFromBytes(b.data.Bytes()) + } +} + +// NewArray creates a Uint64 array from the memory buffers used by the builder and resets the Uint64Builder +// so it can be used to build a new array. +func (b *Uint64Builder) NewArray() Interface { + return b.NewUint64Array() +} + +// NewUint64Array creates a Uint64 array from the memory buffers used by the builder and resets the Uint64Builder +// so it can be used to build a new array. +func (b *Uint64Builder) NewUint64Array() (a *Uint64) { + data := b.newData() + a = NewUint64Data(data) + data.Release() + return +} + +func (b *Uint64Builder) newData() (data *Data) { + bytesRequired := arrow.Uint64Traits.BytesRequired(b.length) + if bytesRequired > 0 && bytesRequired < b.data.Len() { + // trim buffers + b.data.Resize(bytesRequired) + } + data = NewData(arrow.PrimitiveTypes.Uint64, b.length, []*memory.Buffer{b.nullBitmap, b.data}, nil, b.nulls, 0) + b.reset() + + if b.data != nil { + b.data.Release() + b.data = nil + b.rawData = nil + } + + return +} + +type Float64Builder struct { + builder + + data *memory.Buffer + rawData []float64 +} + +func NewFloat64Builder(mem memory.Allocator) *Float64Builder { + return &Float64Builder{builder: builder{refCount: 1, mem: mem}} +} + +// Release decreases the reference count by 1. +// When the reference count goes to zero, the memory is freed. +func (b *Float64Builder) Release() { + debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases") + + if atomic.AddInt64(&b.refCount, -1) == 0 { + if b.nullBitmap != nil { + b.nullBitmap.Release() + b.nullBitmap = nil + } + if b.data != nil { + b.data.Release() + b.data = nil + b.rawData = nil + } + } +} + +func (b *Float64Builder) Append(v float64) { + b.Reserve(1) + b.UnsafeAppend(v) +} + +func (b *Float64Builder) AppendNull() { + b.Reserve(1) + b.UnsafeAppendBoolToBitmap(false) +} + +func (b *Float64Builder) UnsafeAppend(v float64) { + bitutil.SetBit(b.nullBitmap.Bytes(), b.length) + b.rawData[b.length] = v + b.length++ +} + +func (b *Float64Builder) UnsafeAppendBoolToBitmap(isValid bool) { + if isValid { + bitutil.SetBit(b.nullBitmap.Bytes(), b.length) + } else { + b.nulls++ + } + b.length++ +} + +// AppendValues will append the values in the v slice. The valid slice determines which values +// in v are valid (not null). The valid slice must either be empty or be equal in length to v. If empty, +// all values in v are appended and considered valid. +func (b *Float64Builder) AppendValues(v []float64, valid []bool) { + if len(v) != len(valid) && len(valid) != 0 { + panic("len(v) != len(valid) && len(valid) != 0") + } + + if len(v) == 0 { + return + } + + b.Reserve(len(v)) + arrow.Float64Traits.Copy(b.rawData[b.length:], v) + b.builder.unsafeAppendBoolsToBitmap(valid, len(v)) +} + +func (b *Float64Builder) init(capacity int) { + b.builder.init(capacity) + + b.data = memory.NewResizableBuffer(b.mem) + bytesN := arrow.Float64Traits.BytesRequired(capacity) + b.data.Resize(bytesN) + b.rawData = arrow.Float64Traits.CastFromBytes(b.data.Bytes()) +} + +// Reserve ensures there is enough space for appending n elements +// by checking the capacity and calling Resize if necessary. +func (b *Float64Builder) Reserve(n int) { + b.builder.reserve(n, b.Resize) +} + +// Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(), +// additional memory will be allocated. If n is smaller, the allocated memory may reduced. +func (b *Float64Builder) Resize(n int) { + nBuilder := n + if n < minBuilderCapacity { + n = minBuilderCapacity + } + + if b.capacity == 0 { + b.init(n) + } else { + b.builder.resize(nBuilder, b.init) + b.data.Resize(arrow.Float64Traits.BytesRequired(n)) + b.rawData = arrow.Float64Traits.CastFromBytes(b.data.Bytes()) + } +} + +// NewArray creates a Float64 array from the memory buffers used by the builder and resets the Float64Builder +// so it can be used to build a new array. +func (b *Float64Builder) NewArray() Interface { + return b.NewFloat64Array() +} + +// NewFloat64Array creates a Float64 array from the memory buffers used by the builder and resets the Float64Builder +// so it can be used to build a new array. +func (b *Float64Builder) NewFloat64Array() (a *Float64) { + data := b.newData() + a = NewFloat64Data(data) + data.Release() + return +} + +func (b *Float64Builder) newData() (data *Data) { + bytesRequired := arrow.Float64Traits.BytesRequired(b.length) + if bytesRequired > 0 && bytesRequired < b.data.Len() { + // trim buffers + b.data.Resize(bytesRequired) + } + data = NewData(arrow.PrimitiveTypes.Float64, b.length, []*memory.Buffer{b.nullBitmap, b.data}, nil, b.nulls, 0) + b.reset() + + if b.data != nil { + b.data.Release() + b.data = nil + b.rawData = nil + } + + return +} + +type Int32Builder struct { + builder + + data *memory.Buffer + rawData []int32 +} + +func NewInt32Builder(mem memory.Allocator) *Int32Builder { + return &Int32Builder{builder: builder{refCount: 1, mem: mem}} +} + +// Release decreases the reference count by 1. +// When the reference count goes to zero, the memory is freed. +func (b *Int32Builder) Release() { + debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases") + + if atomic.AddInt64(&b.refCount, -1) == 0 { + if b.nullBitmap != nil { + b.nullBitmap.Release() + b.nullBitmap = nil + } + if b.data != nil { + b.data.Release() + b.data = nil + b.rawData = nil + } + } +} + +func (b *Int32Builder) Append(v int32) { + b.Reserve(1) + b.UnsafeAppend(v) +} + +func (b *Int32Builder) AppendNull() { + b.Reserve(1) + b.UnsafeAppendBoolToBitmap(false) +} + +func (b *Int32Builder) UnsafeAppend(v int32) { + bitutil.SetBit(b.nullBitmap.Bytes(), b.length) + b.rawData[b.length] = v + b.length++ +} + +func (b *Int32Builder) UnsafeAppendBoolToBitmap(isValid bool) { + if isValid { + bitutil.SetBit(b.nullBitmap.Bytes(), b.length) + } else { + b.nulls++ + } + b.length++ +} + +// AppendValues will append the values in the v slice. The valid slice determines which values +// in v are valid (not null). The valid slice must either be empty or be equal in length to v. If empty, +// all values in v are appended and considered valid. +func (b *Int32Builder) AppendValues(v []int32, valid []bool) { + if len(v) != len(valid) && len(valid) != 0 { + panic("len(v) != len(valid) && len(valid) != 0") + } + + if len(v) == 0 { + return + } + + b.Reserve(len(v)) + arrow.Int32Traits.Copy(b.rawData[b.length:], v) + b.builder.unsafeAppendBoolsToBitmap(valid, len(v)) +} + +func (b *Int32Builder) init(capacity int) { + b.builder.init(capacity) + + b.data = memory.NewResizableBuffer(b.mem) + bytesN := arrow.Int32Traits.BytesRequired(capacity) + b.data.Resize(bytesN) + b.rawData = arrow.Int32Traits.CastFromBytes(b.data.Bytes()) +} + +// Reserve ensures there is enough space for appending n elements +// by checking the capacity and calling Resize if necessary. +func (b *Int32Builder) Reserve(n int) { + b.builder.reserve(n, b.Resize) +} + +// Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(), +// additional memory will be allocated. If n is smaller, the allocated memory may reduced. +func (b *Int32Builder) Resize(n int) { + nBuilder := n + if n < minBuilderCapacity { + n = minBuilderCapacity + } + + if b.capacity == 0 { + b.init(n) + } else { + b.builder.resize(nBuilder, b.init) + b.data.Resize(arrow.Int32Traits.BytesRequired(n)) + b.rawData = arrow.Int32Traits.CastFromBytes(b.data.Bytes()) + } +} + +// NewArray creates a Int32 array from the memory buffers used by the builder and resets the Int32Builder +// so it can be used to build a new array. +func (b *Int32Builder) NewArray() Interface { + return b.NewInt32Array() +} + +// NewInt32Array creates a Int32 array from the memory buffers used by the builder and resets the Int32Builder +// so it can be used to build a new array. +func (b *Int32Builder) NewInt32Array() (a *Int32) { + data := b.newData() + a = NewInt32Data(data) + data.Release() + return +} + +func (b *Int32Builder) newData() (data *Data) { + bytesRequired := arrow.Int32Traits.BytesRequired(b.length) + if bytesRequired > 0 && bytesRequired < b.data.Len() { + // trim buffers + b.data.Resize(bytesRequired) + } + data = NewData(arrow.PrimitiveTypes.Int32, b.length, []*memory.Buffer{b.nullBitmap, b.data}, nil, b.nulls, 0) + b.reset() + + if b.data != nil { + b.data.Release() + b.data = nil + b.rawData = nil + } + + return +} + +type Uint32Builder struct { + builder + + data *memory.Buffer + rawData []uint32 +} + +func NewUint32Builder(mem memory.Allocator) *Uint32Builder { + return &Uint32Builder{builder: builder{refCount: 1, mem: mem}} +} + +// Release decreases the reference count by 1. +// When the reference count goes to zero, the memory is freed. +func (b *Uint32Builder) Release() { + debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases") + + if atomic.AddInt64(&b.refCount, -1) == 0 { + if b.nullBitmap != nil { + b.nullBitmap.Release() + b.nullBitmap = nil + } + if b.data != nil { + b.data.Release() + b.data = nil + b.rawData = nil + } + } +} + +func (b *Uint32Builder) Append(v uint32) { + b.Reserve(1) + b.UnsafeAppend(v) +} + +func (b *Uint32Builder) AppendNull() { + b.Reserve(1) + b.UnsafeAppendBoolToBitmap(false) +} + +func (b *Uint32Builder) UnsafeAppend(v uint32) { + bitutil.SetBit(b.nullBitmap.Bytes(), b.length) + b.rawData[b.length] = v + b.length++ +} + +func (b *Uint32Builder) UnsafeAppendBoolToBitmap(isValid bool) { + if isValid { + bitutil.SetBit(b.nullBitmap.Bytes(), b.length) + } else { + b.nulls++ + } + b.length++ +} + +// AppendValues will append the values in the v slice. The valid slice determines which values +// in v are valid (not null). The valid slice must either be empty or be equal in length to v. If empty, +// all values in v are appended and considered valid. +func (b *Uint32Builder) AppendValues(v []uint32, valid []bool) { + if len(v) != len(valid) && len(valid) != 0 { + panic("len(v) != len(valid) && len(valid) != 0") + } + + if len(v) == 0 { + return + } + + b.Reserve(len(v)) + arrow.Uint32Traits.Copy(b.rawData[b.length:], v) + b.builder.unsafeAppendBoolsToBitmap(valid, len(v)) +} + +func (b *Uint32Builder) init(capacity int) { + b.builder.init(capacity) + + b.data = memory.NewResizableBuffer(b.mem) + bytesN := arrow.Uint32Traits.BytesRequired(capacity) + b.data.Resize(bytesN) + b.rawData = arrow.Uint32Traits.CastFromBytes(b.data.Bytes()) +} + +// Reserve ensures there is enough space for appending n elements +// by checking the capacity and calling Resize if necessary. +func (b *Uint32Builder) Reserve(n int) { + b.builder.reserve(n, b.Resize) +} + +// Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(), +// additional memory will be allocated. If n is smaller, the allocated memory may reduced. +func (b *Uint32Builder) Resize(n int) { + nBuilder := n + if n < minBuilderCapacity { + n = minBuilderCapacity + } + + if b.capacity == 0 { + b.init(n) + } else { + b.builder.resize(nBuilder, b.init) + b.data.Resize(arrow.Uint32Traits.BytesRequired(n)) + b.rawData = arrow.Uint32Traits.CastFromBytes(b.data.Bytes()) + } +} + +// NewArray creates a Uint32 array from the memory buffers used by the builder and resets the Uint32Builder +// so it can be used to build a new array. +func (b *Uint32Builder) NewArray() Interface { + return b.NewUint32Array() +} + +// NewUint32Array creates a Uint32 array from the memory buffers used by the builder and resets the Uint32Builder +// so it can be used to build a new array. +func (b *Uint32Builder) NewUint32Array() (a *Uint32) { + data := b.newData() + a = NewUint32Data(data) + data.Release() + return +} + +func (b *Uint32Builder) newData() (data *Data) { + bytesRequired := arrow.Uint32Traits.BytesRequired(b.length) + if bytesRequired > 0 && bytesRequired < b.data.Len() { + // trim buffers + b.data.Resize(bytesRequired) + } + data = NewData(arrow.PrimitiveTypes.Uint32, b.length, []*memory.Buffer{b.nullBitmap, b.data}, nil, b.nulls, 0) + b.reset() + + if b.data != nil { + b.data.Release() + b.data = nil + b.rawData = nil + } + + return +} + +type Float32Builder struct { + builder + + data *memory.Buffer + rawData []float32 +} + +func NewFloat32Builder(mem memory.Allocator) *Float32Builder { + return &Float32Builder{builder: builder{refCount: 1, mem: mem}} +} + +// Release decreases the reference count by 1. +// When the reference count goes to zero, the memory is freed. +func (b *Float32Builder) Release() { + debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases") + + if atomic.AddInt64(&b.refCount, -1) == 0 { + if b.nullBitmap != nil { + b.nullBitmap.Release() + b.nullBitmap = nil + } + if b.data != nil { + b.data.Release() + b.data = nil + b.rawData = nil + } + } +} + +func (b *Float32Builder) Append(v float32) { + b.Reserve(1) + b.UnsafeAppend(v) +} + +func (b *Float32Builder) AppendNull() { + b.Reserve(1) + b.UnsafeAppendBoolToBitmap(false) +} + +func (b *Float32Builder) UnsafeAppend(v float32) { + bitutil.SetBit(b.nullBitmap.Bytes(), b.length) + b.rawData[b.length] = v + b.length++ +} + +func (b *Float32Builder) UnsafeAppendBoolToBitmap(isValid bool) { + if isValid { + bitutil.SetBit(b.nullBitmap.Bytes(), b.length) + } else { + b.nulls++ + } + b.length++ +} + +// AppendValues will append the values in the v slice. The valid slice determines which values +// in v are valid (not null). The valid slice must either be empty or be equal in length to v. If empty, +// all values in v are appended and considered valid. +func (b *Float32Builder) AppendValues(v []float32, valid []bool) { + if len(v) != len(valid) && len(valid) != 0 { + panic("len(v) != len(valid) && len(valid) != 0") + } + + if len(v) == 0 { + return + } + + b.Reserve(len(v)) + arrow.Float32Traits.Copy(b.rawData[b.length:], v) + b.builder.unsafeAppendBoolsToBitmap(valid, len(v)) +} + +func (b *Float32Builder) init(capacity int) { + b.builder.init(capacity) + + b.data = memory.NewResizableBuffer(b.mem) + bytesN := arrow.Float32Traits.BytesRequired(capacity) + b.data.Resize(bytesN) + b.rawData = arrow.Float32Traits.CastFromBytes(b.data.Bytes()) +} + +// Reserve ensures there is enough space for appending n elements +// by checking the capacity and calling Resize if necessary. +func (b *Float32Builder) Reserve(n int) { + b.builder.reserve(n, b.Resize) +} + +// Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(), +// additional memory will be allocated. If n is smaller, the allocated memory may reduced. +func (b *Float32Builder) Resize(n int) { + nBuilder := n + if n < minBuilderCapacity { + n = minBuilderCapacity + } + + if b.capacity == 0 { + b.init(n) + } else { + b.builder.resize(nBuilder, b.init) + b.data.Resize(arrow.Float32Traits.BytesRequired(n)) + b.rawData = arrow.Float32Traits.CastFromBytes(b.data.Bytes()) + } +} + +// NewArray creates a Float32 array from the memory buffers used by the builder and resets the Float32Builder +// so it can be used to build a new array. +func (b *Float32Builder) NewArray() Interface { + return b.NewFloat32Array() +} + +// NewFloat32Array creates a Float32 array from the memory buffers used by the builder and resets the Float32Builder +// so it can be used to build a new array. +func (b *Float32Builder) NewFloat32Array() (a *Float32) { + data := b.newData() + a = NewFloat32Data(data) + data.Release() + return +} + +func (b *Float32Builder) newData() (data *Data) { + bytesRequired := arrow.Float32Traits.BytesRequired(b.length) + if bytesRequired > 0 && bytesRequired < b.data.Len() { + // trim buffers + b.data.Resize(bytesRequired) + } + data = NewData(arrow.PrimitiveTypes.Float32, b.length, []*memory.Buffer{b.nullBitmap, b.data}, nil, b.nulls, 0) + b.reset() + + if b.data != nil { + b.data.Release() + b.data = nil + b.rawData = nil + } + + return +} + +type Int16Builder struct { + builder + + data *memory.Buffer + rawData []int16 +} + +func NewInt16Builder(mem memory.Allocator) *Int16Builder { + return &Int16Builder{builder: builder{refCount: 1, mem: mem}} +} + +// Release decreases the reference count by 1. +// When the reference count goes to zero, the memory is freed. +func (b *Int16Builder) Release() { + debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases") + + if atomic.AddInt64(&b.refCount, -1) == 0 { + if b.nullBitmap != nil { + b.nullBitmap.Release() + b.nullBitmap = nil + } + if b.data != nil { + b.data.Release() + b.data = nil + b.rawData = nil + } + } +} + +func (b *Int16Builder) Append(v int16) { + b.Reserve(1) + b.UnsafeAppend(v) +} + +func (b *Int16Builder) AppendNull() { + b.Reserve(1) + b.UnsafeAppendBoolToBitmap(false) +} + +func (b *Int16Builder) UnsafeAppend(v int16) { + bitutil.SetBit(b.nullBitmap.Bytes(), b.length) + b.rawData[b.length] = v + b.length++ +} + +func (b *Int16Builder) UnsafeAppendBoolToBitmap(isValid bool) { + if isValid { + bitutil.SetBit(b.nullBitmap.Bytes(), b.length) + } else { + b.nulls++ + } + b.length++ +} + +// AppendValues will append the values in the v slice. The valid slice determines which values +// in v are valid (not null). The valid slice must either be empty or be equal in length to v. If empty, +// all values in v are appended and considered valid. +func (b *Int16Builder) AppendValues(v []int16, valid []bool) { + if len(v) != len(valid) && len(valid) != 0 { + panic("len(v) != len(valid) && len(valid) != 0") + } + + if len(v) == 0 { + return + } + + b.Reserve(len(v)) + arrow.Int16Traits.Copy(b.rawData[b.length:], v) + b.builder.unsafeAppendBoolsToBitmap(valid, len(v)) +} + +func (b *Int16Builder) init(capacity int) { + b.builder.init(capacity) + + b.data = memory.NewResizableBuffer(b.mem) + bytesN := arrow.Int16Traits.BytesRequired(capacity) + b.data.Resize(bytesN) + b.rawData = arrow.Int16Traits.CastFromBytes(b.data.Bytes()) +} + +// Reserve ensures there is enough space for appending n elements +// by checking the capacity and calling Resize if necessary. +func (b *Int16Builder) Reserve(n int) { + b.builder.reserve(n, b.Resize) +} + +// Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(), +// additional memory will be allocated. If n is smaller, the allocated memory may reduced. +func (b *Int16Builder) Resize(n int) { + nBuilder := n + if n < minBuilderCapacity { + n = minBuilderCapacity + } + + if b.capacity == 0 { + b.init(n) + } else { + b.builder.resize(nBuilder, b.init) + b.data.Resize(arrow.Int16Traits.BytesRequired(n)) + b.rawData = arrow.Int16Traits.CastFromBytes(b.data.Bytes()) + } +} + +// NewArray creates a Int16 array from the memory buffers used by the builder and resets the Int16Builder +// so it can be used to build a new array. +func (b *Int16Builder) NewArray() Interface { + return b.NewInt16Array() +} + +// NewInt16Array creates a Int16 array from the memory buffers used by the builder and resets the Int16Builder +// so it can be used to build a new array. +func (b *Int16Builder) NewInt16Array() (a *Int16) { + data := b.newData() + a = NewInt16Data(data) + data.Release() + return +} + +func (b *Int16Builder) newData() (data *Data) { + bytesRequired := arrow.Int16Traits.BytesRequired(b.length) + if bytesRequired > 0 && bytesRequired < b.data.Len() { + // trim buffers + b.data.Resize(bytesRequired) + } + data = NewData(arrow.PrimitiveTypes.Int16, b.length, []*memory.Buffer{b.nullBitmap, b.data}, nil, b.nulls, 0) + b.reset() + + if b.data != nil { + b.data.Release() + b.data = nil + b.rawData = nil + } + + return +} + +type Uint16Builder struct { + builder + + data *memory.Buffer + rawData []uint16 +} + +func NewUint16Builder(mem memory.Allocator) *Uint16Builder { + return &Uint16Builder{builder: builder{refCount: 1, mem: mem}} +} + +// Release decreases the reference count by 1. +// When the reference count goes to zero, the memory is freed. +func (b *Uint16Builder) Release() { + debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases") + + if atomic.AddInt64(&b.refCount, -1) == 0 { + if b.nullBitmap != nil { + b.nullBitmap.Release() + b.nullBitmap = nil + } + if b.data != nil { + b.data.Release() + b.data = nil + b.rawData = nil + } + } +} + +func (b *Uint16Builder) Append(v uint16) { + b.Reserve(1) + b.UnsafeAppend(v) +} + +func (b *Uint16Builder) AppendNull() { + b.Reserve(1) + b.UnsafeAppendBoolToBitmap(false) +} + +func (b *Uint16Builder) UnsafeAppend(v uint16) { + bitutil.SetBit(b.nullBitmap.Bytes(), b.length) + b.rawData[b.length] = v + b.length++ +} + +func (b *Uint16Builder) UnsafeAppendBoolToBitmap(isValid bool) { + if isValid { + bitutil.SetBit(b.nullBitmap.Bytes(), b.length) + } else { + b.nulls++ + } + b.length++ +} + +// AppendValues will append the values in the v slice. The valid slice determines which values +// in v are valid (not null). The valid slice must either be empty or be equal in length to v. If empty, +// all values in v are appended and considered valid. +func (b *Uint16Builder) AppendValues(v []uint16, valid []bool) { + if len(v) != len(valid) && len(valid) != 0 { + panic("len(v) != len(valid) && len(valid) != 0") + } + + if len(v) == 0 { + return + } + + b.Reserve(len(v)) + arrow.Uint16Traits.Copy(b.rawData[b.length:], v) + b.builder.unsafeAppendBoolsToBitmap(valid, len(v)) +} + +func (b *Uint16Builder) init(capacity int) { + b.builder.init(capacity) + + b.data = memory.NewResizableBuffer(b.mem) + bytesN := arrow.Uint16Traits.BytesRequired(capacity) + b.data.Resize(bytesN) + b.rawData = arrow.Uint16Traits.CastFromBytes(b.data.Bytes()) +} + +// Reserve ensures there is enough space for appending n elements +// by checking the capacity and calling Resize if necessary. +func (b *Uint16Builder) Reserve(n int) { + b.builder.reserve(n, b.Resize) +} + +// Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(), +// additional memory will be allocated. If n is smaller, the allocated memory may reduced. +func (b *Uint16Builder) Resize(n int) { + nBuilder := n + if n < minBuilderCapacity { + n = minBuilderCapacity + } + + if b.capacity == 0 { + b.init(n) + } else { + b.builder.resize(nBuilder, b.init) + b.data.Resize(arrow.Uint16Traits.BytesRequired(n)) + b.rawData = arrow.Uint16Traits.CastFromBytes(b.data.Bytes()) + } +} + +// NewArray creates a Uint16 array from the memory buffers used by the builder and resets the Uint16Builder +// so it can be used to build a new array. +func (b *Uint16Builder) NewArray() Interface { + return b.NewUint16Array() +} + +// NewUint16Array creates a Uint16 array from the memory buffers used by the builder and resets the Uint16Builder +// so it can be used to build a new array. +func (b *Uint16Builder) NewUint16Array() (a *Uint16) { + data := b.newData() + a = NewUint16Data(data) + data.Release() + return +} + +func (b *Uint16Builder) newData() (data *Data) { + bytesRequired := arrow.Uint16Traits.BytesRequired(b.length) + if bytesRequired > 0 && bytesRequired < b.data.Len() { + // trim buffers + b.data.Resize(bytesRequired) + } + data = NewData(arrow.PrimitiveTypes.Uint16, b.length, []*memory.Buffer{b.nullBitmap, b.data}, nil, b.nulls, 0) + b.reset() + + if b.data != nil { + b.data.Release() + b.data = nil + b.rawData = nil + } + + return +} + +type Int8Builder struct { + builder + + data *memory.Buffer + rawData []int8 +} + +func NewInt8Builder(mem memory.Allocator) *Int8Builder { + return &Int8Builder{builder: builder{refCount: 1, mem: mem}} +} + +// Release decreases the reference count by 1. +// When the reference count goes to zero, the memory is freed. +func (b *Int8Builder) Release() { + debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases") + + if atomic.AddInt64(&b.refCount, -1) == 0 { + if b.nullBitmap != nil { + b.nullBitmap.Release() + b.nullBitmap = nil + } + if b.data != nil { + b.data.Release() + b.data = nil + b.rawData = nil + } + } +} + +func (b *Int8Builder) Append(v int8) { + b.Reserve(1) + b.UnsafeAppend(v) +} + +func (b *Int8Builder) AppendNull() { + b.Reserve(1) + b.UnsafeAppendBoolToBitmap(false) +} + +func (b *Int8Builder) UnsafeAppend(v int8) { + bitutil.SetBit(b.nullBitmap.Bytes(), b.length) + b.rawData[b.length] = v + b.length++ +} + +func (b *Int8Builder) UnsafeAppendBoolToBitmap(isValid bool) { + if isValid { + bitutil.SetBit(b.nullBitmap.Bytes(), b.length) + } else { + b.nulls++ + } + b.length++ +} + +// AppendValues will append the values in the v slice. The valid slice determines which values +// in v are valid (not null). The valid slice must either be empty or be equal in length to v. If empty, +// all values in v are appended and considered valid. +func (b *Int8Builder) AppendValues(v []int8, valid []bool) { + if len(v) != len(valid) && len(valid) != 0 { + panic("len(v) != len(valid) && len(valid) != 0") + } + + if len(v) == 0 { + return + } + + b.Reserve(len(v)) + arrow.Int8Traits.Copy(b.rawData[b.length:], v) + b.builder.unsafeAppendBoolsToBitmap(valid, len(v)) +} + +func (b *Int8Builder) init(capacity int) { + b.builder.init(capacity) + + b.data = memory.NewResizableBuffer(b.mem) + bytesN := arrow.Int8Traits.BytesRequired(capacity) + b.data.Resize(bytesN) + b.rawData = arrow.Int8Traits.CastFromBytes(b.data.Bytes()) +} + +// Reserve ensures there is enough space for appending n elements +// by checking the capacity and calling Resize if necessary. +func (b *Int8Builder) Reserve(n int) { + b.builder.reserve(n, b.Resize) +} + +// Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(), +// additional memory will be allocated. If n is smaller, the allocated memory may reduced. +func (b *Int8Builder) Resize(n int) { + nBuilder := n + if n < minBuilderCapacity { + n = minBuilderCapacity + } + + if b.capacity == 0 { + b.init(n) + } else { + b.builder.resize(nBuilder, b.init) + b.data.Resize(arrow.Int8Traits.BytesRequired(n)) + b.rawData = arrow.Int8Traits.CastFromBytes(b.data.Bytes()) + } +} + +// NewArray creates a Int8 array from the memory buffers used by the builder and resets the Int8Builder +// so it can be used to build a new array. +func (b *Int8Builder) NewArray() Interface { + return b.NewInt8Array() +} + +// NewInt8Array creates a Int8 array from the memory buffers used by the builder and resets the Int8Builder +// so it can be used to build a new array. +func (b *Int8Builder) NewInt8Array() (a *Int8) { + data := b.newData() + a = NewInt8Data(data) + data.Release() + return +} + +func (b *Int8Builder) newData() (data *Data) { + bytesRequired := arrow.Int8Traits.BytesRequired(b.length) + if bytesRequired > 0 && bytesRequired < b.data.Len() { + // trim buffers + b.data.Resize(bytesRequired) + } + data = NewData(arrow.PrimitiveTypes.Int8, b.length, []*memory.Buffer{b.nullBitmap, b.data}, nil, b.nulls, 0) + b.reset() + + if b.data != nil { + b.data.Release() + b.data = nil + b.rawData = nil + } + + return +} + +type Uint8Builder struct { + builder + + data *memory.Buffer + rawData []uint8 +} + +func NewUint8Builder(mem memory.Allocator) *Uint8Builder { + return &Uint8Builder{builder: builder{refCount: 1, mem: mem}} +} + +// Release decreases the reference count by 1. +// When the reference count goes to zero, the memory is freed. +func (b *Uint8Builder) Release() { + debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases") + + if atomic.AddInt64(&b.refCount, -1) == 0 { + if b.nullBitmap != nil { + b.nullBitmap.Release() + b.nullBitmap = nil + } + if b.data != nil { + b.data.Release() + b.data = nil + b.rawData = nil + } + } +} + +func (b *Uint8Builder) Append(v uint8) { + b.Reserve(1) + b.UnsafeAppend(v) +} + +func (b *Uint8Builder) AppendNull() { + b.Reserve(1) + b.UnsafeAppendBoolToBitmap(false) +} + +func (b *Uint8Builder) UnsafeAppend(v uint8) { + bitutil.SetBit(b.nullBitmap.Bytes(), b.length) + b.rawData[b.length] = v + b.length++ +} + +func (b *Uint8Builder) UnsafeAppendBoolToBitmap(isValid bool) { + if isValid { + bitutil.SetBit(b.nullBitmap.Bytes(), b.length) + } else { + b.nulls++ + } + b.length++ +} + +// AppendValues will append the values in the v slice. The valid slice determines which values +// in v are valid (not null). The valid slice must either be empty or be equal in length to v. If empty, +// all values in v are appended and considered valid. +func (b *Uint8Builder) AppendValues(v []uint8, valid []bool) { + if len(v) != len(valid) && len(valid) != 0 { + panic("len(v) != len(valid) && len(valid) != 0") + } + + if len(v) == 0 { + return + } + + b.Reserve(len(v)) + arrow.Uint8Traits.Copy(b.rawData[b.length:], v) + b.builder.unsafeAppendBoolsToBitmap(valid, len(v)) +} + +func (b *Uint8Builder) init(capacity int) { + b.builder.init(capacity) + + b.data = memory.NewResizableBuffer(b.mem) + bytesN := arrow.Uint8Traits.BytesRequired(capacity) + b.data.Resize(bytesN) + b.rawData = arrow.Uint8Traits.CastFromBytes(b.data.Bytes()) +} + +// Reserve ensures there is enough space for appending n elements +// by checking the capacity and calling Resize if necessary. +func (b *Uint8Builder) Reserve(n int) { + b.builder.reserve(n, b.Resize) +} + +// Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(), +// additional memory will be allocated. If n is smaller, the allocated memory may reduced. +func (b *Uint8Builder) Resize(n int) { + nBuilder := n + if n < minBuilderCapacity { + n = minBuilderCapacity + } + + if b.capacity == 0 { + b.init(n) + } else { + b.builder.resize(nBuilder, b.init) + b.data.Resize(arrow.Uint8Traits.BytesRequired(n)) + b.rawData = arrow.Uint8Traits.CastFromBytes(b.data.Bytes()) + } +} + +// NewArray creates a Uint8 array from the memory buffers used by the builder and resets the Uint8Builder +// so it can be used to build a new array. +func (b *Uint8Builder) NewArray() Interface { + return b.NewUint8Array() +} + +// NewUint8Array creates a Uint8 array from the memory buffers used by the builder and resets the Uint8Builder +// so it can be used to build a new array. +func (b *Uint8Builder) NewUint8Array() (a *Uint8) { + data := b.newData() + a = NewUint8Data(data) + data.Release() + return +} + +func (b *Uint8Builder) newData() (data *Data) { + bytesRequired := arrow.Uint8Traits.BytesRequired(b.length) + if bytesRequired > 0 && bytesRequired < b.data.Len() { + // trim buffers + b.data.Resize(bytesRequired) + } + data = NewData(arrow.PrimitiveTypes.Uint8, b.length, []*memory.Buffer{b.nullBitmap, b.data}, nil, b.nulls, 0) + b.reset() + + if b.data != nil { + b.data.Release() + b.data = nil + b.rawData = nil + } + + return +} + +type TimestampBuilder struct { + builder + + dtype *arrow.TimestampType + data *memory.Buffer + rawData []arrow.Timestamp +} + +func NewTimestampBuilder(mem memory.Allocator, dtype *arrow.TimestampType) *TimestampBuilder { + return &TimestampBuilder{builder: builder{refCount: 1, mem: mem}, dtype: dtype} +} + +// Release decreases the reference count by 1. +// When the reference count goes to zero, the memory is freed. +func (b *TimestampBuilder) Release() { + debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases") + + if atomic.AddInt64(&b.refCount, -1) == 0 { + if b.nullBitmap != nil { + b.nullBitmap.Release() + b.nullBitmap = nil + } + if b.data != nil { + b.data.Release() + b.data = nil + b.rawData = nil + } + } +} + +func (b *TimestampBuilder) Append(v arrow.Timestamp) { + b.Reserve(1) + b.UnsafeAppend(v) +} + +func (b *TimestampBuilder) AppendNull() { + b.Reserve(1) + b.UnsafeAppendBoolToBitmap(false) +} + +func (b *TimestampBuilder) UnsafeAppend(v arrow.Timestamp) { + bitutil.SetBit(b.nullBitmap.Bytes(), b.length) + b.rawData[b.length] = v + b.length++ +} + +func (b *TimestampBuilder) UnsafeAppendBoolToBitmap(isValid bool) { + if isValid { + bitutil.SetBit(b.nullBitmap.Bytes(), b.length) + } else { + b.nulls++ + } + b.length++ +} + +// AppendValues will append the values in the v slice. The valid slice determines which values +// in v are valid (not null). The valid slice must either be empty or be equal in length to v. If empty, +// all values in v are appended and considered valid. +func (b *TimestampBuilder) AppendValues(v []arrow.Timestamp, valid []bool) { + if len(v) != len(valid) && len(valid) != 0 { + panic("len(v) != len(valid) && len(valid) != 0") + } + + if len(v) == 0 { + return + } + + b.Reserve(len(v)) + arrow.TimestampTraits.Copy(b.rawData[b.length:], v) + b.builder.unsafeAppendBoolsToBitmap(valid, len(v)) +} + +func (b *TimestampBuilder) init(capacity int) { + b.builder.init(capacity) + + b.data = memory.NewResizableBuffer(b.mem) + bytesN := arrow.TimestampTraits.BytesRequired(capacity) + b.data.Resize(bytesN) + b.rawData = arrow.TimestampTraits.CastFromBytes(b.data.Bytes()) +} + +// Reserve ensures there is enough space for appending n elements +// by checking the capacity and calling Resize if necessary. +func (b *TimestampBuilder) Reserve(n int) { + b.builder.reserve(n, b.Resize) +} + +// Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(), +// additional memory will be allocated. If n is smaller, the allocated memory may reduced. +func (b *TimestampBuilder) Resize(n int) { + nBuilder := n + if n < minBuilderCapacity { + n = minBuilderCapacity + } + + if b.capacity == 0 { + b.init(n) + } else { + b.builder.resize(nBuilder, b.init) + b.data.Resize(arrow.TimestampTraits.BytesRequired(n)) + b.rawData = arrow.TimestampTraits.CastFromBytes(b.data.Bytes()) + } +} + +// NewArray creates a Timestamp array from the memory buffers used by the builder and resets the TimestampBuilder +// so it can be used to build a new array. +func (b *TimestampBuilder) NewArray() Interface { + return b.NewTimestampArray() +} + +// NewTimestampArray creates a Timestamp array from the memory buffers used by the builder and resets the TimestampBuilder +// so it can be used to build a new array. +func (b *TimestampBuilder) NewTimestampArray() (a *Timestamp) { + data := b.newData() + a = NewTimestampData(data) + data.Release() + return +} + +func (b *TimestampBuilder) newData() (data *Data) { + bytesRequired := arrow.TimestampTraits.BytesRequired(b.length) + if bytesRequired > 0 && bytesRequired < b.data.Len() { + // trim buffers + b.data.Resize(bytesRequired) + } + data = NewData(b.dtype, b.length, []*memory.Buffer{b.nullBitmap, b.data}, nil, b.nulls, 0) + b.reset() + + if b.data != nil { + b.data.Release() + b.data = nil + b.rawData = nil + } + + return +} + +type Time32Builder struct { + builder + + dtype *arrow.Time32Type + data *memory.Buffer + rawData []arrow.Time32 +} + +func NewTime32Builder(mem memory.Allocator, dtype *arrow.Time32Type) *Time32Builder { + return &Time32Builder{builder: builder{refCount: 1, mem: mem}, dtype: dtype} +} + +// Release decreases the reference count by 1. +// When the reference count goes to zero, the memory is freed. +func (b *Time32Builder) Release() { + debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases") + + if atomic.AddInt64(&b.refCount, -1) == 0 { + if b.nullBitmap != nil { + b.nullBitmap.Release() + b.nullBitmap = nil + } + if b.data != nil { + b.data.Release() + b.data = nil + b.rawData = nil + } + } +} + +func (b *Time32Builder) Append(v arrow.Time32) { + b.Reserve(1) + b.UnsafeAppend(v) +} + +func (b *Time32Builder) AppendNull() { + b.Reserve(1) + b.UnsafeAppendBoolToBitmap(false) +} + +func (b *Time32Builder) UnsafeAppend(v arrow.Time32) { + bitutil.SetBit(b.nullBitmap.Bytes(), b.length) + b.rawData[b.length] = v + b.length++ +} + +func (b *Time32Builder) UnsafeAppendBoolToBitmap(isValid bool) { + if isValid { + bitutil.SetBit(b.nullBitmap.Bytes(), b.length) + } else { + b.nulls++ + } + b.length++ +} + +// AppendValues will append the values in the v slice. The valid slice determines which values +// in v are valid (not null). The valid slice must either be empty or be equal in length to v. If empty, +// all values in v are appended and considered valid. +func (b *Time32Builder) AppendValues(v []arrow.Time32, valid []bool) { + if len(v) != len(valid) && len(valid) != 0 { + panic("len(v) != len(valid) && len(valid) != 0") + } + + if len(v) == 0 { + return + } + + b.Reserve(len(v)) + arrow.Time32Traits.Copy(b.rawData[b.length:], v) + b.builder.unsafeAppendBoolsToBitmap(valid, len(v)) +} + +func (b *Time32Builder) init(capacity int) { + b.builder.init(capacity) + + b.data = memory.NewResizableBuffer(b.mem) + bytesN := arrow.Time32Traits.BytesRequired(capacity) + b.data.Resize(bytesN) + b.rawData = arrow.Time32Traits.CastFromBytes(b.data.Bytes()) +} + +// Reserve ensures there is enough space for appending n elements +// by checking the capacity and calling Resize if necessary. +func (b *Time32Builder) Reserve(n int) { + b.builder.reserve(n, b.Resize) +} + +// Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(), +// additional memory will be allocated. If n is smaller, the allocated memory may reduced. +func (b *Time32Builder) Resize(n int) { + nBuilder := n + if n < minBuilderCapacity { + n = minBuilderCapacity + } + + if b.capacity == 0 { + b.init(n) + } else { + b.builder.resize(nBuilder, b.init) + b.data.Resize(arrow.Time32Traits.BytesRequired(n)) + b.rawData = arrow.Time32Traits.CastFromBytes(b.data.Bytes()) + } +} + +// NewArray creates a Time32 array from the memory buffers used by the builder and resets the Time32Builder +// so it can be used to build a new array. +func (b *Time32Builder) NewArray() Interface { + return b.NewTime32Array() +} + +// NewTime32Array creates a Time32 array from the memory buffers used by the builder and resets the Time32Builder +// so it can be used to build a new array. +func (b *Time32Builder) NewTime32Array() (a *Time32) { + data := b.newData() + a = NewTime32Data(data) + data.Release() + return +} + +func (b *Time32Builder) newData() (data *Data) { + bytesRequired := arrow.Time32Traits.BytesRequired(b.length) + if bytesRequired > 0 && bytesRequired < b.data.Len() { + // trim buffers + b.data.Resize(bytesRequired) + } + data = NewData(b.dtype, b.length, []*memory.Buffer{b.nullBitmap, b.data}, nil, b.nulls, 0) + b.reset() + + if b.data != nil { + b.data.Release() + b.data = nil + b.rawData = nil + } + + return +} + +type Time64Builder struct { + builder + + dtype *arrow.Time64Type + data *memory.Buffer + rawData []arrow.Time64 +} + +func NewTime64Builder(mem memory.Allocator, dtype *arrow.Time64Type) *Time64Builder { + return &Time64Builder{builder: builder{refCount: 1, mem: mem}, dtype: dtype} +} + +// Release decreases the reference count by 1. +// When the reference count goes to zero, the memory is freed. +func (b *Time64Builder) Release() { + debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases") + + if atomic.AddInt64(&b.refCount, -1) == 0 { + if b.nullBitmap != nil { + b.nullBitmap.Release() + b.nullBitmap = nil + } + if b.data != nil { + b.data.Release() + b.data = nil + b.rawData = nil + } + } +} + +func (b *Time64Builder) Append(v arrow.Time64) { + b.Reserve(1) + b.UnsafeAppend(v) +} + +func (b *Time64Builder) AppendNull() { + b.Reserve(1) + b.UnsafeAppendBoolToBitmap(false) +} + +func (b *Time64Builder) UnsafeAppend(v arrow.Time64) { + bitutil.SetBit(b.nullBitmap.Bytes(), b.length) + b.rawData[b.length] = v + b.length++ +} + +func (b *Time64Builder) UnsafeAppendBoolToBitmap(isValid bool) { + if isValid { + bitutil.SetBit(b.nullBitmap.Bytes(), b.length) + } else { + b.nulls++ + } + b.length++ +} + +// AppendValues will append the values in the v slice. The valid slice determines which values +// in v are valid (not null). The valid slice must either be empty or be equal in length to v. If empty, +// all values in v are appended and considered valid. +func (b *Time64Builder) AppendValues(v []arrow.Time64, valid []bool) { + if len(v) != len(valid) && len(valid) != 0 { + panic("len(v) != len(valid) && len(valid) != 0") + } + + if len(v) == 0 { + return + } + + b.Reserve(len(v)) + arrow.Time64Traits.Copy(b.rawData[b.length:], v) + b.builder.unsafeAppendBoolsToBitmap(valid, len(v)) +} + +func (b *Time64Builder) init(capacity int) { + b.builder.init(capacity) + + b.data = memory.NewResizableBuffer(b.mem) + bytesN := arrow.Time64Traits.BytesRequired(capacity) + b.data.Resize(bytesN) + b.rawData = arrow.Time64Traits.CastFromBytes(b.data.Bytes()) +} + +// Reserve ensures there is enough space for appending n elements +// by checking the capacity and calling Resize if necessary. +func (b *Time64Builder) Reserve(n int) { + b.builder.reserve(n, b.Resize) +} + +// Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(), +// additional memory will be allocated. If n is smaller, the allocated memory may reduced. +func (b *Time64Builder) Resize(n int) { + nBuilder := n + if n < minBuilderCapacity { + n = minBuilderCapacity + } + + if b.capacity == 0 { + b.init(n) + } else { + b.builder.resize(nBuilder, b.init) + b.data.Resize(arrow.Time64Traits.BytesRequired(n)) + b.rawData = arrow.Time64Traits.CastFromBytes(b.data.Bytes()) + } +} + +// NewArray creates a Time64 array from the memory buffers used by the builder and resets the Time64Builder +// so it can be used to build a new array. +func (b *Time64Builder) NewArray() Interface { + return b.NewTime64Array() +} + +// NewTime64Array creates a Time64 array from the memory buffers used by the builder and resets the Time64Builder +// so it can be used to build a new array. +func (b *Time64Builder) NewTime64Array() (a *Time64) { + data := b.newData() + a = NewTime64Data(data) + data.Release() + return +} + +func (b *Time64Builder) newData() (data *Data) { + bytesRequired := arrow.Time64Traits.BytesRequired(b.length) + if bytesRequired > 0 && bytesRequired < b.data.Len() { + // trim buffers + b.data.Resize(bytesRequired) + } + data = NewData(b.dtype, b.length, []*memory.Buffer{b.nullBitmap, b.data}, nil, b.nulls, 0) + b.reset() + + if b.data != nil { + b.data.Release() + b.data = nil + b.rawData = nil + } + + return +} + +type Date32Builder struct { + builder + + data *memory.Buffer + rawData []arrow.Date32 +} + +func NewDate32Builder(mem memory.Allocator) *Date32Builder { + return &Date32Builder{builder: builder{refCount: 1, mem: mem}} +} + +// Release decreases the reference count by 1. +// When the reference count goes to zero, the memory is freed. +func (b *Date32Builder) Release() { + debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases") + + if atomic.AddInt64(&b.refCount, -1) == 0 { + if b.nullBitmap != nil { + b.nullBitmap.Release() + b.nullBitmap = nil + } + if b.data != nil { + b.data.Release() + b.data = nil + b.rawData = nil + } + } +} + +func (b *Date32Builder) Append(v arrow.Date32) { + b.Reserve(1) + b.UnsafeAppend(v) +} + +func (b *Date32Builder) AppendNull() { + b.Reserve(1) + b.UnsafeAppendBoolToBitmap(false) +} + +func (b *Date32Builder) UnsafeAppend(v arrow.Date32) { + bitutil.SetBit(b.nullBitmap.Bytes(), b.length) + b.rawData[b.length] = v + b.length++ +} + +func (b *Date32Builder) UnsafeAppendBoolToBitmap(isValid bool) { + if isValid { + bitutil.SetBit(b.nullBitmap.Bytes(), b.length) + } else { + b.nulls++ + } + b.length++ +} + +// AppendValues will append the values in the v slice. The valid slice determines which values +// in v are valid (not null). The valid slice must either be empty or be equal in length to v. If empty, +// all values in v are appended and considered valid. +func (b *Date32Builder) AppendValues(v []arrow.Date32, valid []bool) { + if len(v) != len(valid) && len(valid) != 0 { + panic("len(v) != len(valid) && len(valid) != 0") + } + + if len(v) == 0 { + return + } + + b.Reserve(len(v)) + arrow.Date32Traits.Copy(b.rawData[b.length:], v) + b.builder.unsafeAppendBoolsToBitmap(valid, len(v)) +} + +func (b *Date32Builder) init(capacity int) { + b.builder.init(capacity) + + b.data = memory.NewResizableBuffer(b.mem) + bytesN := arrow.Date32Traits.BytesRequired(capacity) + b.data.Resize(bytesN) + b.rawData = arrow.Date32Traits.CastFromBytes(b.data.Bytes()) +} + +// Reserve ensures there is enough space for appending n elements +// by checking the capacity and calling Resize if necessary. +func (b *Date32Builder) Reserve(n int) { + b.builder.reserve(n, b.Resize) +} + +// Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(), +// additional memory will be allocated. If n is smaller, the allocated memory may reduced. +func (b *Date32Builder) Resize(n int) { + nBuilder := n + if n < minBuilderCapacity { + n = minBuilderCapacity + } + + if b.capacity == 0 { + b.init(n) + } else { + b.builder.resize(nBuilder, b.init) + b.data.Resize(arrow.Date32Traits.BytesRequired(n)) + b.rawData = arrow.Date32Traits.CastFromBytes(b.data.Bytes()) + } +} + +// NewArray creates a Date32 array from the memory buffers used by the builder and resets the Date32Builder +// so it can be used to build a new array. +func (b *Date32Builder) NewArray() Interface { + return b.NewDate32Array() +} + +// NewDate32Array creates a Date32 array from the memory buffers used by the builder and resets the Date32Builder +// so it can be used to build a new array. +func (b *Date32Builder) NewDate32Array() (a *Date32) { + data := b.newData() + a = NewDate32Data(data) + data.Release() + return +} + +func (b *Date32Builder) newData() (data *Data) { + bytesRequired := arrow.Date32Traits.BytesRequired(b.length) + if bytesRequired > 0 && bytesRequired < b.data.Len() { + // trim buffers + b.data.Resize(bytesRequired) + } + data = NewData(arrow.PrimitiveTypes.Date32, b.length, []*memory.Buffer{b.nullBitmap, b.data}, nil, b.nulls, 0) + b.reset() + + if b.data != nil { + b.data.Release() + b.data = nil + b.rawData = nil + } + + return +} + +type Date64Builder struct { + builder + + data *memory.Buffer + rawData []arrow.Date64 +} + +func NewDate64Builder(mem memory.Allocator) *Date64Builder { + return &Date64Builder{builder: builder{refCount: 1, mem: mem}} +} + +// Release decreases the reference count by 1. +// When the reference count goes to zero, the memory is freed. +func (b *Date64Builder) Release() { + debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases") + + if atomic.AddInt64(&b.refCount, -1) == 0 { + if b.nullBitmap != nil { + b.nullBitmap.Release() + b.nullBitmap = nil + } + if b.data != nil { + b.data.Release() + b.data = nil + b.rawData = nil + } + } +} + +func (b *Date64Builder) Append(v arrow.Date64) { + b.Reserve(1) + b.UnsafeAppend(v) +} + +func (b *Date64Builder) AppendNull() { + b.Reserve(1) + b.UnsafeAppendBoolToBitmap(false) +} + +func (b *Date64Builder) UnsafeAppend(v arrow.Date64) { + bitutil.SetBit(b.nullBitmap.Bytes(), b.length) + b.rawData[b.length] = v + b.length++ +} + +func (b *Date64Builder) UnsafeAppendBoolToBitmap(isValid bool) { + if isValid { + bitutil.SetBit(b.nullBitmap.Bytes(), b.length) + } else { + b.nulls++ + } + b.length++ +} + +// AppendValues will append the values in the v slice. The valid slice determines which values +// in v are valid (not null). The valid slice must either be empty or be equal in length to v. If empty, +// all values in v are appended and considered valid. +func (b *Date64Builder) AppendValues(v []arrow.Date64, valid []bool) { + if len(v) != len(valid) && len(valid) != 0 { + panic("len(v) != len(valid) && len(valid) != 0") + } + + if len(v) == 0 { + return + } + + b.Reserve(len(v)) + arrow.Date64Traits.Copy(b.rawData[b.length:], v) + b.builder.unsafeAppendBoolsToBitmap(valid, len(v)) +} + +func (b *Date64Builder) init(capacity int) { + b.builder.init(capacity) + + b.data = memory.NewResizableBuffer(b.mem) + bytesN := arrow.Date64Traits.BytesRequired(capacity) + b.data.Resize(bytesN) + b.rawData = arrow.Date64Traits.CastFromBytes(b.data.Bytes()) +} + +// Reserve ensures there is enough space for appending n elements +// by checking the capacity and calling Resize if necessary. +func (b *Date64Builder) Reserve(n int) { + b.builder.reserve(n, b.Resize) +} + +// Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(), +// additional memory will be allocated. If n is smaller, the allocated memory may reduced. +func (b *Date64Builder) Resize(n int) { + nBuilder := n + if n < minBuilderCapacity { + n = minBuilderCapacity + } + + if b.capacity == 0 { + b.init(n) + } else { + b.builder.resize(nBuilder, b.init) + b.data.Resize(arrow.Date64Traits.BytesRequired(n)) + b.rawData = arrow.Date64Traits.CastFromBytes(b.data.Bytes()) + } +} + +// NewArray creates a Date64 array from the memory buffers used by the builder and resets the Date64Builder +// so it can be used to build a new array. +func (b *Date64Builder) NewArray() Interface { + return b.NewDate64Array() +} + +// NewDate64Array creates a Date64 array from the memory buffers used by the builder and resets the Date64Builder +// so it can be used to build a new array. +func (b *Date64Builder) NewDate64Array() (a *Date64) { + data := b.newData() + a = NewDate64Data(data) + data.Release() + return +} + +func (b *Date64Builder) newData() (data *Data) { + bytesRequired := arrow.Date64Traits.BytesRequired(b.length) + if bytesRequired > 0 && bytesRequired < b.data.Len() { + // trim buffers + b.data.Resize(bytesRequired) + } + data = NewData(arrow.PrimitiveTypes.Date64, b.length, []*memory.Buffer{b.nullBitmap, b.data}, nil, b.nulls, 0) + b.reset() + + if b.data != nil { + b.data.Release() + b.data = nil + b.rawData = nil + } + + return +} + +type DurationBuilder struct { + builder + + dtype *arrow.DurationType + data *memory.Buffer + rawData []arrow.Duration +} + +func NewDurationBuilder(mem memory.Allocator, dtype *arrow.DurationType) *DurationBuilder { + return &DurationBuilder{builder: builder{refCount: 1, mem: mem}, dtype: dtype} +} + +// Release decreases the reference count by 1. +// When the reference count goes to zero, the memory is freed. +func (b *DurationBuilder) Release() { + debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases") + + if atomic.AddInt64(&b.refCount, -1) == 0 { + if b.nullBitmap != nil { + b.nullBitmap.Release() + b.nullBitmap = nil + } + if b.data != nil { + b.data.Release() + b.data = nil + b.rawData = nil + } + } +} + +func (b *DurationBuilder) Append(v arrow.Duration) { + b.Reserve(1) + b.UnsafeAppend(v) +} + +func (b *DurationBuilder) AppendNull() { + b.Reserve(1) + b.UnsafeAppendBoolToBitmap(false) +} + +func (b *DurationBuilder) UnsafeAppend(v arrow.Duration) { + bitutil.SetBit(b.nullBitmap.Bytes(), b.length) + b.rawData[b.length] = v + b.length++ +} + +func (b *DurationBuilder) UnsafeAppendBoolToBitmap(isValid bool) { + if isValid { + bitutil.SetBit(b.nullBitmap.Bytes(), b.length) + } else { + b.nulls++ + } + b.length++ +} + +// AppendValues will append the values in the v slice. The valid slice determines which values +// in v are valid (not null). The valid slice must either be empty or be equal in length to v. If empty, +// all values in v are appended and considered valid. +func (b *DurationBuilder) AppendValues(v []arrow.Duration, valid []bool) { + if len(v) != len(valid) && len(valid) != 0 { + panic("len(v) != len(valid) && len(valid) != 0") + } + + if len(v) == 0 { + return + } + + b.Reserve(len(v)) + arrow.DurationTraits.Copy(b.rawData[b.length:], v) + b.builder.unsafeAppendBoolsToBitmap(valid, len(v)) +} + +func (b *DurationBuilder) init(capacity int) { + b.builder.init(capacity) + + b.data = memory.NewResizableBuffer(b.mem) + bytesN := arrow.DurationTraits.BytesRequired(capacity) + b.data.Resize(bytesN) + b.rawData = arrow.DurationTraits.CastFromBytes(b.data.Bytes()) +} + +// Reserve ensures there is enough space for appending n elements +// by checking the capacity and calling Resize if necessary. +func (b *DurationBuilder) Reserve(n int) { + b.builder.reserve(n, b.Resize) +} + +// Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(), +// additional memory will be allocated. If n is smaller, the allocated memory may reduced. +func (b *DurationBuilder) Resize(n int) { + nBuilder := n + if n < minBuilderCapacity { + n = minBuilderCapacity + } + + if b.capacity == 0 { + b.init(n) + } else { + b.builder.resize(nBuilder, b.init) + b.data.Resize(arrow.DurationTraits.BytesRequired(n)) + b.rawData = arrow.DurationTraits.CastFromBytes(b.data.Bytes()) + } +} + +// NewArray creates a Duration array from the memory buffers used by the builder and resets the DurationBuilder +// so it can be used to build a new array. +func (b *DurationBuilder) NewArray() Interface { + return b.NewDurationArray() +} + +// NewDurationArray creates a Duration array from the memory buffers used by the builder and resets the DurationBuilder +// so it can be used to build a new array. +func (b *DurationBuilder) NewDurationArray() (a *Duration) { + data := b.newData() + a = NewDurationData(data) + data.Release() + return +} + +func (b *DurationBuilder) newData() (data *Data) { + bytesRequired := arrow.DurationTraits.BytesRequired(b.length) + if bytesRequired > 0 && bytesRequired < b.data.Len() { + // trim buffers + b.data.Resize(bytesRequired) + } + data = NewData(b.dtype, b.length, []*memory.Buffer{b.nullBitmap, b.data}, nil, b.nulls, 0) + b.reset() + + if b.data != nil { + b.data.Release() + b.data = nil + b.rawData = nil + } + + return +} + +var ( + _ Builder = (*Int64Builder)(nil) + _ Builder = (*Uint64Builder)(nil) + _ Builder = (*Float64Builder)(nil) + _ Builder = (*Int32Builder)(nil) + _ Builder = (*Uint32Builder)(nil) + _ Builder = (*Float32Builder)(nil) + _ Builder = (*Int16Builder)(nil) + _ Builder = (*Uint16Builder)(nil) + _ Builder = (*Int8Builder)(nil) + _ Builder = (*Uint8Builder)(nil) + _ Builder = (*TimestampBuilder)(nil) + _ Builder = (*Time32Builder)(nil) + _ Builder = (*Time64Builder)(nil) + _ Builder = (*Date32Builder)(nil) + _ Builder = (*Date64Builder)(nil) + _ Builder = (*DurationBuilder)(nil) +) diff --git a/vendor/github.com/apache/arrow/go/arrow/array/numericbuilder.gen.go.tmpl b/vendor/github.com/apache/arrow/go/arrow/array/numericbuilder.gen.go.tmpl new file mode 100644 index 00000000000..e47e9b1d7e9 --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/array/numericbuilder.gen.go.tmpl @@ -0,0 +1,182 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package array + +import ( + "github.com/apache/arrow/go/arrow" + "github.com/apache/arrow/go/arrow/internal/bitutil" + "github.com/apache/arrow/go/arrow/internal/debug" + "github.com/apache/arrow/go/arrow/memory" +) + +{{range .In}} + +type {{.Name}}Builder struct { + builder + +{{if .Opt.Parametric -}} + dtype *arrow.{{.Name}}Type +{{end -}} + data *memory.Buffer + rawData []{{or .QualifiedType .Type}} +} + +{{if .Opt.Parametric}} +func New{{.Name}}Builder(mem memory.Allocator, dtype *arrow.{{.Name}}Type) *{{.Name}}Builder { + return &{{.Name}}Builder{builder: builder{refCount:1, mem: mem}, dtype: dtype} +} +{{else}} +func New{{.Name}}Builder(mem memory.Allocator) *{{.Name}}Builder { + return &{{.Name}}Builder{builder: builder{refCount:1, mem: mem}} +} +{{end}} + +// Release decreases the reference count by 1. +// When the reference count goes to zero, the memory is freed. +func (b *{{.Name}}Builder) Release() { + debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases") + + if atomic.AddInt64(&b.refCount, -1) == 0 { + if b.nullBitmap != nil { + b.nullBitmap.Release() + b.nullBitmap = nil + } + if b.data != nil { + b.data.Release() + b.data = nil + b.rawData = nil + } + } +} + +func (b *{{.Name}}Builder) Append(v {{or .QualifiedType .Type}}) { + b.Reserve(1) + b.UnsafeAppend(v) +} + +func (b *{{.Name}}Builder) AppendNull() { + b.Reserve(1) + b.UnsafeAppendBoolToBitmap(false) +} + +func (b *{{.Name}}Builder) UnsafeAppend(v {{or .QualifiedType .Type}}) { + bitutil.SetBit(b.nullBitmap.Bytes(), b.length) + b.rawData[b.length] = v + b.length++ +} + +func (b *{{.Name}}Builder) UnsafeAppendBoolToBitmap(isValid bool) { + if isValid { + bitutil.SetBit(b.nullBitmap.Bytes(), b.length) + } else { + b.nulls++ + } + b.length++ +} + +// AppendValues will append the values in the v slice. The valid slice determines which values +// in v are valid (not null). The valid slice must either be empty or be equal in length to v. If empty, +// all values in v are appended and considered valid. +func (b *{{.Name}}Builder) AppendValues(v []{{or .QualifiedType .Type}}, valid []bool) { + if len(v) != len(valid) && len(valid) != 0 { + panic("len(v) != len(valid) && len(valid) != 0") + } + + if len(v) == 0 { + return + } + + b.Reserve(len(v)) + arrow.{{.Name}}Traits.Copy(b.rawData[b.length:], v) + b.builder.unsafeAppendBoolsToBitmap(valid, len(v)) +} + +func (b *{{.Name}}Builder) init(capacity int) { + b.builder.init(capacity) + + b.data = memory.NewResizableBuffer(b.mem) + bytesN := arrow.{{.Name}}Traits.BytesRequired(capacity) + b.data.Resize(bytesN) + b.rawData = arrow.{{.Name}}Traits.CastFromBytes(b.data.Bytes()) +} + +// Reserve ensures there is enough space for appending n elements +// by checking the capacity and calling Resize if necessary. +func (b *{{.Name}}Builder) Reserve(n int) { + b.builder.reserve(n, b.Resize) +} + +// Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(), +// additional memory will be allocated. If n is smaller, the allocated memory may reduced. +func (b *{{.Name}}Builder) Resize(n int) { + nBuilder := n + if n < minBuilderCapacity { + n = minBuilderCapacity + } + + if b.capacity == 0 { + b.init(n) + } else { + b.builder.resize(nBuilder, b.init) + b.data.Resize(arrow.{{.Name}}Traits.BytesRequired(n)) + b.rawData = arrow.{{.Name}}Traits.CastFromBytes(b.data.Bytes()) + } +} + +// NewArray creates a {{.Name}} array from the memory buffers used by the builder and resets the {{.Name}}Builder +// so it can be used to build a new array. +func (b *{{.Name}}Builder) NewArray() Interface { + return b.New{{.Name}}Array() +} + +// New{{.Name}}Array creates a {{.Name}} array from the memory buffers used by the builder and resets the {{.Name}}Builder +// so it can be used to build a new array. +func (b *{{.Name}}Builder) New{{.Name}}Array() (a *{{.Name}}) { + data := b.newData() + a = New{{.Name}}Data(data) + data.Release() + return +} + +func (b *{{.Name}}Builder) newData() (data *Data) { + bytesRequired := arrow.{{.Name}}Traits.BytesRequired(b.length) + if bytesRequired > 0 && bytesRequired < b.data.Len() { + // trim buffers + b.data.Resize(bytesRequired) + } +{{if .Opt.Parametric -}} + data = NewData(b.dtype, b.length, []*memory.Buffer{b.nullBitmap, b.data}, nil, b.nulls, 0) +{{else -}} + data = NewData(arrow.PrimitiveTypes.{{.Name}}, b.length, []*memory.Buffer{b.nullBitmap, b.data}, nil, b.nulls, 0) +{{end -}} + b.reset() + + if b.data != nil { + b.data.Release() + b.data = nil + b.rawData = nil + } + + return +} +{{end}} + +var ( +{{- range .In}} + _ Builder = (*{{.Name}}Builder)(nil) +{{- end}} +) diff --git a/vendor/github.com/apache/arrow/go/arrow/array/numericbuilder.gen_test.go.tmpl b/vendor/github.com/apache/arrow/go/arrow/array/numericbuilder.gen_test.go.tmpl new file mode 100644 index 00000000000..e4f78108830 --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/array/numericbuilder.gen_test.go.tmpl @@ -0,0 +1,216 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package array_test + +import ( + "testing" + + "github.com/apache/arrow/go/arrow" + "github.com/apache/arrow/go/arrow/array" + "github.com/apache/arrow/go/arrow/memory" + "github.com/stretchr/testify/assert" +) + +{{range .In}} +func TestNew{{.Name}}Builder(t *testing.T) { + mem := memory.NewCheckedAllocator(memory.NewGoAllocator()) + defer mem.AssertSize(t, 0) + +{{if .Opt.Parametric -}} + dtype := &arrow.{{.Name}}Type{Unit: arrow.Second} + ab := array.New{{.Name}}Builder(mem, dtype) +{{else}} + ab := array.New{{.Name}}Builder(mem) +{{end -}} + defer ab.Release() + + ab.Retain() + ab.Release() + + ab.Append(1) + ab.Append(2) + ab.Append(3) + ab.AppendNull() + ab.Append(5) + ab.Append(6) + ab.AppendNull() + ab.Append(8) + ab.Append(9) + ab.Append(10) + + // check state of builder before New{{.Name}}Array + assert.Equal(t, 10, ab.Len(), "unexpected Len()") + assert.Equal(t, 2, ab.NullN(), "unexpected NullN()") + + a := ab.New{{.Name}}Array() + + // check state of builder after New{{.Name}}Array + assert.Zero(t, ab.Len(), "unexpected ArrayBuilder.Len(), New{{.Name}}Array did not reset state") + assert.Zero(t, ab.Cap(), "unexpected ArrayBuilder.Cap(), New{{.Name}}Array did not reset state") + assert.Zero(t, ab.NullN(), "unexpected ArrayBuilder.NullN(), New{{.Name}}Array did not reset state") + + // check state of array + assert.Equal(t, 2, a.NullN(), "unexpected null count") + assert.Equal(t, []{{or .QualifiedType .Type}}{1, 2, 3, 0, 5, 6, 0, 8, 9, 10}, a.{{.Name}}Values(), "unexpected {{.Name}}Values") + assert.Equal(t, []byte{0xb7}, a.NullBitmapBytes()[:1]) // 4 bytes due to minBuilderCapacity + assert.Len(t, a.{{.Name}}Values(), 10, "unexpected length of {{.Name}}Values") + + a.Release() + + ab.Append(7) + ab.Append(8) + + a = ab.New{{.Name}}Array() + + assert.Equal(t, 0, a.NullN()) + assert.Equal(t, []{{or .QualifiedType .Type}}{7, 8}, a.{{.Name}}Values()) + assert.Len(t, a.{{.Name}}Values(), 2) + + a.Release() + + var ( + want = []{{or .QualifiedType .Type}}{1, 2, 3, 4} + valids = []bool{true, true, false, true} + ) + + ab.AppendValues(want, valids) + a = ab.New{{.Name}}Array() + + sub := array.MakeFromData(a.Data()) + defer sub.Release() + + if got, want := sub.DataType().ID(), a.DataType().ID(); got != want { + t.Fatalf("invalid type: got=%q, want=%q", got, want) + } + + if _, ok := sub.(*array.{{.Name}}); !ok { + t.Fatalf("could not type-assert to array.{{.Name}}") + } + + if got, want := a.String(), `[1 2 (null) 4]`; got != want { + t.Fatalf("got=%q, want=%q", got, want) + } + + slice := array.NewSliceData(a.Data(), 2, 4) + defer slice.Release() + + sub1 := array.MakeFromData(slice) + defer sub1.Release() + + v, ok := sub1.(*array.{{.Name}}) + if !ok { + t.Fatalf("could not type-assert to array.{{.Name}}") + } + + if got, want := v.String(), `[(null) 4]`; got != want { + t.Fatalf("got=%q, want=%q", got, want) + } + + a.Release() +} + +func Test{{.Name}}Builder_AppendValues(t *testing.T) { + mem := memory.NewCheckedAllocator(memory.NewGoAllocator()) + defer mem.AssertSize(t, 0) + +{{if .Opt.Parametric -}} + dtype := &arrow.{{.Name}}Type{Unit: arrow.Second} + ab := array.New{{.Name}}Builder(mem, dtype) +{{else}} + ab := array.New{{.Name}}Builder(mem) +{{end -}} + defer ab.Release() + + exp := []{{or .QualifiedType .Type}}{0, 1, 2, 3} + ab.AppendValues(exp, nil) + a := ab.New{{.Name}}Array() + assert.Equal(t, exp, a.{{.Name}}Values()) + + a.Release() +} + +func Test{{.Name}}Builder_Empty(t *testing.T) { + mem := memory.NewCheckedAllocator(memory.NewGoAllocator()) + defer mem.AssertSize(t, 0) + +{{if .Opt.Parametric -}} + dtype := &arrow.{{.Name}}Type{Unit: arrow.Second} + ab := array.New{{.Name}}Builder(mem, dtype) +{{else}} + ab := array.New{{.Name}}Builder(mem) +{{end -}} + defer ab.Release() + + exp := []{{or .QualifiedType .Type}}{0, 1, 2, 3} + + ab.AppendValues([]{{or .QualifiedType .Type}}{}, nil) + a := ab.New{{.Name}}Array() + assert.Zero(t, a.Len()) + a.Release() + + ab.AppendValues(nil, nil) + a = ab.New{{.Name}}Array() + assert.Zero(t, a.Len()) + a.Release() + + ab.AppendValues([]{{or .QualifiedType .Type}}{}, nil) + ab.AppendValues(exp, nil) + a = ab.New{{.Name}}Array() + assert.Equal(t, exp, a.{{.Name}}Values()) + a.Release() + + ab.AppendValues(exp, nil) + ab.AppendValues([]{{or .QualifiedType .Type}}{}, nil) + a = ab.New{{.Name}}Array() + assert.Equal(t, exp, a.{{.Name}}Values()) + a.Release() +} + +func Test{{.Name}}Builder_Resize(t *testing.T) { + mem := memory.NewCheckedAllocator(memory.NewGoAllocator()) + defer mem.AssertSize(t, 0) + +{{if .Opt.Parametric -}} + dtype := &arrow.{{.Name}}Type{Unit: arrow.Second} + ab := array.New{{.Name}}Builder(mem, dtype) +{{else}} + ab := array.New{{.Name}}Builder(mem) +{{end -}} + defer ab.Release() + + assert.Equal(t, 0, ab.Cap()) + assert.Equal(t, 0, ab.Len()) + + ab.Reserve(63) + assert.Equal(t, 64, ab.Cap()) + assert.Equal(t, 0, ab.Len()) + + for i := 0; i < 63; i++ { + ab.Append(0) + } + assert.Equal(t, 64, ab.Cap()) + assert.Equal(t, 63, ab.Len()) + + ab.Resize(5) + assert.Equal(t, 5, ab.Len()) + + ab.Resize(32) + assert.Equal(t, 5, ab.Len()) +} +{{end}} + + diff --git a/vendor/github.com/apache/arrow/go/arrow/array/record.go b/vendor/github.com/apache/arrow/go/arrow/array/record.go new file mode 100644 index 00000000000..f7b7bd75f70 --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/array/record.go @@ -0,0 +1,345 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package array + +import ( + "fmt" + "strings" + "sync/atomic" + + "github.com/apache/arrow/go/arrow" + "github.com/apache/arrow/go/arrow/internal/debug" + "github.com/apache/arrow/go/arrow/memory" +) + +// RecordReader reads a stream of records. +type RecordReader interface { + Retain() + Release() + + Schema() *arrow.Schema + + Next() bool + Record() Record +} + +// simpleRecords is a simple iterator over a collection of records. +type simpleRecords struct { + refCount int64 + + schema *arrow.Schema + recs []Record + cur Record +} + +// NewRecordReader returns a simple iterator over the given slice of records. +func NewRecordReader(schema *arrow.Schema, recs []Record) (*simpleRecords, error) { + rs := &simpleRecords{ + refCount: 1, + schema: schema, + recs: recs, + cur: nil, + } + + for _, rec := range rs.recs { + rec.Retain() + } + + for _, rec := range recs { + if !rec.Schema().Equal(rs.schema) { + rs.Release() + return nil, fmt.Errorf("arrow/array: mismatch schema") + } + } + + return rs, nil +} + +// Retain increases the reference count by 1. +// Retain may be called simultaneously from multiple goroutines. +func (rs *simpleRecords) Retain() { + atomic.AddInt64(&rs.refCount, 1) +} + +// Release decreases the reference count by 1. +// When the reference count goes to zero, the memory is freed. +// Release may be called simultaneously from multiple goroutines. +func (rs *simpleRecords) Release() { + debug.Assert(atomic.LoadInt64(&rs.refCount) > 0, "too many releases") + + if atomic.AddInt64(&rs.refCount, -1) == 0 { + if rs.cur != nil { + rs.cur.Release() + } + for _, rec := range rs.recs { + rec.Release() + } + rs.recs = nil + } +} + +func (rs *simpleRecords) Schema() *arrow.Schema { return rs.schema } +func (rs *simpleRecords) Record() Record { return rs.cur } +func (rs *simpleRecords) Next() bool { + if len(rs.recs) == 0 { + return false + } + if rs.cur != nil { + rs.cur.Release() + } + rs.cur = rs.recs[0] + rs.recs = rs.recs[1:] + return true +} + +// Record is a collection of equal-length arrays +// matching a particular Schema. +type Record interface { + Release() + Retain() + + Schema() *arrow.Schema + + NumRows() int64 + NumCols() int64 + + Columns() []Interface + Column(i int) Interface + ColumnName(i int) string + + // NewSlice constructs a zero-copy slice of the record with the indicated + // indices i and j, corresponding to array[i:j]. + // The returned record must be Release()'d after use. + // + // NewSlice panics if the slice is outside the valid range of the record array. + // NewSlice panics if j < i. + NewSlice(i, j int64) Record +} + +// simpleRecord is a basic, non-lazy in-memory record batch. +type simpleRecord struct { + refCount int64 + + schema *arrow.Schema + + rows int64 + arrs []Interface +} + +// NewRecord returns a basic, non-lazy in-memory record batch. +// +// NewRecord panics if the columns and schema are inconsistent. +// NewRecord panics if rows is larger than the height of the columns. +func NewRecord(schema *arrow.Schema, cols []Interface, nrows int64) *simpleRecord { + rec := &simpleRecord{ + refCount: 1, + schema: schema, + rows: nrows, + arrs: make([]Interface, len(cols)), + } + copy(rec.arrs, cols) + for _, arr := range rec.arrs { + arr.Retain() + } + + if rec.rows < 0 { + switch len(rec.arrs) { + case 0: + rec.rows = 0 + default: + rec.rows = int64(rec.arrs[0].Len()) + } + } + + err := rec.validate() + if err != nil { + rec.Release() + panic(err) + } + + return rec +} + +func (rec *simpleRecord) validate() error { + if len(rec.arrs) != len(rec.schema.Fields()) { + return fmt.Errorf("arrow/array: number of columns/fields mismatch") + } + + for i, arr := range rec.arrs { + f := rec.schema.Field(i) + if int64(arr.Len()) < rec.rows { + return fmt.Errorf("arrow/array: mismatch number of rows in column %q: got=%d, want=%d", + f.Name, + arr.Len(), rec.rows, + ) + } + if !arrow.TypeEquals(f.Type, arr.DataType()) { + return fmt.Errorf("arrow/array: column %q type mismatch: got=%v, want=%v", + f.Name, + arr.DataType(), f.Type, + ) + } + } + return nil +} + +// Retain increases the reference count by 1. +// Retain may be called simultaneously from multiple goroutines. +func (rec *simpleRecord) Retain() { + atomic.AddInt64(&rec.refCount, 1) +} + +// Release decreases the reference count by 1. +// When the reference count goes to zero, the memory is freed. +// Release may be called simultaneously from multiple goroutines. +func (rec *simpleRecord) Release() { + debug.Assert(atomic.LoadInt64(&rec.refCount) > 0, "too many releases") + + if atomic.AddInt64(&rec.refCount, -1) == 0 { + for _, arr := range rec.arrs { + arr.Release() + } + rec.arrs = nil + } +} + +func (rec *simpleRecord) Schema() *arrow.Schema { return rec.schema } +func (rec *simpleRecord) NumRows() int64 { return rec.rows } +func (rec *simpleRecord) NumCols() int64 { return int64(len(rec.arrs)) } +func (rec *simpleRecord) Columns() []Interface { return rec.arrs } +func (rec *simpleRecord) Column(i int) Interface { return rec.arrs[i] } +func (rec *simpleRecord) ColumnName(i int) string { return rec.schema.Field(i).Name } + +// NewSlice constructs a zero-copy slice of the record with the indicated +// indices i and j, corresponding to array[i:j]. +// The returned record must be Release()'d after use. +// +// NewSlice panics if the slice is outside the valid range of the record array. +// NewSlice panics if j < i. +func (rec *simpleRecord) NewSlice(i, j int64) Record { + arrs := make([]Interface, len(rec.arrs)) + for ii, arr := range rec.arrs { + arrs[ii] = NewSlice(arr, i, j) + } + defer func() { + for _, arr := range arrs { + arr.Release() + } + }() + return NewRecord(rec.schema, arrs, j-i) +} + +func (rec *simpleRecord) String() string { + o := new(strings.Builder) + fmt.Fprintf(o, "record:\n %v\n", rec.schema) + fmt.Fprintf(o, " rows: %d\n", rec.rows) + for i, col := range rec.arrs { + fmt.Fprintf(o, " col[%d][%s]: %v\n", i, rec.schema.Field(i).Name, col) + } + + return o.String() +} + +// RecordBuilder eases the process of building a Record, iteratively, from +// a known Schema. +type RecordBuilder struct { + refCount int64 + mem memory.Allocator + schema *arrow.Schema + fields []Builder +} + +// NewRecordBuilder returns a builder, using the provided memory allocator and a schema. +func NewRecordBuilder(mem memory.Allocator, schema *arrow.Schema) *RecordBuilder { + b := &RecordBuilder{ + refCount: 1, + mem: mem, + schema: schema, + fields: make([]Builder, len(schema.Fields())), + } + + for i, f := range schema.Fields() { + b.fields[i] = newBuilder(b.mem, f.Type) + } + + return b +} + +// Retain increases the reference count by 1. +// Retain may be called simultaneously from multiple goroutines. +func (b *RecordBuilder) Retain() { + atomic.AddInt64(&b.refCount, 1) +} + +// Release decreases the reference count by 1. +func (b *RecordBuilder) Release() { + debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases") + + for _, f := range b.fields { + f.Release() + } + + if atomic.AddInt64(&b.refCount, -1) == 0 { + b.fields = nil + } +} + +func (b *RecordBuilder) Schema() *arrow.Schema { return b.schema } +func (b *RecordBuilder) Fields() []Builder { return b.fields } +func (b *RecordBuilder) Field(i int) Builder { return b.fields[i] } + +func (b *RecordBuilder) Reserve(size int) { + for _, f := range b.fields { + f.Reserve(size) + } +} + +// NewRecord creates a new record from the memory buffers and resets the +// RecordBuilder so it can be used to build a new record. +// +// The returned Record must be Release()'d after use. +// +// NewRecord panics if the fields' builder do not have the same length. +func (b *RecordBuilder) NewRecord() Record { + cols := make([]Interface, len(b.fields)) + rows := int64(0) + + defer func(cols []Interface) { + for _, col := range cols { + if col == nil { + continue + } + col.Release() + } + }(cols) + + for i, f := range b.fields { + cols[i] = f.NewArray() + irow := int64(cols[i].Len()) + if i > 0 && irow != rows { + panic(fmt.Errorf("arrow/array: field %d has %d rows. want=%d", i, irow, rows)) + } + rows = irow + } + + return NewRecord(b.schema, cols, rows) +} + +var ( + _ Record = (*simpleRecord)(nil) + _ RecordReader = (*simpleRecords)(nil) +) diff --git a/vendor/github.com/apache/arrow/go/arrow/array/string.go b/vendor/github.com/apache/arrow/go/arrow/array/string.go new file mode 100644 index 00000000000..b7e607cfe6f --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/array/string.go @@ -0,0 +1,194 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package array + +import ( + "fmt" + "math" + "strings" + "unsafe" + + "github.com/apache/arrow/go/arrow" + "github.com/apache/arrow/go/arrow/memory" +) + +const ( + stringArrayMaximumCapacity = math.MaxInt32 +) + +// A type which represents an immutable sequence of variable-length UTF-8 strings. +type String struct { + array + offsets []int32 + values string +} + +// NewStringData constructs a new String array from data. +func NewStringData(data *Data) *String { + a := &String{} + a.refCount = 1 + a.setData(data) + return a +} + +// Value returns the slice at index i. This value should not be mutated. +func (a *String) Value(i int) string { + i = i + a.array.data.offset + return a.values[a.offsets[i]:a.offsets[i+1]] +} +func (a *String) ValueOffset(i int) int { return int(a.offsets[i]) } + +func (a *String) String() string { + o := new(strings.Builder) + o.WriteString("[") + for i := 0; i < a.Len(); i++ { + if i > 0 { + o.WriteString(" ") + } + switch { + case a.IsNull(i): + o.WriteString("(null)") + default: + fmt.Fprintf(o, "%q", a.Value(i)) + } + } + o.WriteString("]") + return o.String() +} + +func (a *String) setData(data *Data) { + if len(data.buffers) != 3 { + panic("arrow/array: len(data.buffers) != 3") + } + + a.array.setData(data) + + if vdata := data.buffers[2]; vdata != nil { + b := vdata.Bytes() + a.values = *(*string)(unsafe.Pointer(&b)) + } + + if offsets := data.buffers[1]; offsets != nil { + a.offsets = arrow.Int32Traits.CastFromBytes(offsets.Bytes()) + } +} + +func arrayEqualString(left, right *String) bool { + for i := 0; i < left.Len(); i++ { + if left.IsNull(i) { + continue + } + if left.Value(i) != right.Value(i) { + return false + } + } + return true +} + +// A StringBuilder is used to build a String array using the Append methods. +type StringBuilder struct { + builder *BinaryBuilder +} + +func NewStringBuilder(mem memory.Allocator) *StringBuilder { + b := &StringBuilder{ + builder: NewBinaryBuilder(mem, arrow.BinaryTypes.String), + } + return b +} + +// Release decreases the reference count by 1. +// When the reference count goes to zero, the memory is freed. +// Release may be called simultaneously from multiple goroutines. +func (b *StringBuilder) Release() { + b.builder.Release() +} + +// Retain increases the reference count by 1. +// Retain may be called simultaneously from multiple goroutines. +func (b *StringBuilder) Retain() { + b.builder.Retain() +} + +// +// Len returns the number of elements in the array builder. +func (b *StringBuilder) Len() int { return b.builder.Len() } + +// Cap returns the total number of elements that can be stored without allocating additional memory. +func (b *StringBuilder) Cap() int { return b.builder.Cap() } + +// NullN returns the number of null values in the array builder. +func (b *StringBuilder) NullN() int { return b.builder.NullN() } + +func (b *StringBuilder) Append(v string) { + b.builder.Append([]byte(v)) +} + +func (b *StringBuilder) AppendNull() { + b.builder.AppendNull() +} + +// AppendValues will append the values in the v slice. The valid slice determines which values +// in v are valid (not null). The valid slice must either be empty or be equal in length to v. If empty, +// all values in v are appended and considered valid. +func (b *StringBuilder) AppendValues(v []string, valid []bool) { + b.builder.AppendStringValues(v, valid) +} + +func (b *StringBuilder) Value(i int) string { + return string(b.builder.Value(i)) +} + +func (b *StringBuilder) init(capacity int) { + b.builder.init(capacity) +} + +func (b *StringBuilder) resize(newBits int, init func(int)) { + b.builder.resize(newBits, init) +} + +// Reserve ensures there is enough space for appending n elements +// by checking the capacity and calling Resize if necessary. +func (b *StringBuilder) Reserve(n int) { + b.builder.Reserve(n) +} + +// Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(), +// additional memory will be allocated. If n is smaller, the allocated memory may reduced. +func (b *StringBuilder) Resize(n int) { + b.builder.Resize(n) +} + +// NewArray creates a String array from the memory buffers used by the builder and resets the StringBuilder +// so it can be used to build a new array. +func (b *StringBuilder) NewArray() Interface { + return b.NewStringArray() +} + +// NewStringArray creates a String array from the memory buffers used by the builder and resets the StringBuilder +// so it can be used to build a new array. +func (b *StringBuilder) NewStringArray() (a *String) { + data := b.builder.newData() + a = NewStringData(data) + data.Release() + return +} + +var ( + _ Interface = (*String)(nil) + _ Builder = (*StringBuilder)(nil) +) diff --git a/vendor/github.com/apache/arrow/go/arrow/array/struct.go b/vendor/github.com/apache/arrow/go/arrow/array/struct.go new file mode 100644 index 00000000000..b70d9539a48 --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/array/struct.go @@ -0,0 +1,234 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package array + +import ( + "fmt" + "strings" + "sync/atomic" + + "github.com/apache/arrow/go/arrow" + "github.com/apache/arrow/go/arrow/internal/bitutil" + "github.com/apache/arrow/go/arrow/internal/debug" + "github.com/apache/arrow/go/arrow/memory" +) + +// Struct represents an ordered sequence of relative types. +type Struct struct { + array + fields []Interface +} + +// NewStructData returns a new Struct array value from data. +func NewStructData(data *Data) *Struct { + a := &Struct{} + a.refCount = 1 + a.setData(data) + return a +} + +func (a *Struct) NumField() int { return len(a.fields) } +func (a *Struct) Field(i int) Interface { return a.fields[i] } + +func (a *Struct) String() string { + o := new(strings.Builder) + o.WriteString("{") + for i, v := range a.fields { + if i > 0 { + o.WriteString(" ") + } + fmt.Fprintf(o, "%v", v) + } + o.WriteString("}") + return o.String() +} + +func (a *Struct) setData(data *Data) { + a.array.setData(data) + a.fields = make([]Interface, len(data.childData)) + for i, child := range data.childData { + a.fields[i] = MakeFromData(child) + } +} + +func arrayEqualStruct(left, right *Struct) bool { + for i, lf := range left.fields { + rf := right.fields[i] + if !ArrayEqual(lf, rf) { + return false + } + } + return true +} + +func (a *Struct) Retain() { + a.array.Retain() + for _, f := range a.fields { + f.Retain() + } +} + +func (a *Struct) Release() { + a.array.Release() + for _, f := range a.fields { + f.Release() + } +} + +type StructBuilder struct { + builder + + dtype arrow.DataType + fields []Builder +} + +// NewStructBuilder returns a builder, using the provided memory allocator. +func NewStructBuilder(mem memory.Allocator, dtype *arrow.StructType) *StructBuilder { + b := &StructBuilder{ + builder: builder{refCount: 1, mem: mem}, + dtype: dtype, + fields: make([]Builder, len(dtype.Fields())), + } + for i, f := range dtype.Fields() { + b.fields[i] = newBuilder(b.mem, f.Type) + } + return b +} + +// Release decreases the reference count by 1. +// When the reference count goes to zero, the memory is freed. +func (b *StructBuilder) Release() { + debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases") + + if atomic.AddInt64(&b.refCount, -1) == 0 { + if b.nullBitmap != nil { + b.nullBitmap.Release() + b.nullBitmap = nil + } + } + + for _, f := range b.fields { + f.Release() + } +} + +func (b *StructBuilder) Append(v bool) { + b.Reserve(1) + b.unsafeAppendBoolToBitmap(v) + if !v { + for _, f := range b.fields { + f.AppendNull() + } + } +} + +func (b *StructBuilder) AppendValues(valids []bool) { + b.Reserve(len(valids)) + b.builder.unsafeAppendBoolsToBitmap(valids, len(valids)) +} + +func (b *StructBuilder) AppendNull() { b.Append(false) } + +func (b *StructBuilder) unsafeAppend(v bool) { + bitutil.SetBit(b.nullBitmap.Bytes(), b.length) + b.length++ +} + +func (b *StructBuilder) unsafeAppendBoolToBitmap(isValid bool) { + if isValid { + bitutil.SetBit(b.nullBitmap.Bytes(), b.length) + } else { + b.nulls++ + } + b.length++ +} + +func (b *StructBuilder) init(capacity int) { + b.builder.init(capacity) + for _, f := range b.fields { + f.init(capacity) + } +} + +// Reserve ensures there is enough space for appending n elements +// by checking the capacity and calling Resize if necessary. +func (b *StructBuilder) Reserve(n int) { + b.builder.reserve(n, b.Resize) +} + +// Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(), +// additional memory will be allocated. If n is smaller, the allocated memory may reduced. +func (b *StructBuilder) Resize(n int) { + if n < minBuilderCapacity { + n = minBuilderCapacity + } + + if b.capacity == 0 { + b.init(n) + } else { + b.builder.resize(n, b.builder.init) + for _, f := range b.fields { + f.resize(n, f.init) + } + } +} + +func (b *StructBuilder) NumField() int { return len(b.fields) } +func (b *StructBuilder) FieldBuilder(i int) Builder { return b.fields[i] } + +// NewArray creates a Struct array from the memory buffers used by the builder and resets the StructBuilder +// so it can be used to build a new array. +func (b *StructBuilder) NewArray() Interface { + return b.NewStructArray() +} + +// NewStructArray creates a Struct array from the memory buffers used by the builder and resets the StructBuilder +// so it can be used to build a new array. +func (b *StructBuilder) NewStructArray() (a *Struct) { + data := b.newData() + a = NewStructData(data) + data.Release() + return +} + +func (b *StructBuilder) newData() (data *Data) { + fields := make([]*Data, len(b.fields)) + for i, f := range b.fields { + arr := f.NewArray() + defer arr.Release() + fields[i] = arr.Data() + } + + data = NewData( + b.dtype, b.length, + []*memory.Buffer{ + b.nullBitmap, + nil, // FIXME(sbinet) + }, + fields, + b.nulls, + 0, + ) + b.reset() + + return +} + +var ( + _ Interface = (*Struct)(nil) + _ Builder = (*StructBuilder)(nil) +) diff --git a/vendor/github.com/apache/arrow/go/arrow/array/table.go b/vendor/github.com/apache/arrow/go/arrow/array/table.go new file mode 100644 index 00000000000..4d8bb45e2f8 --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/array/table.go @@ -0,0 +1,455 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package array + +import ( + "errors" + "fmt" + "math" + "sync/atomic" + + "github.com/apache/arrow/go/arrow" + "github.com/apache/arrow/go/arrow/internal/debug" +) + +// Table represents a logical sequence of chunked arrays. +type Table interface { + Schema() *arrow.Schema + NumRows() int64 + NumCols() int64 + Column(i int) *Column + + Retain() + Release() +} + +// Column is an immutable column data structure consisting of +// a field (type metadata) and a chunked data array. +type Column struct { + field arrow.Field + data *Chunked +} + +// NewColumn returns a column from a field and a chunked data array. +// +// NewColumn panics if the field's data type is inconsistent with the data type +// of the chunked data array. +func NewColumn(field arrow.Field, chunks *Chunked) *Column { + col := Column{ + field: field, + data: chunks, + } + col.data.Retain() + + if !arrow.TypeEquals(col.data.DataType(), col.field.Type) { + col.data.Release() + panic("arrow/array: inconsistent data type") + } + + return &col +} + +// Retain increases the reference count by 1. +// Retain may be called simultaneously from multiple goroutines. +func (col *Column) Retain() { + col.data.Retain() +} + +// Release decreases the reference count by 1. +// When the reference count goes to zero, the memory is freed. +// Release may be called simultaneously from multiple goroutines. +func (col *Column) Release() { + col.data.Release() +} + +func (col *Column) Len() int { return col.data.Len() } +func (col *Column) NullN() int { return col.data.NullN() } +func (col *Column) Data() *Chunked { return col.data } +func (col *Column) Field() arrow.Field { return col.field } +func (col *Column) Name() string { return col.field.Name } +func (col *Column) DataType() arrow.DataType { return col.field.Type } + +// NewSlice returns a new zero-copy slice of the column with the indicated +// indices i and j, corresponding to the column's array[i:j]. +// The returned column must be Release()'d after use. +// +// NewSlice panics if the slice is outside the valid range of the column's array. +// NewSlice panics if j < i. +func (col *Column) NewSlice(i, j int64) *Column { + return &Column{ + field: col.field, + data: col.data.NewSlice(i, j), + } +} + +// Chunked manages a collection of primitives arrays as one logical large array. +type Chunked struct { + chunks []Interface + + refCount int64 + + length int + nulls int + dtype arrow.DataType +} + +// NewChunked returns a new chunked array from the slice of arrays. +// +// NewChunked panics if the chunks do not have the same data type. +func NewChunked(dtype arrow.DataType, chunks []Interface) *Chunked { + arr := &Chunked{ + chunks: make([]Interface, len(chunks)), + refCount: 1, + dtype: dtype, + } + for i, chunk := range chunks { + if !arrow.TypeEquals(chunk.DataType(), dtype) { + panic("arrow/array: mismatch data type") + } + chunk.Retain() + arr.chunks[i] = chunk + arr.length += chunk.Len() + arr.nulls += chunk.NullN() + } + return arr +} + +// Retain increases the reference count by 1. +// Retain may be called simultaneously from multiple goroutines. +func (a *Chunked) Retain() { + atomic.AddInt64(&a.refCount, 1) +} + +// Release decreases the reference count by 1. +// When the reference count goes to zero, the memory is freed. +// Release may be called simultaneously from multiple goroutines. +func (a *Chunked) Release() { + debug.Assert(atomic.LoadInt64(&a.refCount) > 0, "too many releases") + + if atomic.AddInt64(&a.refCount, -1) == 0 { + for _, arr := range a.chunks { + arr.Release() + } + a.chunks = nil + a.length = 0 + a.nulls = 0 + } +} + +func (a *Chunked) Len() int { return a.length } +func (a *Chunked) NullN() int { return a.nulls } +func (a *Chunked) DataType() arrow.DataType { return a.dtype } +func (a *Chunked) Chunks() []Interface { return a.chunks } +func (a *Chunked) Chunk(i int) Interface { return a.chunks[i] } + +// NewSlice constructs a zero-copy slice of the chunked array with the indicated +// indices i and j, corresponding to array[i:j]. +// The returned chunked array must be Release()'d after use. +// +// NewSlice panics if the slice is outside the valid range of the input array. +// NewSlice panics if j < i. +func (a *Chunked) NewSlice(i, j int64) *Chunked { + if j > int64(a.length) || i > j || i > int64(a.length) { + panic("arrow/array: index out of range") + } + + var ( + cur = 0 + beg = i + sz = j - i + chunks = make([]Interface, 0, len(a.chunks)) + ) + + for cur < len(a.chunks) && beg >= int64(a.chunks[cur].Len()) { + beg -= int64(a.chunks[cur].Len()) + cur++ + } + + for cur < len(a.chunks) && sz > 0 { + arr := a.chunks[cur] + end := beg + sz + if end > int64(arr.Len()) { + end = int64(arr.Len()) + } + chunks = append(chunks, NewSlice(arr, beg, end)) + sz -= int64(arr.Len()) - beg + beg = 0 + cur++ + } + chunks = chunks[:len(chunks):len(chunks)] + defer func() { + for _, chunk := range chunks { + chunk.Release() + } + }() + + return NewChunked(a.dtype, chunks) +} + +// simpleTable is a basic, non-lazy in-memory table. +type simpleTable struct { + refCount int64 + + rows int64 + cols []Column + + schema *arrow.Schema +} + +// NewTable returns a new basic, non-lazy in-memory table. +// If rows is negative, the number of rows will be inferred from the height +// of the columns. +// +// NewTable panics if the columns and schema are inconsistent. +// NewTable panics if rows is larger than the height of the columns. +func NewTable(schema *arrow.Schema, cols []Column, rows int64) *simpleTable { + tbl := simpleTable{ + refCount: 1, + rows: rows, + cols: cols, + schema: schema, + } + + if tbl.rows < 0 { + switch len(tbl.cols) { + case 0: + tbl.rows = 0 + default: + tbl.rows = int64(tbl.cols[0].Len()) + } + } + + // validate the table and its constituents. + // note we retain the columns after having validated the table + // in case the validation fails and panics (and would otherwise leak + // a ref-count on the columns.) + tbl.validate() + + for i := range tbl.cols { + tbl.cols[i].Retain() + } + + return &tbl +} + +// NewTableFromRecords returns a new basic, non-lazy in-memory table. +// +// NewTableFromRecords panics if the records and schema are inconsistent. +func NewTableFromRecords(schema *arrow.Schema, recs []Record) *simpleTable { + arrs := make([]Interface, len(recs)) + cols := make([]Column, len(schema.Fields())) + + defer func(cols []Column) { + for i := range cols { + cols[i].Release() + } + }(cols) + + for i := range cols { + field := schema.Field(i) + for j, rec := range recs { + arrs[j] = rec.Column(i) + } + chunk := NewChunked(field.Type, arrs) + cols[i] = *NewColumn(field, chunk) + chunk.Release() + } + + return NewTable(schema, cols, -1) +} + +func (tbl *simpleTable) Schema() *arrow.Schema { return tbl.schema } +func (tbl *simpleTable) NumRows() int64 { return tbl.rows } +func (tbl *simpleTable) NumCols() int64 { return int64(len(tbl.cols)) } +func (tbl *simpleTable) Column(i int) *Column { return &tbl.cols[i] } + +func (tbl *simpleTable) validate() { + if len(tbl.cols) != len(tbl.schema.Fields()) { + panic(errors.New("arrow/array: table schema mismatch")) + } + for i, col := range tbl.cols { + if !col.field.Equal(tbl.schema.Field(i)) { + panic(fmt.Errorf("arrow/array: column field %q is inconsistent with schema", col.Name())) + } + + if int64(col.Len()) < tbl.rows { + panic(fmt.Errorf("arrow/array: column %q expected length >= %d but got length %d", col.Name(), tbl.rows, col.Len())) + } + } +} + +// Retain increases the reference count by 1. +// Retain may be called simultaneously from multiple goroutines. +func (tbl *simpleTable) Retain() { + atomic.AddInt64(&tbl.refCount, 1) +} + +// Release decreases the reference count by 1. +// When the reference count goes to zero, the memory is freed. +// Release may be called simultaneously from multiple goroutines. +func (tbl *simpleTable) Release() { + debug.Assert(atomic.LoadInt64(&tbl.refCount) > 0, "too many releases") + + if atomic.AddInt64(&tbl.refCount, -1) == 0 { + for i := range tbl.cols { + tbl.cols[i].Release() + } + tbl.cols = nil + } +} + +// TableReader is a Record iterator over a (possibly chunked) Table +type TableReader struct { + refCount int64 + + tbl Table + cur int64 // current row + max int64 // total number of rows + rec Record // current Record + chksz int64 // chunk size + + chunks []*Chunked + slots []int // chunk indices + offsets []int64 // chunk offsets +} + +// NewTableReader returns a new TableReader to iterate over the (possibly chunked) Table. +// if chunkSize is <= 0, the biggest possible chunk will be selected. +func NewTableReader(tbl Table, chunkSize int64) *TableReader { + ncols := tbl.NumCols() + tr := &TableReader{ + refCount: 1, + tbl: tbl, + cur: 0, + max: int64(tbl.NumRows()), + chksz: chunkSize, + chunks: make([]*Chunked, ncols), + slots: make([]int, ncols), + offsets: make([]int64, ncols), + } + tr.tbl.Retain() + + if tr.chksz <= 0 { + tr.chksz = math.MaxInt64 + } + + for i := range tr.chunks { + col := tr.tbl.Column(i) + tr.chunks[i] = col.Data() + tr.chunks[i].Retain() + } + return tr +} + +func (tr *TableReader) Schema() *arrow.Schema { return tr.tbl.Schema() } +func (tr *TableReader) Record() Record { return tr.rec } + +func (tr *TableReader) Next() bool { + if tr.cur >= tr.max { + return false + } + + if tr.rec != nil { + tr.rec.Release() + } + + // determine the minimum contiguous slice across all columns + chunksz := imin64(tr.max, tr.chksz) + chunks := make([]Interface, len(tr.chunks)) + for i := range chunks { + j := tr.slots[i] + chunk := tr.chunks[i].Chunk(j) + remain := int64(chunk.Len()) - tr.offsets[i] + if remain < chunksz { + chunksz = remain + } + + chunks[i] = chunk + } + + // slice the chunks, advance each chunk slot as appropriate. + batch := make([]Interface, len(tr.chunks)) + for i, chunk := range chunks { + var slice Interface + offset := tr.offsets[i] + switch int64(chunk.Len()) - offset { + case chunksz: + tr.slots[i]++ + tr.offsets[i] = 0 + if offset > 0 { + // need to slice + slice = NewSlice(chunk, offset, offset+chunksz) + } else { + // no need to slice + slice = chunk + slice.Retain() + } + default: + tr.offsets[i] += chunksz + slice = NewSlice(chunk, offset, offset+chunksz) + } + batch[i] = slice + } + + tr.cur += chunksz + tr.rec = NewRecord(tr.tbl.Schema(), batch, chunksz) + + for _, arr := range batch { + arr.Release() + } + + return true +} + +// Retain increases the reference count by 1. +// Retain may be called simultaneously from multiple goroutines. +func (tr *TableReader) Retain() { + atomic.AddInt64(&tr.refCount, 1) +} + +// Release decreases the reference count by 1. +// When the reference count goes to zero, the memory is freed. +// Release may be called simultaneously from multiple goroutines. +func (tr *TableReader) Release() { + debug.Assert(atomic.LoadInt64(&tr.refCount) > 0, "too many releases") + + if atomic.AddInt64(&tr.refCount, -1) == 0 { + tr.tbl.Release() + for _, chk := range tr.chunks { + chk.Release() + } + if tr.rec != nil { + tr.rec.Release() + } + tr.tbl = nil + tr.chunks = nil + tr.slots = nil + tr.offsets = nil + } +} + +func imin64(a, b int64) int64 { + if a < b { + return a + } + return b +} + +var ( + _ Table = (*simpleTable)(nil) + _ RecordReader = (*TableReader)(nil) +) diff --git a/vendor/github.com/apache/arrow/go/arrow/array/util.go b/vendor/github.com/apache/arrow/go/arrow/array/util.go new file mode 100644 index 00000000000..c8d7e17c2bd --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/array/util.go @@ -0,0 +1,24 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package array + +func min(a, b int) int { + if a < b { + return a + } + return b +} diff --git a/vendor/github.com/apache/arrow/go/arrow/arrio/arrio.go b/vendor/github.com/apache/arrow/go/arrow/arrio/arrio.go new file mode 100644 index 00000000000..60cfb78b106 --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/arrio/arrio.go @@ -0,0 +1,91 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package arrio exposes functions to manipulate records, exposing and using +// interfaces not unlike the ones defined in the stdlib io package. +package arrio // import "github.com/apache/arrow/go/arrow/arrio" + +import ( + "io" + + "github.com/apache/arrow/go/arrow/array" +) + +// Reader is the interface that wraps the Read method. +type Reader interface { + // Read reads the current record from the underlying stream and an error, if any. + // When the Reader reaches the end of the underlying stream, it returns (nil, io.EOF). + Read() (array.Record, error) +} + +// ReaderAt is the interface that wraps the ReadAt method. +type ReaderAt interface { + // ReadAt reads the i-th record from the underlying stream and an error, if any. + ReadAt(i int64) (array.Record, error) +} + +// Writer is the interface that wraps the Write method. +type Writer interface { + Write(rec array.Record) error +} + +// Copy copies all the records available from src to dst. +// Copy returns the number of records copied and the first error +// encountered while copying, if any. +// +// A successful Copy returns err == nil, not err == EOF. Because Copy is +// defined to read from src until EOF, it does not treat an EOF from Read as an +// error to be reported. +func Copy(dst Writer, src Reader) (n int64, err error) { + for { + rec, err := src.Read() + if err != nil { + if err == io.EOF { + return n, nil + } + return n, err + } + err = dst.Write(rec) + if err != nil { + return n, err + } + n++ + } +} + +// CopyN copies n records (or until an error) from src to dst. It returns the +// number of records copied and the earliest error encountered while copying. On +// return, written == n if and only if err == nil. +func CopyN(dst Writer, src Reader, n int64) (written int64, err error) { + for ; written < n; written++ { + rec, err := src.Read() + if err != nil { + if err == io.EOF && written == n { + return written, nil + } + return written, err + } + err = dst.Write(rec) + if err != nil { + return written, err + } + } + + if written != n && err == nil { + err = io.EOF + } + return written, err +} diff --git a/vendor/github.com/apache/arrow/go/arrow/compare.go b/vendor/github.com/apache/arrow/go/arrow/compare.go new file mode 100644 index 00000000000..17bbd63664e --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/compare.go @@ -0,0 +1,79 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package arrow + +import ( + "reflect" +) + +type typeEqualsConfig struct { + metadata bool +} + +// TypeEqualsOption is a functional option type used for configuring type +// equality checks. +type TypeEqualsOption func(*typeEqualsConfig) + +// CheckMetadata is an option for TypeEquals that allows checking for metadata +// equality besides type equality. It only makes sense for STRUCT type. +func CheckMetadata() TypeEqualsOption { + return func(cfg *typeEqualsConfig) { + cfg.metadata = true + } +} + +// TypeEquals checks if two DataType are the same, optionally checking metadata +// equality for STRUCT types. +func TypeEquals(left, right DataType, opts ...TypeEqualsOption) bool { + var cfg typeEqualsConfig + for _, opt := range opts { + opt(&cfg) + } + + switch { + case left == nil || right == nil: + return false + case left.ID() != right.ID(): + return false + } + + // StructType is the only type that has metadata. + l, ok := left.(*StructType) + if !ok || cfg.metadata { + return reflect.DeepEqual(left, right) + } + + r := right.(*StructType) + switch { + case len(l.fields) != len(r.fields): + return false + case !reflect.DeepEqual(l.index, r.index): + return false + } + for i := range l.fields { + leftField, rightField := l.fields[i], r.fields[i] + switch { + case leftField.Name != rightField.Name: + return false + case leftField.Nullable != rightField.Nullable: + return false + case !TypeEquals(leftField.Type, rightField.Type, opts...): + return false + } + } + return true +} diff --git a/vendor/github.com/apache/arrow/go/arrow/datatype.go b/vendor/github.com/apache/arrow/go/arrow/datatype.go new file mode 100644 index 00000000000..5a9936ec95b --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/datatype.go @@ -0,0 +1,143 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package arrow + +// Type is a logical type. They can be expressed as +// either a primitive physical type (bytes or bits of some fixed size), a +// nested type consisting of other data types, or another data type (e.g. a +// timestamp encoded as an int64) +type Type int + +const ( + // NULL type having no physical storage + NULL Type = iota + + // BOOL is a 1 bit, LSB bit-packed ordering + BOOL + + // UINT8 is an Unsigned 8-bit little-endian integer + UINT8 + + // INT8 is a Signed 8-bit little-endian integer + INT8 + + // UINT16 is an Unsigned 16-bit little-endian integer + UINT16 + + // INT16 is a Signed 16-bit little-endian integer + INT16 + + // UINT32 is an Unsigned 32-bit little-endian integer + UINT32 + + // INT32 is a Signed 32-bit little-endian integer + INT32 + + // UINT64 is an Unsigned 64-bit little-endian integer + UINT64 + + // INT64 is a Signed 64-bit little-endian integer + INT64 + + // FLOAT16 is a 2-byte floating point value + FLOAT16 + + // FLOAT32 is a 4-byte floating point value + FLOAT32 + + // FLOAT64 is an 8-byte floating point value + FLOAT64 + + // STRING is a UTF8 variable-length string + STRING + + // BINARY is a Variable-length byte type (no guarantee of UTF8-ness) + BINARY + + // FIXED_SIZE_BINARY is a binary where each value occupies the same number of bytes + FIXED_SIZE_BINARY + + // DATE32 is int32 days since the UNIX epoch + DATE32 + + // DATE64 is int64 milliseconds since the UNIX epoch + DATE64 + + // TIMESTAMP is an exact timestamp encoded with int64 since UNIX epoch + // Default unit millisecond + TIMESTAMP + + // TIME32 is a signed 32-bit integer, representing either seconds or + // milliseconds since midnight + TIME32 + + // TIME64 is a signed 64-bit integer, representing either microseconds or + // nanoseconds since midnight + TIME64 + + // INTERVAL is YEAR_MONTH or DAY_TIME interval in SQL style + INTERVAL + + // DECIMAL is a precision- and scale-based decimal type. Storage type depends on the + // parameters. + DECIMAL + + // LIST is a list of some logical data type + LIST + + // STRUCT of logical types + STRUCT + + // UNION of logical types + UNION + + // DICTIONARY aka Category type + DICTIONARY + + // MAP is a repeated struct logical type + MAP + + // Custom data type, implemented by user + EXTENSION + + // Fixed size list of some logical type + FIXED_SIZE_LIST + + // Measure of elapsed time in either seconds, milliseconds, microseconds + // or nanoseconds. + DURATION +) + +// DataType is the representation of an Arrow type. +type DataType interface { + ID() Type + // Name is name of the data type. + Name() string +} + +// FixedWidthDataType is the representation of an Arrow type that +// requires a fixed number of bits in memory for each element. +type FixedWidthDataType interface { + DataType + // BitWidth returns the number of bits required to store a single element of this data type in memory. + BitWidth() int +} + +type BinaryDataType interface { + DataType + binary() +} diff --git a/vendor/github.com/apache/arrow/go/arrow/datatype_binary.go b/vendor/github.com/apache/arrow/go/arrow/datatype_binary.go new file mode 100644 index 00000000000..4fc0162734e --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/datatype_binary.go @@ -0,0 +1,41 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package arrow + +type BinaryType struct{} + +func (t *BinaryType) ID() Type { return BINARY } +func (t *BinaryType) Name() string { return "binary" } +func (t *BinaryType) String() string { return "binary" } +func (t *BinaryType) binary() {} + +type StringType struct{} + +func (t *StringType) ID() Type { return STRING } +func (t *StringType) Name() string { return "utf8" } +func (t *StringType) String() string { return "utf8" } +func (t *StringType) binary() {} + +var ( + BinaryTypes = struct { + Binary BinaryDataType + String BinaryDataType + }{ + Binary: &BinaryType{}, + String: &StringType{}, + } +) diff --git a/vendor/github.com/apache/arrow/go/arrow/datatype_fixedwidth.go b/vendor/github.com/apache/arrow/go/arrow/datatype_fixedwidth.go new file mode 100644 index 00000000000..79bb878f36a --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/datatype_fixedwidth.go @@ -0,0 +1,213 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package arrow + +import ( + "fmt" + "strconv" +) + +type BooleanType struct{} + +func (t *BooleanType) ID() Type { return BOOL } +func (t *BooleanType) Name() string { return "bool" } +func (t *BooleanType) String() string { return "bool" } + +// BitWidth returns the number of bits required to store a single element of this data type in memory. +func (t *BooleanType) BitWidth() int { return 1 } + +type FixedSizeBinaryType struct { + ByteWidth int +} + +func (*FixedSizeBinaryType) ID() Type { return FIXED_SIZE_BINARY } +func (*FixedSizeBinaryType) Name() string { return "fixed_size_binary" } +func (t *FixedSizeBinaryType) BitWidth() int { return 8 * t.ByteWidth } + +func (t *FixedSizeBinaryType) String() string { + return "fixed_size_binary[" + strconv.Itoa(t.ByteWidth) + "]" +} + +type ( + Timestamp int64 + Time32 int32 + Time64 int64 + TimeUnit int + Date32 int32 + Date64 int64 + Duration int64 +) + +const ( + Nanosecond TimeUnit = iota + Microsecond + Millisecond + Second +) + +func (u TimeUnit) String() string { return [...]string{"ns", "us", "ms", "s"}[uint(u)&3] } + +// TimestampType is encoded as a 64-bit signed integer since the UNIX epoch (2017-01-01T00:00:00Z). +// The zero-value is a nanosecond and time zone neutral. Time zone neutral can be +// considered UTC without having "UTC" as a time zone. +type TimestampType struct { + Unit TimeUnit + TimeZone string +} + +func (*TimestampType) ID() Type { return TIMESTAMP } +func (*TimestampType) Name() string { return "timestamp" } +func (t *TimestampType) String() string { + switch len(t.TimeZone) { + case 0: + return "timestamp[" + t.Unit.String() + "]" + default: + return "timestamp[" + t.Unit.String() + ", tz=" + t.TimeZone + "]" + } +} + +// BitWidth returns the number of bits required to store a single element of this data type in memory. +func (*TimestampType) BitWidth() int { return 64 } + +// Time32Type is encoded as a 32-bit signed integer, representing either seconds or milliseconds since midnight. +type Time32Type struct { + Unit TimeUnit +} + +func (*Time32Type) ID() Type { return TIME32 } +func (*Time32Type) Name() string { return "time32" } +func (*Time32Type) BitWidth() int { return 32 } +func (t *Time32Type) String() string { return "time32[" + t.Unit.String() + "]" } + +// Time64Type is encoded as a 64-bit signed integer, representing either microseconds or nanoseconds since midnight. +type Time64Type struct { + Unit TimeUnit +} + +func (*Time64Type) ID() Type { return TIME64 } +func (*Time64Type) Name() string { return "time64" } +func (*Time64Type) BitWidth() int { return 64 } +func (t *Time64Type) String() string { return "time64[" + t.Unit.String() + "]" } + +// DurationType is encoded as a 64-bit signed integer, representing an amount +// of elapsed time without any relation to a calendar artifact. +type DurationType struct { + Unit TimeUnit +} + +func (*DurationType) ID() Type { return DURATION } +func (*DurationType) Name() string { return "duration" } +func (*DurationType) BitWidth() int { return 64 } +func (t *DurationType) String() string { return "duration[" + t.Unit.String() + "]" } + +// Float16Type represents a floating point value encoded with a 16-bit precision. +type Float16Type struct{} + +func (t *Float16Type) ID() Type { return FLOAT16 } +func (t *Float16Type) Name() string { return "float16" } +func (t *Float16Type) String() string { return "float16" } + +// BitWidth returns the number of bits required to store a single element of this data type in memory. +func (t *Float16Type) BitWidth() int { return 16 } + +// Decimal128Type represents a fixed-size 128-bit decimal type. +type Decimal128Type struct { + Precision int32 + Scale int32 +} + +func (*Decimal128Type) ID() Type { return DECIMAL } +func (*Decimal128Type) Name() string { return "decimal" } +func (*Decimal128Type) BitWidth() int { return 16 } +func (t *Decimal128Type) String() string { + return fmt.Sprintf("%s(%d, %d)", t.Name(), t.Precision, t.Scale) +} + +// MonthInterval represents a number of months. +type MonthInterval int32 + +// MonthIntervalType is encoded as a 32-bit signed integer, +// representing a number of months. +type MonthIntervalType struct{} + +func (*MonthIntervalType) ID() Type { return INTERVAL } +func (*MonthIntervalType) Name() string { return "month_interval" } +func (*MonthIntervalType) String() string { return "month_interval" } + +// BitWidth returns the number of bits required to store a single element of this data type in memory. +func (t *MonthIntervalType) BitWidth() int { return 32 } + +// DayTimeInterval represents a number of days and milliseconds (fraction of day). +type DayTimeInterval struct { + Days int32 `json:"days"` + Milliseconds int32 `json:"milliseconds"` +} + +// DayTimeIntervalType is encoded as a pair of 32-bit signed integer, +// representing a number of days and milliseconds (fraction of day). +type DayTimeIntervalType struct{} + +func (*DayTimeIntervalType) ID() Type { return INTERVAL } +func (*DayTimeIntervalType) Name() string { return "day_time_interval" } +func (*DayTimeIntervalType) String() string { return "day_time_interval" } + +// BitWidth returns the number of bits required to store a single element of this data type in memory. +func (t *DayTimeIntervalType) BitWidth() int { return 64 } + +var ( + FixedWidthTypes = struct { + Boolean FixedWidthDataType + Date32 FixedWidthDataType + Date64 FixedWidthDataType + DayTimeInterval FixedWidthDataType + Duration_s FixedWidthDataType + Duration_ms FixedWidthDataType + Duration_us FixedWidthDataType + Duration_ns FixedWidthDataType + Float16 FixedWidthDataType + MonthInterval FixedWidthDataType + Time32s FixedWidthDataType + Time32ms FixedWidthDataType + Time64us FixedWidthDataType + Time64ns FixedWidthDataType + Timestamp_s FixedWidthDataType + Timestamp_ms FixedWidthDataType + Timestamp_us FixedWidthDataType + Timestamp_ns FixedWidthDataType + }{ + Boolean: &BooleanType{}, + Date32: &Date32Type{}, + Date64: &Date64Type{}, + DayTimeInterval: &DayTimeIntervalType{}, + Duration_s: &DurationType{Unit: Second}, + Duration_ms: &DurationType{Unit: Millisecond}, + Duration_us: &DurationType{Unit: Microsecond}, + Duration_ns: &DurationType{Unit: Nanosecond}, + Float16: &Float16Type{}, + MonthInterval: &MonthIntervalType{}, + Time32s: &Time32Type{Unit: Second}, + Time32ms: &Time32Type{Unit: Millisecond}, + Time64us: &Time64Type{Unit: Microsecond}, + Time64ns: &Time64Type{Unit: Nanosecond}, + Timestamp_s: &TimestampType{Unit: Second, TimeZone: "UTC"}, + Timestamp_ms: &TimestampType{Unit: Millisecond, TimeZone: "UTC"}, + Timestamp_us: &TimestampType{Unit: Microsecond, TimeZone: "UTC"}, + Timestamp_ns: &TimestampType{Unit: Nanosecond, TimeZone: "UTC"}, + } + + _ FixedWidthDataType = (*FixedSizeBinaryType)(nil) +) diff --git a/vendor/github.com/apache/arrow/go/arrow/datatype_nested.go b/vendor/github.com/apache/arrow/go/arrow/datatype_nested.go new file mode 100644 index 00000000000..c6173fe87cb --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/datatype_nested.go @@ -0,0 +1,180 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package arrow + +import ( + "fmt" + "reflect" + "strings" +) + +// ListType describes a nested type in which each array slot contains +// a variable-size sequence of values, all having the same relative type. +type ListType struct { + elem DataType // DataType of the list's elements +} + +// ListOf returns the list type with element type t. +// For example, if t represents int32, ListOf(t) represents []int32. +// +// ListOf panics if t is nil or invalid. +func ListOf(t DataType) *ListType { + if t == nil { + panic("arrow: nil DataType") + } + return &ListType{elem: t} +} + +func (*ListType) ID() Type { return LIST } +func (*ListType) Name() string { return "list" } +func (t *ListType) String() string { return fmt.Sprintf("list", t.elem) } + +// Elem returns the ListType's element type. +func (t *ListType) Elem() DataType { return t.elem } + +// FixedSizeListType describes a nested type in which each array slot contains +// a fixed-size sequence of values, all having the same relative type. +type FixedSizeListType struct { + n int32 // number of elements in the list + elem DataType // DataType of the list's elements +} + +// FixedSizeListOf returns the list type with element type t. +// For example, if t represents int32, FixedSizeListOf(10, t) represents [10]int32. +// +// FixedSizeListOf panics if t is nil or invalid. +// FixedSizeListOf panics if n is <= 0. +func FixedSizeListOf(n int32, t DataType) *FixedSizeListType { + if t == nil { + panic("arrow: nil DataType") + } + if n <= 0 { + panic("arrow: invalid size") + } + return &FixedSizeListType{elem: t, n: n} +} + +func (*FixedSizeListType) ID() Type { return FIXED_SIZE_LIST } +func (*FixedSizeListType) Name() string { return "fixed_size_list" } +func (t *FixedSizeListType) String() string { + return fmt.Sprintf("fixed_size_list[%d]", t.elem, t.n) +} + +// Elem returns the FixedSizeListType's element type. +func (t *FixedSizeListType) Elem() DataType { return t.elem } + +// Len returns the FixedSizeListType's size. +func (t *FixedSizeListType) Len() int32 { return t.n } + +// StructType describes a nested type parameterized by an ordered sequence +// of relative types, called its fields. +type StructType struct { + fields []Field + index map[string]int + meta Metadata +} + +// StructOf returns the struct type with fields fs. +// +// StructOf panics if there are duplicated fields. +// StructOf panics if there is a field with an invalid DataType. +func StructOf(fs ...Field) *StructType { + n := len(fs) + if n == 0 { + return &StructType{} + } + + t := &StructType{ + fields: make([]Field, n), + index: make(map[string]int, n), + } + for i, f := range fs { + if f.Type == nil { + panic("arrow: field with nil DataType") + } + t.fields[i] = Field{ + Name: f.Name, + Type: f.Type, + Nullable: f.Nullable, + Metadata: f.Metadata.clone(), + } + if _, dup := t.index[f.Name]; dup { + panic(fmt.Errorf("arrow: duplicate field with name %q", f.Name)) + } + t.index[f.Name] = i + } + + return t +} + +func (*StructType) ID() Type { return STRUCT } +func (*StructType) Name() string { return "struct" } + +func (t *StructType) String() string { + o := new(strings.Builder) + o.WriteString("struct<") + for i, f := range t.fields { + if i > 0 { + o.WriteString(", ") + } + o.WriteString(fmt.Sprintf("%s: %v", f.Name, f.Type)) + } + o.WriteString(">") + return o.String() +} + +func (t *StructType) Fields() []Field { return t.fields } +func (t *StructType) Field(i int) Field { return t.fields[i] } + +func (t *StructType) FieldByName(name string) (Field, bool) { + i, ok := t.index[name] + if !ok { + return Field{}, false + } + return t.fields[i], true +} + +type Field struct { + Name string // Field name + Type DataType // The field's data type + Nullable bool // Fields can be nullable + Metadata Metadata // The field's metadata, if any +} + +func (f Field) HasMetadata() bool { return f.Metadata.Len() != 0 } + +func (f Field) Equal(o Field) bool { + return reflect.DeepEqual(f, o) +} + +func (f Field) String() string { + o := new(strings.Builder) + nullable := "" + if f.Nullable { + nullable = ", nullable" + } + fmt.Fprintf(o, "%s: type=%v%v", f.Name, f.Type, nullable) + if f.HasMetadata() { + fmt.Fprintf(o, "\n%*.smetadata: %v", len(f.Name)+2, "", f.Metadata) + } + return o.String() +} + +var ( + _ DataType = (*ListType)(nil) + _ DataType = (*StructType)(nil) +) diff --git a/vendor/github.com/apache/arrow/go/arrow/datatype_null.go b/vendor/github.com/apache/arrow/go/arrow/datatype_null.go new file mode 100644 index 00000000000..5882dfe39b1 --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/datatype_null.go @@ -0,0 +1,29 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package arrow + +// NullType describes a degenerate array, with zero physical storage. +type NullType struct{} + +func (*NullType) ID() Type { return NULL } +func (*NullType) Name() string { return "null" } +func (*NullType) String() string { return "null" } + +var ( + Null *NullType + _ DataType = Null +) diff --git a/vendor/github.com/apache/arrow/go/arrow/datatype_numeric.gen.go b/vendor/github.com/apache/arrow/go/arrow/datatype_numeric.gen.go new file mode 100644 index 00000000000..d5b34d7d4fa --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/datatype_numeric.gen.go @@ -0,0 +1,134 @@ +// Code generated by datatype_numeric.gen.go.tmpl. DO NOT EDIT. + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package arrow + +type Int8Type struct{} + +func (t *Int8Type) ID() Type { return INT8 } +func (t *Int8Type) Name() string { return "int8" } +func (t *Int8Type) String() string { return "int8" } +func (t *Int8Type) BitWidth() int { return 8 } + +type Int16Type struct{} + +func (t *Int16Type) ID() Type { return INT16 } +func (t *Int16Type) Name() string { return "int16" } +func (t *Int16Type) String() string { return "int16" } +func (t *Int16Type) BitWidth() int { return 16 } + +type Int32Type struct{} + +func (t *Int32Type) ID() Type { return INT32 } +func (t *Int32Type) Name() string { return "int32" } +func (t *Int32Type) String() string { return "int32" } +func (t *Int32Type) BitWidth() int { return 32 } + +type Int64Type struct{} + +func (t *Int64Type) ID() Type { return INT64 } +func (t *Int64Type) Name() string { return "int64" } +func (t *Int64Type) String() string { return "int64" } +func (t *Int64Type) BitWidth() int { return 64 } + +type Uint8Type struct{} + +func (t *Uint8Type) ID() Type { return UINT8 } +func (t *Uint8Type) Name() string { return "uint8" } +func (t *Uint8Type) String() string { return "uint8" } +func (t *Uint8Type) BitWidth() int { return 8 } + +type Uint16Type struct{} + +func (t *Uint16Type) ID() Type { return UINT16 } +func (t *Uint16Type) Name() string { return "uint16" } +func (t *Uint16Type) String() string { return "uint16" } +func (t *Uint16Type) BitWidth() int { return 16 } + +type Uint32Type struct{} + +func (t *Uint32Type) ID() Type { return UINT32 } +func (t *Uint32Type) Name() string { return "uint32" } +func (t *Uint32Type) String() string { return "uint32" } +func (t *Uint32Type) BitWidth() int { return 32 } + +type Uint64Type struct{} + +func (t *Uint64Type) ID() Type { return UINT64 } +func (t *Uint64Type) Name() string { return "uint64" } +func (t *Uint64Type) String() string { return "uint64" } +func (t *Uint64Type) BitWidth() int { return 64 } + +type Float32Type struct{} + +func (t *Float32Type) ID() Type { return FLOAT32 } +func (t *Float32Type) Name() string { return "float32" } +func (t *Float32Type) String() string { return "float32" } +func (t *Float32Type) BitWidth() int { return 32 } + +type Float64Type struct{} + +func (t *Float64Type) ID() Type { return FLOAT64 } +func (t *Float64Type) Name() string { return "float64" } +func (t *Float64Type) String() string { return "float64" } +func (t *Float64Type) BitWidth() int { return 64 } + +type Date32Type struct{} + +func (t *Date32Type) ID() Type { return DATE32 } +func (t *Date32Type) Name() string { return "date32" } +func (t *Date32Type) String() string { return "date32" } +func (t *Date32Type) BitWidth() int { return 32 } + +type Date64Type struct{} + +func (t *Date64Type) ID() Type { return DATE64 } +func (t *Date64Type) Name() string { return "date64" } +func (t *Date64Type) String() string { return "date64" } +func (t *Date64Type) BitWidth() int { return 64 } + +var ( + PrimitiveTypes = struct { + Int8 DataType + Int16 DataType + Int32 DataType + Int64 DataType + Uint8 DataType + Uint16 DataType + Uint32 DataType + Uint64 DataType + Float32 DataType + Float64 DataType + Date32 DataType + Date64 DataType + }{ + + Int8: &Int8Type{}, + Int16: &Int16Type{}, + Int32: &Int32Type{}, + Int64: &Int64Type{}, + Uint8: &Uint8Type{}, + Uint16: &Uint16Type{}, + Uint32: &Uint32Type{}, + Uint64: &Uint64Type{}, + Float32: &Float32Type{}, + Float64: &Float64Type{}, + Date32: &Date32Type{}, + Date64: &Date64Type{}, + } +) diff --git a/vendor/github.com/apache/arrow/go/arrow/datatype_numeric.gen.go.tmpl b/vendor/github.com/apache/arrow/go/arrow/datatype_numeric.gen.go.tmpl new file mode 100644 index 00000000000..193cf09e45d --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/datatype_numeric.gen.go.tmpl @@ -0,0 +1,40 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package arrow + +{{range .In}} +type {{.Name}}Type struct {} + +func (t *{{.Name}}Type) ID() Type { return {{.Name|upper}} } +func (t *{{.Name}}Type) Name() string { return "{{.Name|lower}}" } +func (t *{{.Name}}Type) String() string { return "{{.Name|lower}}" } +func (t *{{.Name}}Type) BitWidth() int { return {{.Size}} } + + +{{end}} + +var ( + PrimitiveTypes = struct { +{{range .In}} + {{.Name}} DataType +{{- end}} + }{ +{{range .In}} + {{.Name}}: &{{.Name}}Type{}, +{{- end}} + } +) diff --git a/vendor/github.com/apache/arrow/go/arrow/datatype_numeric.gen.go.tmpldata b/vendor/github.com/apache/arrow/go/arrow/datatype_numeric.gen.go.tmpldata new file mode 100644 index 00000000000..9badc6ee2b2 --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/datatype_numeric.gen.go.tmpldata @@ -0,0 +1,62 @@ +[ + { + "Name": "Int8", + "Type": "int8", + "Size": 8 + }, + { + "Name": "Int16", + "Type": "int16", + "Size": 16 + }, + { + "Name": "Int32", + "Type": "int32", + "Size": 32 + }, + { + "Name": "Int64", + "Type": "int64", + "Size": 64 + }, + { + "Name": "Uint8", + "Type": "uint8", + "Size": 8 + }, + { + "Name": "Uint16", + "Type": "uint16", + "Size": 16 + }, + { + "Name": "Uint32", + "Type": "uint32", + "Size": 32 + }, + { + "Name": "Uint64", + "Type": "uint64", + "Size": 64 + }, + { + "Name": "Float32", + "Type": "float32", + "Size": 32 + }, + { + "Name": "Float64", + "Type": "float64", + "Size": 64 + }, + { + "Name": "Date32", + "Type": "date32", + "Size": 32 + }, + { + "Name": "Date64", + "Type": "date64", + "Size": 64 + } +] diff --git a/vendor/github.com/apache/arrow/go/arrow/decimal128/decimal128.go b/vendor/github.com/apache/arrow/go/arrow/decimal128/decimal128.go new file mode 100644 index 00000000000..2f1b181772d --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/decimal128/decimal128.go @@ -0,0 +1,73 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package decimal128 // import "github.com/apache/arrow/go/arrow/decimal128" + +var ( + MaxDecimal128 = New(542101086242752217, 687399551400673280-1) +) + +// Num represents a signed 128-bit integer in two's complement. +// Calculations wrap around and overflow is ignored. +// +// For a discussion of the algorithms, look at Knuth's volume 2, +// Semi-numerical Algorithms section 4.3.1. +// +// Adapted from the Apache ORC C++ implementation +type Num struct { + lo uint64 // low bits + hi int64 // high bits +} + +// New returns a new signed 128-bit integer value. +func New(hi int64, lo uint64) Num { + return Num{lo: lo, hi: hi} +} + +// FromU64 returns a new signed 128-bit integer value from the provided uint64 one. +func FromU64(v uint64) Num { + return New(0, v) +} + +// FromI64 returns a new signed 128-bit integer value from the provided int64 one. +func FromI64(v int64) Num { + switch { + case v > 0: + return New(0, uint64(v)) + case v < 0: + return New(-1, uint64(v)) + default: + return Num{} + } +} + +// LowBits returns the low bits of the two's complement representation of the number. +func (n Num) LowBits() uint64 { return n.lo } + +// HighBits returns the high bits of the two's complement representation of the number. +func (n Num) HighBits() int64 { return n.hi } + +// Sign returns: +// +// -1 if x < 0 +// 0 if x == 0 +// +1 if x > 0 +func (n Num) Sign() int { + if n == (Num{}) { + return 0 + } + return int(1 | (n.hi >> 63)) +} diff --git a/vendor/github.com/apache/arrow/go/arrow/doc.go b/vendor/github.com/apache/arrow/go/arrow/doc.go new file mode 100644 index 00000000000..10ddda97cbb --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/doc.go @@ -0,0 +1,39 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/* +Package arrow provides an implementation of Apache Arrow. + +Apache Arrow is a cross-language development platform for in-memory data. It specifies a standardized +language-independent columnar memory format for flat and hierarchical data, organized for efficient analytic +operations on modern hardware. It also provides computational libraries and zero-copy streaming +messaging and inter-process communication. + +Basics + +The fundamental data structure in Arrow is an Array, which holds a sequence of values of the same type. An array +consists of memory holding the data and an additional validity bitmap that indicates if the corresponding entry in the +array is valid (not null). If the array has no null entries, it is possible to omit this bitmap. + +*/ +package arrow + +//go:generate go run _tools/tmpl/main.go -i -data=numeric.tmpldata type_traits_numeric.gen.go.tmpl type_traits_numeric.gen_test.go.tmpl array/numeric.gen.go.tmpl array/numericbuilder.gen.go.tmpl array/bufferbuilder_numeric.gen.go.tmpl +//go:generate go run _tools/tmpl/main.go -i -data=datatype_numeric.gen.go.tmpldata datatype_numeric.gen.go.tmpl tensor/numeric.gen.go.tmpl tensor/numeric.gen_test.go.tmpl +//go:generate go run ./gen-flatbuffers.go + +// stringer +//go:generate stringer -type=Type diff --git a/vendor/github.com/apache/arrow/go/arrow/float16/float16.go b/vendor/github.com/apache/arrow/go/arrow/float16/float16.go new file mode 100644 index 00000000000..e15ee2c20d7 --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/float16/float16.go @@ -0,0 +1,70 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package float16 // import "github.com/apache/arrow/go/arrow/float16" + +import ( + "math" + "strconv" +) + +// Num represents a half-precision floating point value (float16) +// stored on 16 bits. +// +// See https://en.wikipedia.org/wiki/Half-precision_floating-point_format for more informations. +type Num struct { + bits uint16 +} + +// New creates a new half-precision floating point value from the provided +// float32 value. +func New(f float32) Num { + b := math.Float32bits(f) + sn := uint16((b >> 31) & 0x1) + exp := (b >> 23) & 0xff + res := int16(exp) - 127 + 15 + fc := uint16(b>>13) & 0x3ff + switch { + case exp == 0: + res = 0 + case exp == 0xff: + res = 0x1f + case res > 0x1e: + res = 0x1f + fc = 0 + case res < 0x01: + res = 0 + fc = 0 + } + return Num{bits: (sn << 15) | uint16(res<<10) | fc} +} + +func (f Num) Float32() float32 { + sn := uint32((f.bits >> 15) & 0x1) + exp := (f.bits >> 10) & 0x1f + res := uint32(exp) + 127 - 15 + fc := uint32(f.bits & 0x3ff) + switch { + case exp == 0: + res = 0 + case exp == 0x1f: + res = 0xff + } + return math.Float32frombits((sn << 31) | (res << 23) | (fc << 13)) +} + +func (f Num) Uint16() uint16 { return f.bits } +func (f Num) String() string { return strconv.FormatFloat(float64(f.Float32()), 'g', -1, 32) } diff --git a/vendor/github.com/apache/arrow/go/arrow/go.mod b/vendor/github.com/apache/arrow/go/arrow/go.mod new file mode 100644 index 00000000000..fa0950452bf --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/go.mod @@ -0,0 +1,27 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +module github.com/apache/arrow/go/arrow + +go 1.12 + +require ( + github.com/davecgh/go-spew v1.1.0 // indirect + github.com/google/flatbuffers v1.11.0 + github.com/pkg/errors v0.8.1 + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/stretchr/testify v1.2.0 +) diff --git a/vendor/github.com/apache/arrow/go/arrow/go.sum b/vendor/github.com/apache/arrow/go/arrow/go.sum new file mode 100644 index 00000000000..ce4ea3667d0 --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/go.sum @@ -0,0 +1,10 @@ +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/flatbuffers v1.11.0 h1:O7CEyB8Cb3/DmtxODGtLHcEvpr81Jm5qLg/hsHnxA2A= +github.com/google/flatbuffers v1.11.0/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.2.0 h1:LThGCOvhuJic9Gyd1VBCkhyUXmO8vKaBFvBsJ2k03rg= +github.com/stretchr/testify v1.2.0/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= diff --git a/vendor/github.com/apache/arrow/go/arrow/internal/bitutil/bitutil.go b/vendor/github.com/apache/arrow/go/arrow/internal/bitutil/bitutil.go new file mode 100644 index 00000000000..b547d618a5a --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/internal/bitutil/bitutil.go @@ -0,0 +1,156 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bitutil + +import ( + "math/bits" + "reflect" + "unsafe" +) + +var ( + BitMask = [8]byte{1, 2, 4, 8, 16, 32, 64, 128} + FlippedBitMask = [8]byte{254, 253, 251, 247, 239, 223, 191, 127} +) + +// IsMultipleOf8 returns whether v is a multiple of 8. +func IsMultipleOf8(v int64) bool { return v&7 == 0 } + +func BytesForBits(bits int64) int64 { return (bits + 7) >> 3 } + +// NextPowerOf2 rounds x to the next power of two. +func NextPowerOf2(x int) int { return 1 << uint(bits.Len(uint(x))) } + +// CeilByte rounds size to the next multiple of 8. +func CeilByte(size int) int { return (size + 7) &^ 7 } + +// CeilByte64 rounds size to the next multiple of 8. +func CeilByte64(size int64) int64 { return (size + 7) &^ 7 } + +// BitIsSet returns true if the bit at index i in buf is set (1). +func BitIsSet(buf []byte, i int) bool { return (buf[uint(i)/8] & BitMask[byte(i)%8]) != 0 } + +// BitIsNotSet returns true if the bit at index i in buf is not set (0). +func BitIsNotSet(buf []byte, i int) bool { return (buf[uint(i)/8] & BitMask[byte(i)%8]) == 0 } + +// SetBit sets the bit at index i in buf to 1. +func SetBit(buf []byte, i int) { buf[uint(i)/8] |= BitMask[byte(i)%8] } + +// ClearBit sets the bit at index i in buf to 0. +func ClearBit(buf []byte, i int) { buf[uint(i)/8] &= FlippedBitMask[byte(i)%8] } + +// SetBitTo sets the bit at index i in buf to val. +func SetBitTo(buf []byte, i int, val bool) { + if val { + SetBit(buf, i) + } else { + ClearBit(buf, i) + } +} + +// CountSetBits counts the number of 1's in buf up to n bits. +func CountSetBits(buf []byte, offset, n int) int { + if offset > 0 { + return countSetBitsWithOffset(buf, offset, n) + } + + count := 0 + + uint64Bytes := n / uint64SizeBits * 8 + for _, v := range bytesToUint64(buf[:uint64Bytes]) { + count += bits.OnesCount64(v) + } + + for _, v := range buf[uint64Bytes : n/8] { + count += bits.OnesCount8(v) + } + + // tail bits + for i := n &^ 0x7; i < n; i++ { + if BitIsSet(buf, i) { + count++ + } + } + + return count +} + +func countSetBitsWithOffset(buf []byte, offset, n int) int { + count := 0 + + beg := offset + end := offset + n + + begU8 := roundUp(beg, uint64SizeBits) + + init := min(n, begU8-beg) + for i := offset; i < beg+init; i++ { + if BitIsSet(buf, i) { + count++ + } + } + + nU64 := (n - init) / uint64SizeBits + begU64 := begU8 / uint64SizeBits + endU64 := begU64 + nU64 + bufU64 := bytesToUint64(buf) + if begU64 < len(bufU64) { + for _, v := range bufU64[begU64:endU64] { + count += bits.OnesCount64(v) + } + } + + // FIXME: use a fallback to bits.OnesCount8 + // before counting the tail bits. + + tail := beg + init + nU64*uint64SizeBits + for i := tail; i < end; i++ { + if BitIsSet(buf, i) { + count++ + } + } + + return count +} + +func roundUp(v, f int) int { + return (v + (f - 1)) / f * f +} + +func min(a, b int) int { + if a < b { + return a + } + return b +} + +const ( + uint64SizeBytes = int(unsafe.Sizeof(uint64(0))) + uint64SizeBits = uint64SizeBytes * 8 +) + +func bytesToUint64(b []byte) []uint64 { + h := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + + var res []uint64 + s := (*reflect.SliceHeader)(unsafe.Pointer(&res)) + s.Data = h.Data + s.Len = h.Len / uint64SizeBytes + s.Cap = h.Cap / uint64SizeBytes + + return res +} diff --git a/vendor/github.com/apache/arrow/go/arrow/internal/cpu/README.md b/vendor/github.com/apache/arrow/go/arrow/internal/cpu/README.md new file mode 100644 index 00000000000..2619b38c11e --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/internal/cpu/README.md @@ -0,0 +1,42 @@ + + +# Package cpu + +Copied from Go src/internal/cpu + +## Extras + +### Intel + +The `INTEL_DISABLE_EXT` environment variable can control which CPU extensions are available for +the running process. It should be a comma-separate list of upper-case strings as follows + +| Flag | Description | +| -------- | ----------- | +| `ALL` | Disable all CPU extensions and fall back to Go implementation | +| `AVX2` | Disable AVX2 optimizations | +| `AVX` | Disable AVX optimizations | +| `SSE` | Disable all SSE optimizations | +| `SSE4` | Disable SSE42, SSE41 optimizations | +| `SSSE3` | Disable supplemental SSE3 optimizations | +| `SSE3` | Disable SSE3 optimizations | +| `SSE2` | Disable SSE2 optimizations | + +Any unrecognized flags will be ignored and therefore it is possible to leave the environment variable with a bogus value such as `NONE` when experimenting. \ No newline at end of file diff --git a/vendor/github.com/apache/arrow/go/arrow/internal/cpu/cpu.go b/vendor/github.com/apache/arrow/go/arrow/internal/cpu/cpu.go new file mode 100644 index 00000000000..22fc561002d --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/internal/cpu/cpu.go @@ -0,0 +1,77 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package cpu implements processor feature detection +// used by the Go standard library. +package cpu + +var X86 x86 + +// The booleans in x86 contain the correspondingly named cpuid feature bit. +// HasAVX and HasAVX2 are only set if the OS does support XMM and YMM registers +// in addition to the cpuid feature bit being set. +// The struct is padded to avoid false sharing. +type x86 struct { + _ [CacheLineSize]byte + HasAES bool + HasADX bool + HasAVX bool + HasAVX2 bool + HasBMI1 bool + HasBMI2 bool + HasERMS bool + HasFMA bool + HasOSXSAVE bool + HasPCLMULQDQ bool + HasPOPCNT bool + HasSSE2 bool + HasSSE3 bool + HasSSSE3 bool + HasSSE41 bool + HasSSE42 bool + _ [CacheLineSize]byte +} + +var PPC64 ppc64 + +// For ppc64x, it is safe to check only for ISA level starting on ISA v3.00, +// since there are no optional categories. There are some exceptions that also +// require kernel support to work (darn, scv), so there are capability bits for +// those as well. The minimum processor requirement is POWER8 (ISA 2.07), so we +// maintain some of the old capability checks for optional categories for +// safety. +// The struct is padded to avoid false sharing. +type ppc64 struct { + _ [CacheLineSize]byte + HasVMX bool // Vector unit (Altivec) + HasDFP bool // Decimal Floating Point unit + HasVSX bool // Vector-scalar unit + HasHTM bool // Hardware Transactional Memory + HasISEL bool // Integer select + HasVCRYPTO bool // Vector cryptography + HasHTMNOSC bool // HTM: kernel-aborted transaction in syscalls + HasDARN bool // Hardware random number generator (requires kernel enablement) + HasSCV bool // Syscall vectored (requires kernel enablement) + IsPOWER8 bool // ISA v2.07 (POWER8) + IsPOWER9 bool // ISA v3.00 (POWER9) + _ [CacheLineSize]byte +} + +var ARM64 arm64 + +// The booleans in arm64 contain the correspondingly named cpu feature bit. +// The struct is padded to avoid false sharing. +type arm64 struct { + _ [CacheLineSize]byte + HasFP bool + HasASIMD bool + HasEVTSTRM bool + HasAES bool + HasPMULL bool + HasSHA1 bool + HasSHA2 bool + HasCRC32 bool + HasATOMICS bool + _ [CacheLineSize]byte +} diff --git a/vendor/github.com/apache/arrow/go/arrow/internal/cpu/cpu_x86.go b/vendor/github.com/apache/arrow/go/arrow/internal/cpu/cpu_x86.go new file mode 100644 index 00000000000..777269f1a25 --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/internal/cpu/cpu_x86.go @@ -0,0 +1,107 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build 386 amd64 amd64p32 + +package cpu + +import ( + "os" + "strings" +) + +const CacheLineSize = 64 + +// cpuid is implemented in cpu_x86.s. +func cpuid(eaxArg, ecxArg uint32) (eax, ebx, ecx, edx uint32) + +// xgetbv with ecx = 0 is implemented in cpu_x86.s. +func xgetbv() (eax, edx uint32) + +func init() { + maxID, _, _, _ := cpuid(0, 0) + + if maxID < 1 { + return + } + + _, _, ecx1, edx1 := cpuid(1, 0) + X86.HasSSE2 = isSet(26, edx1) + + X86.HasSSE3 = isSet(0, ecx1) + X86.HasPCLMULQDQ = isSet(1, ecx1) + X86.HasSSSE3 = isSet(9, ecx1) + X86.HasFMA = isSet(12, ecx1) + X86.HasSSE41 = isSet(19, ecx1) + X86.HasSSE42 = isSet(20, ecx1) + X86.HasPOPCNT = isSet(23, ecx1) + X86.HasAES = isSet(25, ecx1) + X86.HasOSXSAVE = isSet(27, ecx1) + + osSupportsAVX := false + // For XGETBV, OSXSAVE bit is required and sufficient. + if X86.HasOSXSAVE { + eax, _ := xgetbv() + // Check if XMM and YMM registers have OS support. + osSupportsAVX = isSet(1, eax) && isSet(2, eax) + } + + X86.HasAVX = isSet(28, ecx1) && osSupportsAVX + + if maxID < 7 { + return + } + + _, ebx7, _, _ := cpuid(7, 0) + X86.HasBMI1 = isSet(3, ebx7) + X86.HasAVX2 = isSet(5, ebx7) && osSupportsAVX + X86.HasBMI2 = isSet(8, ebx7) + X86.HasERMS = isSet(9, ebx7) + X86.HasADX = isSet(19, ebx7) + + // NOTE(sgc): added ability to disable extension via environment + checkEnvironment() +} +func checkEnvironment() { + if ext, ok := os.LookupEnv("INTEL_DISABLE_EXT"); ok { + exts := strings.Split(ext, ",") + + for _, x := range exts { + switch x { + case "ALL": + X86.HasAVX2 = false + X86.HasAVX = false + X86.HasSSE42 = false + X86.HasSSE41 = false + X86.HasSSSE3 = false + X86.HasSSE3 = false + X86.HasSSE2 = false + + case "AVX2": + X86.HasAVX2 = false + case "AVX": + X86.HasAVX = false + case "SSE": + X86.HasSSE42 = false + X86.HasSSE41 = false + X86.HasSSSE3 = false + X86.HasSSE3 = false + X86.HasSSE2 = false + case "SSE4": + X86.HasSSE42 = false + X86.HasSSE41 = false + case "SSSE3": + X86.HasSSSE3 = false + case "SSE3": + X86.HasSSE3 = false + case "SSE2": + X86.HasSSE2 = false + } + } + } +} + +func isSet(bitpos uint, value uint32) bool { + return value&(1< with values [[1, 2, 3], null, [4], [5, 6], null] +/// would have {length: 5, null_count: 2} for its List node, and {length: 6, +/// null_count: 0} for its Int16 node, as separate FieldNode structs +type FieldNode struct { + _tab flatbuffers.Struct +} + +func (rcv *FieldNode) Init(buf []byte, i flatbuffers.UOffsetT) { + rcv._tab.Bytes = buf + rcv._tab.Pos = i +} + +func (rcv *FieldNode) Table() flatbuffers.Table { + return rcv._tab.Table +} + +/// The number of value slots in the Arrow array at this level of a nested +/// tree +func (rcv *FieldNode) Length() int64 { + return rcv._tab.GetInt64(rcv._tab.Pos + flatbuffers.UOffsetT(0)) +} +/// The number of value slots in the Arrow array at this level of a nested +/// tree +func (rcv *FieldNode) MutateLength(n int64) bool { + return rcv._tab.MutateInt64(rcv._tab.Pos+flatbuffers.UOffsetT(0), n) +} + +/// The number of observed nulls. Fields with null_count == 0 may choose not +/// to write their physical validity bitmap out as a materialized buffer, +/// instead setting the length of the bitmap buffer to 0. +func (rcv *FieldNode) NullCount() int64 { + return rcv._tab.GetInt64(rcv._tab.Pos + flatbuffers.UOffsetT(8)) +} +/// The number of observed nulls. Fields with null_count == 0 may choose not +/// to write their physical validity bitmap out as a materialized buffer, +/// instead setting the length of the bitmap buffer to 0. +func (rcv *FieldNode) MutateNullCount(n int64) bool { + return rcv._tab.MutateInt64(rcv._tab.Pos+flatbuffers.UOffsetT(8), n) +} + +func CreateFieldNode(builder *flatbuffers.Builder, length int64, nullCount int64) flatbuffers.UOffsetT { + builder.Prep(8, 16) + builder.PrependInt64(nullCount) + builder.PrependInt64(length) + return builder.Offset() +} diff --git a/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/FixedSizeBinary.go b/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/FixedSizeBinary.go new file mode 100644 index 00000000000..4e660d5077f --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/FixedSizeBinary.go @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by the FlatBuffers compiler. DO NOT EDIT. + +package flatbuf + +import ( + flatbuffers "github.com/google/flatbuffers/go" +) + +type FixedSizeBinary struct { + _tab flatbuffers.Table +} + +func GetRootAsFixedSizeBinary(buf []byte, offset flatbuffers.UOffsetT) *FixedSizeBinary { + n := flatbuffers.GetUOffsetT(buf[offset:]) + x := &FixedSizeBinary{} + x.Init(buf, n+offset) + return x +} + +func (rcv *FixedSizeBinary) Init(buf []byte, i flatbuffers.UOffsetT) { + rcv._tab.Bytes = buf + rcv._tab.Pos = i +} + +func (rcv *FixedSizeBinary) Table() flatbuffers.Table { + return rcv._tab +} + +/// Number of bytes per value +func (rcv *FixedSizeBinary) ByteWidth() int32 { + o := flatbuffers.UOffsetT(rcv._tab.Offset(4)) + if o != 0 { + return rcv._tab.GetInt32(o + rcv._tab.Pos) + } + return 0 +} + +/// Number of bytes per value +func (rcv *FixedSizeBinary) MutateByteWidth(n int32) bool { + return rcv._tab.MutateInt32Slot(4, n) +} + +func FixedSizeBinaryStart(builder *flatbuffers.Builder) { + builder.StartObject(1) +} +func FixedSizeBinaryAddByteWidth(builder *flatbuffers.Builder, byteWidth int32) { + builder.PrependInt32Slot(0, byteWidth, 0) +} +func FixedSizeBinaryEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { + return builder.EndObject() +} diff --git a/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/FixedSizeList.go b/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/FixedSizeList.go new file mode 100644 index 00000000000..dabf5cc8581 --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/FixedSizeList.go @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by the FlatBuffers compiler. DO NOT EDIT. + +package flatbuf + +import ( + flatbuffers "github.com/google/flatbuffers/go" +) + +type FixedSizeList struct { + _tab flatbuffers.Table +} + +func GetRootAsFixedSizeList(buf []byte, offset flatbuffers.UOffsetT) *FixedSizeList { + n := flatbuffers.GetUOffsetT(buf[offset:]) + x := &FixedSizeList{} + x.Init(buf, n+offset) + return x +} + +func (rcv *FixedSizeList) Init(buf []byte, i flatbuffers.UOffsetT) { + rcv._tab.Bytes = buf + rcv._tab.Pos = i +} + +func (rcv *FixedSizeList) Table() flatbuffers.Table { + return rcv._tab +} + +/// Number of list items per value +func (rcv *FixedSizeList) ListSize() int32 { + o := flatbuffers.UOffsetT(rcv._tab.Offset(4)) + if o != 0 { + return rcv._tab.GetInt32(o + rcv._tab.Pos) + } + return 0 +} + +/// Number of list items per value +func (rcv *FixedSizeList) MutateListSize(n int32) bool { + return rcv._tab.MutateInt32Slot(4, n) +} + +func FixedSizeListStart(builder *flatbuffers.Builder) { + builder.StartObject(1) +} +func FixedSizeListAddListSize(builder *flatbuffers.Builder, listSize int32) { + builder.PrependInt32Slot(0, listSize, 0) +} +func FixedSizeListEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { + return builder.EndObject() +} diff --git a/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/FloatingPoint.go b/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/FloatingPoint.go new file mode 100644 index 00000000000..975a52dbe0a --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/FloatingPoint.go @@ -0,0 +1,65 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by the FlatBuffers compiler. DO NOT EDIT. + +package flatbuf + +import ( + flatbuffers "github.com/google/flatbuffers/go" +) + +type FloatingPoint struct { + _tab flatbuffers.Table +} + +func GetRootAsFloatingPoint(buf []byte, offset flatbuffers.UOffsetT) *FloatingPoint { + n := flatbuffers.GetUOffsetT(buf[offset:]) + x := &FloatingPoint{} + x.Init(buf, n+offset) + return x +} + +func (rcv *FloatingPoint) Init(buf []byte, i flatbuffers.UOffsetT) { + rcv._tab.Bytes = buf + rcv._tab.Pos = i +} + +func (rcv *FloatingPoint) Table() flatbuffers.Table { + return rcv._tab +} + +func (rcv *FloatingPoint) Precision() Precision { + o := flatbuffers.UOffsetT(rcv._tab.Offset(4)) + if o != 0 { + return rcv._tab.GetInt16(o + rcv._tab.Pos) + } + return 0 +} + +func (rcv *FloatingPoint) MutatePrecision(n Precision) bool { + return rcv._tab.MutateInt16Slot(4, n) +} + +func FloatingPointStart(builder *flatbuffers.Builder) { + builder.StartObject(1) +} +func FloatingPointAddPrecision(builder *flatbuffers.Builder, precision int16) { + builder.PrependInt16Slot(0, precision, 0) +} +func FloatingPointEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { + return builder.EndObject() +} diff --git a/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/Footer.go b/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/Footer.go new file mode 100644 index 00000000000..6802e77c032 --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/Footer.go @@ -0,0 +1,134 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by the FlatBuffers compiler. DO NOT EDIT. + +package flatbuf + +import ( + flatbuffers "github.com/google/flatbuffers/go" +) + +/// ---------------------------------------------------------------------- +/// Arrow File metadata +/// +type Footer struct { + _tab flatbuffers.Table +} + +func GetRootAsFooter(buf []byte, offset flatbuffers.UOffsetT) *Footer { + n := flatbuffers.GetUOffsetT(buf[offset:]) + x := &Footer{} + x.Init(buf, n+offset) + return x +} + +func (rcv *Footer) Init(buf []byte, i flatbuffers.UOffsetT) { + rcv._tab.Bytes = buf + rcv._tab.Pos = i +} + +func (rcv *Footer) Table() flatbuffers.Table { + return rcv._tab +} + +func (rcv *Footer) Version() MetadataVersion { + o := flatbuffers.UOffsetT(rcv._tab.Offset(4)) + if o != 0 { + return rcv._tab.GetInt16(o + rcv._tab.Pos) + } + return 0 +} + +func (rcv *Footer) MutateVersion(n MetadataVersion) bool { + return rcv._tab.MutateInt16Slot(4, n) +} + +func (rcv *Footer) Schema(obj *Schema) *Schema { + o := flatbuffers.UOffsetT(rcv._tab.Offset(6)) + if o != 0 { + x := rcv._tab.Indirect(o + rcv._tab.Pos) + if obj == nil { + obj = new(Schema) + } + obj.Init(rcv._tab.Bytes, x) + return obj + } + return nil +} + +func (rcv *Footer) Dictionaries(obj *Block, j int) bool { + o := flatbuffers.UOffsetT(rcv._tab.Offset(8)) + if o != 0 { + x := rcv._tab.Vector(o) + x += flatbuffers.UOffsetT(j) * 24 + obj.Init(rcv._tab.Bytes, x) + return true + } + return false +} + +func (rcv *Footer) DictionariesLength() int { + o := flatbuffers.UOffsetT(rcv._tab.Offset(8)) + if o != 0 { + return rcv._tab.VectorLen(o) + } + return 0 +} + +func (rcv *Footer) RecordBatches(obj *Block, j int) bool { + o := flatbuffers.UOffsetT(rcv._tab.Offset(10)) + if o != 0 { + x := rcv._tab.Vector(o) + x += flatbuffers.UOffsetT(j) * 24 + obj.Init(rcv._tab.Bytes, x) + return true + } + return false +} + +func (rcv *Footer) RecordBatchesLength() int { + o := flatbuffers.UOffsetT(rcv._tab.Offset(10)) + if o != 0 { + return rcv._tab.VectorLen(o) + } + return 0 +} + +func FooterStart(builder *flatbuffers.Builder) { + builder.StartObject(4) +} +func FooterAddVersion(builder *flatbuffers.Builder, version int16) { + builder.PrependInt16Slot(0, version, 0) +} +func FooterAddSchema(builder *flatbuffers.Builder, schema flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(1, flatbuffers.UOffsetT(schema), 0) +} +func FooterAddDictionaries(builder *flatbuffers.Builder, dictionaries flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(2, flatbuffers.UOffsetT(dictionaries), 0) +} +func FooterStartDictionariesVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT { + return builder.StartVector(24, numElems, 8) +} +func FooterAddRecordBatches(builder *flatbuffers.Builder, recordBatches flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(3, flatbuffers.UOffsetT(recordBatches), 0) +} +func FooterStartRecordBatchesVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT { + return builder.StartVector(24, numElems, 8) +} +func FooterEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { + return builder.EndObject() +} diff --git a/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/Int.go b/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/Int.go new file mode 100644 index 00000000000..9f4b1911705 --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/Int.go @@ -0,0 +1,80 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by the FlatBuffers compiler. DO NOT EDIT. + +package flatbuf + +import ( + flatbuffers "github.com/google/flatbuffers/go" +) + +type Int struct { + _tab flatbuffers.Table +} + +func GetRootAsInt(buf []byte, offset flatbuffers.UOffsetT) *Int { + n := flatbuffers.GetUOffsetT(buf[offset:]) + x := &Int{} + x.Init(buf, n+offset) + return x +} + +func (rcv *Int) Init(buf []byte, i flatbuffers.UOffsetT) { + rcv._tab.Bytes = buf + rcv._tab.Pos = i +} + +func (rcv *Int) Table() flatbuffers.Table { + return rcv._tab +} + +func (rcv *Int) BitWidth() int32 { + o := flatbuffers.UOffsetT(rcv._tab.Offset(4)) + if o != 0 { + return rcv._tab.GetInt32(o + rcv._tab.Pos) + } + return 0 +} + +func (rcv *Int) MutateBitWidth(n int32) bool { + return rcv._tab.MutateInt32Slot(4, n) +} + +func (rcv *Int) IsSigned() bool { + o := flatbuffers.UOffsetT(rcv._tab.Offset(6)) + if o != 0 { + return rcv._tab.GetBool(o + rcv._tab.Pos) + } + return false +} + +func (rcv *Int) MutateIsSigned(n bool) bool { + return rcv._tab.MutateBoolSlot(6, n) +} + +func IntStart(builder *flatbuffers.Builder) { + builder.StartObject(2) +} +func IntAddBitWidth(builder *flatbuffers.Builder, bitWidth int32) { + builder.PrependInt32Slot(0, bitWidth, 0) +} +func IntAddIsSigned(builder *flatbuffers.Builder, isSigned bool) { + builder.PrependBoolSlot(1, isSigned, false) +} +func IntEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { + return builder.EndObject() +} diff --git a/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/Interval.go b/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/Interval.go new file mode 100644 index 00000000000..21970ef1080 --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/Interval.go @@ -0,0 +1,65 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by the FlatBuffers compiler. DO NOT EDIT. + +package flatbuf + +import ( + flatbuffers "github.com/google/flatbuffers/go" +) + +type Interval struct { + _tab flatbuffers.Table +} + +func GetRootAsInterval(buf []byte, offset flatbuffers.UOffsetT) *Interval { + n := flatbuffers.GetUOffsetT(buf[offset:]) + x := &Interval{} + x.Init(buf, n+offset) + return x +} + +func (rcv *Interval) Init(buf []byte, i flatbuffers.UOffsetT) { + rcv._tab.Bytes = buf + rcv._tab.Pos = i +} + +func (rcv *Interval) Table() flatbuffers.Table { + return rcv._tab +} + +func (rcv *Interval) Unit() IntervalUnit { + o := flatbuffers.UOffsetT(rcv._tab.Offset(4)) + if o != 0 { + return rcv._tab.GetInt16(o + rcv._tab.Pos) + } + return 0 +} + +func (rcv *Interval) MutateUnit(n IntervalUnit) bool { + return rcv._tab.MutateInt16Slot(4, n) +} + +func IntervalStart(builder *flatbuffers.Builder) { + builder.StartObject(1) +} +func IntervalAddUnit(builder *flatbuffers.Builder, unit int16) { + builder.PrependInt16Slot(0, unit, 0) +} +func IntervalEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { + return builder.EndObject() +} diff --git a/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/IntervalUnit.go b/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/IntervalUnit.go new file mode 100644 index 00000000000..62baea99117 --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/IntervalUnit.go @@ -0,0 +1,31 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by the FlatBuffers compiler. DO NOT EDIT. + +package flatbuf + +type IntervalUnit = int16 +const ( + IntervalUnitYEAR_MONTH IntervalUnit = 0 + IntervalUnitDAY_TIME IntervalUnit = 1 +) + +var EnumNamesIntervalUnit = map[IntervalUnit]string{ + IntervalUnitYEAR_MONTH:"YEAR_MONTH", + IntervalUnitDAY_TIME:"DAY_TIME", +} + diff --git a/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/KeyValue.go b/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/KeyValue.go new file mode 100644 index 00000000000..c1b85318ecd --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/KeyValue.go @@ -0,0 +1,75 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by the FlatBuffers compiler. DO NOT EDIT. + +package flatbuf + +import ( + flatbuffers "github.com/google/flatbuffers/go" +) + +/// ---------------------------------------------------------------------- +/// user defined key value pairs to add custom metadata to arrow +/// key namespacing is the responsibility of the user +type KeyValue struct { + _tab flatbuffers.Table +} + +func GetRootAsKeyValue(buf []byte, offset flatbuffers.UOffsetT) *KeyValue { + n := flatbuffers.GetUOffsetT(buf[offset:]) + x := &KeyValue{} + x.Init(buf, n+offset) + return x +} + +func (rcv *KeyValue) Init(buf []byte, i flatbuffers.UOffsetT) { + rcv._tab.Bytes = buf + rcv._tab.Pos = i +} + +func (rcv *KeyValue) Table() flatbuffers.Table { + return rcv._tab +} + +func (rcv *KeyValue) Key() []byte { + o := flatbuffers.UOffsetT(rcv._tab.Offset(4)) + if o != 0 { + return rcv._tab.ByteVector(o + rcv._tab.Pos) + } + return nil +} + +func (rcv *KeyValue) Value() []byte { + o := flatbuffers.UOffsetT(rcv._tab.Offset(6)) + if o != 0 { + return rcv._tab.ByteVector(o + rcv._tab.Pos) + } + return nil +} + +func KeyValueStart(builder *flatbuffers.Builder) { + builder.StartObject(2) +} +func KeyValueAddKey(builder *flatbuffers.Builder, key flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(0, flatbuffers.UOffsetT(key), 0) +} +func KeyValueAddValue(builder *flatbuffers.Builder, value flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(1, flatbuffers.UOffsetT(value), 0) +} +func KeyValueEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { + return builder.EndObject() +} diff --git a/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/List.go b/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/List.go new file mode 100644 index 00000000000..ba84319d3f6 --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/List.go @@ -0,0 +1,50 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by the FlatBuffers compiler. DO NOT EDIT. + +package flatbuf + +import ( + flatbuffers "github.com/google/flatbuffers/go" +) + +type List struct { + _tab flatbuffers.Table +} + +func GetRootAsList(buf []byte, offset flatbuffers.UOffsetT) *List { + n := flatbuffers.GetUOffsetT(buf[offset:]) + x := &List{} + x.Init(buf, n+offset) + return x +} + +func (rcv *List) Init(buf []byte, i flatbuffers.UOffsetT) { + rcv._tab.Bytes = buf + rcv._tab.Pos = i +} + +func (rcv *List) Table() flatbuffers.Table { + return rcv._tab +} + +func ListStart(builder *flatbuffers.Builder) { + builder.StartObject(0) +} +func ListEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { + return builder.EndObject() +} diff --git a/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/Map.go b/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/Map.go new file mode 100644 index 00000000000..d548358a198 --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/Map.go @@ -0,0 +1,91 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by the FlatBuffers compiler. DO NOT EDIT. + +package flatbuf + +import ( + flatbuffers "github.com/google/flatbuffers/go" +) + +/// A Map is a logical nested type that is represented as +/// +/// List> +/// +/// In this layout, the keys and values are each respectively contiguous. We do +/// not constrain the key and value types, so the application is responsible +/// for ensuring that the keys are hashable and unique. Whether the keys are sorted +/// may be set in the metadata for this field +/// +/// In a Field with Map type, the Field has a child Struct field, which then +/// has two children: key type and the second the value type. The names of the +/// child fields may be respectively "entry", "key", and "value", but this is +/// not enforced +/// +/// Map +/// - child[0] entry: Struct +/// - child[0] key: K +/// - child[1] value: V +/// +/// Neither the "entry" field nor the "key" field may be nullable. +/// +/// The metadata is structured so that Arrow systems without special handling +/// for Map can make Map an alias for List. The "layout" attribute for the Map +/// field must have the same contents as a List. +type Map struct { + _tab flatbuffers.Table +} + +func GetRootAsMap(buf []byte, offset flatbuffers.UOffsetT) *Map { + n := flatbuffers.GetUOffsetT(buf[offset:]) + x := &Map{} + x.Init(buf, n+offset) + return x +} + +func (rcv *Map) Init(buf []byte, i flatbuffers.UOffsetT) { + rcv._tab.Bytes = buf + rcv._tab.Pos = i +} + +func (rcv *Map) Table() flatbuffers.Table { + return rcv._tab +} + +/// Set to true if the keys within each value are sorted +func (rcv *Map) KeysSorted() bool { + o := flatbuffers.UOffsetT(rcv._tab.Offset(4)) + if o != 0 { + return rcv._tab.GetBool(o + rcv._tab.Pos) + } + return false +} + +/// Set to true if the keys within each value are sorted +func (rcv *Map) MutateKeysSorted(n bool) bool { + return rcv._tab.MutateBoolSlot(4, n) +} + +func MapStart(builder *flatbuffers.Builder) { + builder.StartObject(1) +} +func MapAddKeysSorted(builder *flatbuffers.Builder, keysSorted bool) { + builder.PrependBoolSlot(0, keysSorted, false) +} +func MapEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { + return builder.EndObject() +} diff --git a/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/Message.go b/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/Message.go new file mode 100644 index 00000000000..76fcc389d1d --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/Message.go @@ -0,0 +1,133 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by the FlatBuffers compiler. DO NOT EDIT. + +package flatbuf + +import ( + flatbuffers "github.com/google/flatbuffers/go" +) + +type Message struct { + _tab flatbuffers.Table +} + +func GetRootAsMessage(buf []byte, offset flatbuffers.UOffsetT) *Message { + n := flatbuffers.GetUOffsetT(buf[offset:]) + x := &Message{} + x.Init(buf, n+offset) + return x +} + +func (rcv *Message) Init(buf []byte, i flatbuffers.UOffsetT) { + rcv._tab.Bytes = buf + rcv._tab.Pos = i +} + +func (rcv *Message) Table() flatbuffers.Table { + return rcv._tab +} + +func (rcv *Message) Version() MetadataVersion { + o := flatbuffers.UOffsetT(rcv._tab.Offset(4)) + if o != 0 { + return rcv._tab.GetInt16(o + rcv._tab.Pos) + } + return 0 +} + +func (rcv *Message) MutateVersion(n MetadataVersion) bool { + return rcv._tab.MutateInt16Slot(4, n) +} + +func (rcv *Message) HeaderType() byte { + o := flatbuffers.UOffsetT(rcv._tab.Offset(6)) + if o != 0 { + return rcv._tab.GetByte(o + rcv._tab.Pos) + } + return 0 +} + +func (rcv *Message) MutateHeaderType(n byte) bool { + return rcv._tab.MutateByteSlot(6, n) +} + +func (rcv *Message) Header(obj *flatbuffers.Table) bool { + o := flatbuffers.UOffsetT(rcv._tab.Offset(8)) + if o != 0 { + rcv._tab.Union(obj, o) + return true + } + return false +} + +func (rcv *Message) BodyLength() int64 { + o := flatbuffers.UOffsetT(rcv._tab.Offset(10)) + if o != 0 { + return rcv._tab.GetInt64(o + rcv._tab.Pos) + } + return 0 +} + +func (rcv *Message) MutateBodyLength(n int64) bool { + return rcv._tab.MutateInt64Slot(10, n) +} + +func (rcv *Message) CustomMetadata(obj *KeyValue, j int) bool { + o := flatbuffers.UOffsetT(rcv._tab.Offset(12)) + if o != 0 { + x := rcv._tab.Vector(o) + x += flatbuffers.UOffsetT(j) * 4 + x = rcv._tab.Indirect(x) + obj.Init(rcv._tab.Bytes, x) + return true + } + return false +} + +func (rcv *Message) CustomMetadataLength() int { + o := flatbuffers.UOffsetT(rcv._tab.Offset(12)) + if o != 0 { + return rcv._tab.VectorLen(o) + } + return 0 +} + +func MessageStart(builder *flatbuffers.Builder) { + builder.StartObject(5) +} +func MessageAddVersion(builder *flatbuffers.Builder, version int16) { + builder.PrependInt16Slot(0, version, 0) +} +func MessageAddHeaderType(builder *flatbuffers.Builder, headerType byte) { + builder.PrependByteSlot(1, headerType, 0) +} +func MessageAddHeader(builder *flatbuffers.Builder, header flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(2, flatbuffers.UOffsetT(header), 0) +} +func MessageAddBodyLength(builder *flatbuffers.Builder, bodyLength int64) { + builder.PrependInt64Slot(3, bodyLength, 0) +} +func MessageAddCustomMetadata(builder *flatbuffers.Builder, customMetadata flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(4, flatbuffers.UOffsetT(customMetadata), 0) +} +func MessageStartCustomMetadataVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT { + return builder.StartVector(4, numElems, 4) +} +func MessageEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { + return builder.EndObject() +} diff --git a/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/MessageHeader.go b/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/MessageHeader.go new file mode 100644 index 00000000000..18730bed59f --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/MessageHeader.go @@ -0,0 +1,47 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by the FlatBuffers compiler. DO NOT EDIT. + +package flatbuf + +/// ---------------------------------------------------------------------- +/// The root Message type +/// This union enables us to easily send different message types without +/// redundant storage, and in the future we can easily add new message types. +/// +/// Arrow implementations do not need to implement all of the message types, +/// which may include experimental metadata types. For maximum compatibility, +/// it is best to send data using RecordBatch +type MessageHeader = byte +const ( + MessageHeaderNONE MessageHeader = 0 + MessageHeaderSchema MessageHeader = 1 + MessageHeaderDictionaryBatch MessageHeader = 2 + MessageHeaderRecordBatch MessageHeader = 3 + MessageHeaderTensor MessageHeader = 4 + MessageHeaderSparseTensor MessageHeader = 5 +) + +var EnumNamesMessageHeader = map[MessageHeader]string{ + MessageHeaderNONE:"NONE", + MessageHeaderSchema:"Schema", + MessageHeaderDictionaryBatch:"DictionaryBatch", + MessageHeaderRecordBatch:"RecordBatch", + MessageHeaderTensor:"Tensor", + MessageHeaderSparseTensor:"SparseTensor", +} + diff --git a/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/MetadataVersion.go b/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/MetadataVersion.go new file mode 100644 index 00000000000..0094430c750 --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/MetadataVersion.go @@ -0,0 +1,39 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by the FlatBuffers compiler. DO NOT EDIT. + +package flatbuf + +type MetadataVersion = int16 +const ( + /// 0.1.0 + MetadataVersionV1 MetadataVersion = 0 + /// 0.2.0 + MetadataVersionV2 MetadataVersion = 1 + /// 0.3.0 -> 0.7.1 + MetadataVersionV3 MetadataVersion = 2 + /// >= 0.8.0 + MetadataVersionV4 MetadataVersion = 3 +) + +var EnumNamesMetadataVersion = map[MetadataVersion]string{ + MetadataVersionV1:"V1", + MetadataVersionV2:"V2", + MetadataVersionV3:"V3", + MetadataVersionV4:"V4", +} + diff --git a/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/Null.go b/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/Null.go new file mode 100644 index 00000000000..3c3eb4bda36 --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/Null.go @@ -0,0 +1,51 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by the FlatBuffers compiler. DO NOT EDIT. + +package flatbuf + +import ( + flatbuffers "github.com/google/flatbuffers/go" +) + +/// These are stored in the flatbuffer in the Type union below +type Null struct { + _tab flatbuffers.Table +} + +func GetRootAsNull(buf []byte, offset flatbuffers.UOffsetT) *Null { + n := flatbuffers.GetUOffsetT(buf[offset:]) + x := &Null{} + x.Init(buf, n+offset) + return x +} + +func (rcv *Null) Init(buf []byte, i flatbuffers.UOffsetT) { + rcv._tab.Bytes = buf + rcv._tab.Pos = i +} + +func (rcv *Null) Table() flatbuffers.Table { + return rcv._tab +} + +func NullStart(builder *flatbuffers.Builder) { + builder.StartObject(0) +} +func NullEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { + return builder.EndObject() +} diff --git a/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/Precision.go b/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/Precision.go new file mode 100644 index 00000000000..417d1e788e8 --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/Precision.go @@ -0,0 +1,33 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by the FlatBuffers compiler. DO NOT EDIT. + +package flatbuf + +type Precision = int16 +const ( + PrecisionHALF Precision = 0 + PrecisionSINGLE Precision = 1 + PrecisionDOUBLE Precision = 2 +) + +var EnumNamesPrecision = map[Precision]string{ + PrecisionHALF:"HALF", + PrecisionSINGLE:"SINGLE", + PrecisionDOUBLE:"DOUBLE", +} + diff --git a/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/RecordBatch.go b/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/RecordBatch.go new file mode 100644 index 00000000000..b997bdf16c7 --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/RecordBatch.go @@ -0,0 +1,136 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by the FlatBuffers compiler. DO NOT EDIT. + +package flatbuf + +import ( + flatbuffers "github.com/google/flatbuffers/go" +) + +/// A data header describing the shared memory layout of a "record" or "row" +/// batch. Some systems call this a "row batch" internally and others a "record +/// batch". +type RecordBatch struct { + _tab flatbuffers.Table +} + +func GetRootAsRecordBatch(buf []byte, offset flatbuffers.UOffsetT) *RecordBatch { + n := flatbuffers.GetUOffsetT(buf[offset:]) + x := &RecordBatch{} + x.Init(buf, n+offset) + return x +} + +func (rcv *RecordBatch) Init(buf []byte, i flatbuffers.UOffsetT) { + rcv._tab.Bytes = buf + rcv._tab.Pos = i +} + +func (rcv *RecordBatch) Table() flatbuffers.Table { + return rcv._tab +} + +/// number of records / rows. The arrays in the batch should all have this +/// length +func (rcv *RecordBatch) Length() int64 { + o := flatbuffers.UOffsetT(rcv._tab.Offset(4)) + if o != 0 { + return rcv._tab.GetInt64(o + rcv._tab.Pos) + } + return 0 +} + +/// number of records / rows. The arrays in the batch should all have this +/// length +func (rcv *RecordBatch) MutateLength(n int64) bool { + return rcv._tab.MutateInt64Slot(4, n) +} + +/// Nodes correspond to the pre-ordered flattened logical schema +func (rcv *RecordBatch) Nodes(obj *FieldNode, j int) bool { + o := flatbuffers.UOffsetT(rcv._tab.Offset(6)) + if o != 0 { + x := rcv._tab.Vector(o) + x += flatbuffers.UOffsetT(j) * 16 + obj.Init(rcv._tab.Bytes, x) + return true + } + return false +} + +func (rcv *RecordBatch) NodesLength() int { + o := flatbuffers.UOffsetT(rcv._tab.Offset(6)) + if o != 0 { + return rcv._tab.VectorLen(o) + } + return 0 +} + +/// Nodes correspond to the pre-ordered flattened logical schema +/// Buffers correspond to the pre-ordered flattened buffer tree +/// +/// The number of buffers appended to this list depends on the schema. For +/// example, most primitive arrays will have 2 buffers, 1 for the validity +/// bitmap and 1 for the values. For struct arrays, there will only be a +/// single buffer for the validity (nulls) bitmap +func (rcv *RecordBatch) Buffers(obj *Buffer, j int) bool { + o := flatbuffers.UOffsetT(rcv._tab.Offset(8)) + if o != 0 { + x := rcv._tab.Vector(o) + x += flatbuffers.UOffsetT(j) * 16 + obj.Init(rcv._tab.Bytes, x) + return true + } + return false +} + +func (rcv *RecordBatch) BuffersLength() int { + o := flatbuffers.UOffsetT(rcv._tab.Offset(8)) + if o != 0 { + return rcv._tab.VectorLen(o) + } + return 0 +} + +/// Buffers correspond to the pre-ordered flattened buffer tree +/// +/// The number of buffers appended to this list depends on the schema. For +/// example, most primitive arrays will have 2 buffers, 1 for the validity +/// bitmap and 1 for the values. For struct arrays, there will only be a +/// single buffer for the validity (nulls) bitmap +func RecordBatchStart(builder *flatbuffers.Builder) { + builder.StartObject(3) +} +func RecordBatchAddLength(builder *flatbuffers.Builder, length int64) { + builder.PrependInt64Slot(0, length, 0) +} +func RecordBatchAddNodes(builder *flatbuffers.Builder, nodes flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(1, flatbuffers.UOffsetT(nodes), 0) +} +func RecordBatchStartNodesVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT { + return builder.StartVector(16, numElems, 8) +} +func RecordBatchAddBuffers(builder *flatbuffers.Builder, buffers flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(2, flatbuffers.UOffsetT(buffers), 0) +} +func RecordBatchStartBuffersVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT { + return builder.StartVector(16, numElems, 8) +} +func RecordBatchEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { + return builder.EndObject() +} diff --git a/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/Schema.go b/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/Schema.go new file mode 100644 index 00000000000..d3e27373b2e --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/Schema.go @@ -0,0 +1,125 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by the FlatBuffers compiler. DO NOT EDIT. + +package flatbuf + +import ( + flatbuffers "github.com/google/flatbuffers/go" +) + +/// ---------------------------------------------------------------------- +/// A Schema describes the columns in a row batch +type Schema struct { + _tab flatbuffers.Table +} + +func GetRootAsSchema(buf []byte, offset flatbuffers.UOffsetT) *Schema { + n := flatbuffers.GetUOffsetT(buf[offset:]) + x := &Schema{} + x.Init(buf, n+offset) + return x +} + +func (rcv *Schema) Init(buf []byte, i flatbuffers.UOffsetT) { + rcv._tab.Bytes = buf + rcv._tab.Pos = i +} + +func (rcv *Schema) Table() flatbuffers.Table { + return rcv._tab +} + +/// endianness of the buffer +/// it is Little Endian by default +/// if endianness doesn't match the underlying system then the vectors need to be converted +func (rcv *Schema) Endianness() Endianness { + o := flatbuffers.UOffsetT(rcv._tab.Offset(4)) + if o != 0 { + return rcv._tab.GetInt16(o + rcv._tab.Pos) + } + return 0 +} + +/// endianness of the buffer +/// it is Little Endian by default +/// if endianness doesn't match the underlying system then the vectors need to be converted +func (rcv *Schema) MutateEndianness(n Endianness) bool { + return rcv._tab.MutateInt16Slot(4, n) +} + +func (rcv *Schema) Fields(obj *Field, j int) bool { + o := flatbuffers.UOffsetT(rcv._tab.Offset(6)) + if o != 0 { + x := rcv._tab.Vector(o) + x += flatbuffers.UOffsetT(j) * 4 + x = rcv._tab.Indirect(x) + obj.Init(rcv._tab.Bytes, x) + return true + } + return false +} + +func (rcv *Schema) FieldsLength() int { + o := flatbuffers.UOffsetT(rcv._tab.Offset(6)) + if o != 0 { + return rcv._tab.VectorLen(o) + } + return 0 +} + +func (rcv *Schema) CustomMetadata(obj *KeyValue, j int) bool { + o := flatbuffers.UOffsetT(rcv._tab.Offset(8)) + if o != 0 { + x := rcv._tab.Vector(o) + x += flatbuffers.UOffsetT(j) * 4 + x = rcv._tab.Indirect(x) + obj.Init(rcv._tab.Bytes, x) + return true + } + return false +} + +func (rcv *Schema) CustomMetadataLength() int { + o := flatbuffers.UOffsetT(rcv._tab.Offset(8)) + if o != 0 { + return rcv._tab.VectorLen(o) + } + return 0 +} + +func SchemaStart(builder *flatbuffers.Builder) { + builder.StartObject(3) +} +func SchemaAddEndianness(builder *flatbuffers.Builder, endianness int16) { + builder.PrependInt16Slot(0, endianness, 0) +} +func SchemaAddFields(builder *flatbuffers.Builder, fields flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(1, flatbuffers.UOffsetT(fields), 0) +} +func SchemaStartFieldsVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT { + return builder.StartVector(4, numElems, 4) +} +func SchemaAddCustomMetadata(builder *flatbuffers.Builder, customMetadata flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(2, flatbuffers.UOffsetT(customMetadata), 0) +} +func SchemaStartCustomMetadataVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT { + return builder.StartVector(4, numElems, 4) +} +func SchemaEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { + return builder.EndObject() +} diff --git a/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/SparseMatrixIndexCSR.go b/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/SparseMatrixIndexCSR.go new file mode 100644 index 00000000000..c09006c2906 --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/SparseMatrixIndexCSR.go @@ -0,0 +1,145 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by the FlatBuffers compiler. DO NOT EDIT. + +package flatbuf + +import ( + flatbuffers "github.com/google/flatbuffers/go" +) + +/// Compressed Sparse Row format, that is matrix-specific. +type SparseMatrixIndexCSR struct { + _tab flatbuffers.Table +} + +func GetRootAsSparseMatrixIndexCSR(buf []byte, offset flatbuffers.UOffsetT) *SparseMatrixIndexCSR { + n := flatbuffers.GetUOffsetT(buf[offset:]) + x := &SparseMatrixIndexCSR{} + x.Init(buf, n+offset) + return x +} + +func (rcv *SparseMatrixIndexCSR) Init(buf []byte, i flatbuffers.UOffsetT) { + rcv._tab.Bytes = buf + rcv._tab.Pos = i +} + +func (rcv *SparseMatrixIndexCSR) Table() flatbuffers.Table { + return rcv._tab +} + +/// indptrBuffer stores the location and size of indptr array that +/// represents the range of the rows. +/// The i-th row spans from indptr[i] to indptr[i+1] in the data. +/// The length of this array is 1 + (the number of rows), and the type +/// of index value is long. +/// +/// For example, let X be the following 6x4 matrix: +/// +/// X := [[0, 1, 2, 0], +/// [0, 0, 3, 0], +/// [0, 4, 0, 5], +/// [0, 0, 0, 0], +/// [6, 0, 7, 8], +/// [0, 9, 0, 0]]. +/// +/// The array of non-zero values in X is: +/// +/// values(X) = [1, 2, 3, 4, 5, 6, 7, 8, 9]. +/// +/// And the indptr of X is: +/// +/// indptr(X) = [0, 2, 3, 5, 5, 8, 10]. +func (rcv *SparseMatrixIndexCSR) IndptrBuffer(obj *Buffer) *Buffer { + o := flatbuffers.UOffsetT(rcv._tab.Offset(4)) + if o != 0 { + x := o + rcv._tab.Pos + if obj == nil { + obj = new(Buffer) + } + obj.Init(rcv._tab.Bytes, x) + return obj + } + return nil +} + +/// indptrBuffer stores the location and size of indptr array that +/// represents the range of the rows. +/// The i-th row spans from indptr[i] to indptr[i+1] in the data. +/// The length of this array is 1 + (the number of rows), and the type +/// of index value is long. +/// +/// For example, let X be the following 6x4 matrix: +/// +/// X := [[0, 1, 2, 0], +/// [0, 0, 3, 0], +/// [0, 4, 0, 5], +/// [0, 0, 0, 0], +/// [6, 0, 7, 8], +/// [0, 9, 0, 0]]. +/// +/// The array of non-zero values in X is: +/// +/// values(X) = [1, 2, 3, 4, 5, 6, 7, 8, 9]. +/// +/// And the indptr of X is: +/// +/// indptr(X) = [0, 2, 3, 5, 5, 8, 10]. +/// indicesBuffer stores the location and size of the array that +/// contains the column indices of the corresponding non-zero values. +/// The type of index value is long. +/// +/// For example, the indices of the above X is: +/// +/// indices(X) = [1, 2, 2, 1, 3, 0, 2, 3, 1]. +/// +/// Note that the indices are sorted in lexicographical order for each row. +func (rcv *SparseMatrixIndexCSR) IndicesBuffer(obj *Buffer) *Buffer { + o := flatbuffers.UOffsetT(rcv._tab.Offset(6)) + if o != 0 { + x := o + rcv._tab.Pos + if obj == nil { + obj = new(Buffer) + } + obj.Init(rcv._tab.Bytes, x) + return obj + } + return nil +} + +/// indicesBuffer stores the location and size of the array that +/// contains the column indices of the corresponding non-zero values. +/// The type of index value is long. +/// +/// For example, the indices of the above X is: +/// +/// indices(X) = [1, 2, 2, 1, 3, 0, 2, 3, 1]. +/// +/// Note that the indices are sorted in lexicographical order for each row. +func SparseMatrixIndexCSRStart(builder *flatbuffers.Builder) { + builder.StartObject(2) +} +func SparseMatrixIndexCSRAddIndptrBuffer(builder *flatbuffers.Builder, indptrBuffer flatbuffers.UOffsetT) { + builder.PrependStructSlot(0, flatbuffers.UOffsetT(indptrBuffer), 0) +} +func SparseMatrixIndexCSRAddIndicesBuffer(builder *flatbuffers.Builder, indicesBuffer flatbuffers.UOffsetT) { + builder.PrependStructSlot(1, flatbuffers.UOffsetT(indicesBuffer), 0) +} +func SparseMatrixIndexCSREnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { + return builder.EndObject() +} diff --git a/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/SparseTensor.go b/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/SparseTensor.go new file mode 100644 index 00000000000..389832b95a1 --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/SparseTensor.go @@ -0,0 +1,175 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by the FlatBuffers compiler. DO NOT EDIT. + +package flatbuf + +import ( + flatbuffers "github.com/google/flatbuffers/go" +) + +type SparseTensor struct { + _tab flatbuffers.Table +} + +func GetRootAsSparseTensor(buf []byte, offset flatbuffers.UOffsetT) *SparseTensor { + n := flatbuffers.GetUOffsetT(buf[offset:]) + x := &SparseTensor{} + x.Init(buf, n+offset) + return x +} + +func (rcv *SparseTensor) Init(buf []byte, i flatbuffers.UOffsetT) { + rcv._tab.Bytes = buf + rcv._tab.Pos = i +} + +func (rcv *SparseTensor) Table() flatbuffers.Table { + return rcv._tab +} + +func (rcv *SparseTensor) TypeType() byte { + o := flatbuffers.UOffsetT(rcv._tab.Offset(4)) + if o != 0 { + return rcv._tab.GetByte(o + rcv._tab.Pos) + } + return 0 +} + +func (rcv *SparseTensor) MutateTypeType(n byte) bool { + return rcv._tab.MutateByteSlot(4, n) +} + +/// The type of data contained in a value cell. +/// Currently only fixed-width value types are supported, +/// no strings or nested types. +func (rcv *SparseTensor) Type(obj *flatbuffers.Table) bool { + o := flatbuffers.UOffsetT(rcv._tab.Offset(6)) + if o != 0 { + rcv._tab.Union(obj, o) + return true + } + return false +} + +/// The type of data contained in a value cell. +/// Currently only fixed-width value types are supported, +/// no strings or nested types. +/// The dimensions of the tensor, optionally named. +func (rcv *SparseTensor) Shape(obj *TensorDim, j int) bool { + o := flatbuffers.UOffsetT(rcv._tab.Offset(8)) + if o != 0 { + x := rcv._tab.Vector(o) + x += flatbuffers.UOffsetT(j) * 4 + x = rcv._tab.Indirect(x) + obj.Init(rcv._tab.Bytes, x) + return true + } + return false +} + +func (rcv *SparseTensor) ShapeLength() int { + o := flatbuffers.UOffsetT(rcv._tab.Offset(8)) + if o != 0 { + return rcv._tab.VectorLen(o) + } + return 0 +} + +/// The dimensions of the tensor, optionally named. +/// The number of non-zero values in a sparse tensor. +func (rcv *SparseTensor) NonZeroLength() int64 { + o := flatbuffers.UOffsetT(rcv._tab.Offset(10)) + if o != 0 { + return rcv._tab.GetInt64(o + rcv._tab.Pos) + } + return 0 +} + +/// The number of non-zero values in a sparse tensor. +func (rcv *SparseTensor) MutateNonZeroLength(n int64) bool { + return rcv._tab.MutateInt64Slot(10, n) +} + +func (rcv *SparseTensor) SparseIndexType() byte { + o := flatbuffers.UOffsetT(rcv._tab.Offset(12)) + if o != 0 { + return rcv._tab.GetByte(o + rcv._tab.Pos) + } + return 0 +} + +func (rcv *SparseTensor) MutateSparseIndexType(n byte) bool { + return rcv._tab.MutateByteSlot(12, n) +} + +/// Sparse tensor index +func (rcv *SparseTensor) SparseIndex(obj *flatbuffers.Table) bool { + o := flatbuffers.UOffsetT(rcv._tab.Offset(14)) + if o != 0 { + rcv._tab.Union(obj, o) + return true + } + return false +} + +/// Sparse tensor index +/// The location and size of the tensor's data +func (rcv *SparseTensor) Data(obj *Buffer) *Buffer { + o := flatbuffers.UOffsetT(rcv._tab.Offset(16)) + if o != 0 { + x := o + rcv._tab.Pos + if obj == nil { + obj = new(Buffer) + } + obj.Init(rcv._tab.Bytes, x) + return obj + } + return nil +} + +/// The location and size of the tensor's data +func SparseTensorStart(builder *flatbuffers.Builder) { + builder.StartObject(7) +} +func SparseTensorAddTypeType(builder *flatbuffers.Builder, typeType byte) { + builder.PrependByteSlot(0, typeType, 0) +} +func SparseTensorAddType(builder *flatbuffers.Builder, type_ flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(1, flatbuffers.UOffsetT(type_), 0) +} +func SparseTensorAddShape(builder *flatbuffers.Builder, shape flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(2, flatbuffers.UOffsetT(shape), 0) +} +func SparseTensorStartShapeVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT { + return builder.StartVector(4, numElems, 4) +} +func SparseTensorAddNonZeroLength(builder *flatbuffers.Builder, nonZeroLength int64) { + builder.PrependInt64Slot(3, nonZeroLength, 0) +} +func SparseTensorAddSparseIndexType(builder *flatbuffers.Builder, sparseIndexType byte) { + builder.PrependByteSlot(4, sparseIndexType, 0) +} +func SparseTensorAddSparseIndex(builder *flatbuffers.Builder, sparseIndex flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(5, flatbuffers.UOffsetT(sparseIndex), 0) +} +func SparseTensorAddData(builder *flatbuffers.Builder, data flatbuffers.UOffsetT) { + builder.PrependStructSlot(6, flatbuffers.UOffsetT(data), 0) +} +func SparseTensorEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { + return builder.EndObject() +} diff --git a/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/SparseTensorIndex.go b/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/SparseTensorIndex.go new file mode 100644 index 00000000000..4b82de7efec --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/SparseTensorIndex.go @@ -0,0 +1,33 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by the FlatBuffers compiler. DO NOT EDIT. + +package flatbuf + +type SparseTensorIndex = byte +const ( + SparseTensorIndexNONE SparseTensorIndex = 0 + SparseTensorIndexSparseTensorIndexCOO SparseTensorIndex = 1 + SparseTensorIndexSparseMatrixIndexCSR SparseTensorIndex = 2 +) + +var EnumNamesSparseTensorIndex = map[SparseTensorIndex]string{ + SparseTensorIndexNONE:"NONE", + SparseTensorIndexSparseTensorIndexCOO:"SparseTensorIndexCOO", + SparseTensorIndexSparseMatrixIndexCSR:"SparseMatrixIndexCSR", +} + diff --git a/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/SparseTensorIndexCOO.go b/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/SparseTensorIndexCOO.go new file mode 100644 index 00000000000..6d8a0e8fb79 --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/SparseTensorIndexCOO.go @@ -0,0 +1,115 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by the FlatBuffers compiler. DO NOT EDIT. + +package flatbuf + +import ( + flatbuffers "github.com/google/flatbuffers/go" +) + +/// ---------------------------------------------------------------------- +/// EXPERIMENTAL: Data structures for sparse tensors +/// Coodinate format of sparse tensor index. +type SparseTensorIndexCOO struct { + _tab flatbuffers.Table +} + +func GetRootAsSparseTensorIndexCOO(buf []byte, offset flatbuffers.UOffsetT) *SparseTensorIndexCOO { + n := flatbuffers.GetUOffsetT(buf[offset:]) + x := &SparseTensorIndexCOO{} + x.Init(buf, n+offset) + return x +} + +func (rcv *SparseTensorIndexCOO) Init(buf []byte, i flatbuffers.UOffsetT) { + rcv._tab.Bytes = buf + rcv._tab.Pos = i +} + +func (rcv *SparseTensorIndexCOO) Table() flatbuffers.Table { + return rcv._tab +} + +/// COO's index list are represented as a NxM matrix, +/// where N is the number of non-zero values, +/// and M is the number of dimensions of a sparse tensor. +/// indicesBuffer stores the location and size of this index matrix. +/// The type of index value is long, so the stride for the index matrix is unnecessary. +/// +/// For example, let X be a 2x3x4x5 tensor, and it has the following 6 non-zero values: +/// +/// X[0, 1, 2, 0] := 1 +/// X[1, 1, 2, 3] := 2 +/// X[0, 2, 1, 0] := 3 +/// X[0, 1, 3, 0] := 4 +/// X[0, 1, 2, 1] := 5 +/// X[1, 2, 0, 4] := 6 +/// +/// In COO format, the index matrix of X is the following 4x6 matrix: +/// +/// [[0, 0, 0, 0, 1, 1], +/// [1, 1, 1, 2, 1, 2], +/// [2, 2, 3, 1, 2, 0], +/// [0, 1, 0, 0, 3, 4]] +/// +/// Note that the indices are sorted in lexicographical order. +func (rcv *SparseTensorIndexCOO) IndicesBuffer(obj *Buffer) *Buffer { + o := flatbuffers.UOffsetT(rcv._tab.Offset(4)) + if o != 0 { + x := o + rcv._tab.Pos + if obj == nil { + obj = new(Buffer) + } + obj.Init(rcv._tab.Bytes, x) + return obj + } + return nil +} + +/// COO's index list are represented as a NxM matrix, +/// where N is the number of non-zero values, +/// and M is the number of dimensions of a sparse tensor. +/// indicesBuffer stores the location and size of this index matrix. +/// The type of index value is long, so the stride for the index matrix is unnecessary. +/// +/// For example, let X be a 2x3x4x5 tensor, and it has the following 6 non-zero values: +/// +/// X[0, 1, 2, 0] := 1 +/// X[1, 1, 2, 3] := 2 +/// X[0, 2, 1, 0] := 3 +/// X[0, 1, 3, 0] := 4 +/// X[0, 1, 2, 1] := 5 +/// X[1, 2, 0, 4] := 6 +/// +/// In COO format, the index matrix of X is the following 4x6 matrix: +/// +/// [[0, 0, 0, 0, 1, 1], +/// [1, 1, 1, 2, 1, 2], +/// [2, 2, 3, 1, 2, 0], +/// [0, 1, 0, 0, 3, 4]] +/// +/// Note that the indices are sorted in lexicographical order. +func SparseTensorIndexCOOStart(builder *flatbuffers.Builder) { + builder.StartObject(1) +} +func SparseTensorIndexCOOAddIndicesBuffer(builder *flatbuffers.Builder, indicesBuffer flatbuffers.UOffsetT) { + builder.PrependStructSlot(0, flatbuffers.UOffsetT(indicesBuffer), 0) +} +func SparseTensorIndexCOOEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { + return builder.EndObject() +} diff --git a/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/Struct_.go b/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/Struct_.go new file mode 100644 index 00000000000..427e7060382 --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/Struct_.go @@ -0,0 +1,53 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by the FlatBuffers compiler. DO NOT EDIT. + +package flatbuf + +import ( + flatbuffers "github.com/google/flatbuffers/go" +) + +/// A Struct_ in the flatbuffer metadata is the same as an Arrow Struct +/// (according to the physical memory layout). We used Struct_ here as +/// Struct is a reserved word in Flatbuffers +type Struct_ struct { + _tab flatbuffers.Table +} + +func GetRootAsStruct_(buf []byte, offset flatbuffers.UOffsetT) *Struct_ { + n := flatbuffers.GetUOffsetT(buf[offset:]) + x := &Struct_{} + x.Init(buf, n+offset) + return x +} + +func (rcv *Struct_) Init(buf []byte, i flatbuffers.UOffsetT) { + rcv._tab.Bytes = buf + rcv._tab.Pos = i +} + +func (rcv *Struct_) Table() flatbuffers.Table { + return rcv._tab +} + +func Struct_Start(builder *flatbuffers.Builder) { + builder.StartObject(0) +} +func Struct_End(builder *flatbuffers.Builder) flatbuffers.UOffsetT { + return builder.EndObject() +} diff --git a/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/Tensor.go b/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/Tensor.go new file mode 100644 index 00000000000..f8341e02c0b --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/Tensor.go @@ -0,0 +1,161 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by the FlatBuffers compiler. DO NOT EDIT. + +package flatbuf + +import ( + flatbuffers "github.com/google/flatbuffers/go" +) + +type Tensor struct { + _tab flatbuffers.Table +} + +func GetRootAsTensor(buf []byte, offset flatbuffers.UOffsetT) *Tensor { + n := flatbuffers.GetUOffsetT(buf[offset:]) + x := &Tensor{} + x.Init(buf, n+offset) + return x +} + +func (rcv *Tensor) Init(buf []byte, i flatbuffers.UOffsetT) { + rcv._tab.Bytes = buf + rcv._tab.Pos = i +} + +func (rcv *Tensor) Table() flatbuffers.Table { + return rcv._tab +} + +func (rcv *Tensor) TypeType() byte { + o := flatbuffers.UOffsetT(rcv._tab.Offset(4)) + if o != 0 { + return rcv._tab.GetByte(o + rcv._tab.Pos) + } + return 0 +} + +func (rcv *Tensor) MutateTypeType(n byte) bool { + return rcv._tab.MutateByteSlot(4, n) +} + +/// The type of data contained in a value cell. Currently only fixed-width +/// value types are supported, no strings or nested types +func (rcv *Tensor) Type(obj *flatbuffers.Table) bool { + o := flatbuffers.UOffsetT(rcv._tab.Offset(6)) + if o != 0 { + rcv._tab.Union(obj, o) + return true + } + return false +} + +/// The type of data contained in a value cell. Currently only fixed-width +/// value types are supported, no strings or nested types +/// The dimensions of the tensor, optionally named +func (rcv *Tensor) Shape(obj *TensorDim, j int) bool { + o := flatbuffers.UOffsetT(rcv._tab.Offset(8)) + if o != 0 { + x := rcv._tab.Vector(o) + x += flatbuffers.UOffsetT(j) * 4 + x = rcv._tab.Indirect(x) + obj.Init(rcv._tab.Bytes, x) + return true + } + return false +} + +func (rcv *Tensor) ShapeLength() int { + o := flatbuffers.UOffsetT(rcv._tab.Offset(8)) + if o != 0 { + return rcv._tab.VectorLen(o) + } + return 0 +} + +/// The dimensions of the tensor, optionally named +/// Non-negative byte offsets to advance one value cell along each dimension +func (rcv *Tensor) Strides(j int) int64 { + o := flatbuffers.UOffsetT(rcv._tab.Offset(10)) + if o != 0 { + a := rcv._tab.Vector(o) + return rcv._tab.GetInt64(a + flatbuffers.UOffsetT(j*8)) + } + return 0 +} + +func (rcv *Tensor) StridesLength() int { + o := flatbuffers.UOffsetT(rcv._tab.Offset(10)) + if o != 0 { + return rcv._tab.VectorLen(o) + } + return 0 +} + +/// Non-negative byte offsets to advance one value cell along each dimension +func (rcv *Tensor) MutateStrides(j int, n int64) bool { + o := flatbuffers.UOffsetT(rcv._tab.Offset(10)) + if o != 0 { + a := rcv._tab.Vector(o) + return rcv._tab.MutateInt64(a+flatbuffers.UOffsetT(j*8), n) + } + return false +} + +/// The location and size of the tensor's data +func (rcv *Tensor) Data(obj *Buffer) *Buffer { + o := flatbuffers.UOffsetT(rcv._tab.Offset(12)) + if o != 0 { + x := o + rcv._tab.Pos + if obj == nil { + obj = new(Buffer) + } + obj.Init(rcv._tab.Bytes, x) + return obj + } + return nil +} + +/// The location and size of the tensor's data +func TensorStart(builder *flatbuffers.Builder) { + builder.StartObject(5) +} +func TensorAddTypeType(builder *flatbuffers.Builder, typeType byte) { + builder.PrependByteSlot(0, typeType, 0) +} +func TensorAddType(builder *flatbuffers.Builder, type_ flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(1, flatbuffers.UOffsetT(type_), 0) +} +func TensorAddShape(builder *flatbuffers.Builder, shape flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(2, flatbuffers.UOffsetT(shape), 0) +} +func TensorStartShapeVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT { + return builder.StartVector(4, numElems, 4) +} +func TensorAddStrides(builder *flatbuffers.Builder, strides flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(3, flatbuffers.UOffsetT(strides), 0) +} +func TensorStartStridesVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT { + return builder.StartVector(8, numElems, 8) +} +func TensorAddData(builder *flatbuffers.Builder, data flatbuffers.UOffsetT) { + builder.PrependStructSlot(4, flatbuffers.UOffsetT(data), 0) +} +func TensorEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { + return builder.EndObject() +} diff --git a/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/TensorDim.go b/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/TensorDim.go new file mode 100644 index 00000000000..14b82120887 --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/TensorDim.go @@ -0,0 +1,83 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by the FlatBuffers compiler. DO NOT EDIT. + +package flatbuf + +import ( + flatbuffers "github.com/google/flatbuffers/go" +) + +/// ---------------------------------------------------------------------- +/// Data structures for dense tensors +/// Shape data for a single axis in a tensor +type TensorDim struct { + _tab flatbuffers.Table +} + +func GetRootAsTensorDim(buf []byte, offset flatbuffers.UOffsetT) *TensorDim { + n := flatbuffers.GetUOffsetT(buf[offset:]) + x := &TensorDim{} + x.Init(buf, n+offset) + return x +} + +func (rcv *TensorDim) Init(buf []byte, i flatbuffers.UOffsetT) { + rcv._tab.Bytes = buf + rcv._tab.Pos = i +} + +func (rcv *TensorDim) Table() flatbuffers.Table { + return rcv._tab +} + +/// Length of dimension +func (rcv *TensorDim) Size() int64 { + o := flatbuffers.UOffsetT(rcv._tab.Offset(4)) + if o != 0 { + return rcv._tab.GetInt64(o + rcv._tab.Pos) + } + return 0 +} + +/// Length of dimension +func (rcv *TensorDim) MutateSize(n int64) bool { + return rcv._tab.MutateInt64Slot(4, n) +} + +/// Name of the dimension, optional +func (rcv *TensorDim) Name() []byte { + o := flatbuffers.UOffsetT(rcv._tab.Offset(6)) + if o != 0 { + return rcv._tab.ByteVector(o + rcv._tab.Pos) + } + return nil +} + +/// Name of the dimension, optional +func TensorDimStart(builder *flatbuffers.Builder) { + builder.StartObject(2) +} +func TensorDimAddSize(builder *flatbuffers.Builder, size int64) { + builder.PrependInt64Slot(0, size, 0) +} +func TensorDimAddName(builder *flatbuffers.Builder, name flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(1, flatbuffers.UOffsetT(name), 0) +} +func TensorDimEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { + return builder.EndObject() +} diff --git a/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/Time.go b/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/Time.go new file mode 100644 index 00000000000..f3fa99f07b8 --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/Time.go @@ -0,0 +1,83 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by the FlatBuffers compiler. DO NOT EDIT. + +package flatbuf + +import ( + flatbuffers "github.com/google/flatbuffers/go" +) + +/// Time type. The physical storage type depends on the unit +/// - SECOND and MILLISECOND: 32 bits +/// - MICROSECOND and NANOSECOND: 64 bits +type Time struct { + _tab flatbuffers.Table +} + +func GetRootAsTime(buf []byte, offset flatbuffers.UOffsetT) *Time { + n := flatbuffers.GetUOffsetT(buf[offset:]) + x := &Time{} + x.Init(buf, n+offset) + return x +} + +func (rcv *Time) Init(buf []byte, i flatbuffers.UOffsetT) { + rcv._tab.Bytes = buf + rcv._tab.Pos = i +} + +func (rcv *Time) Table() flatbuffers.Table { + return rcv._tab +} + +func (rcv *Time) Unit() TimeUnit { + o := flatbuffers.UOffsetT(rcv._tab.Offset(4)) + if o != 0 { + return rcv._tab.GetInt16(o + rcv._tab.Pos) + } + return 1 +} + +func (rcv *Time) MutateUnit(n TimeUnit) bool { + return rcv._tab.MutateInt16Slot(4, n) +} + +func (rcv *Time) BitWidth() int32 { + o := flatbuffers.UOffsetT(rcv._tab.Offset(6)) + if o != 0 { + return rcv._tab.GetInt32(o + rcv._tab.Pos) + } + return 32 +} + +func (rcv *Time) MutateBitWidth(n int32) bool { + return rcv._tab.MutateInt32Slot(6, n) +} + +func TimeStart(builder *flatbuffers.Builder) { + builder.StartObject(2) +} +func TimeAddUnit(builder *flatbuffers.Builder, unit int16) { + builder.PrependInt16Slot(0, unit, 1) +} +func TimeAddBitWidth(builder *flatbuffers.Builder, bitWidth int32) { + builder.PrependInt32Slot(1, bitWidth, 32) +} +func TimeEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { + return builder.EndObject() +} diff --git a/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/TimeUnit.go b/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/TimeUnit.go new file mode 100644 index 00000000000..cb0c7427d6c --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/TimeUnit.go @@ -0,0 +1,35 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by the FlatBuffers compiler. DO NOT EDIT. + +package flatbuf + +type TimeUnit = int16 +const ( + TimeUnitSECOND TimeUnit = 0 + TimeUnitMILLISECOND TimeUnit = 1 + TimeUnitMICROSECOND TimeUnit = 2 + TimeUnitNANOSECOND TimeUnit = 3 +) + +var EnumNamesTimeUnit = map[TimeUnit]string{ + TimeUnitSECOND:"SECOND", + TimeUnitMILLISECOND:"MILLISECOND", + TimeUnitMICROSECOND:"MICROSECOND", + TimeUnitNANOSECOND:"NANOSECOND", +} + diff --git a/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/Timestamp.go b/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/Timestamp.go new file mode 100644 index 00000000000..dd98c8226ec --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/Timestamp.go @@ -0,0 +1,122 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by the FlatBuffers compiler. DO NOT EDIT. + +package flatbuf + +import ( + flatbuffers "github.com/google/flatbuffers/go" +) + +/// Time elapsed from the Unix epoch, 00:00:00.000 on 1 January 1970, excluding +/// leap seconds, as a 64-bit integer. Note that UNIX time does not include +/// leap seconds. +/// +/// The Timestamp metadata supports both "time zone naive" and "time zone +/// aware" timestamps. Read about the timezone attribute for more detail +type Timestamp struct { + _tab flatbuffers.Table +} + +func GetRootAsTimestamp(buf []byte, offset flatbuffers.UOffsetT) *Timestamp { + n := flatbuffers.GetUOffsetT(buf[offset:]) + x := &Timestamp{} + x.Init(buf, n+offset) + return x +} + +func (rcv *Timestamp) Init(buf []byte, i flatbuffers.UOffsetT) { + rcv._tab.Bytes = buf + rcv._tab.Pos = i +} + +func (rcv *Timestamp) Table() flatbuffers.Table { + return rcv._tab +} + +func (rcv *Timestamp) Unit() TimeUnit { + o := flatbuffers.UOffsetT(rcv._tab.Offset(4)) + if o != 0 { + return rcv._tab.GetInt16(o + rcv._tab.Pos) + } + return 0 +} + +func (rcv *Timestamp) MutateUnit(n TimeUnit) bool { + return rcv._tab.MutateInt16Slot(4, n) +} + +/// The time zone is a string indicating the name of a time zone, one of: +/// +/// * As used in the Olson time zone database (the "tz database" or +/// "tzdata"), such as "America/New_York" +/// * An absolute time zone offset of the form +XX:XX or -XX:XX, such as +07:30 +/// +/// Whether a timezone string is present indicates different semantics about +/// the data: +/// +/// * If the time zone is null or equal to an empty string, the data is "time +/// zone naive" and shall be displayed *as is* to the user, not localized +/// to the locale of the user. This data can be though of as UTC but +/// without having "UTC" as the time zone, it is not considered to be +/// localized to any time zone +/// +/// * If the time zone is set to a valid value, values can be displayed as +/// "localized" to that time zone, even though the underlying 64-bit +/// integers are identical to the same data stored in UTC. Converting +/// between time zones is a metadata-only operation and does not change the +/// underlying values +func (rcv *Timestamp) Timezone() []byte { + o := flatbuffers.UOffsetT(rcv._tab.Offset(6)) + if o != 0 { + return rcv._tab.ByteVector(o + rcv._tab.Pos) + } + return nil +} + +/// The time zone is a string indicating the name of a time zone, one of: +/// +/// * As used in the Olson time zone database (the "tz database" or +/// "tzdata"), such as "America/New_York" +/// * An absolute time zone offset of the form +XX:XX or -XX:XX, such as +07:30 +/// +/// Whether a timezone string is present indicates different semantics about +/// the data: +/// +/// * If the time zone is null or equal to an empty string, the data is "time +/// zone naive" and shall be displayed *as is* to the user, not localized +/// to the locale of the user. This data can be though of as UTC but +/// without having "UTC" as the time zone, it is not considered to be +/// localized to any time zone +/// +/// * If the time zone is set to a valid value, values can be displayed as +/// "localized" to that time zone, even though the underlying 64-bit +/// integers are identical to the same data stored in UTC. Converting +/// between time zones is a metadata-only operation and does not change the +/// underlying values +func TimestampStart(builder *flatbuffers.Builder) { + builder.StartObject(2) +} +func TimestampAddUnit(builder *flatbuffers.Builder, unit int16) { + builder.PrependInt16Slot(0, unit, 0) +} +func TimestampAddTimezone(builder *flatbuffers.Builder, timezone flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(1, flatbuffers.UOffsetT(timezone), 0) +} +func TimestampEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { + return builder.EndObject() +} diff --git a/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/Type.go b/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/Type.go new file mode 100644 index 00000000000..6fcbd25ce88 --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/Type.go @@ -0,0 +1,68 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by the FlatBuffers compiler. DO NOT EDIT. + +package flatbuf + +/// ---------------------------------------------------------------------- +/// Top-level Type value, enabling extensible type-specific metadata. We can +/// add new logical types to Type without breaking backwards compatibility +type Type = byte +const ( + TypeNONE Type = 0 + TypeNull Type = 1 + TypeInt Type = 2 + TypeFloatingPoint Type = 3 + TypeBinary Type = 4 + TypeUtf8 Type = 5 + TypeBool Type = 6 + TypeDecimal Type = 7 + TypeDate Type = 8 + TypeTime Type = 9 + TypeTimestamp Type = 10 + TypeInterval Type = 11 + TypeList Type = 12 + TypeStruct_ Type = 13 + TypeUnion Type = 14 + TypeFixedSizeBinary Type = 15 + TypeFixedSizeList Type = 16 + TypeMap Type = 17 + TypeDuration Type = 18 +) + +var EnumNamesType = map[Type]string{ + TypeNONE:"NONE", + TypeNull:"Null", + TypeInt:"Int", + TypeFloatingPoint:"FloatingPoint", + TypeBinary:"Binary", + TypeUtf8:"Utf8", + TypeBool:"Bool", + TypeDecimal:"Decimal", + TypeDate:"Date", + TypeTime:"Time", + TypeTimestamp:"Timestamp", + TypeInterval:"Interval", + TypeList:"List", + TypeStruct_:"Struct_", + TypeUnion:"Union", + TypeFixedSizeBinary:"FixedSizeBinary", + TypeFixedSizeList:"FixedSizeList", + TypeMap:"Map", + TypeDuration:"Duration", +} + diff --git a/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/Union.go b/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/Union.go new file mode 100644 index 00000000000..1eae176e630 --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/Union.go @@ -0,0 +1,101 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by the FlatBuffers compiler. DO NOT EDIT. + +package flatbuf + +import ( + flatbuffers "github.com/google/flatbuffers/go" +) + +/// A union is a complex type with children in Field +/// By default ids in the type vector refer to the offsets in the children +/// optionally typeIds provides an indirection between the child offset and the type id +/// for each child typeIds[offset] is the id used in the type vector +type Union struct { + _tab flatbuffers.Table +} + +func GetRootAsUnion(buf []byte, offset flatbuffers.UOffsetT) *Union { + n := flatbuffers.GetUOffsetT(buf[offset:]) + x := &Union{} + x.Init(buf, n+offset) + return x +} + +func (rcv *Union) Init(buf []byte, i flatbuffers.UOffsetT) { + rcv._tab.Bytes = buf + rcv._tab.Pos = i +} + +func (rcv *Union) Table() flatbuffers.Table { + return rcv._tab +} + +func (rcv *Union) Mode() UnionMode { + o := flatbuffers.UOffsetT(rcv._tab.Offset(4)) + if o != 0 { + return rcv._tab.GetInt16(o + rcv._tab.Pos) + } + return 0 +} + +func (rcv *Union) MutateMode(n UnionMode) bool { + return rcv._tab.MutateInt16Slot(4, n) +} + +func (rcv *Union) TypeIds(j int) int32 { + o := flatbuffers.UOffsetT(rcv._tab.Offset(6)) + if o != 0 { + a := rcv._tab.Vector(o) + return rcv._tab.GetInt32(a + flatbuffers.UOffsetT(j*4)) + } + return 0 +} + +func (rcv *Union) TypeIdsLength() int { + o := flatbuffers.UOffsetT(rcv._tab.Offset(6)) + if o != 0 { + return rcv._tab.VectorLen(o) + } + return 0 +} + +func (rcv *Union) MutateTypeIds(j int, n int32) bool { + o := flatbuffers.UOffsetT(rcv._tab.Offset(6)) + if o != 0 { + a := rcv._tab.Vector(o) + return rcv._tab.MutateInt32(a+flatbuffers.UOffsetT(j*4), n) + } + return false +} + +func UnionStart(builder *flatbuffers.Builder) { + builder.StartObject(2) +} +func UnionAddMode(builder *flatbuffers.Builder, mode int16) { + builder.PrependInt16Slot(0, mode, 0) +} +func UnionAddTypeIds(builder *flatbuffers.Builder, typeIds flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(1, flatbuffers.UOffsetT(typeIds), 0) +} +func UnionStartTypeIdsVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT { + return builder.StartVector(4, numElems, 4) +} +func UnionEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { + return builder.EndObject() +} diff --git a/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/UnionMode.go b/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/UnionMode.go new file mode 100644 index 00000000000..4357682ad4d --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/UnionMode.go @@ -0,0 +1,31 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by the FlatBuffers compiler. DO NOT EDIT. + +package flatbuf + +type UnionMode = int16 +const ( + UnionModeSparse UnionMode = 0 + UnionModeDense UnionMode = 1 +) + +var EnumNamesUnionMode = map[UnionMode]string{ + UnionModeSparse:"Sparse", + UnionModeDense:"Dense", +} + diff --git a/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/Utf8.go b/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/Utf8.go new file mode 100644 index 00000000000..4ff365a3750 --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/Utf8.go @@ -0,0 +1,51 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by the FlatBuffers compiler. DO NOT EDIT. + +package flatbuf + +import ( + flatbuffers "github.com/google/flatbuffers/go" +) + +/// Unicode with UTF-8 encoding +type Utf8 struct { + _tab flatbuffers.Table +} + +func GetRootAsUtf8(buf []byte, offset flatbuffers.UOffsetT) *Utf8 { + n := flatbuffers.GetUOffsetT(buf[offset:]) + x := &Utf8{} + x.Init(buf, n+offset) + return x +} + +func (rcv *Utf8) Init(buf []byte, i flatbuffers.UOffsetT) { + rcv._tab.Bytes = buf + rcv._tab.Pos = i +} + +func (rcv *Utf8) Table() flatbuffers.Table { + return rcv._tab +} + +func Utf8Start(builder *flatbuffers.Builder) { + builder.StartObject(0) +} +func Utf8End(builder *flatbuffers.Builder) flatbuffers.UOffsetT { + return builder.EndObject() +} diff --git a/vendor/github.com/apache/arrow/go/arrow/ipc/dict.go b/vendor/github.com/apache/arrow/go/arrow/ipc/dict.go new file mode 100644 index 00000000000..802a5acf157 --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/ipc/dict.go @@ -0,0 +1,85 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ipc // import "github.com/apache/arrow/go/arrow/ipc" + +import ( + "github.com/apache/arrow/go/arrow" + "github.com/apache/arrow/go/arrow/array" + "github.com/pkg/errors" +) + +type dictMap map[int64]array.Interface +type dictTypeMap map[int64]arrow.Field + +type dictMemo struct { + dict2id map[array.Interface]int64 + id2dict dictMap // map of dictionary ID to dictionary array +} + +func newMemo() dictMemo { + return dictMemo{ + dict2id: make(map[array.Interface]int64), + id2dict: make(dictMap), + } +} + +func (memo *dictMemo) Len() int { return len(memo.id2dict) } + +func (memo *dictMemo) delete() { + for id, v := range memo.id2dict { + delete(memo.id2dict, id) + delete(memo.dict2id, v) + v.Release() + } +} + +func (memo dictMemo) Dict(id int64) (array.Interface, bool) { + v, ok := memo.id2dict[id] + return v, ok +} + +func (memo *dictMemo) ID(v array.Interface) int64 { + id, ok := memo.dict2id[v] + if ok { + return id + } + + v.Retain() + id = int64(len(memo.dict2id)) + memo.dict2id[v] = id + memo.id2dict[id] = v + return id +} + +func (memo dictMemo) HasDict(v array.Interface) bool { + _, ok := memo.dict2id[v] + return ok +} + +func (memo dictMemo) HasID(id int64) bool { + _, ok := memo.id2dict[id] + return ok +} + +func (memo *dictMemo) Add(id int64, v array.Interface) { + if _, dup := memo.id2dict[id]; dup { + panic(errors.Errorf("arrow/ipc: duplicate id=%d", id)) + } + v.Retain() + memo.id2dict[id] = v + memo.dict2id[v] = id +} diff --git a/vendor/github.com/apache/arrow/go/arrow/ipc/file_reader.go b/vendor/github.com/apache/arrow/go/arrow/ipc/file_reader.go new file mode 100644 index 00000000000..5de7efb4ec7 --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/ipc/file_reader.go @@ -0,0 +1,573 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ipc // import "github.com/apache/arrow/go/arrow/ipc" + +import ( + "bytes" + "encoding/binary" + "fmt" + "io" + + "github.com/apache/arrow/go/arrow" + "github.com/apache/arrow/go/arrow/array" + "github.com/apache/arrow/go/arrow/internal/bitutil" + "github.com/apache/arrow/go/arrow/internal/flatbuf" + "github.com/apache/arrow/go/arrow/memory" + "github.com/pkg/errors" +) + +// FileReader is an Arrow file reader. +type FileReader struct { + r ReadAtSeeker + + footer struct { + offset int64 + buffer *memory.Buffer + data *flatbuf.Footer + } + + fields dictTypeMap + memo dictMemo + + schema *arrow.Schema + record array.Record + + irec int // current record index. used for the arrio.Reader interface + err error // last error +} + +// NewFileReader opens an Arrow file using the provided reader r. +func NewFileReader(r ReadAtSeeker, opts ...Option) (*FileReader, error) { + var ( + cfg = newConfig(opts...) + err error + + f = FileReader{ + r: r, + fields: make(dictTypeMap), + memo: newMemo(), + } + ) + + if cfg.footer.offset <= 0 { + cfg.footer.offset, err = f.r.Seek(0, io.SeekEnd) + if err != nil { + return nil, errors.Wrap(err, "arrow/ipc: could retrieve footer offset") + } + } + f.footer.offset = cfg.footer.offset + + err = f.readFooter() + if err != nil { + return nil, errors.Wrap(err, "arrow/ipc: could not decode footer") + } + + err = f.readSchema() + if err != nil { + return nil, errors.Wrap(err, "arrow/ipc: could not decode schema") + } + + if cfg.schema != nil && !cfg.schema.Equal(f.schema) { + return nil, errors.Errorf("arrow/ipc: inconsitent schema for reading (got: %v, want: %v)", f.schema, cfg.schema) + } + + return &f, err +} + +func (f *FileReader) readFooter() error { + var err error + + if f.footer.offset <= int64(len(Magic)*2+4) { + return fmt.Errorf("arrow/ipc: file too small (size=%d)", f.footer.offset) + } + + eof := int64(len(Magic) + 4) + buf := make([]byte, eof) + n, err := f.r.ReadAt(buf, f.footer.offset-eof) + if err != nil { + return errors.Wrap(err, "arrow/ipc: could not read footer") + } + if n != len(buf) { + return errors.Errorf("arrow/ipc: could not read %d bytes from end of file", len(buf)) + } + + if !bytes.Equal(buf[4:], Magic) { + return errNotArrowFile + } + + size := int64(binary.LittleEndian.Uint32(buf[:4])) + if size <= 0 || size+int64(len(Magic)*2+4) > f.footer.offset { + return errInconsistentFileMetadata + } + + buf = make([]byte, size) + n, err = f.r.ReadAt(buf, f.footer.offset-size-eof) + if err != nil { + return errors.Wrap(err, "arrow/ipc: could not read footer data") + } + if n != len(buf) { + return errors.Errorf("arrow/ipc: could not read %d bytes from footer data", len(buf)) + } + + f.footer.buffer = memory.NewBufferBytes(buf) + f.footer.data = flatbuf.GetRootAsFooter(buf, 0) + return err +} + +func (f *FileReader) readSchema() error { + var err error + f.fields, err = dictTypesFromFB(f.footer.data.Schema(nil)) + if err != nil { + return errors.Wrap(err, "arrow/ipc: could not load dictionary types from file") + } + + for i := 0; i < f.NumDictionaries(); i++ { + blk, err := f.dict(i) + if err != nil { + return errors.Wrapf(err, "arrow/ipc: could read dictionary[%d]", i) + } + switch { + case !bitutil.IsMultipleOf8(blk.Offset): + return errors.Errorf("arrow/ipc: invalid file offset=%d for dictionary %d", blk.Offset, i) + case !bitutil.IsMultipleOf8(int64(blk.Meta)): + return errors.Errorf("arrow/ipc: invalid file metadata=%d position for dictionary %d", blk.Meta, i) + case !bitutil.IsMultipleOf8(blk.Body): + return errors.Errorf("arrow/ipc: invalid file body=%d position for dictionary %d", blk.Body, i) + } + + msg, err := blk.NewMessage() + if err != nil { + return err + } + defer msg.Release() + + id, dict, err := readDictionary(msg.meta, f.fields, f.r) + if err != nil { + return errors.Wrapf(err, "arrow/ipc: could not read dictionary %d from file", i) + } + f.memo.Add(id, dict) + dict.Release() // memo.Add increases ref-count of dict. + } + + schema := f.footer.data.Schema(nil) + if schema == nil { + return errors.New("arrow/ipc: could not load schema from flatbuffer data") + } + f.schema, err = schemaFromFB(schema, &f.memo) + if err != nil { + return errors.Wrap(err, "arrow/ipc: could not read schema") + } + + return err +} + +func (f *FileReader) block(i int) (fileBlock, error) { + var blk flatbuf.Block + if !f.footer.data.RecordBatches(&blk, i) { + return fileBlock{}, errors.Errorf("arrow/ipc: could not extract file block %d", i) + } + + return fileBlock{ + Offset: blk.Offset(), + Meta: blk.MetaDataLength(), + Body: blk.BodyLength(), + r: f.r, + }, nil +} + +func (f *FileReader) dict(i int) (fileBlock, error) { + var blk flatbuf.Block + if !f.footer.data.Dictionaries(&blk, i) { + return fileBlock{}, errors.Errorf("arrow/ipc: could not extract dictionary block %d", i) + } + + return fileBlock{ + Offset: blk.Offset(), + Meta: blk.MetaDataLength(), + Body: blk.BodyLength(), + r: f.r, + }, nil +} + +func (f *FileReader) Schema() *arrow.Schema { + return f.schema +} + +func (f *FileReader) NumDictionaries() int { + if f.footer.data == nil { + return 0 + } + return f.footer.data.DictionariesLength() +} + +func (f *FileReader) NumRecords() int { + return f.footer.data.RecordBatchesLength() +} + +func (f *FileReader) Version() MetadataVersion { + return MetadataVersion(f.footer.data.Version()) +} + +// Close cleans up resources used by the File. +// Close does not close the underlying reader. +func (f *FileReader) Close() error { + if f.footer.data != nil { + f.footer.data = nil + } + + if f.footer.buffer != nil { + f.footer.buffer.Release() + f.footer.buffer = nil + } + + if f.record != nil { + f.record.Release() + f.record = nil + } + return nil +} + +// Record returns the i-th record from the file. +// The returned value is valid until the next call to Record. +// Users need to call Retain on that Record to keep it valid for longer. +func (f *FileReader) Record(i int) (array.Record, error) { + if i < 0 || i > f.NumRecords() { + panic("arrow/ipc: record index out of bounds") + } + + blk, err := f.block(i) + if err != nil { + return nil, err + } + switch { + case !bitutil.IsMultipleOf8(blk.Offset): + return nil, errors.Errorf("arrow/ipc: invalid file offset=%d for record %d", blk.Offset, i) + case !bitutil.IsMultipleOf8(int64(blk.Meta)): + return nil, errors.Errorf("arrow/ipc: invalid file metadata=%d position for record %d", blk.Meta, i) + case !bitutil.IsMultipleOf8(blk.Body): + return nil, errors.Errorf("arrow/ipc: invalid file body=%d position for record %d", blk.Body, i) + } + + msg, err := blk.NewMessage() + if err != nil { + return nil, err + } + defer msg.Release() + + if msg.Type() != MessageRecordBatch { + return nil, errors.Errorf("arrow/ipc: message %d is not a Record", i) + } + + if f.record != nil { + f.record.Release() + } + + f.record = newRecord(f.schema, msg.meta, bytes.NewReader(msg.body.Bytes())) + return f.record, nil +} + +// Read reads the current record from the underlying stream and an error, if any. +// When the Reader reaches the end of the underlying stream, it returns (nil, io.EOF). +// +// The returned record value is valid until the next call to Read. +// Users need to call Retain on that Record to keep it valid for longer. +func (f *FileReader) Read() (rec array.Record, err error) { + if f.irec == f.NumRecords() { + return nil, io.EOF + } + rec, f.err = f.Record(f.irec) + f.irec++ + return rec, f.err +} + +// ReadAt reads the i-th record from the underlying stream and an error, if any. +func (f *FileReader) ReadAt(i int64) (array.Record, error) { + return f.Record(int(i)) +} + +func newRecord(schema *arrow.Schema, meta *memory.Buffer, body ReadAtSeeker) array.Record { + var ( + msg = flatbuf.GetRootAsMessage(meta.Bytes(), 0) + md flatbuf.RecordBatch + ) + initFB(&md, msg.Header) + rows := md.Length() + + ctx := &arrayLoaderContext{ + src: ipcSource{ + meta: &md, + r: body, + }, + max: kMaxNestingDepth, + } + + cols := make([]array.Interface, len(schema.Fields())) + for i, field := range schema.Fields() { + cols[i] = ctx.loadArray(field.Type) + } + + return array.NewRecord(schema, cols, rows) +} + +type ipcSource struct { + meta *flatbuf.RecordBatch + r ReadAtSeeker +} + +func (src *ipcSource) buffer(i int) *memory.Buffer { + var buf flatbuf.Buffer + if !src.meta.Buffers(&buf, i) { + panic("buffer index out of bound") + } + if buf.Length() == 0 { + return memory.NewBufferBytes(nil) + } + + raw := make([]byte, buf.Length()) + _, err := src.r.ReadAt(raw, buf.Offset()) + if err != nil { + panic(err) + } + + return memory.NewBufferBytes(raw) +} + +func (src *ipcSource) fieldMetadata(i int) *flatbuf.FieldNode { + var node flatbuf.FieldNode + if !src.meta.Nodes(&node, i) { + panic("field metadata out of bound") + } + return &node +} + +type arrayLoaderContext struct { + src ipcSource + ifield int + ibuffer int + max int +} + +func (ctx *arrayLoaderContext) field() *flatbuf.FieldNode { + field := ctx.src.fieldMetadata(ctx.ifield) + ctx.ifield++ + return field +} + +func (ctx *arrayLoaderContext) buffer() *memory.Buffer { + buf := ctx.src.buffer(ctx.ibuffer) + ctx.ibuffer++ + return buf +} + +func (ctx *arrayLoaderContext) loadArray(dt arrow.DataType) array.Interface { + switch dt := dt.(type) { + case *arrow.NullType: + return ctx.loadNull() + + case *arrow.BooleanType, + *arrow.Int8Type, *arrow.Int16Type, *arrow.Int32Type, *arrow.Int64Type, + *arrow.Uint8Type, *arrow.Uint16Type, *arrow.Uint32Type, *arrow.Uint64Type, + *arrow.Float16Type, *arrow.Float32Type, *arrow.Float64Type, + *arrow.Decimal128Type, + *arrow.Time32Type, *arrow.Time64Type, + *arrow.TimestampType, + *arrow.Date32Type, *arrow.Date64Type, + *arrow.MonthIntervalType, *arrow.DayTimeIntervalType, + *arrow.DurationType: + return ctx.loadPrimitive(dt) + + case *arrow.BinaryType, *arrow.StringType: + return ctx.loadBinary(dt) + + case *arrow.FixedSizeBinaryType: + return ctx.loadFixedSizeBinary(dt) + + case *arrow.ListType: + return ctx.loadList(dt) + + case *arrow.FixedSizeListType: + return ctx.loadFixedSizeList(dt) + + case *arrow.StructType: + return ctx.loadStruct(dt) + + default: + panic(errors.Errorf("array type %T not handled yet", dt)) + } +} + +func (ctx *arrayLoaderContext) loadCommon(nbufs int) (*flatbuf.FieldNode, []*memory.Buffer) { + buffers := make([]*memory.Buffer, 0, nbufs) + field := ctx.field() + + var buf *memory.Buffer + switch field.NullCount() { + case 0: + ctx.ibuffer++ + default: + buf = ctx.buffer() + } + buffers = append(buffers, buf) + + return field, buffers +} + +func (ctx *arrayLoaderContext) loadChild(dt arrow.DataType) array.Interface { + if ctx.max == 0 { + panic("arrow/ipc: nested type limit reached") + } + ctx.max-- + sub := ctx.loadArray(dt) + ctx.max++ + return sub +} + +func (ctx *arrayLoaderContext) loadNull() array.Interface { + field, buffers := ctx.loadCommon(1) + buffers = append(buffers, ctx.buffer()) + + data := array.NewData(arrow.Null, int(field.Length()), buffers, nil, int(field.NullCount()), 0) + defer data.Release() + + return array.MakeFromData(data) +} + +func (ctx *arrayLoaderContext) loadPrimitive(dt arrow.DataType) array.Interface { + field, buffers := ctx.loadCommon(2) + + switch field.Length() { + case 0: + buffers = append(buffers, nil) + ctx.ibuffer++ + default: + buffers = append(buffers, ctx.buffer()) + } + + data := array.NewData(dt, int(field.Length()), buffers, nil, int(field.NullCount()), 0) + defer data.Release() + + return array.MakeFromData(data) +} + +func (ctx *arrayLoaderContext) loadBinary(dt arrow.DataType) array.Interface { + field, buffers := ctx.loadCommon(3) + buffers = append(buffers, ctx.buffer(), ctx.buffer()) + + data := array.NewData(dt, int(field.Length()), buffers, nil, int(field.NullCount()), 0) + defer data.Release() + + return array.MakeFromData(data) +} + +func (ctx *arrayLoaderContext) loadFixedSizeBinary(dt *arrow.FixedSizeBinaryType) array.Interface { + field, buffers := ctx.loadCommon(2) + buffers = append(buffers, ctx.buffer()) + + data := array.NewData(dt, int(field.Length()), buffers, nil, int(field.NullCount()), 0) + defer data.Release() + + return array.MakeFromData(data) +} + +func (ctx *arrayLoaderContext) loadList(dt *arrow.ListType) array.Interface { + field, buffers := ctx.loadCommon(2) + buffers = append(buffers, ctx.buffer()) + + sub := ctx.loadChild(dt.Elem()) + defer sub.Release() + + data := array.NewData(dt, int(field.Length()), buffers, []*array.Data{sub.Data()}, int(field.NullCount()), 0) + defer data.Release() + + return array.NewListData(data) +} + +func (ctx *arrayLoaderContext) loadFixedSizeList(dt *arrow.FixedSizeListType) array.Interface { + field, buffers := ctx.loadCommon(1) + + sub := ctx.loadChild(dt.Elem()) + defer sub.Release() + + data := array.NewData(dt, int(field.Length()), buffers, []*array.Data{sub.Data()}, int(field.NullCount()), 0) + defer data.Release() + + return array.NewFixedSizeListData(data) +} + +func (ctx *arrayLoaderContext) loadStruct(dt *arrow.StructType) array.Interface { + field, buffers := ctx.loadCommon(1) + + arrs := make([]array.Interface, len(dt.Fields())) + subs := make([]*array.Data, len(dt.Fields())) + for i, f := range dt.Fields() { + arrs[i] = ctx.loadChild(f.Type) + subs[i] = arrs[i].Data() + } + defer func() { + for i := range arrs { + arrs[i].Release() + } + }() + + data := array.NewData(dt, int(field.Length()), buffers, subs, int(field.NullCount()), 0) + defer data.Release() + + return array.NewStructData(data) +} + +func readDictionary(meta *memory.Buffer, types dictTypeMap, r ReadAtSeeker) (int64, array.Interface, error) { + // msg := flatbuf.GetRootAsMessage(meta.Bytes(), 0) + // var dictBatch flatbuf.DictionaryBatch + // initFB(&dictBatch, msg.Header) + // + // id := dictBatch.Id() + // v, ok := types[id] + // if !ok { + // return id, nil, errors.Errorf("arrow/ipc: no type metadata for dictionary with ID=%d", id) + // } + // + // fields := []arrow.Field{v} + // + // // we need a schema for the record batch. + // schema := arrow.NewSchema(fields, nil) + // + // // the dictionary is embedded in a record batch with a single column. + // recBatch := dictBatch.Data(nil) + // + // var ( + // batchMeta *memory.Buffer + // body *memory.Buffer + // ) + // + // + // ctx := &arrayLoaderContext{ + // src: ipcSource{ + // meta: &md, + // r: bytes.NewReader(body.Bytes()), + // }, + // max: kMaxNestingDepth, + // } + // + // cols := make([]array.Interface, len(schema.Fields())) + // for i, field := range schema.Fields() { + // cols[i] = ctx.loadArray(field.Type) + // } + // + // batch := array.NewRecord(schema, cols, rows) + + panic("not implemented") +} diff --git a/vendor/github.com/apache/arrow/go/arrow/ipc/file_writer.go b/vendor/github.com/apache/arrow/go/arrow/ipc/file_writer.go new file mode 100644 index 00000000000..1fb6f529302 --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/ipc/file_writer.go @@ -0,0 +1,333 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ipc // import "github.com/apache/arrow/go/arrow/ipc" + +import ( + "encoding/binary" + "io" + + "github.com/apache/arrow/go/arrow" + "github.com/apache/arrow/go/arrow/array" + "github.com/apache/arrow/go/arrow/internal/bitutil" + "github.com/apache/arrow/go/arrow/internal/flatbuf" + "github.com/apache/arrow/go/arrow/memory" + "github.com/pkg/errors" +) + +type payloadWriter interface { + start() error + write(payload) error + Close() error +} + +type pwriter struct { + w io.WriteSeeker + pos int64 + + schema *arrow.Schema + dicts []fileBlock + recs []fileBlock +} + +func (w *pwriter) start() error { + var err error + + err = w.updatePos() + if err != nil { + return errors.Wrap(err, "arrow/ipc: could not update position while in start") + } + + // only necessary to align to 8-byte boundary at the start of the file + _, err = w.Write(Magic) + if err != nil { + return errors.Wrap(err, "arrow/ipc: could not write magic Arrow bytes") + } + + err = w.align(kArrowIPCAlignment) + if err != nil { + return errors.Wrap(err, "arrow/ipc: could not align start block") + } + + return err +} + +func (w *pwriter) write(p payload) error { + blk := fileBlock{Offset: w.pos, Meta: 0, Body: p.size} + n, err := writeIPCPayload(w, p) + if err != nil { + return err + } + + blk.Meta = int32(n) + + err = w.updatePos() + if err != nil { + return errors.Wrap(err, "arrow/ipc: could not update position while in write-payload") + } + + switch byte(p.msg) { + case flatbuf.MessageHeaderDictionaryBatch: + w.dicts = append(w.dicts, blk) + case flatbuf.MessageHeaderRecordBatch: + w.recs = append(w.recs, blk) + } + + return nil +} + +func (w *pwriter) Close() error { + var err error + + // write file footer + err = w.updatePos() + if err != nil { + return errors.Wrap(err, "arrow/ipc: could not update position while in close") + } + + pos := w.pos + err = writeFileFooter(w.schema, w.dicts, w.recs, w) + if err != nil { + return errors.Wrap(err, "arrow/ipc: could not write file footer") + } + + // write file footer length + err = w.updatePos() // not strictly needed as we passed w to writeFileFooter... + if err != nil { + return errors.Wrap(err, "arrow/ipc: could not compute file footer length") + } + + size := w.pos - pos + if size <= 0 { + return errors.Errorf("arrow/ipc: invalid file footer size (size=%d)", size) + } + + buf := make([]byte, 4) + binary.LittleEndian.PutUint32(buf, uint32(size)) + _, err = w.Write(buf) + if err != nil { + return errors.Wrap(err, "arrow/ipc: could not write file footer size") + } + + _, err = w.Write(Magic) + if err != nil { + return errors.Wrap(err, "arrow/ipc: could not write Arrow magic bytes") + } + + return nil +} + +func (w *pwriter) updatePos() error { + var err error + w.pos, err = w.w.Seek(0, io.SeekCurrent) + return err +} + +func (w *pwriter) align(align int32) error { + remainder := paddedLength(w.pos, align) - w.pos + if remainder == 0 { + return nil + } + + _, err := w.Write(paddingBytes[:int(remainder)]) + return err +} + +func (w *pwriter) Write(p []byte) (int, error) { + n, err := w.w.Write(p) + w.pos += int64(n) + return n, err +} + +func writeIPCPayload(w io.Writer, p payload) (int, error) { + n, err := writeMessage(p.meta, kArrowIPCAlignment, w) + if err != nil { + return n, err + } + + // now write the buffers + for _, buf := range p.body { + var ( + size int64 + padding int64 + ) + + // the buffer might be null if we are handling zero row lengths. + if buf != nil { + size = int64(buf.Len()) + padding = bitutil.CeilByte64(size) - size + } + + if size > 0 { + _, err = w.Write(buf.Bytes()) + if err != nil { + return n, errors.Wrap(err, "arrow/ipc: could not write payload message body") + } + } + + if padding > 0 { + _, err = w.Write(paddingBytes[:padding]) + if err != nil { + return n, errors.Wrap(err, "arrow/ipc: could not write payload message padding") + } + } + } + + return n, err +} + +type payload struct { + msg MessageType + meta *memory.Buffer + body []*memory.Buffer + size int64 // length of body +} + +func (p *payload) Release() { + if p.meta != nil { + p.meta.Release() + p.meta = nil + } + for i, b := range p.body { + if b == nil { + continue + } + b.Release() + p.body[i] = nil + } +} + +type payloads []payload + +func (ps payloads) Release() { + for i := range ps { + ps[i].Release() + } +} + +// FileWriter is an Arrow file writer. +type FileWriter struct { + w io.WriteSeeker + + mem memory.Allocator + + header struct { + started bool + offset int64 + } + + footer struct { + written bool + } + + pw payloadWriter + + schema *arrow.Schema +} + +// NewFileWriter opens an Arrow file using the provided writer w. +func NewFileWriter(w io.WriteSeeker, opts ...Option) (*FileWriter, error) { + var ( + cfg = newConfig(opts...) + err error + ) + + f := FileWriter{ + w: w, + pw: &pwriter{w: w, schema: cfg.schema, pos: -1}, + mem: cfg.alloc, + schema: cfg.schema, + } + + pos, err := f.w.Seek(0, io.SeekCurrent) + if err != nil { + return nil, errors.Errorf("arrow/ipc: could not seek current position: %v", err) + } + f.header.offset = pos + + return &f, err +} + +func (f *FileWriter) Close() error { + err := f.checkStarted() + if err != nil { + return errors.Wrap(err, "arrow/ipc: could not write empty file") + } + + if f.footer.written { + return nil + } + + err = f.pw.Close() + if err != nil { + return errors.Wrap(err, "arrow/ipc: could not close payload writer") + } + f.footer.written = true + + return nil +} + +func (f *FileWriter) Write(rec array.Record) error { + schema := rec.Schema() + if schema == nil || !schema.Equal(f.schema) { + return errInconsistentSchema + } + + if err := f.checkStarted(); err != nil { + return errors.Wrap(err, "arrow/ipc: could not write header") + } + + const allow64b = true + var ( + data = payload{msg: MessageRecordBatch} + enc = newRecordEncoder(f.mem, 0, kMaxNestingDepth, allow64b) + ) + defer data.Release() + + if err := enc.Encode(&data, rec); err != nil { + return errors.Wrap(err, "arrow/ipc: could not encode record to payload") + } + + return f.pw.write(data) +} + +func (f *FileWriter) checkStarted() error { + if !f.header.started { + return f.start() + } + return nil +} + +func (f *FileWriter) start() error { + f.header.started = true + err := f.pw.start() + if err != nil { + return err + } + + // write out schema payloads + ps := payloadsFromSchema(f.schema, f.mem, nil) + defer ps.Release() + + for _, data := range ps { + err = f.pw.write(data) + if err != nil { + return err + } + } + + return nil +} diff --git a/vendor/github.com/apache/arrow/go/arrow/ipc/ipc.go b/vendor/github.com/apache/arrow/go/arrow/ipc/ipc.go new file mode 100644 index 00000000000..e6889743dfd --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/ipc/ipc.go @@ -0,0 +1,113 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ipc // import "github.com/apache/arrow/go/arrow/ipc" + +import ( + "io" + + "github.com/apache/arrow/go/arrow" + "github.com/apache/arrow/go/arrow/arrio" + "github.com/apache/arrow/go/arrow/memory" +) + +const ( + errNotArrowFile = errString("arrow/ipc: not an Arrow file") + errInconsistentFileMetadata = errString("arrow/ipc: file is smaller than indicated metadata size") + errInconsistentSchema = errString("arrow/ipc: tried to write record batch with different schema") + errMaxRecursion = errString("arrow/ipc: max recursion depth reached") + errBigArray = errString("arrow/ipc: array larger than 2^31-1 in length") + + kArrowAlignment = 64 // buffers are padded to 64b boundaries (for SIMD) + kTensorAlignment = 64 // tensors are padded to 64b boundaries + kArrowIPCAlignment = 8 // align on 8b boundaries in IPC +) + +var ( + paddingBytes [kArrowAlignment]byte + kEOS = [4]byte{0, 0, 0, 0} // end of stream message +) + +func paddedLength(nbytes int64, alignment int32) int64 { + align := int64(alignment) + return ((nbytes + align - 1) / align) * align +} + +type errString string + +func (s errString) Error() string { + return string(s) +} + +type ReadAtSeeker interface { + io.Reader + io.Seeker + io.ReaderAt +} + +type config struct { + alloc memory.Allocator + schema *arrow.Schema + footer struct { + offset int64 + } +} + +func newConfig(opts ...Option) *config { + cfg := &config{ + alloc: memory.NewGoAllocator(), + } + + for _, opt := range opts { + opt(cfg) + } + + return cfg +} + +// Option is a functional option to configure opening or creating Arrow files +// and streams. +type Option func(*config) + +// WithFooterOffset specifies the Arrow footer position in bytes. +func WithFooterOffset(offset int64) Option { + return func(cfg *config) { + cfg.footer.offset = offset + } +} + +// WithAllocator specifies the Arrow memory allocator used while building records. +func WithAllocator(mem memory.Allocator) Option { + return func(cfg *config) { + cfg.alloc = mem + } +} + +// WithSchema specifies the Arrow schema to be used for reading or writing. +func WithSchema(schema *arrow.Schema) Option { + return func(cfg *config) { + cfg.schema = schema + } +} + +var ( + _ arrio.Reader = (*Reader)(nil) + _ arrio.Writer = (*Writer)(nil) + _ arrio.Reader = (*FileReader)(nil) + _ arrio.Writer = (*FileWriter)(nil) + + _ arrio.ReaderAt = (*FileReader)(nil) +) diff --git a/vendor/github.com/apache/arrow/go/arrow/ipc/message.go b/vendor/github.com/apache/arrow/go/arrow/ipc/message.go new file mode 100644 index 00000000000..bb12dbbc518 --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/ipc/message.go @@ -0,0 +1,215 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ipc // import "github.com/apache/arrow/go/arrow/ipc" + +import ( + "encoding/binary" + "fmt" + "io" + "sync/atomic" + + "github.com/apache/arrow/go/arrow/internal/debug" + "github.com/apache/arrow/go/arrow/internal/flatbuf" + "github.com/apache/arrow/go/arrow/memory" + "github.com/pkg/errors" +) + +// MetadataVersion represents the Arrow metadata version. +type MetadataVersion flatbuf.MetadataVersion + +const ( + MetadataV1 = MetadataVersion(flatbuf.MetadataVersionV1) // version for Arrow-0.1.0 + MetadataV2 = MetadataVersion(flatbuf.MetadataVersionV2) // version for Arrow-0.2.0 + MetadataV3 = MetadataVersion(flatbuf.MetadataVersionV3) // version for Arrow-0.3.0 to 0.7.1 + MetadataV4 = MetadataVersion(flatbuf.MetadataVersionV4) // version for >= Arrow-0.8.0 +) + +func (m MetadataVersion) String() string { + if v, ok := flatbuf.EnumNamesMetadataVersion[int16(m)]; ok { + return v + } + return fmt.Sprintf("MetadataVersion(%d)", int16(m)) +} + +// MessageType represents the type of Message in an Arrow format. +type MessageType flatbuf.MessageHeader + +const ( + MessageNone = MessageType(flatbuf.MessageHeaderNONE) + MessageSchema = MessageType(flatbuf.MessageHeaderSchema) + MessageDictionaryBatch = MessageType(flatbuf.MessageHeaderDictionaryBatch) + MessageRecordBatch = MessageType(flatbuf.MessageHeaderRecordBatch) + MessageTensor = MessageType(flatbuf.MessageHeaderTensor) + MessageSparseTensor = MessageType(flatbuf.MessageHeaderSparseTensor) +) + +func (m MessageType) String() string { + if v, ok := flatbuf.EnumNamesMessageHeader[byte(m)]; ok { + return v + } + return fmt.Sprintf("MessageType(%d)", int(m)) +} + +const ( + // maxNestingDepth is an arbitrary value to catch user mistakes. + // For deeply nested schemas, it is expected the user will indicate + // explicitly the maximum allowed recursion depth. + maxNestingDepth = 64 +) + +// Message is an IPC message, including metadata and body. +type Message struct { + refCount int64 + msg *flatbuf.Message + meta *memory.Buffer + body *memory.Buffer +} + +// NewMessage creates a new message from the metadata and body buffers. +// NewMessage panics if any of these buffers is nil. +func NewMessage(meta, body *memory.Buffer) *Message { + if meta == nil || body == nil { + panic("arrow/ipc: nil buffers") + } + meta.Retain() + body.Retain() + return &Message{ + refCount: 1, + msg: flatbuf.GetRootAsMessage(meta.Bytes(), 0), + meta: meta, + body: body, + } +} + +func newMessageFromFB(meta *flatbuf.Message, body *memory.Buffer) *Message { + if meta == nil || body == nil { + panic("arrow/ipc: nil buffers") + } + body.Retain() + return &Message{ + refCount: 1, + msg: meta, + meta: memory.NewBufferBytes(meta.Table().Bytes), + body: body, + } +} + +// Retain increases the reference count by 1. +// Retain may be called simultaneously from multiple goroutines. +func (msg *Message) Retain() { + atomic.AddInt64(&msg.refCount, 1) +} + +// Release decreases the reference count by 1. +// Release may be called simultaneously from multiple goroutines. +// When the reference count goes to zero, the memory is freed. +func (msg *Message) Release() { + debug.Assert(atomic.LoadInt64(&msg.refCount) > 0, "too many releases") + + if atomic.AddInt64(&msg.refCount, -1) == 0 { + msg.meta.Release() + msg.body.Release() + msg.msg = nil + msg.meta = nil + msg.body = nil + } +} + +func (msg *Message) Version() MetadataVersion { + return MetadataVersion(msg.msg.Version()) +} + +func (msg *Message) Type() MessageType { + return MessageType(msg.msg.HeaderType()) +} + +func (msg *Message) BodyLen() int64 { + return msg.msg.BodyLength() +} + +// MessageReader reads messages from an io.Reader. +type MessageReader struct { + r io.Reader + + refCount int64 + msg *Message +} + +// NewMessageReader returns a reader that reads messages from an input stream. +func NewMessageReader(r io.Reader) *MessageReader { + return &MessageReader{r: r, refCount: 1} +} + +// Retain increases the reference count by 1. +// Retain may be called simultaneously from multiple goroutines. +func (r *MessageReader) Retain() { + atomic.AddInt64(&r.refCount, 1) +} + +// Release decreases the reference count by 1. +// When the reference count goes to zero, the memory is freed. +// Release may be called simultaneously from multiple goroutines. +func (r *MessageReader) Release() { + debug.Assert(atomic.LoadInt64(&r.refCount) > 0, "too many releases") + + if atomic.AddInt64(&r.refCount, -1) == 0 { + if r.msg != nil { + r.msg.Release() + r.msg = nil + } + } +} + +// Message returns the current message that has been extracted from the +// underlying stream. +// It is valid until the next call to Message. +func (r *MessageReader) Message() (*Message, error) { + var buf = make([]byte, 4) + _, err := io.ReadFull(r.r, buf) + if err != nil { + return nil, errors.Wrap(err, "arrow/ipc: could not read message length") + } + msgLen := int32(binary.LittleEndian.Uint32(buf)) + if msgLen == 0 { + // optional 0 EOS control message + return nil, io.EOF // FIXME(sbinet): send nil instead? or a special EOS error? + } + + buf = make([]byte, msgLen) + _, err = io.ReadFull(r.r, buf) + if err != nil { + return nil, errors.Wrap(err, "arrow/ipc: could not read message metadata") + } + + meta := flatbuf.GetRootAsMessage(buf, 0) + bodyLen := meta.BodyLength() + + buf = make([]byte, bodyLen) + _, err = io.ReadFull(r.r, buf) + if err != nil { + return nil, errors.Wrap(err, "arrow/ipc: could not read message body") + } + body := memory.NewBufferBytes(buf) + + if r.msg != nil { + r.msg.Release() + r.msg = nil + } + r.msg = newMessageFromFB(meta, body) + + return r.msg, nil +} diff --git a/vendor/github.com/apache/arrow/go/arrow/ipc/metadata.go b/vendor/github.com/apache/arrow/go/arrow/ipc/metadata.go new file mode 100644 index 00000000000..7b4e3dad433 --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/ipc/metadata.go @@ -0,0 +1,1040 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ipc // import "github.com/apache/arrow/go/arrow/ipc" + +import ( + "encoding/binary" + "fmt" + "io" + "sort" + + "github.com/apache/arrow/go/arrow" + "github.com/apache/arrow/go/arrow/internal/flatbuf" + "github.com/apache/arrow/go/arrow/memory" + flatbuffers "github.com/google/flatbuffers/go" + "github.com/pkg/errors" +) + +// Magic string identifying an Apache Arrow file. +var Magic = []byte("ARROW1") + +const ( + currentMetadataVersion = MetadataV4 + minMetadataVersion = MetadataV4 + + kExtensionTypeKeyName = "arrow_extension_name" + kExtensionDataKeyName = "arrow_extension_data" + + // ARROW-109: We set this number arbitrarily to help catch user mistakes. For + // deeply nested schemas, it is expected the user will indicate explicitly the + // maximum allowed recursion depth + kMaxNestingDepth = 64 +) + +type startVecFunc func(b *flatbuffers.Builder, n int) flatbuffers.UOffsetT + +type fieldMetadata struct { + Len int64 + Nulls int64 + Offset int64 +} + +type bufferMetadata struct { + Offset int64 // relative offset into the memory page to the starting byte of the buffer + Len int64 // absolute length in bytes of the buffer +} + +type fileBlock struct { + Offset int64 + Meta int32 + Body int64 + + r io.ReaderAt +} + +func fileBlocksToFB(b *flatbuffers.Builder, blocks []fileBlock, start startVecFunc) flatbuffers.UOffsetT { + start(b, len(blocks)) + for i := len(blocks) - 1; i >= 0; i-- { + blk := blocks[i] + flatbuf.CreateBlock(b, blk.Offset, blk.Meta, blk.Body) + } + + return b.EndVector(len(blocks)) +} + +func (blk fileBlock) NewMessage() (*Message, error) { + var ( + err error + buf []byte + r = blk.section() + ) + + buf = make([]byte, blk.Meta) + _, err = io.ReadFull(r, buf) + if err != nil { + return nil, errors.Wrap(err, "arrow/ipc: could not read message metadata") + } + meta := memory.NewBufferBytes(buf[4:]) // drop buf-size already known from blk.Meta + + buf = make([]byte, blk.Body) + _, err = io.ReadFull(r, buf) + if err != nil { + return nil, errors.Wrap(err, "arrow/ipc: could not read message body") + } + body := memory.NewBufferBytes(buf) + + return NewMessage(meta, body), nil +} + +func (blk fileBlock) section() io.Reader { + return io.NewSectionReader(blk.r, blk.Offset, int64(blk.Meta)+blk.Body) +} + +func unitFromFB(unit flatbuf.TimeUnit) arrow.TimeUnit { + switch unit { + case flatbuf.TimeUnitSECOND: + return arrow.Second + case flatbuf.TimeUnitMILLISECOND: + return arrow.Millisecond + case flatbuf.TimeUnitMICROSECOND: + return arrow.Microsecond + case flatbuf.TimeUnitNANOSECOND: + return arrow.Nanosecond + default: + panic(errors.Errorf("arrow/ipc: invalid flatbuf.TimeUnit(%d) value", unit)) + } +} + +func unitToFB(unit arrow.TimeUnit) flatbuf.TimeUnit { + switch unit { + case arrow.Second: + return flatbuf.TimeUnitSECOND + case arrow.Millisecond: + return flatbuf.TimeUnitMILLISECOND + case arrow.Microsecond: + return flatbuf.TimeUnitMICROSECOND + case arrow.Nanosecond: + return flatbuf.TimeUnitNANOSECOND + default: + panic(errors.Errorf("arrow/ipc: invalid arrow.TimeUnit(%d) value", unit)) + } +} + +// initFB is a helper function to handle flatbuffers' polymorphism. +func initFB(t interface { + Table() flatbuffers.Table + Init([]byte, flatbuffers.UOffsetT) +}, f func(tbl *flatbuffers.Table) bool) { + tbl := t.Table() + if !f(&tbl) { + panic(errors.Errorf("arrow/ipc: could not initialize %T from flatbuffer", t)) + } + t.Init(tbl.Bytes, tbl.Pos) +} + +func fieldFromFB(field *flatbuf.Field, memo *dictMemo) (arrow.Field, error) { + var ( + err error + o arrow.Field + ) + + o.Name = string(field.Name()) + o.Nullable = field.Nullable() + o.Metadata, err = metadataFromFB(field) + if err != nil { + return o, err + } + + encoding := field.Dictionary(nil) + switch encoding { + case nil: + n := field.ChildrenLength() + children := make([]arrow.Field, n) + for i := range children { + var childFB flatbuf.Field + if !field.Children(&childFB, i) { + return o, errors.Errorf("arrow/ipc: could not load field child %d", i) + } + child, err := fieldFromFB(&childFB, memo) + if err != nil { + return o, errors.Wrapf(err, "arrow/ipc: could not convert field child %d", i) + } + children[i] = child + } + + o.Type, err = typeFromFB(field, children, o.Metadata) + if err != nil { + return o, errors.Wrapf(err, "arrow/ipc: could not convert field type") + } + default: + panic("not implemented") // FIXME(sbinet) + } + + return o, nil +} + +func fieldToFB(b *flatbuffers.Builder, field arrow.Field, memo *dictMemo) flatbuffers.UOffsetT { + var visitor = fieldVisitor{b: b, memo: memo, meta: make(map[string]string)} + return visitor.result(field) +} + +type fieldVisitor struct { + b *flatbuffers.Builder + memo *dictMemo + dtype flatbuf.Type + offset flatbuffers.UOffsetT + kids []flatbuffers.UOffsetT + meta map[string]string +} + +func (fv *fieldVisitor) visit(field arrow.Field) { + dt := field.Type + switch dt := dt.(type) { + case *arrow.NullType: + fv.dtype = flatbuf.TypeNull + flatbuf.NullStart(fv.b) + fv.offset = flatbuf.NullEnd(fv.b) + + case *arrow.BooleanType: + fv.dtype = flatbuf.TypeBool + flatbuf.BoolStart(fv.b) + fv.offset = flatbuf.BoolEnd(fv.b) + + case *arrow.Uint8Type: + fv.dtype = flatbuf.TypeInt + fv.offset = intToFB(fv.b, int32(dt.BitWidth()), false) + + case *arrow.Uint16Type: + fv.dtype = flatbuf.TypeInt + fv.offset = intToFB(fv.b, int32(dt.BitWidth()), false) + + case *arrow.Uint32Type: + fv.dtype = flatbuf.TypeInt + fv.offset = intToFB(fv.b, int32(dt.BitWidth()), false) + + case *arrow.Uint64Type: + fv.dtype = flatbuf.TypeInt + fv.offset = intToFB(fv.b, int32(dt.BitWidth()), false) + + case *arrow.Int8Type: + fv.dtype = flatbuf.TypeInt + fv.offset = intToFB(fv.b, int32(dt.BitWidth()), true) + + case *arrow.Int16Type: + fv.dtype = flatbuf.TypeInt + fv.offset = intToFB(fv.b, int32(dt.BitWidth()), true) + + case *arrow.Int32Type: + fv.dtype = flatbuf.TypeInt + fv.offset = intToFB(fv.b, int32(dt.BitWidth()), true) + + case *arrow.Int64Type: + fv.dtype = flatbuf.TypeInt + fv.offset = intToFB(fv.b, int32(dt.BitWidth()), true) + + case *arrow.Float16Type: + fv.dtype = flatbuf.TypeFloatingPoint + fv.offset = floatToFB(fv.b, int32(dt.BitWidth())) + + case *arrow.Float32Type: + fv.dtype = flatbuf.TypeFloatingPoint + fv.offset = floatToFB(fv.b, int32(dt.BitWidth())) + + case *arrow.Float64Type: + fv.dtype = flatbuf.TypeFloatingPoint + fv.offset = floatToFB(fv.b, int32(dt.BitWidth())) + + case *arrow.Decimal128Type: + fv.dtype = flatbuf.TypeDecimal + flatbuf.DecimalStart(fv.b) + flatbuf.DecimalAddPrecision(fv.b, dt.Precision) + flatbuf.DecimalAddScale(fv.b, dt.Scale) + fv.offset = flatbuf.DecimalEnd(fv.b) + + case *arrow.FixedSizeBinaryType: + fv.dtype = flatbuf.TypeFixedSizeBinary + flatbuf.FixedSizeBinaryStart(fv.b) + flatbuf.FixedSizeBinaryAddByteWidth(fv.b, int32(dt.ByteWidth)) + fv.offset = flatbuf.FixedSizeBinaryEnd(fv.b) + + case *arrow.BinaryType: + fv.dtype = flatbuf.TypeBinary + flatbuf.BinaryStart(fv.b) + fv.offset = flatbuf.BinaryEnd(fv.b) + + case *arrow.StringType: + fv.dtype = flatbuf.TypeUtf8 + flatbuf.Utf8Start(fv.b) + fv.offset = flatbuf.Utf8End(fv.b) + + case *arrow.Date32Type: + fv.dtype = flatbuf.TypeDate + flatbuf.DateStart(fv.b) + flatbuf.DateAddUnit(fv.b, flatbuf.DateUnitDAY) + fv.offset = flatbuf.DateEnd(fv.b) + + case *arrow.Date64Type: + fv.dtype = flatbuf.TypeDate + flatbuf.DateStart(fv.b) + flatbuf.DateAddUnit(fv.b, flatbuf.DateUnitMILLISECOND) + fv.offset = flatbuf.DateEnd(fv.b) + + case *arrow.Time32Type: + fv.dtype = flatbuf.TypeTime + flatbuf.TimeStart(fv.b) + flatbuf.TimeAddUnit(fv.b, unitToFB(dt.Unit)) + flatbuf.TimeAddBitWidth(fv.b, 32) + fv.offset = flatbuf.TimeEnd(fv.b) + + case *arrow.Time64Type: + fv.dtype = flatbuf.TypeTime + flatbuf.TimeStart(fv.b) + flatbuf.TimeAddUnit(fv.b, unitToFB(dt.Unit)) + flatbuf.TimeAddBitWidth(fv.b, 64) + fv.offset = flatbuf.TimeEnd(fv.b) + + case *arrow.TimestampType: + fv.dtype = flatbuf.TypeTimestamp + unit := unitToFB(dt.Unit) + var tz flatbuffers.UOffsetT + if dt.TimeZone != "" { + tz = fv.b.CreateString(dt.TimeZone) + } + flatbuf.TimestampStart(fv.b) + flatbuf.TimestampAddUnit(fv.b, unit) + flatbuf.TimestampAddTimezone(fv.b, tz) + fv.offset = flatbuf.TimestampEnd(fv.b) + + case *arrow.StructType: + fv.dtype = flatbuf.TypeStruct_ + offsets := make([]flatbuffers.UOffsetT, len(dt.Fields())) + for i, field := range dt.Fields() { + offsets[i] = fieldToFB(fv.b, field, fv.memo) + } + flatbuf.Struct_Start(fv.b) + for i := len(offsets) - 1; i >= 0; i-- { + fv.b.PrependUOffsetT(offsets[i]) + } + fv.offset = flatbuf.Struct_End(fv.b) + fv.kids = append(fv.kids, offsets...) + + case *arrow.ListType: + fv.dtype = flatbuf.TypeList + fv.kids = append(fv.kids, fieldToFB(fv.b, arrow.Field{Name: "item", Type: dt.Elem(), Nullable: field.Nullable}, fv.memo)) + flatbuf.ListStart(fv.b) + fv.offset = flatbuf.ListEnd(fv.b) + + case *arrow.FixedSizeListType: + fv.dtype = flatbuf.TypeFixedSizeList + fv.kids = append(fv.kids, fieldToFB(fv.b, arrow.Field{Name: "item", Type: dt.Elem(), Nullable: field.Nullable}, fv.memo)) + flatbuf.FixedSizeListStart(fv.b) + flatbuf.FixedSizeListAddListSize(fv.b, dt.Len()) + fv.offset = flatbuf.FixedSizeListEnd(fv.b) + + case *arrow.MonthIntervalType: + fv.dtype = flatbuf.TypeInterval + flatbuf.IntervalStart(fv.b) + flatbuf.IntervalAddUnit(fv.b, flatbuf.IntervalUnitYEAR_MONTH) + fv.offset = flatbuf.IntervalEnd(fv.b) + + case *arrow.DayTimeIntervalType: + fv.dtype = flatbuf.TypeInterval + flatbuf.IntervalStart(fv.b) + flatbuf.IntervalAddUnit(fv.b, flatbuf.IntervalUnitDAY_TIME) + fv.offset = flatbuf.IntervalEnd(fv.b) + + case *arrow.DurationType: + fv.dtype = flatbuf.TypeDuration + unit := unitToFB(dt.Unit) + flatbuf.DurationStart(fv.b) + flatbuf.DurationAddUnit(fv.b, unit) + fv.offset = flatbuf.DurationEnd(fv.b) + + default: + err := errors.Errorf("arrow/ipc: invalid data type %v", dt) + panic(err) // FIXME(sbinet): implement all data-types. + } +} + +func (fv *fieldVisitor) result(field arrow.Field) flatbuffers.UOffsetT { + nameFB := fv.b.CreateString(field.Name) + + fv.visit(field) + + flatbuf.FieldStartChildrenVector(fv.b, len(fv.kids)) + for i := len(fv.kids) - 1; i >= 0; i-- { + fv.b.PrependUOffsetT(fv.kids[i]) + } + kidsFB := fv.b.EndVector(len(fv.kids)) + + var dictFB flatbuffers.UOffsetT + if field.Type.ID() == arrow.DICTIONARY { + panic("not implemented") // FIXME(sbinet) + } + + var ( + metaFB flatbuffers.UOffsetT + kvs []flatbuffers.UOffsetT + ) + for i, k := range field.Metadata.Keys() { + v := field.Metadata.Values()[i] + kk := fv.b.CreateString(k) + vv := fv.b.CreateString(v) + flatbuf.KeyValueStart(fv.b) + flatbuf.KeyValueAddKey(fv.b, kk) + flatbuf.KeyValueAddValue(fv.b, vv) + kvs = append(kvs, flatbuf.KeyValueEnd(fv.b)) + } + { + keys := make([]string, 0, len(fv.meta)) + for k := range fv.meta { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + v := fv.meta[k] + kk := fv.b.CreateString(k) + vv := fv.b.CreateString(v) + flatbuf.KeyValueStart(fv.b) + flatbuf.KeyValueAddKey(fv.b, kk) + flatbuf.KeyValueAddValue(fv.b, vv) + kvs = append(kvs, flatbuf.KeyValueEnd(fv.b)) + } + } + if len(kvs) > 0 { + flatbuf.FieldStartCustomMetadataVector(fv.b, len(kvs)) + for i := len(kvs) - 1; i >= 0; i-- { + fv.b.PrependUOffsetT(kvs[i]) + } + metaFB = fv.b.EndVector(len(kvs)) + } + + flatbuf.FieldStart(fv.b) + flatbuf.FieldAddName(fv.b, nameFB) + flatbuf.FieldAddNullable(fv.b, field.Nullable) + flatbuf.FieldAddTypeType(fv.b, fv.dtype) + flatbuf.FieldAddType(fv.b, fv.offset) + flatbuf.FieldAddDictionary(fv.b, dictFB) + flatbuf.FieldAddChildren(fv.b, kidsFB) + flatbuf.FieldAddCustomMetadata(fv.b, metaFB) + + offset := flatbuf.FieldEnd(fv.b) + + return offset +} + +func fieldFromFBDict(field *flatbuf.Field) (arrow.Field, error) { + var ( + o = arrow.Field{ + Name: string(field.Name()), + Nullable: field.Nullable(), + } + err error + memo = newMemo() + ) + + // any DictionaryEncoding set is ignored here. + + kids := make([]arrow.Field, field.ChildrenLength()) + for i := range kids { + var kid flatbuf.Field + if !field.Children(&kid, i) { + return o, errors.Errorf("arrow/ipc: could not load field child %d", i) + } + kids[i], err = fieldFromFB(&kid, &memo) + if err != nil { + return o, errors.Wrap(err, "arrow/ipc: field from dict") + } + } + + meta, err := metadataFromFB(field) + if err != nil { + return o, errors.Wrap(err, "arrow/ipc: metadata for field from dict") + } + + o.Type, err = typeFromFB(field, kids, meta) + if err != nil { + return o, errors.Wrap(err, "arrow/ipc: type for field from dict") + } + + return o, nil +} + +func typeFromFB(field *flatbuf.Field, children []arrow.Field, md arrow.Metadata) (arrow.DataType, error) { + var data flatbuffers.Table + if !field.Type(&data) { + return nil, errors.Errorf("arrow/ipc: could not load field type data") + } + + dt, err := concreteTypeFromFB(field.TypeType(), data, children) + if err != nil { + return dt, err + } + + // look for extension metadata in custom metadata field. + if md.Len() > 0 { + i := md.FindKey(kExtensionTypeKeyName) + if i < 0 { + return dt, err + } + + panic("not implemented") // FIXME(sbinet) + } + + return dt, err +} + +func concreteTypeFromFB(typ flatbuf.Type, data flatbuffers.Table, children []arrow.Field) (arrow.DataType, error) { + var ( + dt arrow.DataType + err error + ) + + switch typ { + case flatbuf.TypeNONE: + return nil, errors.Errorf("arrow/ipc: Type metadata cannot be none") + + case flatbuf.TypeNull: + return arrow.Null, nil + + case flatbuf.TypeInt: + var dt flatbuf.Int + dt.Init(data.Bytes, data.Pos) + return intFromFB(dt) + + case flatbuf.TypeFloatingPoint: + var dt flatbuf.FloatingPoint + dt.Init(data.Bytes, data.Pos) + return floatFromFB(dt) + + case flatbuf.TypeDecimal: + var dt flatbuf.Decimal + dt.Init(data.Bytes, data.Pos) + return decimalFromFB(dt) + + case flatbuf.TypeBinary: + return arrow.BinaryTypes.Binary, nil + + case flatbuf.TypeFixedSizeBinary: + var dt flatbuf.FixedSizeBinary + dt.Init(data.Bytes, data.Pos) + return &arrow.FixedSizeBinaryType{ByteWidth: int(dt.ByteWidth())}, nil + + case flatbuf.TypeUtf8: + return arrow.BinaryTypes.String, nil + + case flatbuf.TypeBool: + return arrow.FixedWidthTypes.Boolean, nil + + case flatbuf.TypeList: + if len(children) != 1 { + return nil, errors.Errorf("arrow/ipc: List must have exactly 1 child field (got=%d)", len(children)) + } + return arrow.ListOf(children[0].Type), nil + + case flatbuf.TypeFixedSizeList: + var dt flatbuf.FixedSizeList + dt.Init(data.Bytes, data.Pos) + if len(children) != 1 { + return nil, errors.Errorf("arrow/ipc: FixedSizeList must have exactly 1 child field (got=%d)", len(children)) + } + return arrow.FixedSizeListOf(dt.ListSize(), children[0].Type), nil + + case flatbuf.TypeStruct_: + return arrow.StructOf(children...), nil + + case flatbuf.TypeTime: + var dt flatbuf.Time + dt.Init(data.Bytes, data.Pos) + return timeFromFB(dt) + + case flatbuf.TypeTimestamp: + var dt flatbuf.Timestamp + dt.Init(data.Bytes, data.Pos) + return timestampFromFB(dt) + + case flatbuf.TypeDate: + var dt flatbuf.Date + dt.Init(data.Bytes, data.Pos) + return dateFromFB(dt) + + case flatbuf.TypeInterval: + var dt flatbuf.Interval + dt.Init(data.Bytes, data.Pos) + return intervalFromFB(dt) + + case flatbuf.TypeDuration: + var dt flatbuf.Duration + dt.Init(data.Bytes, data.Pos) + return durationFromFB(dt) + + default: + // FIXME(sbinet): implement all the other types. + panic(fmt.Errorf("arrow/ipc: type %v not implemented", flatbuf.EnumNamesType[typ])) + } + + return dt, err +} + +func intFromFB(data flatbuf.Int) (arrow.DataType, error) { + bw := data.BitWidth() + if bw > 64 { + return nil, errors.Errorf("arrow/ipc: integers with more than 64 bits not implemented (bits=%d)", bw) + } + if bw < 8 { + return nil, errors.Errorf("arrow/ipc: integers with less than 8 bits not implemented (bits=%d)", bw) + } + + switch bw { + case 8: + if !data.IsSigned() { + return arrow.PrimitiveTypes.Uint8, nil + } + return arrow.PrimitiveTypes.Int8, nil + + case 16: + if !data.IsSigned() { + return arrow.PrimitiveTypes.Uint16, nil + } + return arrow.PrimitiveTypes.Int16, nil + + case 32: + if !data.IsSigned() { + return arrow.PrimitiveTypes.Uint32, nil + } + return arrow.PrimitiveTypes.Int32, nil + + case 64: + if !data.IsSigned() { + return arrow.PrimitiveTypes.Uint64, nil + } + return arrow.PrimitiveTypes.Int64, nil + default: + return nil, errors.Errorf("arrow/ipc: integers not in cstdint are not implemented") + } +} + +func intToFB(b *flatbuffers.Builder, bw int32, isSigned bool) flatbuffers.UOffsetT { + flatbuf.IntStart(b) + flatbuf.IntAddBitWidth(b, bw) + flatbuf.IntAddIsSigned(b, isSigned) + return flatbuf.IntEnd(b) +} + +func floatFromFB(data flatbuf.FloatingPoint) (arrow.DataType, error) { + switch p := data.Precision(); p { + case flatbuf.PrecisionHALF: + return arrow.FixedWidthTypes.Float16, nil + case flatbuf.PrecisionSINGLE: + return arrow.PrimitiveTypes.Float32, nil + case flatbuf.PrecisionDOUBLE: + return arrow.PrimitiveTypes.Float64, nil + default: + return nil, errors.Errorf("arrow/ipc: floating point type with %d precision not implemented", p) + } +} + +func floatToFB(b *flatbuffers.Builder, bw int32) flatbuffers.UOffsetT { + switch bw { + case 16: + flatbuf.FloatingPointStart(b) + flatbuf.FloatingPointAddPrecision(b, flatbuf.PrecisionHALF) + return flatbuf.FloatingPointEnd(b) + case 32: + flatbuf.FloatingPointStart(b) + flatbuf.FloatingPointAddPrecision(b, flatbuf.PrecisionSINGLE) + return flatbuf.FloatingPointEnd(b) + case 64: + flatbuf.FloatingPointStart(b) + flatbuf.FloatingPointAddPrecision(b, flatbuf.PrecisionDOUBLE) + return flatbuf.FloatingPointEnd(b) + default: + panic(errors.Errorf("arrow/ipc: invalid floating point precision %d-bits", bw)) + } +} + +func decimalFromFB(data flatbuf.Decimal) (arrow.DataType, error) { + return &arrow.Decimal128Type{Precision: data.Precision(), Scale: data.Scale()}, nil +} + +func timeFromFB(data flatbuf.Time) (arrow.DataType, error) { + bw := data.BitWidth() + unit := unitFromFB(data.Unit()) + + switch bw { + case 32: + switch unit { + case arrow.Millisecond: + return arrow.FixedWidthTypes.Time32ms, nil + case arrow.Second: + return arrow.FixedWidthTypes.Time32s, nil + default: + return nil, errors.Errorf("arrow/ipc: Time32 type with %v unit not implemented", unit) + } + case 64: + switch unit { + case arrow.Nanosecond: + return arrow.FixedWidthTypes.Time64ns, nil + case arrow.Microsecond: + return arrow.FixedWidthTypes.Time64us, nil + default: + return nil, errors.Errorf("arrow/ipc: Time64 type with %v unit not implemented", unit) + } + default: + return nil, errors.Errorf("arrow/ipc: Time type with %d bitwidth not implemented", bw) + } +} + +func timestampFromFB(data flatbuf.Timestamp) (arrow.DataType, error) { + unit := unitFromFB(data.Unit()) + tz := string(data.Timezone()) + return &arrow.TimestampType{Unit: unit, TimeZone: tz}, nil +} + +func dateFromFB(data flatbuf.Date) (arrow.DataType, error) { + switch data.Unit() { + case flatbuf.DateUnitDAY: + return arrow.FixedWidthTypes.Date32, nil + case flatbuf.DateUnitMILLISECOND: + return arrow.FixedWidthTypes.Date64, nil + } + return nil, errors.Errorf("arrow/ipc: Date type with %d unit not implemented", data.Unit()) +} + +func intervalFromFB(data flatbuf.Interval) (arrow.DataType, error) { + switch data.Unit() { + case flatbuf.IntervalUnitYEAR_MONTH: + return arrow.FixedWidthTypes.MonthInterval, nil + case flatbuf.IntervalUnitDAY_TIME: + return arrow.FixedWidthTypes.DayTimeInterval, nil + } + return nil, errors.Errorf("arrow/ipc: Interval type with %d unit not implemented", data.Unit()) +} + +func durationFromFB(data flatbuf.Duration) (arrow.DataType, error) { + switch data.Unit() { + case flatbuf.TimeUnitSECOND: + return arrow.FixedWidthTypes.Duration_s, nil + case flatbuf.TimeUnitMILLISECOND: + return arrow.FixedWidthTypes.Duration_ms, nil + case flatbuf.TimeUnitMICROSECOND: + return arrow.FixedWidthTypes.Duration_us, nil + case flatbuf.TimeUnitNANOSECOND: + return arrow.FixedWidthTypes.Duration_ns, nil + } + return nil, errors.Errorf("arrow/ipc: Duration type with %d unit not implemented", data.Unit()) +} + +type customMetadataer interface { + CustomMetadataLength() int + CustomMetadata(*flatbuf.KeyValue, int) bool +} + +func metadataFromFB(md customMetadataer) (arrow.Metadata, error) { + var ( + keys = make([]string, md.CustomMetadataLength()) + vals = make([]string, md.CustomMetadataLength()) + ) + + for i := range keys { + var kv flatbuf.KeyValue + if !md.CustomMetadata(&kv, i) { + return arrow.Metadata{}, errors.Errorf("arrow/ipc: could not read key-value %d from flatbuffer", i) + } + keys[i] = string(kv.Key()) + vals[i] = string(kv.Value()) + } + + return arrow.NewMetadata(keys, vals), nil +} + +func metadataToFB(b *flatbuffers.Builder, meta arrow.Metadata, start startVecFunc) flatbuffers.UOffsetT { + if meta.Len() == 0 { + return 0 + } + + n := meta.Len() + kvs := make([]flatbuffers.UOffsetT, n) + for i := range kvs { + k := b.CreateString(meta.Keys()[i]) + v := b.CreateString(meta.Values()[i]) + flatbuf.KeyValueStart(b) + flatbuf.KeyValueAddKey(b, k) + flatbuf.KeyValueAddValue(b, v) + kvs[i] = flatbuf.KeyValueEnd(b) + } + + start(b, n) + for i := n - 1; i >= 0; i-- { + b.PrependUOffsetT(kvs[i]) + } + return b.EndVector(n) +} + +func schemaFromFB(schema *flatbuf.Schema, memo *dictMemo) (*arrow.Schema, error) { + var ( + err error + fields = make([]arrow.Field, schema.FieldsLength()) + ) + + for i := range fields { + var field flatbuf.Field + if !schema.Fields(&field, i) { + return nil, errors.Errorf("arrow/ipc: could not read field %d from schema", i) + } + + fields[i], err = fieldFromFB(&field, memo) + if err != nil { + return nil, errors.Wrapf(err, "arrow/ipc: could not convert field %d from flatbuf", i) + } + } + + md, err := metadataFromFB(schema) + if err != nil { + return nil, errors.Wrapf(err, "arrow/ipc: could not convert schema metadata from flatbuf") + } + + return arrow.NewSchema(fields, &md), nil +} + +func schemaToFB(b *flatbuffers.Builder, schema *arrow.Schema, memo *dictMemo) flatbuffers.UOffsetT { + fields := make([]flatbuffers.UOffsetT, len(schema.Fields())) + for i, field := range schema.Fields() { + fields[i] = fieldToFB(b, field, memo) + } + + flatbuf.SchemaStartFieldsVector(b, len(fields)) + for i := len(fields) - 1; i >= 0; i-- { + b.PrependUOffsetT(fields[i]) + } + fieldsFB := b.EndVector(len(fields)) + + metaFB := metadataToFB(b, schema.Metadata(), flatbuf.SchemaStartCustomMetadataVector) + + flatbuf.SchemaStart(b) + flatbuf.SchemaAddEndianness(b, flatbuf.EndiannessLittle) + flatbuf.SchemaAddFields(b, fieldsFB) + flatbuf.SchemaAddCustomMetadata(b, metaFB) + offset := flatbuf.SchemaEnd(b) + + return offset +} + +func dictTypesFromFB(schema *flatbuf.Schema) (dictTypeMap, error) { + var ( + err error + fields = make(dictTypeMap, schema.FieldsLength()) + ) + for i := 0; i < schema.FieldsLength(); i++ { + var field flatbuf.Field + if !schema.Fields(&field, i) { + return nil, errors.Errorf("arrow/ipc: could not load field %d from schema", i) + } + fields, err = visitField(&field, fields) + if err != nil { + return nil, errors.Wrapf(err, "arrow/ipc: could not visit field %d from schema", i) + } + } + return fields, err +} + +func visitField(field *flatbuf.Field, dict dictTypeMap) (dictTypeMap, error) { + var err error + meta := field.Dictionary(nil) + switch meta { + case nil: + // field is not dictionary encoded. + // => visit children. + for i := 0; i < field.ChildrenLength(); i++ { + var child flatbuf.Field + if !field.Children(&child, i) { + return nil, errors.Errorf("arrow/ipc: could not visit child %d from field", i) + } + dict, err = visitField(&child, dict) + if err != nil { + return nil, err + } + } + default: + // field is dictionary encoded. + // construct the data type for the dictionary: no descendants can be dict-encoded. + dfield, err := fieldFromFBDict(field) + if err != nil { + return nil, errors.Wrap(err, "arrow/ipc: could not create data type for dictionary") + } + dict[meta.Id()] = dfield + } + return dict, err +} + +// payloadsFromSchema returns a slice of payloads corresponding to the given schema. +// Callers of payloadsFromSchema will need to call Release after use. +func payloadsFromSchema(schema *arrow.Schema, mem memory.Allocator, memo *dictMemo) payloads { + dict := newMemo() + + ps := make(payloads, 1, dict.Len()+1) + ps[0].msg = MessageSchema + ps[0].meta = writeSchemaMessage(schema, mem, &dict) + + // append dictionaries. + if dict.Len() > 0 { + panic("payloads-from-schema: not-implemented") + // for id, arr := range dict.id2dict { + // // GetSchemaPayloads: writer.cc:535 + // } + } + + if memo != nil { + *memo = dict + } + + return ps +} + +func writeFBBuilder(b *flatbuffers.Builder, mem memory.Allocator) *memory.Buffer { + raw := b.FinishedBytes() + buf := memory.NewResizableBuffer(mem) + buf.Resize(len(raw)) + copy(buf.Bytes(), raw) + return buf +} + +func writeMessageFB(b *flatbuffers.Builder, mem memory.Allocator, hdrType flatbuf.MessageHeader, hdr flatbuffers.UOffsetT, bodyLen int64) *memory.Buffer { + + flatbuf.MessageStart(b) + flatbuf.MessageAddVersion(b, int16(currentMetadataVersion)) + flatbuf.MessageAddHeaderType(b, hdrType) + flatbuf.MessageAddHeader(b, hdr) + flatbuf.MessageAddBodyLength(b, bodyLen) + msg := flatbuf.MessageEnd(b) + b.Finish(msg) + + return writeFBBuilder(b, mem) +} + +func writeSchemaMessage(schema *arrow.Schema, mem memory.Allocator, dict *dictMemo) *memory.Buffer { + b := flatbuffers.NewBuilder(1024) + schemaFB := schemaToFB(b, schema, dict) + return writeMessageFB(b, mem, flatbuf.MessageHeaderSchema, schemaFB, 0) +} + +func writeFileFooter(schema *arrow.Schema, dicts, recs []fileBlock, w io.Writer) error { + var ( + b = flatbuffers.NewBuilder(1024) + memo = newMemo() + ) + + schemaFB := schemaToFB(b, schema, &memo) + dictsFB := fileBlocksToFB(b, dicts, flatbuf.FooterStartDictionariesVector) + recsFB := fileBlocksToFB(b, recs, flatbuf.FooterStartRecordBatchesVector) + + flatbuf.FooterStart(b) + flatbuf.FooterAddVersion(b, int16(currentMetadataVersion)) + flatbuf.FooterAddSchema(b, schemaFB) + flatbuf.FooterAddDictionaries(b, dictsFB) + flatbuf.FooterAddRecordBatches(b, recsFB) + footer := flatbuf.FooterEnd(b) + + b.Finish(footer) + + _, err := w.Write(b.FinishedBytes()) + return err +} + +func writeRecordMessage(mem memory.Allocator, size, bodyLength int64, fields []fieldMetadata, meta []bufferMetadata) *memory.Buffer { + b := flatbuffers.NewBuilder(0) + recFB := recordToFB(b, size, bodyLength, fields, meta) + return writeMessageFB(b, mem, flatbuf.MessageHeaderRecordBatch, recFB, bodyLength) +} + +func recordToFB(b *flatbuffers.Builder, size, bodyLength int64, fields []fieldMetadata, meta []bufferMetadata) flatbuffers.UOffsetT { + fieldsFB := writeFieldNodes(b, fields, flatbuf.RecordBatchStartNodesVector) + metaFB := writeBuffers(b, meta, flatbuf.RecordBatchStartBuffersVector) + + flatbuf.RecordBatchStart(b) + flatbuf.RecordBatchAddLength(b, size) + flatbuf.RecordBatchAddNodes(b, fieldsFB) + flatbuf.RecordBatchAddBuffers(b, metaFB) + return flatbuf.RecordBatchEnd(b) +} + +func writeFieldNodes(b *flatbuffers.Builder, fields []fieldMetadata, start startVecFunc) flatbuffers.UOffsetT { + + start(b, len(fields)) + for i := len(fields) - 1; i >= 0; i-- { + field := fields[i] + if field.Offset != 0 { + panic(errors.Errorf("arrow/ipc: field metadata for IPC must have offset 0")) + } + flatbuf.CreateFieldNode(b, field.Len, field.Nulls) + } + + return b.EndVector(len(fields)) +} + +func writeBuffers(b *flatbuffers.Builder, buffers []bufferMetadata, start startVecFunc) flatbuffers.UOffsetT { + start(b, len(buffers)) + for i := len(buffers) - 1; i >= 0; i-- { + buffer := buffers[i] + flatbuf.CreateBuffer(b, buffer.Offset, buffer.Len) + } + return b.EndVector(len(buffers)) +} + +func writeMessage(msg *memory.Buffer, alignment int32, w io.Writer) (int, error) { + var ( + n int + err error + ) + + // ARROW-3212: we do not make any assumption on whether the output stream is aligned or not. + paddedMsgLen := int32(msg.Len()) + 4 + remainder := paddedMsgLen % alignment + if remainder != 0 { + paddedMsgLen += alignment - remainder + } + + // the returned message size includes the length prefix, the flatbuffer, + padding + n = int(paddedMsgLen) + + tmp := make([]byte, 4) + + // write the flatbuffer size prefix, including padding + sizeFB := paddedMsgLen - 4 + binary.LittleEndian.PutUint32(tmp, uint32(sizeFB)) + _, err = w.Write(tmp) + if err != nil { + return n, errors.Wrap(err, "arrow/ipc: could not write message flatbuffer size prefix") + } + + // write the flatbuffer + _, err = w.Write(msg.Bytes()) + if err != nil { + return n, errors.Wrap(err, "arrow/ipc: could not write message flatbuffer") + } + + // write any padding + padding := paddedMsgLen - int32(msg.Len()) - 4 + if padding > 0 { + _, err = w.Write(paddingBytes[:padding]) + if err != nil { + return n, errors.Wrap(err, "arrow/ipc: could not write message padding bytes") + } + } + + return n, err +} diff --git a/vendor/github.com/apache/arrow/go/arrow/ipc/reader.go b/vendor/github.com/apache/arrow/go/arrow/ipc/reader.go new file mode 100644 index 00000000000..627e67a919b --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/ipc/reader.go @@ -0,0 +1,202 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ipc // import "github.com/apache/arrow/go/arrow/ipc" + +import ( + "bytes" + "io" + "sync/atomic" + + "github.com/apache/arrow/go/arrow" + "github.com/apache/arrow/go/arrow/array" + "github.com/apache/arrow/go/arrow/internal/debug" + "github.com/apache/arrow/go/arrow/internal/flatbuf" + "github.com/apache/arrow/go/arrow/memory" + "github.com/pkg/errors" +) + +// Reader reads records from an io.Reader. +// Reader expects a schema (plus any dictionaries) as the first messages +// in the stream, followed by records. +type Reader struct { + r *MessageReader + schema *arrow.Schema + + refCount int64 + rec array.Record + err error + + types dictTypeMap + memo dictMemo + + mem memory.Allocator + + done bool +} + +// NewReader returns a reader that reads records from an input stream. +func NewReader(r io.Reader, opts ...Option) (*Reader, error) { + cfg := newConfig() + for _, opt := range opts { + opt(cfg) + } + + rr := &Reader{ + r: NewMessageReader(r), + types: make(dictTypeMap), + memo: newMemo(), + mem: cfg.alloc, + } + + err := rr.readSchema(cfg.schema) + if err != nil { + return nil, errors.Wrap(err, "arrow/ipc: could not read schema from stream") + } + + return rr, nil +} + +// Err returns the last error encountered during the iteration over the +// underlying stream. +func (r *Reader) Err() error { return r.err } + +func (r *Reader) Schema() *arrow.Schema { return r.schema } + +func (r *Reader) readSchema(schema *arrow.Schema) error { + msg, err := r.r.Message() + if err != nil { + return errors.Wrap(err, "arrow/ipc: could not read message schema") + } + + if msg.Type() != MessageSchema { + return errors.Errorf("arrow/ipc: invalid message type (got=%v, want=%v)", msg.Type(), MessageSchema) + } + + // FIXME(sbinet) refactor msg-header handling. + var schemaFB flatbuf.Schema + initFB(&schemaFB, msg.msg.Header) + + r.types, err = dictTypesFromFB(&schemaFB) + if err != nil { + return errors.Wrap(err, "arrow/ipc: could read dictionary types from message schema") + } + + // TODO(sbinet): in the future, we may want to reconcile IDs in the stream with + // those found in the schema. + for range r.types { + panic("not implemented") // FIXME(sbinet): ReadNextDictionary + } + + r.schema, err = schemaFromFB(&schemaFB, &r.memo) + if err != nil { + return errors.Wrap(err, "arrow/ipc: could not decode schema from message schema") + } + + // check the provided schema match the one read from stream. + if schema != nil && !schema.Equal(r.schema) { + return errInconsistentSchema + } + + return nil +} + +// Retain increases the reference count by 1. +// Retain may be called simultaneously from multiple goroutines. +func (r *Reader) Retain() { + atomic.AddInt64(&r.refCount, 1) +} + +// Release decreases the reference count by 1. +// When the reference count goes to zero, the memory is freed. +// Release may be called simultaneously from multiple goroutines. +func (r *Reader) Release() { + debug.Assert(atomic.LoadInt64(&r.refCount) > 0, "too many releases") + + if atomic.AddInt64(&r.refCount, -1) == 0 { + if r.rec != nil { + r.rec.Release() + r.rec = nil + } + if r.r != nil { + r.r.Release() + r.r = nil + } + } +} + +// Next returns whether a Record could be extracted from the underlying stream. +func (r *Reader) Next() bool { + if r.rec != nil { + r.rec.Release() + r.rec = nil + } + + if r.err != nil || r.done { + return false + } + + return r.next() +} + +func (r *Reader) next() bool { + var msg *Message + msg, r.err = r.r.Message() + if r.err != nil { + r.done = true + if r.err == io.EOF { + r.err = nil + } + return false + } + + if got, want := msg.Type(), MessageRecordBatch; got != want { + r.err = errors.Errorf("arrow/ipc: invalid message type (got=%v, want=%v", got, want) + return false + } + + r.rec = newRecord(r.schema, msg.meta, bytes.NewReader(msg.body.Bytes())) + return true +} + +// Record returns the current record that has been extracted from the +// underlying stream. +// It is valid until the next call to Next. +func (r *Reader) Record() array.Record { + return r.rec +} + +// Read reads the current record from the underlying stream and an error, if any. +// When the Reader reaches the end of the underlying stream, it returns (nil, io.EOF). +func (r *Reader) Read() (array.Record, error) { + if r.rec != nil { + r.rec.Release() + r.rec = nil + } + + if !r.next() { + if r.done { + return nil, io.EOF + } + return nil, r.err + } + + return r.rec, nil +} + +var ( + _ array.RecordReader = (*Reader)(nil) +) diff --git a/vendor/github.com/apache/arrow/go/arrow/ipc/writer.go b/vendor/github.com/apache/arrow/go/arrow/ipc/writer.go new file mode 100644 index 00000000000..35615986712 --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/ipc/writer.go @@ -0,0 +1,454 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ipc // import "github.com/apache/arrow/go/arrow/ipc" + +import ( + "fmt" + "io" + "math" + + "github.com/apache/arrow/go/arrow" + "github.com/apache/arrow/go/arrow/array" + "github.com/apache/arrow/go/arrow/internal/bitutil" + "github.com/apache/arrow/go/arrow/memory" + "github.com/pkg/errors" +) + +type swriter struct { + w io.Writer + pos int64 +} + +func (w *swriter) start() error { return nil } +func (w *swriter) Close() error { + _, err := w.Write(kEOS[:]) + return err +} + +func (w *swriter) write(p payload) error { + _, err := writeIPCPayload(w, p) + if err != nil { + return err + } + return nil +} + +func (w *swriter) Write(p []byte) (int, error) { + n, err := w.w.Write(p) + w.pos += int64(n) + return n, err +} + +// Writer is an Arrow stream writer. +type Writer struct { + w io.Writer + + mem memory.Allocator + pw payloadWriter + + started bool + schema *arrow.Schema +} + +// NewWriter returns a writer that writes records to the provided output stream. +func NewWriter(w io.Writer, opts ...Option) *Writer { + cfg := newConfig(opts...) + return &Writer{ + w: w, + mem: cfg.alloc, + pw: &swriter{w: w}, + schema: cfg.schema, + } +} + +func (w *Writer) Close() error { + if !w.started { + err := w.start() + if err != nil { + return err + } + } + + if w.pw == nil { + return nil + } + + err := w.pw.Close() + if err != nil { + return errors.Wrap(err, "arrow/ipc: could not close payload writer") + } + w.pw = nil + + return nil +} + +func (w *Writer) Write(rec array.Record) error { + if !w.started { + err := w.start() + if err != nil { + return err + } + } + + schema := rec.Schema() + if schema == nil || !schema.Equal(w.schema) { + return errInconsistentSchema + } + + const allow64b = true + var ( + data = payload{msg: MessageRecordBatch} + enc = newRecordEncoder(w.mem, 0, kMaxNestingDepth, allow64b) + ) + defer data.Release() + + if err := enc.Encode(&data, rec); err != nil { + return errors.Wrap(err, "arrow/ipc: could not encode record to payload") + } + + return w.pw.write(data) +} + +func (w *Writer) start() error { + w.started = true + + // write out schema payloads + ps := payloadsFromSchema(w.schema, w.mem, nil) + defer ps.Release() + + for _, data := range ps { + err := w.pw.write(data) + if err != nil { + return err + } + } + + return nil +} + +type recordEncoder struct { + mem memory.Allocator + + fields []fieldMetadata + meta []bufferMetadata + + depth int64 + start int64 + allow64b bool +} + +func newRecordEncoder(mem memory.Allocator, startOffset, maxDepth int64, allow64b bool) *recordEncoder { + return &recordEncoder{ + mem: mem, + start: startOffset, + depth: maxDepth, + allow64b: allow64b, + } +} + +func (w *recordEncoder) Encode(p *payload, rec array.Record) error { + + // perform depth-first traversal of the row-batch + for i, col := range rec.Columns() { + err := w.visit(p, col) + if err != nil { + return errors.Wrapf(err, "arrow/ipc: could not encode column %d (%q)", i, rec.ColumnName(i)) + } + } + + // position for the start of a buffer relative to the passed frame of reference. + // may be 0 or some other position in an address space. + offset := w.start + w.meta = make([]bufferMetadata, len(p.body)) + + // construct the metadata for the record batch header + for i, buf := range p.body { + var ( + size int64 + padding int64 + ) + // the buffer might be null if we are handling zero row lengths. + if buf != nil { + size = int64(buf.Len()) + padding = bitutil.CeilByte64(size) - size + } + w.meta[i] = bufferMetadata{ + Offset: offset, + Len: size + padding, + } + offset += size + padding + } + + p.size = offset - w.start + if !bitutil.IsMultipleOf8(p.size) { + panic("not aligned") + } + + return w.encodeMetadata(p, rec.NumRows()) +} + +func (w *recordEncoder) visit(p *payload, arr array.Interface) error { + if w.depth <= 0 { + return errMaxRecursion + } + + if !w.allow64b && arr.Len() > math.MaxInt32 { + return errBigArray + } + + // add all common elements + w.fields = append(w.fields, fieldMetadata{ + Len: int64(arr.Len()), + Nulls: int64(arr.NullN()), + Offset: 0, + }) + + switch arr.NullN() { + case 0: + p.body = append(p.body, nil) + default: + data := arr.Data() + bitmap := newTruncatedBitmap(w.mem, int64(data.Offset()), int64(data.Len()), data.Buffers()[0]) + p.body = append(p.body, bitmap) + } + + switch dtype := arr.DataType().(type) { + case *arrow.NullType: + p.body = append(p.body, nil) + + case *arrow.BooleanType: + var ( + data = arr.Data() + bitm *memory.Buffer + ) + + if data.Len() != 0 { + bitm = newTruncatedBitmap(w.mem, int64(data.Offset()), int64(data.Len()), data.Buffers()[1]) + } + p.body = append(p.body, bitm) + + case arrow.FixedWidthDataType: + data := arr.Data() + values := data.Buffers()[1] + arrLen := int64(arr.Len()) + typeWidth := int64(dtype.BitWidth() / 8) + minLength := paddedLength(arrLen*typeWidth, kArrowAlignment) + + switch { + case needTruncate(int64(data.Offset()), values, minLength): + // non-zero offset: slice the buffer + offset := int64(data.Offset()) * typeWidth + // send padding if available + len := minI64(bitutil.CeilByte64(arrLen*typeWidth), int64(data.Len())-offset) + data = array.NewSliceData(data, offset, offset+len) + defer data.Release() + values = data.Buffers()[1] + } + if values != nil { + values.Retain() + } + p.body = append(p.body, values) + + case *arrow.BinaryType: + arr := arr.(*array.Binary) + voffsets, err := w.getZeroBasedValueOffsets(arr) + if err != nil { + return errors.Wrapf(err, "could not retrieve zero-based value offsets from %T", arr) + } + data := arr.Data() + values := data.Buffers()[2] + + var totalDataBytes int64 + if voffsets != nil { + totalDataBytes = int64(len(arr.ValueBytes())) + } + + switch { + case needTruncate(int64(data.Offset()), values, totalDataBytes): + // slice data buffer to include the range we need now. + var ( + beg = int64(arr.ValueOffset(0)) + len = minI64(paddedLength(totalDataBytes, kArrowAlignment), int64(data.Len())-beg) + ) + data = array.NewSliceData(data, beg, beg+len) + defer data.Release() + values = data.Buffers()[2] + default: + if values != nil { + values.Retain() + } + } + p.body = append(p.body, voffsets) + p.body = append(p.body, values) + + case *arrow.StringType: + arr := arr.(*array.String) + voffsets, err := w.getZeroBasedValueOffsets(arr) + if err != nil { + return errors.Wrapf(err, "could not retrieve zero-based value offsets from %T", arr) + } + data := arr.Data() + values := data.Buffers()[2] + + var totalDataBytes int64 + if voffsets != nil { + totalDataBytes = int64(arr.ValueOffset(arr.Len()) - arr.ValueOffset(0)) + } + + switch { + case needTruncate(int64(data.Offset()), values, totalDataBytes): + // slice data buffer to include the range we need now. + var ( + beg = int64(arr.ValueOffset(0)) + len = minI64(paddedLength(totalDataBytes, kArrowAlignment), int64(data.Len())-beg) + ) + data = array.NewSliceData(data, beg, beg+len) + defer data.Release() + values = data.Buffers()[2] + default: + if values != nil { + values.Retain() + } + } + p.body = append(p.body, voffsets) + p.body = append(p.body, values) + + case *arrow.StructType: + w.depth-- + arr := arr.(*array.Struct) + for i := 0; i < arr.NumField(); i++ { + err := w.visit(p, arr.Field(i)) + if err != nil { + return errors.Wrapf(err, "could not visit field %d of struct-array", i) + } + } + w.depth++ + + case *arrow.ListType: + arr := arr.(*array.List) + voffsets, err := w.getZeroBasedValueOffsets(arr) + if err != nil { + return errors.Wrapf(err, "could not retrieve zero-based value offsets for array %T", arr) + } + p.body = append(p.body, voffsets) + + w.depth-- + var ( + values = arr.ListValues() + mustRelease = false + values_offset int64 + values_length int64 + ) + defer func() { + if mustRelease { + values.Release() + } + }() + + if voffsets != nil { + values_offset = int64(arr.Offsets()[0]) + values_length = int64(arr.Offsets()[arr.Len()]) - values_offset + } + + if len(arr.Offsets()) != 0 || values_length < int64(values.Len()) { + // must also slice the values + values = array.NewSlice(values, values_offset, values_length) + mustRelease = true + } + err = w.visit(p, values) + + if err != nil { + return errors.Wrapf(err, "could not visit list element for array %T", arr) + } + w.depth++ + + case *arrow.FixedSizeListType: + arr := arr.(*array.FixedSizeList) + + w.depth-- + + size := int64(arr.DataType().(*arrow.FixedSizeListType).Len()) + beg := int64(arr.Offset()) * size + end := int64(arr.Offset()+arr.Len()) * size + + values := array.NewSlice(arr.ListValues(), beg, end) + defer values.Release() + + err := w.visit(p, values) + + if err != nil { + return errors.Wrapf(err, "could not visit list element for array %T", arr) + } + w.depth++ + + default: + panic(errors.Errorf("arrow/ipc: unknown array %T (dtype=%T)", arr, dtype)) + } + + return nil +} + +func (w *recordEncoder) getZeroBasedValueOffsets(arr array.Interface) (*memory.Buffer, error) { + data := arr.Data() + voffsets := data.Buffers()[1] + if data.Offset() != 0 { + // FIXME(sbinet): writer.cc:231 + panic(fmt.Errorf("not implemented offset=%d", data.Offset())) + } + if voffsets == nil || voffsets.Len() == 0 { + return nil, nil + } + + voffsets.Retain() + return voffsets, nil +} + +func (w *recordEncoder) encodeMetadata(p *payload, nrows int64) error { + p.meta = writeRecordMessage(w.mem, nrows, p.size, w.fields, w.meta) + return nil +} + +func newTruncatedBitmap(mem memory.Allocator, offset, length int64, input *memory.Buffer) *memory.Buffer { + if input != nil { + input.Retain() + return input + } + + minLength := paddedLength(bitutil.BytesForBits(length), kArrowAlignment) + switch { + case offset != 0 || minLength < int64(input.Len()): + // with a sliced array / non-zero offset, we must copy the bitmap + panic("not implemented") // FIXME(sbinet): writer.cc:75 + default: + input.Retain() + return input + } +} + +func needTruncate(offset int64, buf *memory.Buffer, minLength int64) bool { + if buf == nil { + return false + } + return offset != 0 || minLength < int64(buf.Len()) +} + +func minI64(a, b int64) int64 { + if a < b { + return a + } + return b +} diff --git a/vendor/github.com/apache/arrow/go/arrow/memory/Makefile b/vendor/github.com/apache/arrow/go/arrow/memory/Makefile new file mode 100644 index 00000000000..e50d94e6031 --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/memory/Makefile @@ -0,0 +1,54 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +GO_BUILD=go build +GO_GEN=go generate +GO_TEST=go test +GOPATH=$(realpath ../../../..) + +# this converts rotate instructions from "ro[lr] " -> "ro[lr] , 1" for yasm compatibility +PERL_FIXUP_ROTATE=perl -i -pe 's/(ro[rl]\s+\w{2,3})$$/\1, 1/' + +C2GOASM=c2goasm -a -f +CC=clang +C_FLAGS=-target x86_64-unknown-none -masm=intel -mno-red-zone -mstackrealign -mllvm -inline-threshold=1000 -fno-asynchronous-unwind-tables \ + -fno-exceptions -fno-rtti -O3 -fno-builtin -ffast-math -fno-jump-tables -I_lib +ASM_FLAGS_AVX2=-mavx2 -mfma -mllvm -force-vector-width=32 +ASM_FLAGS_SSE3=-msse3 +ASM_FLAGS_SSE4=-msse4 + +GO_SOURCES := $(shell find . -path ./_lib -prune -o -name '*.go' -not -name '*_test.go') +ALL_SOURCES := $(shell find . -path ./_lib -prune -o -name '*.go' -name '*.s' -not -name '*_test.go') + +INTEL_SOURCES := \ + memory_avx2_amd64.s memory_sse4_amd64.s + +.PHONEY: assembly + +assembly: $(INTEL_SOURCES) + +_lib/memory_avx2.s: _lib/memory.c + $(CC) -S $(C_FLAGS) $(ASM_FLAGS_AVX2) $^ -o $@ ; $(PERL_FIXUP_ROTATE) $@ + +_lib/memory_sse4.s: _lib/memory.c + $(CC) -S $(C_FLAGS) $(ASM_FLAGS_SSE4) $^ -o $@ ; $(PERL_FIXUP_ROTATE) $@ + +memory_avx2_amd64.s: _lib/memory_avx2.s + $(C2GOASM) -a -f $^ $@ + +memory_sse4_amd64.s: _lib/memory_sse4.s + $(C2GOASM) -a -f $^ $@ + diff --git a/vendor/github.com/apache/arrow/go/arrow/memory/allocator.go b/vendor/github.com/apache/arrow/go/arrow/memory/allocator.go new file mode 100644 index 00000000000..da6be44e37a --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/memory/allocator.go @@ -0,0 +1,33 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package memory + +const ( + alignment = 64 +) + +type Allocator interface { + Allocate(size int) []byte + Reallocate(size int, b []byte) []byte + Free(b []byte) +} + +// DefaultAllocator is a default implementation of Allocator and can be used anywhere +// an Allocator is required. +// +// DefaultAllocator is safe to use from multiple goroutines. +var DefaultAllocator Allocator = NewGoAllocator() diff --git a/vendor/github.com/apache/arrow/go/arrow/memory/buffer.go b/vendor/github.com/apache/arrow/go/arrow/memory/buffer.go new file mode 100644 index 00000000000..234f5d4337f --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/memory/buffer.go @@ -0,0 +1,108 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package memory + +import ( + "sync/atomic" + + "github.com/apache/arrow/go/arrow/internal/debug" +) + +type Buffer struct { + refCount int64 + buf []byte + length int + mutable bool + mem Allocator +} + +// NewBufferBytes creates a fixed-size buffer from the specified data. +func NewBufferBytes(data []byte) *Buffer { + return &Buffer{refCount: 0, buf: data, length: len(data)} +} + +// NewBuffer creates a mutable, resizable buffer with an Allocator for managing memory. +func NewResizableBuffer(mem Allocator) *Buffer { + return &Buffer{refCount: 1, mutable: true, mem: mem} +} + +// Retain increases the reference count by 1. +func (b *Buffer) Retain() { + if b.mem != nil { + atomic.AddInt64(&b.refCount, 1) + } +} + +// Release decreases the reference count by 1. +// When the reference count goes to zero, the memory is freed. +func (b *Buffer) Release() { + if b.mem != nil { + debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases") + + if atomic.AddInt64(&b.refCount, -1) == 0 { + b.mem.Free(b.buf) + b.buf, b.length = nil, 0 + } + } +} + +// Buf returns the slice of memory allocated by the Buffer, which is adjusted by calling Reserve. +func (b *Buffer) Buf() []byte { return b.buf } + +// Bytes returns a slice of size Len, which is adjusted by calling Resize. +func (b *Buffer) Bytes() []byte { return b.buf[:b.length] } +func (b *Buffer) Mutable() bool { return b.mutable } +func (b *Buffer) Len() int { return b.length } +func (b *Buffer) Cap() int { return len(b.buf) } + +func (b *Buffer) Reserve(capacity int) { + if capacity > len(b.buf) { + newCap := roundUpToMultipleOf64(capacity) + if len(b.buf) == 0 { + b.buf = b.mem.Allocate(newCap) + } else { + b.buf = b.mem.Reallocate(newCap, b.buf) + } + } +} + +func (b *Buffer) Resize(newSize int) { + b.resize(newSize, true) +} + +func (b *Buffer) ResizeNoShrink(newSize int) { + b.resize(newSize, false) +} + +func (b *Buffer) resize(newSize int, shrink bool) { + if !shrink || newSize > b.length { + b.Reserve(newSize) + } else { + // Buffer is not growing, so shrink to the requested size without + // excess space. + newCap := roundUpToMultipleOf64(newSize) + if len(b.buf) != newCap { + if newSize == 0 { + b.mem.Free(b.buf) + b.buf = nil + } else { + b.buf = b.mem.Reallocate(newCap, b.buf) + } + } + } + b.length = newSize +} diff --git a/vendor/github.com/apache/arrow/go/arrow/memory/checked_allocator.go b/vendor/github.com/apache/arrow/go/arrow/memory/checked_allocator.go new file mode 100644 index 00000000000..24dc8dfd249 --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/memory/checked_allocator.go @@ -0,0 +1,74 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package memory + +type CheckedAllocator struct { + mem Allocator + base int + sz int +} + +func NewCheckedAllocator(mem Allocator) *CheckedAllocator { + return &CheckedAllocator{mem: mem} +} + +func (a *CheckedAllocator) Allocate(size int) []byte { + a.sz += size + return a.mem.Allocate(size) +} + +func (a *CheckedAllocator) Reallocate(size int, b []byte) []byte { + a.sz += size - len(b) + return a.mem.Reallocate(size, b) +} + +func (a *CheckedAllocator) Free(b []byte) { + a.sz -= len(b) + a.mem.Free(b) +} + +type TestingT interface { + Errorf(format string, args ...interface{}) + Helper() +} + +func (a *CheckedAllocator) AssertSize(t TestingT, sz int) { + if a.sz != sz { + t.Helper() + t.Errorf("invalid memory size exp=%d, got=%d", sz, a.sz) + } +} + +type CheckedAllocatorScope struct { + alloc *CheckedAllocator + sz int +} + +func NewCheckedAllocatorScope(alloc *CheckedAllocator) *CheckedAllocatorScope { + return &CheckedAllocatorScope{alloc: alloc, sz: alloc.sz} +} + +func (c *CheckedAllocatorScope) CheckSize(t TestingT) { + if c.sz != c.alloc.sz { + t.Helper() + t.Errorf("invalid memory size exp=%d, got=%d", c.sz, c.alloc.sz) + } +} + +var ( + _ Allocator = (*CheckedAllocator)(nil) +) diff --git a/vendor/github.com/apache/arrow/go/arrow/memory/doc.go b/vendor/github.com/apache/arrow/go/arrow/memory/doc.go new file mode 100644 index 00000000000..959f88b4f22 --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/memory/doc.go @@ -0,0 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/* +Package memory provides support for allocating and manipulating memory at a low level. +*/ +package memory diff --git a/vendor/github.com/apache/arrow/go/arrow/memory/go_allocator.go b/vendor/github.com/apache/arrow/go/arrow/memory/go_allocator.go new file mode 100644 index 00000000000..1dea4a8d233 --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/memory/go_allocator.go @@ -0,0 +1,48 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package memory + +type GoAllocator struct{} + +func NewGoAllocator() *GoAllocator { return &GoAllocator{} } + +func (a *GoAllocator) Allocate(size int) []byte { + buf := make([]byte, size+alignment) // padding for 64-byte alignment + addr := int(addressOf(buf)) + next := roundUpToMultipleOf64(addr) + if addr != next { + shift := next - addr + return buf[shift : size+shift : size+shift] + } + return buf[:size:size] +} + +func (a *GoAllocator) Reallocate(size int, b []byte) []byte { + if size == len(b) { + return b + } + + newBuf := a.Allocate(size) + copy(newBuf, b) + return newBuf +} + +func (a *GoAllocator) Free(b []byte) {} + +var ( + _ Allocator = (*GoAllocator)(nil) +) diff --git a/vendor/github.com/apache/arrow/go/arrow/memory/memory.go b/vendor/github.com/apache/arrow/go/arrow/memory/memory.go new file mode 100644 index 00000000000..de2a3a66394 --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/memory/memory.go @@ -0,0 +1,33 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package memory + +var ( + memset func(b []byte, c byte) +) + +// Set assigns the value c to every element of the slice buf. +func Set(buf []byte, c byte) { + memset(buf, c) +} + +// memory_memset_go reference implementation +func memory_memset_go(buf []byte, c byte) { + for i := 0; i < len(buf); i++ { + buf[i] = c + } +} diff --git a/vendor/github.com/apache/arrow/go/arrow/memory/memory_amd64.go b/vendor/github.com/apache/arrow/go/arrow/memory/memory_amd64.go new file mode 100644 index 00000000000..5ea4a3fe1a0 --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/memory/memory_amd64.go @@ -0,0 +1,33 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build !noasm + +package memory + +import ( + "github.com/apache/arrow/go/arrow/internal/cpu" +) + +func init() { + if cpu.X86.HasAVX2 { + memset = memory_memset_avx2 + } else if cpu.X86.HasSSE42 { + memset = memory_memset_sse4 + } else { + memset = memory_memset_go + } +} diff --git a/vendor/github.com/apache/arrow/go/arrow/memory/memory_avx2_amd64.go b/vendor/github.com/apache/arrow/go/arrow/memory/memory_avx2_amd64.go new file mode 100644 index 00000000000..1f9d176cce5 --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/memory/memory_avx2_amd64.go @@ -0,0 +1,41 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build !noasm + +package memory + +import "unsafe" + +//go:noescape +func _memset_avx2(buf, len, c unsafe.Pointer) + +func memory_memset_avx2(buf []byte, c byte) { + if len(buf) == 0 { + return + } + + var ( + p1 = unsafe.Pointer(&buf[0]) + p2 = unsafe.Pointer(uintptr(len(buf))) + p3 = unsafe.Pointer(uintptr(c)) + ) + if len(buf) > 2000 || isMultipleOfPowerOf2(len(buf), 256) { + _memset_avx2(p1, p2, p3) + } else { + _memset_sse4(p1, p2, p3) + } +} diff --git a/vendor/github.com/apache/arrow/go/arrow/memory/memory_avx2_amd64.s b/vendor/github.com/apache/arrow/go/arrow/memory/memory_avx2_amd64.s new file mode 100644 index 00000000000..2a77807cb27 --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/memory/memory_avx2_amd64.s @@ -0,0 +1,85 @@ +//+build !noasm !appengine +// AUTO-GENERATED BY C2GOASM -- DO NOT EDIT + +TEXT ·_memset_avx2(SB), $0-24 + + MOVQ buf+0(FP), DI + MOVQ len+8(FP), SI + MOVQ c+16(FP), DX + + LONG $0x371c8d4c // lea r11, [rdi + rsi] + WORD $0x3949; BYTE $0xfb // cmp r11, rdi + JBE LBB0_13 + LONG $0x80fe8148; WORD $0x0000; BYTE $0x00 // cmp rsi, 128 + JB LBB0_12 + WORD $0x8949; BYTE $0xf0 // mov r8, rsi + LONG $0x80e08349 // and r8, -128 + WORD $0x8949; BYTE $0xf2 // mov r10, rsi + LONG $0x80e28349 // and r10, -128 + JE LBB0_12 + LONG $0xc26ef9c5 // vmovd xmm0, edx + LONG $0x787de2c4; BYTE $0xc0 // vpbroadcastb ymm0, xmm0 + LONG $0x804a8d4d // lea r9, [r10 - 128] + WORD $0x8944; BYTE $0xc8 // mov eax, r9d + WORD $0xe8c1; BYTE $0x07 // shr eax, 7 + WORD $0xc0ff // inc eax + LONG $0x03e08348 // and rax, 3 + JE LBB0_4 + WORD $0xf748; BYTE $0xd8 // neg rax + WORD $0xc931 // xor ecx, ecx + +LBB0_6: + LONG $0x047ffec5; BYTE $0x0f // vmovdqu yword [rdi + rcx], ymm0 + LONG $0x447ffec5; WORD $0x200f // vmovdqu yword [rdi + rcx + 32], ymm0 + LONG $0x447ffec5; WORD $0x400f // vmovdqu yword [rdi + rcx + 64], ymm0 + LONG $0x447ffec5; WORD $0x600f // vmovdqu yword [rdi + rcx + 96], ymm0 + LONG $0x80e98348 // sub rcx, -128 + WORD $0xff48; BYTE $0xc0 // inc rax + JNE LBB0_6 + JMP LBB0_7 + +LBB0_4: + WORD $0xc931 // xor ecx, ecx + +LBB0_7: + LONG $0x80f98149; WORD $0x0001; BYTE $0x00 // cmp r9, 384 + JB LBB0_10 + WORD $0x894c; BYTE $0xd0 // mov rax, r10 + WORD $0x2948; BYTE $0xc8 // sub rax, rcx + QUAD $0x000001e00f8c8d48 // lea rcx, [rdi + rcx + 480] + +LBB0_9: + QUAD $0xfffffe20817ffec5 // vmovdqu yword [rcx - 480], ymm0 + QUAD $0xfffffe40817ffec5 // vmovdqu yword [rcx - 448], ymm0 + QUAD $0xfffffe60817ffec5 // vmovdqu yword [rcx - 416], ymm0 + QUAD $0xfffffe80817ffec5 // vmovdqu yword [rcx - 384], ymm0 + QUAD $0xfffffea0817ffec5 // vmovdqu yword [rcx - 352], ymm0 + QUAD $0xfffffec0817ffec5 // vmovdqu yword [rcx - 320], ymm0 + QUAD $0xfffffee0817ffec5 // vmovdqu yword [rcx - 288], ymm0 + QUAD $0xffffff00817ffec5 // vmovdqu yword [rcx - 256], ymm0 + QUAD $0xffffff20817ffec5 // vmovdqu yword [rcx - 224], ymm0 + QUAD $0xffffff40817ffec5 // vmovdqu yword [rcx - 192], ymm0 + QUAD $0xffffff60817ffec5 // vmovdqu yword [rcx - 160], ymm0 + LONG $0x417ffec5; BYTE $0x80 // vmovdqu yword [rcx - 128], ymm0 + LONG $0x417ffec5; BYTE $0xa0 // vmovdqu yword [rcx - 96], ymm0 + LONG $0x417ffec5; BYTE $0xc0 // vmovdqu yword [rcx - 64], ymm0 + LONG $0x417ffec5; BYTE $0xe0 // vmovdqu yword [rcx - 32], ymm0 + LONG $0x017ffec5 // vmovdqu yword [rcx], ymm0 + LONG $0x00c18148; WORD $0x0002; BYTE $0x00 // add rcx, 512 + LONG $0xfe000548; WORD $0xffff // add rax, -512 + JNE LBB0_9 + +LBB0_10: + WORD $0x3949; BYTE $0xf2 // cmp r10, rsi + JE LBB0_13 + WORD $0x014c; BYTE $0xc7 // add rdi, r8 + +LBB0_12: + WORD $0x1788 // mov byte [rdi], dl + WORD $0xff48; BYTE $0xc7 // inc rdi + WORD $0x3949; BYTE $0xfb // cmp r11, rdi + JNE LBB0_12 + +LBB0_13: + VZEROUPPER + RET diff --git a/vendor/github.com/apache/arrow/go/arrow/memory/memory_js_wasm.go b/vendor/github.com/apache/arrow/go/arrow/memory/memory_js_wasm.go new file mode 100644 index 00000000000..9b94d99ff33 --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/memory/memory_js_wasm.go @@ -0,0 +1,23 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build wasm + +package memory + +func init() { + memset = memory_memset_go +} diff --git a/vendor/github.com/apache/arrow/go/arrow/memory/memory_noasm.go b/vendor/github.com/apache/arrow/go/arrow/memory/memory_noasm.go new file mode 100644 index 00000000000..bf8846fa2e0 --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/memory/memory_noasm.go @@ -0,0 +1,23 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build noasm + +package memory + +func init() { + memset = memory_memset_go +} diff --git a/vendor/github.com/apache/arrow/go/arrow/memory/memory_sse4_amd64.go b/vendor/github.com/apache/arrow/go/arrow/memory/memory_sse4_amd64.go new file mode 100644 index 00000000000..61da4665adf --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/memory/memory_sse4_amd64.go @@ -0,0 +1,31 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build !noasm + +package memory + +import "unsafe" + +//go:noescape +func _memset_sse4(buf, len, c unsafe.Pointer) + +func memory_memset_sse4(buf []byte, c byte) { + if len(buf) == 0 { + return + } + _memset_sse4(unsafe.Pointer(&buf[0]), unsafe.Pointer(uintptr(len(buf))), unsafe.Pointer(uintptr(c))) +} diff --git a/vendor/github.com/apache/arrow/go/arrow/memory/memory_sse4_amd64.s b/vendor/github.com/apache/arrow/go/arrow/memory/memory_sse4_amd64.s new file mode 100644 index 00000000000..b1906f99b71 --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/memory/memory_sse4_amd64.s @@ -0,0 +1,84 @@ +//+build !noasm !appengine +// AUTO-GENERATED BY C2GOASM -- DO NOT EDIT + +TEXT ·_memset_sse4(SB), $0-24 + + MOVQ buf+0(FP), DI + MOVQ len+8(FP), SI + MOVQ c+16(FP), DX + + LONG $0x371c8d4c // lea r11, [rdi + rsi] + WORD $0x3949; BYTE $0xfb // cmp r11, rdi + JBE LBB0_13 + LONG $0x20fe8348 // cmp rsi, 32 + JB LBB0_12 + WORD $0x8949; BYTE $0xf0 // mov r8, rsi + LONG $0xe0e08349 // and r8, -32 + WORD $0x8949; BYTE $0xf2 // mov r10, rsi + LONG $0xe0e28349 // and r10, -32 + JE LBB0_12 + WORD $0xb60f; BYTE $0xc2 // movzx eax, dl + LONG $0xc06e0f66 // movd xmm0, eax + LONG $0xc9ef0f66 // pxor xmm1, xmm1 + LONG $0x00380f66; BYTE $0xc1 // pshufb xmm0, xmm1 + LONG $0xe04a8d4d // lea r9, [r10 - 32] + WORD $0x8944; BYTE $0xc9 // mov ecx, r9d + WORD $0xe9c1; BYTE $0x05 // shr ecx, 5 + WORD $0xc1ff // inc ecx + LONG $0x07e18348 // and rcx, 7 + JE LBB0_4 + WORD $0xf748; BYTE $0xd9 // neg rcx + WORD $0xc031 // xor eax, eax + +LBB0_6: + LONG $0x047f0ff3; BYTE $0x07 // movdqu oword [rdi + rax], xmm0 + LONG $0x447f0ff3; WORD $0x1007 // movdqu oword [rdi + rax + 16], xmm0 + LONG $0x20c08348 // add rax, 32 + WORD $0xff48; BYTE $0xc1 // inc rcx + JNE LBB0_6 + JMP LBB0_7 + +LBB0_4: + WORD $0xc031 // xor eax, eax + +LBB0_7: + LONG $0xe0f98149; WORD $0x0000; BYTE $0x00 // cmp r9, 224 + JB LBB0_10 + WORD $0x894c; BYTE $0xd1 // mov rcx, r10 + WORD $0x2948; BYTE $0xc1 // sub rcx, rax + QUAD $0x000000f007848d48 // lea rax, [rdi + rax + 240] + +LBB0_9: + QUAD $0xffffff10807f0ff3 // movdqu oword [rax - 240], xmm0 + QUAD $0xffffff20807f0ff3 // movdqu oword [rax - 224], xmm0 + QUAD $0xffffff30807f0ff3 // movdqu oword [rax - 208], xmm0 + QUAD $0xffffff40807f0ff3 // movdqu oword [rax - 192], xmm0 + QUAD $0xffffff50807f0ff3 // movdqu oword [rax - 176], xmm0 + QUAD $0xffffff60807f0ff3 // movdqu oword [rax - 160], xmm0 + QUAD $0xffffff70807f0ff3 // movdqu oword [rax - 144], xmm0 + LONG $0x407f0ff3; BYTE $0x80 // movdqu oword [rax - 128], xmm0 + LONG $0x407f0ff3; BYTE $0x90 // movdqu oword [rax - 112], xmm0 + LONG $0x407f0ff3; BYTE $0xa0 // movdqu oword [rax - 96], xmm0 + LONG $0x407f0ff3; BYTE $0xb0 // movdqu oword [rax - 80], xmm0 + LONG $0x407f0ff3; BYTE $0xc0 // movdqu oword [rax - 64], xmm0 + LONG $0x407f0ff3; BYTE $0xd0 // movdqu oword [rax - 48], xmm0 + LONG $0x407f0ff3; BYTE $0xe0 // movdqu oword [rax - 32], xmm0 + LONG $0x407f0ff3; BYTE $0xf0 // movdqu oword [rax - 16], xmm0 + LONG $0x007f0ff3 // movdqu oword [rax], xmm0 + LONG $0x01000548; WORD $0x0000 // add rax, 256 + LONG $0x00c18148; WORD $0xffff; BYTE $0xff // add rcx, -256 + JNE LBB0_9 + +LBB0_10: + WORD $0x3949; BYTE $0xf2 // cmp r10, rsi + JE LBB0_13 + WORD $0x014c; BYTE $0xc7 // add rdi, r8 + +LBB0_12: + WORD $0x1788 // mov byte [rdi], dl + WORD $0xff48; BYTE $0xc7 // inc rdi + WORD $0x3949; BYTE $0xfb // cmp r11, rdi + JNE LBB0_12 + +LBB0_13: + RET diff --git a/vendor/github.com/apache/arrow/go/arrow/memory/util.go b/vendor/github.com/apache/arrow/go/arrow/memory/util.go new file mode 100644 index 00000000000..3b0d3a5cb9e --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/memory/util.go @@ -0,0 +1,37 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package memory + +import "unsafe" + +func roundToPowerOf2(v, round int) int { + forceCarry := round - 1 + truncateMask := ^forceCarry + return (v + forceCarry) & truncateMask +} + +func roundUpToMultipleOf64(v int) int { + return roundToPowerOf2(v, 64) +} + +func isMultipleOfPowerOf2(v int, d int) bool { + return (v & (d - 1)) == 0 +} + +func addressOf(b []byte) uintptr { + return uintptr(unsafe.Pointer(&b[0])) +} diff --git a/vendor/github.com/apache/arrow/go/arrow/numeric.schema.json b/vendor/github.com/apache/arrow/go/arrow/numeric.schema.json new file mode 100644 index 00000000000..7fa2800a57a --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/numeric.schema.json @@ -0,0 +1,15 @@ +{ + "title": "templates", + "type": "array", + "items": { + "title": "template", + "type": "object", + "properties": { + "Name": { + "type": "string", + "description": "The name of the template type" + } + }, + "required": ["Name"] + } +} \ No newline at end of file diff --git a/vendor/github.com/apache/arrow/go/arrow/numeric.tmpldata b/vendor/github.com/apache/arrow/go/arrow/numeric.tmpldata new file mode 100644 index 00000000000..127a5a107e2 --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/numeric.tmpldata @@ -0,0 +1,141 @@ +[ + { + "Name": "Int64", + "name": "int64", + "Type": "int64", + "Default": "0", + "Size": "8" + }, + { + "Name": "Uint64", + "name": "uint64", + "Type": "uint64", + "Default": "0", + "Size": "8" + }, + { + "Name": "Float64", + "name": "float64", + "Type": "float64", + "Default": "0", + "Size": "8" + }, + { + "Name": "Int32", + "name": "int32", + "Type": "int32", + "Default": "0", + "Size": "4", + "Opt": { + "BufferBuilder": true + } + }, + { + "Name": "Uint32", + "name": "uint32", + "Type": "uint32", + "Default": "0", + "Size": "4" + }, + { + "Name": "Float32", + "name": "float32", + "Type": "float32", + "Default": "0", + "Size": "4" + }, + { + "Name": "Int16", + "name": "int16", + "Type": "int16", + "Default": "0", + "Size": "2" + }, + { + "Name": "Uint16", + "name": "uint16", + "Type": "uint16", + "Default": "0", + "Size": "2" + }, + { + "Name": "Int8", + "name": "int8", + "Type": "int8", + "Default": "0", + "Size": "1" + }, + { + "Name": "Uint8", + "name": "uint8", + "Type": "uint8", + "Default": "0", + "Size": "1" + }, + { + "Name": "Timestamp", + "name": "timestamp", + "Type": "Timestamp", + "QualifiedType": "arrow.Timestamp", + "InternalType": "int64", + "Default": "0", + "Size": "8", + "Opt": { + "Parametric": true + } + }, + { + "Name": "Time32", + "name": "time32", + "Type": "Time32", + "QualifiedType": "arrow.Time32", + "InternalType": "int32", + "Default": "0", + "Size": "4", + "Opt": { + "Parametric": true + } + }, + { + "Name": "Time64", + "name": "time64", + "Type": "Time64", + "QualifiedType": "arrow.Time64", + "InternalType": "int64", + "Default": "0", + "Size": "8", + "Opt": { + "Parametric": true + } + }, + { + "Name": "Date32", + "name": "date32", + "Type": "Date32", + "QualifiedType": "arrow.Date32", + "InternalType": "int32", + "Default": "0", + "Size": "4" + }, + { + "Name": "Date64", + "name": "date64", + "Type": "Date64", + "QualifiedType": "arrow.Date64", + "InternalType": "int64", + "Default": "0", + "Size": "8" + }, + { + "Name": "Duration", + "name": "duration", + "Type": "Duration", + "QualifiedType": "arrow.Duration", + "InternalType": "int64", + "Default": "0", + "Size": "8", + "Opt": { + "Parametric": true + } + } +] diff --git a/vendor/github.com/apache/arrow/go/arrow/schema.go b/vendor/github.com/apache/arrow/go/arrow/schema.go new file mode 100644 index 00000000000..f86bd2ea4f8 --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/schema.go @@ -0,0 +1,200 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package arrow + +import ( + "fmt" + "sort" + "strings" +) + +type Metadata struct { + keys []string + values []string +} + +func NewMetadata(keys, values []string) Metadata { + if len(keys) != len(values) { + panic("arrow: len mismatch") + } + + n := len(keys) + if n == 0 { + return Metadata{} + } + + md := Metadata{ + keys: make([]string, n), + values: make([]string, n), + } + copy(md.keys, keys) + copy(md.values, values) + return md +} + +func MetadataFrom(kv map[string]string) Metadata { + md := Metadata{ + keys: make([]string, 0, len(kv)), + values: make([]string, 0, len(kv)), + } + for k := range kv { + md.keys = append(md.keys, k) + } + sort.Strings(md.keys) + for _, k := range md.keys { + md.values = append(md.values, kv[k]) + } + return md +} + +func (md Metadata) Len() int { return len(md.keys) } +func (md Metadata) Keys() []string { return md.keys } +func (md Metadata) Values() []string { return md.values } + +func (md Metadata) String() string { + o := new(strings.Builder) + fmt.Fprintf(o, "[") + for i := range md.keys { + if i > 0 { + fmt.Fprintf(o, ", ") + } + fmt.Fprintf(o, "%q: %q", md.keys[i], md.values[i]) + } + fmt.Fprintf(o, "]") + return o.String() +} + +// FindKey returns the index of the key-value pair with the provided key name, +// or -1 if such a key does not exist. +func (md Metadata) FindKey(k string) int { + for i, v := range md.keys { + if v == k { + return i + } + } + return -1 +} + +func (md Metadata) clone() Metadata { + if len(md.keys) == 0 { + return Metadata{} + } + + o := Metadata{ + keys: make([]string, len(md.keys)), + values: make([]string, len(md.values)), + } + copy(o.keys, md.keys) + copy(o.values, md.values) + + return o +} + +// Schema is a sequence of Field values, describing the columns of a table or +// a record batch. +type Schema struct { + fields []Field + index map[string]int + meta Metadata +} + +// NewSchema returns a new Schema value from the slice of fields and metadata. +// +// NewSchema panics if there are duplicated fields. +// NewSchema panics if there is a field with an invalid DataType. +func NewSchema(fields []Field, metadata *Metadata) *Schema { + sc := &Schema{ + fields: make([]Field, 0, len(fields)), + index: make(map[string]int, len(fields)), + } + if metadata != nil { + sc.meta = metadata.clone() + } + for i, field := range fields { + if field.Type == nil { + panic("arrow: field with nil DataType") + } + sc.fields = append(sc.fields, field) + if _, dup := sc.index[field.Name]; dup { + panic(fmt.Errorf("arrow: duplicate field with name %q", field.Name)) + } + sc.index[field.Name] = i + } + return sc +} + +func (sc *Schema) Metadata() Metadata { return sc.meta } +func (sc *Schema) Fields() []Field { return sc.fields } +func (sc *Schema) Field(i int) Field { return sc.fields[i] } + +func (sc *Schema) FieldByName(n string) (Field, bool) { + i, ok := sc.index[n] + if !ok { + return Field{}, ok + } + return sc.fields[i], ok +} + +// FieldIndex returns the index of the named field or -1. +func (sc *Schema) FieldIndex(n string) int { + i, ok := sc.index[n] + if !ok { + return -1 + } + return i +} + +func (sc *Schema) HasField(n string) bool { + return sc.FieldIndex(n) >= 0 +} + +func (sc *Schema) HasMetadata() bool { return len(sc.meta.keys) > 0 } + +// Equal returns whether two schema are equal. +// Equal does not compare the metadata. +func (sc *Schema) Equal(o *Schema) bool { + switch { + case sc == o: + return true + case sc == nil || o == nil: + return false + case len(sc.fields) != len(o.fields): + return false + } + + for i := range sc.fields { + if !sc.fields[i].Equal(o.fields[i]) { + return false + } + } + return true +} + +func (s *Schema) String() string { + o := new(strings.Builder) + fmt.Fprintf(o, "schema:\n fields: %d\n", len(s.Fields())) + for i, f := range s.Fields() { + if i > 0 { + o.WriteString("\n") + } + fmt.Fprintf(o, " - %v", f) + } + if meta := s.Metadata(); meta.Len() > 0 { + fmt.Fprintf(o, "\n metadata: %v", meta) + } + return o.String() +} diff --git a/vendor/github.com/apache/arrow/go/arrow/type_string.go b/vendor/github.com/apache/arrow/go/arrow/type_string.go new file mode 100644 index 00000000000..d81cb7b5b1f --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/type_string.go @@ -0,0 +1,53 @@ +// Code generated by "stringer -type=Type"; DO NOT EDIT. + +package arrow + +import "strconv" + +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[NULL-0] + _ = x[BOOL-1] + _ = x[UINT8-2] + _ = x[INT8-3] + _ = x[UINT16-4] + _ = x[INT16-5] + _ = x[UINT32-6] + _ = x[INT32-7] + _ = x[UINT64-8] + _ = x[INT64-9] + _ = x[FLOAT16-10] + _ = x[FLOAT32-11] + _ = x[FLOAT64-12] + _ = x[STRING-13] + _ = x[BINARY-14] + _ = x[FIXED_SIZE_BINARY-15] + _ = x[DATE32-16] + _ = x[DATE64-17] + _ = x[TIMESTAMP-18] + _ = x[TIME32-19] + _ = x[TIME64-20] + _ = x[INTERVAL-21] + _ = x[DECIMAL-22] + _ = x[LIST-23] + _ = x[STRUCT-24] + _ = x[UNION-25] + _ = x[DICTIONARY-26] + _ = x[MAP-27] + _ = x[EXTENSION-28] + _ = x[FIXED_SIZE_LIST-29] + _ = x[DURATION-30] +} + +const _Type_name = "NULLBOOLUINT8INT8UINT16INT16UINT32INT32UINT64INT64FLOAT16FLOAT32FLOAT64STRINGBINARYFIXED_SIZE_BINARYDATE32DATE64TIMESTAMPTIME32TIME64INTERVALDECIMALLISTSTRUCTUNIONDICTIONARYMAPEXTENSIONFIXED_SIZE_LISTDURATION" + +var _Type_index = [...]uint8{0, 4, 8, 13, 17, 23, 28, 34, 39, 45, 50, 57, 64, 71, 77, 83, 100, 106, 112, 121, 127, 133, 141, 148, 152, 158, 163, 173, 176, 185, 200, 208} + +func (i Type) String() string { + if i < 0 || i >= Type(len(_Type_index)-1) { + return "Type(" + strconv.FormatInt(int64(i), 10) + ")" + } + return _Type_name[_Type_index[i]:_Type_index[i+1]] +} diff --git a/vendor/github.com/apache/arrow/go/arrow/type_traits_boolean.go b/vendor/github.com/apache/arrow/go/arrow/type_traits_boolean.go new file mode 100644 index 00000000000..c30ae0fec4f --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/type_traits_boolean.go @@ -0,0 +1,28 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package arrow + +import ( + "github.com/apache/arrow/go/arrow/internal/bitutil" +) + +type booleanTraits struct{} + +var BooleanTraits booleanTraits + +// BytesRequired returns the number of bytes required to store n elements in memory. +func (booleanTraits) BytesRequired(n int) int { return bitutil.CeilByte(n) / 8 } diff --git a/vendor/github.com/apache/arrow/go/arrow/type_traits_decimal128.go b/vendor/github.com/apache/arrow/go/arrow/type_traits_decimal128.go new file mode 100644 index 00000000000..debe6c7a47d --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/type_traits_decimal128.go @@ -0,0 +1,75 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package arrow + +import ( + "encoding/binary" + "reflect" + "unsafe" + + "github.com/apache/arrow/go/arrow/decimal128" +) + +// Decimal128 traits +var Decimal128Traits decimal128Traits + +const ( + // Decimal128SizeBytes specifies the number of bytes required to store a single decimal128 in memory + Decimal128SizeBytes = int(unsafe.Sizeof(decimal128.Num{})) +) + +type decimal128Traits struct{} + +// BytesRequired returns the number of bytes required to store n elements in memory. +func (decimal128Traits) BytesRequired(n int) int { return Decimal128SizeBytes * n } + +// PutValue +func (decimal128Traits) PutValue(b []byte, v decimal128.Num) { + binary.LittleEndian.PutUint64(b[:8], uint64(v.LowBits())) + binary.LittleEndian.PutUint64(b[8:], uint64(v.HighBits())) +} + +// CastFromBytes reinterprets the slice b to a slice of type uint16. +// +// NOTE: len(b) must be a multiple of Uint16SizeBytes. +func (decimal128Traits) CastFromBytes(b []byte) []decimal128.Num { + h := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + + var res []decimal128.Num + s := (*reflect.SliceHeader)(unsafe.Pointer(&res)) + s.Data = h.Data + s.Len = h.Len / Decimal128SizeBytes + s.Cap = h.Cap / Decimal128SizeBytes + + return res +} + +// CastToBytes reinterprets the slice b to a slice of bytes. +func (decimal128Traits) CastToBytes(b []decimal128.Num) []byte { + h := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + + var res []byte + s := (*reflect.SliceHeader)(unsafe.Pointer(&res)) + s.Data = h.Data + s.Len = h.Len * Decimal128SizeBytes + s.Cap = h.Cap * Decimal128SizeBytes + + return res +} + +// Copy copies src to dst. +func (decimal128Traits) Copy(dst, src []decimal128.Num) { copy(dst, src) } diff --git a/vendor/github.com/apache/arrow/go/arrow/type_traits_float16.go b/vendor/github.com/apache/arrow/go/arrow/type_traits_float16.go new file mode 100644 index 00000000000..f21ea4d3236 --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/type_traits_float16.go @@ -0,0 +1,74 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package arrow + +import ( + "encoding/binary" + "reflect" + "unsafe" + + "github.com/apache/arrow/go/arrow/float16" +) + +// Float16 traits +var Float16Traits float16Traits + +const ( + // Float16SizeBytes specifies the number of bytes required to store a single float16 in memory + Float16SizeBytes = int(unsafe.Sizeof(uint16(0))) +) + +type float16Traits struct{} + +// BytesRequired returns the number of bytes required to store n elements in memory. +func (float16Traits) BytesRequired(n int) int { return Float16SizeBytes * n } + +// PutValue +func (float16Traits) PutValue(b []byte, v float16.Num) { + binary.LittleEndian.PutUint16(b, uint16(v.Uint16())) +} + +// CastFromBytes reinterprets the slice b to a slice of type uint16. +// +// NOTE: len(b) must be a multiple of Uint16SizeBytes. +func (float16Traits) CastFromBytes(b []byte) []float16.Num { + h := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + + var res []float16.Num + s := (*reflect.SliceHeader)(unsafe.Pointer(&res)) + s.Data = h.Data + s.Len = h.Len / Float16SizeBytes + s.Cap = h.Cap / Float16SizeBytes + + return res +} + +// CastToBytes reinterprets the slice b to a slice of bytes. +func (float16Traits) CastToBytes(b []float16.Num) []byte { + h := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + + var res []byte + s := (*reflect.SliceHeader)(unsafe.Pointer(&res)) + s.Data = h.Data + s.Len = h.Len * Float16SizeBytes + s.Cap = h.Cap * Float16SizeBytes + + return res +} + +// Copy copies src to dst. +func (float16Traits) Copy(dst, src []float16.Num) { copy(dst, src) } diff --git a/vendor/github.com/apache/arrow/go/arrow/type_traits_interval.go b/vendor/github.com/apache/arrow/go/arrow/type_traits_interval.go new file mode 100644 index 00000000000..fcff1e6fd67 --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/type_traits_interval.go @@ -0,0 +1,125 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package arrow + +import ( + "encoding/binary" + "reflect" + "unsafe" +) + +var ( + MonthIntervalTraits monthTraits + DayTimeIntervalTraits daytimeTraits +) + +// MonthInterval traits + +const ( + // MonthIntervalSizeBytes specifies the number of bytes required to store a single MonthInterval in memory + MonthIntervalSizeBytes = int(unsafe.Sizeof(MonthInterval(0))) +) + +type monthTraits struct{} + +// BytesRequired returns the number of bytes required to store n elements in memory. +func (monthTraits) BytesRequired(n int) int { return MonthIntervalSizeBytes * n } + +// PutValue +func (monthTraits) PutValue(b []byte, v MonthInterval) { + binary.LittleEndian.PutUint32(b, uint32(v)) +} + +// CastFromBytes reinterprets the slice b to a slice of type MonthInterval. +// +// NOTE: len(b) must be a multiple of MonthIntervalSizeBytes. +func (monthTraits) CastFromBytes(b []byte) []MonthInterval { + h := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + + var res []MonthInterval + s := (*reflect.SliceHeader)(unsafe.Pointer(&res)) + s.Data = h.Data + s.Len = h.Len / MonthIntervalSizeBytes + s.Cap = h.Cap / MonthIntervalSizeBytes + + return res +} + +// CastToBytes reinterprets the slice b to a slice of bytes. +func (monthTraits) CastToBytes(b []MonthInterval) []byte { + h := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + + var res []byte + s := (*reflect.SliceHeader)(unsafe.Pointer(&res)) + s.Data = h.Data + s.Len = h.Len * MonthIntervalSizeBytes + s.Cap = h.Cap * MonthIntervalSizeBytes + + return res +} + +// Copy copies src to dst. +func (monthTraits) Copy(dst, src []MonthInterval) { copy(dst, src) } + +// DayTimeInterval traits + +const ( + // DayTimeIntervalSizeBytes specifies the number of bytes required to store a single DayTimeInterval in memory + DayTimeIntervalSizeBytes = int(unsafe.Sizeof(DayTimeInterval{})) +) + +type daytimeTraits struct{} + +// BytesRequired returns the number of bytes required to store n elements in memory. +func (daytimeTraits) BytesRequired(n int) int { return DayTimeIntervalSizeBytes * n } + +// PutValue +func (daytimeTraits) PutValue(b []byte, v DayTimeInterval) { + binary.LittleEndian.PutUint32(b[0:4], uint32(v.Days)) + binary.LittleEndian.PutUint32(b[4:8], uint32(v.Milliseconds)) +} + +// CastFromBytes reinterprets the slice b to a slice of type DayTimeInterval. +// +// NOTE: len(b) must be a multiple of DayTimeIntervalSizeBytes. +func (daytimeTraits) CastFromBytes(b []byte) []DayTimeInterval { + h := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + + var res []DayTimeInterval + s := (*reflect.SliceHeader)(unsafe.Pointer(&res)) + s.Data = h.Data + s.Len = h.Len / DayTimeIntervalSizeBytes + s.Cap = h.Cap / DayTimeIntervalSizeBytes + + return res +} + +// CastToBytes reinterprets the slice b to a slice of bytes. +func (daytimeTraits) CastToBytes(b []DayTimeInterval) []byte { + h := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + + var res []byte + s := (*reflect.SliceHeader)(unsafe.Pointer(&res)) + s.Data = h.Data + s.Len = h.Len * DayTimeIntervalSizeBytes + s.Cap = h.Cap * DayTimeIntervalSizeBytes + + return res +} + +// Copy copies src to dst. +func (daytimeTraits) Copy(dst, src []DayTimeInterval) { copy(dst, src) } diff --git a/vendor/github.com/apache/arrow/go/arrow/type_traits_numeric.gen.go b/vendor/github.com/apache/arrow/go/arrow/type_traits_numeric.gen.go new file mode 100644 index 00000000000..f98f4947e25 --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/type_traits_numeric.gen.go @@ -0,0 +1,813 @@ +// Code generated by type_traits_numeric.gen.go.tmpl. DO NOT EDIT. + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package arrow // import "github.com/apache/arrow/go/arrow" + +import ( + "encoding/binary" + "math" + "reflect" + "unsafe" +) + +var ( + Int64Traits int64Traits + Uint64Traits uint64Traits + Float64Traits float64Traits + Int32Traits int32Traits + Uint32Traits uint32Traits + Float32Traits float32Traits + Int16Traits int16Traits + Uint16Traits uint16Traits + Int8Traits int8Traits + Uint8Traits uint8Traits + TimestampTraits timestampTraits + Time32Traits time32Traits + Time64Traits time64Traits + Date32Traits date32Traits + Date64Traits date64Traits + DurationTraits durationTraits +) + +// Int64 traits + +const ( + // Int64SizeBytes specifies the number of bytes required to store a single int64 in memory + Int64SizeBytes = int(unsafe.Sizeof(int64(0))) +) + +type int64Traits struct{} + +// BytesRequired returns the number of bytes required to store n elements in memory. +func (int64Traits) BytesRequired(n int) int { return Int64SizeBytes * n } + +// PutValue +func (int64Traits) PutValue(b []byte, v int64) { + binary.LittleEndian.PutUint64(b, uint64(v)) +} + +// CastFromBytes reinterprets the slice b to a slice of type int64. +// +// NOTE: len(b) must be a multiple of Int64SizeBytes. +func (int64Traits) CastFromBytes(b []byte) []int64 { + h := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + + var res []int64 + s := (*reflect.SliceHeader)(unsafe.Pointer(&res)) + s.Data = h.Data + s.Len = h.Len / Int64SizeBytes + s.Cap = h.Cap / Int64SizeBytes + + return res +} + +// CastToBytes reinterprets the slice b to a slice of bytes. +func (int64Traits) CastToBytes(b []int64) []byte { + h := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + + var res []byte + s := (*reflect.SliceHeader)(unsafe.Pointer(&res)) + s.Data = h.Data + s.Len = h.Len * Int64SizeBytes + s.Cap = h.Cap * Int64SizeBytes + + return res +} + +// Copy copies src to dst. +func (int64Traits) Copy(dst, src []int64) { copy(dst, src) } + +// Uint64 traits + +const ( + // Uint64SizeBytes specifies the number of bytes required to store a single uint64 in memory + Uint64SizeBytes = int(unsafe.Sizeof(uint64(0))) +) + +type uint64Traits struct{} + +// BytesRequired returns the number of bytes required to store n elements in memory. +func (uint64Traits) BytesRequired(n int) int { return Uint64SizeBytes * n } + +// PutValue +func (uint64Traits) PutValue(b []byte, v uint64) { + binary.LittleEndian.PutUint64(b, uint64(v)) +} + +// CastFromBytes reinterprets the slice b to a slice of type uint64. +// +// NOTE: len(b) must be a multiple of Uint64SizeBytes. +func (uint64Traits) CastFromBytes(b []byte) []uint64 { + h := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + + var res []uint64 + s := (*reflect.SliceHeader)(unsafe.Pointer(&res)) + s.Data = h.Data + s.Len = h.Len / Uint64SizeBytes + s.Cap = h.Cap / Uint64SizeBytes + + return res +} + +// CastToBytes reinterprets the slice b to a slice of bytes. +func (uint64Traits) CastToBytes(b []uint64) []byte { + h := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + + var res []byte + s := (*reflect.SliceHeader)(unsafe.Pointer(&res)) + s.Data = h.Data + s.Len = h.Len * Uint64SizeBytes + s.Cap = h.Cap * Uint64SizeBytes + + return res +} + +// Copy copies src to dst. +func (uint64Traits) Copy(dst, src []uint64) { copy(dst, src) } + +// Float64 traits + +const ( + // Float64SizeBytes specifies the number of bytes required to store a single float64 in memory + Float64SizeBytes = int(unsafe.Sizeof(float64(0))) +) + +type float64Traits struct{} + +// BytesRequired returns the number of bytes required to store n elements in memory. +func (float64Traits) BytesRequired(n int) int { return Float64SizeBytes * n } + +// PutValue +func (float64Traits) PutValue(b []byte, v float64) { + binary.LittleEndian.PutUint64(b, math.Float64bits(v)) +} + +// CastFromBytes reinterprets the slice b to a slice of type float64. +// +// NOTE: len(b) must be a multiple of Float64SizeBytes. +func (float64Traits) CastFromBytes(b []byte) []float64 { + h := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + + var res []float64 + s := (*reflect.SliceHeader)(unsafe.Pointer(&res)) + s.Data = h.Data + s.Len = h.Len / Float64SizeBytes + s.Cap = h.Cap / Float64SizeBytes + + return res +} + +// CastToBytes reinterprets the slice b to a slice of bytes. +func (float64Traits) CastToBytes(b []float64) []byte { + h := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + + var res []byte + s := (*reflect.SliceHeader)(unsafe.Pointer(&res)) + s.Data = h.Data + s.Len = h.Len * Float64SizeBytes + s.Cap = h.Cap * Float64SizeBytes + + return res +} + +// Copy copies src to dst. +func (float64Traits) Copy(dst, src []float64) { copy(dst, src) } + +// Int32 traits + +const ( + // Int32SizeBytes specifies the number of bytes required to store a single int32 in memory + Int32SizeBytes = int(unsafe.Sizeof(int32(0))) +) + +type int32Traits struct{} + +// BytesRequired returns the number of bytes required to store n elements in memory. +func (int32Traits) BytesRequired(n int) int { return Int32SizeBytes * n } + +// PutValue +func (int32Traits) PutValue(b []byte, v int32) { + binary.LittleEndian.PutUint32(b, uint32(v)) +} + +// CastFromBytes reinterprets the slice b to a slice of type int32. +// +// NOTE: len(b) must be a multiple of Int32SizeBytes. +func (int32Traits) CastFromBytes(b []byte) []int32 { + h := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + + var res []int32 + s := (*reflect.SliceHeader)(unsafe.Pointer(&res)) + s.Data = h.Data + s.Len = h.Len / Int32SizeBytes + s.Cap = h.Cap / Int32SizeBytes + + return res +} + +// CastToBytes reinterprets the slice b to a slice of bytes. +func (int32Traits) CastToBytes(b []int32) []byte { + h := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + + var res []byte + s := (*reflect.SliceHeader)(unsafe.Pointer(&res)) + s.Data = h.Data + s.Len = h.Len * Int32SizeBytes + s.Cap = h.Cap * Int32SizeBytes + + return res +} + +// Copy copies src to dst. +func (int32Traits) Copy(dst, src []int32) { copy(dst, src) } + +// Uint32 traits + +const ( + // Uint32SizeBytes specifies the number of bytes required to store a single uint32 in memory + Uint32SizeBytes = int(unsafe.Sizeof(uint32(0))) +) + +type uint32Traits struct{} + +// BytesRequired returns the number of bytes required to store n elements in memory. +func (uint32Traits) BytesRequired(n int) int { return Uint32SizeBytes * n } + +// PutValue +func (uint32Traits) PutValue(b []byte, v uint32) { + binary.LittleEndian.PutUint32(b, uint32(v)) +} + +// CastFromBytes reinterprets the slice b to a slice of type uint32. +// +// NOTE: len(b) must be a multiple of Uint32SizeBytes. +func (uint32Traits) CastFromBytes(b []byte) []uint32 { + h := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + + var res []uint32 + s := (*reflect.SliceHeader)(unsafe.Pointer(&res)) + s.Data = h.Data + s.Len = h.Len / Uint32SizeBytes + s.Cap = h.Cap / Uint32SizeBytes + + return res +} + +// CastToBytes reinterprets the slice b to a slice of bytes. +func (uint32Traits) CastToBytes(b []uint32) []byte { + h := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + + var res []byte + s := (*reflect.SliceHeader)(unsafe.Pointer(&res)) + s.Data = h.Data + s.Len = h.Len * Uint32SizeBytes + s.Cap = h.Cap * Uint32SizeBytes + + return res +} + +// Copy copies src to dst. +func (uint32Traits) Copy(dst, src []uint32) { copy(dst, src) } + +// Float32 traits + +const ( + // Float32SizeBytes specifies the number of bytes required to store a single float32 in memory + Float32SizeBytes = int(unsafe.Sizeof(float32(0))) +) + +type float32Traits struct{} + +// BytesRequired returns the number of bytes required to store n elements in memory. +func (float32Traits) BytesRequired(n int) int { return Float32SizeBytes * n } + +// PutValue +func (float32Traits) PutValue(b []byte, v float32) { + binary.LittleEndian.PutUint32(b, math.Float32bits(v)) +} + +// CastFromBytes reinterprets the slice b to a slice of type float32. +// +// NOTE: len(b) must be a multiple of Float32SizeBytes. +func (float32Traits) CastFromBytes(b []byte) []float32 { + h := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + + var res []float32 + s := (*reflect.SliceHeader)(unsafe.Pointer(&res)) + s.Data = h.Data + s.Len = h.Len / Float32SizeBytes + s.Cap = h.Cap / Float32SizeBytes + + return res +} + +// CastToBytes reinterprets the slice b to a slice of bytes. +func (float32Traits) CastToBytes(b []float32) []byte { + h := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + + var res []byte + s := (*reflect.SliceHeader)(unsafe.Pointer(&res)) + s.Data = h.Data + s.Len = h.Len * Float32SizeBytes + s.Cap = h.Cap * Float32SizeBytes + + return res +} + +// Copy copies src to dst. +func (float32Traits) Copy(dst, src []float32) { copy(dst, src) } + +// Int16 traits + +const ( + // Int16SizeBytes specifies the number of bytes required to store a single int16 in memory + Int16SizeBytes = int(unsafe.Sizeof(int16(0))) +) + +type int16Traits struct{} + +// BytesRequired returns the number of bytes required to store n elements in memory. +func (int16Traits) BytesRequired(n int) int { return Int16SizeBytes * n } + +// PutValue +func (int16Traits) PutValue(b []byte, v int16) { + binary.LittleEndian.PutUint16(b, uint16(v)) +} + +// CastFromBytes reinterprets the slice b to a slice of type int16. +// +// NOTE: len(b) must be a multiple of Int16SizeBytes. +func (int16Traits) CastFromBytes(b []byte) []int16 { + h := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + + var res []int16 + s := (*reflect.SliceHeader)(unsafe.Pointer(&res)) + s.Data = h.Data + s.Len = h.Len / Int16SizeBytes + s.Cap = h.Cap / Int16SizeBytes + + return res +} + +// CastToBytes reinterprets the slice b to a slice of bytes. +func (int16Traits) CastToBytes(b []int16) []byte { + h := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + + var res []byte + s := (*reflect.SliceHeader)(unsafe.Pointer(&res)) + s.Data = h.Data + s.Len = h.Len * Int16SizeBytes + s.Cap = h.Cap * Int16SizeBytes + + return res +} + +// Copy copies src to dst. +func (int16Traits) Copy(dst, src []int16) { copy(dst, src) } + +// Uint16 traits + +const ( + // Uint16SizeBytes specifies the number of bytes required to store a single uint16 in memory + Uint16SizeBytes = int(unsafe.Sizeof(uint16(0))) +) + +type uint16Traits struct{} + +// BytesRequired returns the number of bytes required to store n elements in memory. +func (uint16Traits) BytesRequired(n int) int { return Uint16SizeBytes * n } + +// PutValue +func (uint16Traits) PutValue(b []byte, v uint16) { + binary.LittleEndian.PutUint16(b, uint16(v)) +} + +// CastFromBytes reinterprets the slice b to a slice of type uint16. +// +// NOTE: len(b) must be a multiple of Uint16SizeBytes. +func (uint16Traits) CastFromBytes(b []byte) []uint16 { + h := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + + var res []uint16 + s := (*reflect.SliceHeader)(unsafe.Pointer(&res)) + s.Data = h.Data + s.Len = h.Len / Uint16SizeBytes + s.Cap = h.Cap / Uint16SizeBytes + + return res +} + +// CastToBytes reinterprets the slice b to a slice of bytes. +func (uint16Traits) CastToBytes(b []uint16) []byte { + h := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + + var res []byte + s := (*reflect.SliceHeader)(unsafe.Pointer(&res)) + s.Data = h.Data + s.Len = h.Len * Uint16SizeBytes + s.Cap = h.Cap * Uint16SizeBytes + + return res +} + +// Copy copies src to dst. +func (uint16Traits) Copy(dst, src []uint16) { copy(dst, src) } + +// Int8 traits + +const ( + // Int8SizeBytes specifies the number of bytes required to store a single int8 in memory + Int8SizeBytes = int(unsafe.Sizeof(int8(0))) +) + +type int8Traits struct{} + +// BytesRequired returns the number of bytes required to store n elements in memory. +func (int8Traits) BytesRequired(n int) int { return Int8SizeBytes * n } + +// PutValue +func (int8Traits) PutValue(b []byte, v int8) { + b[0] = byte(v) +} + +// CastFromBytes reinterprets the slice b to a slice of type int8. +// +// NOTE: len(b) must be a multiple of Int8SizeBytes. +func (int8Traits) CastFromBytes(b []byte) []int8 { + h := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + + var res []int8 + s := (*reflect.SliceHeader)(unsafe.Pointer(&res)) + s.Data = h.Data + s.Len = h.Len / Int8SizeBytes + s.Cap = h.Cap / Int8SizeBytes + + return res +} + +// CastToBytes reinterprets the slice b to a slice of bytes. +func (int8Traits) CastToBytes(b []int8) []byte { + h := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + + var res []byte + s := (*reflect.SliceHeader)(unsafe.Pointer(&res)) + s.Data = h.Data + s.Len = h.Len * Int8SizeBytes + s.Cap = h.Cap * Int8SizeBytes + + return res +} + +// Copy copies src to dst. +func (int8Traits) Copy(dst, src []int8) { copy(dst, src) } + +// Uint8 traits + +const ( + // Uint8SizeBytes specifies the number of bytes required to store a single uint8 in memory + Uint8SizeBytes = int(unsafe.Sizeof(uint8(0))) +) + +type uint8Traits struct{} + +// BytesRequired returns the number of bytes required to store n elements in memory. +func (uint8Traits) BytesRequired(n int) int { return Uint8SizeBytes * n } + +// PutValue +func (uint8Traits) PutValue(b []byte, v uint8) { + b[0] = byte(v) +} + +// CastFromBytes reinterprets the slice b to a slice of type uint8. +// +// NOTE: len(b) must be a multiple of Uint8SizeBytes. +func (uint8Traits) CastFromBytes(b []byte) []uint8 { + h := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + + var res []uint8 + s := (*reflect.SliceHeader)(unsafe.Pointer(&res)) + s.Data = h.Data + s.Len = h.Len / Uint8SizeBytes + s.Cap = h.Cap / Uint8SizeBytes + + return res +} + +// CastToBytes reinterprets the slice b to a slice of bytes. +func (uint8Traits) CastToBytes(b []uint8) []byte { + h := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + + var res []byte + s := (*reflect.SliceHeader)(unsafe.Pointer(&res)) + s.Data = h.Data + s.Len = h.Len * Uint8SizeBytes + s.Cap = h.Cap * Uint8SizeBytes + + return res +} + +// Copy copies src to dst. +func (uint8Traits) Copy(dst, src []uint8) { copy(dst, src) } + +// Timestamp traits + +const ( + // TimestampSizeBytes specifies the number of bytes required to store a single Timestamp in memory + TimestampSizeBytes = int(unsafe.Sizeof(Timestamp(0))) +) + +type timestampTraits struct{} + +// BytesRequired returns the number of bytes required to store n elements in memory. +func (timestampTraits) BytesRequired(n int) int { return TimestampSizeBytes * n } + +// PutValue +func (timestampTraits) PutValue(b []byte, v Timestamp) { + binary.LittleEndian.PutUint64(b, uint64(v)) +} + +// CastFromBytes reinterprets the slice b to a slice of type Timestamp. +// +// NOTE: len(b) must be a multiple of TimestampSizeBytes. +func (timestampTraits) CastFromBytes(b []byte) []Timestamp { + h := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + + var res []Timestamp + s := (*reflect.SliceHeader)(unsafe.Pointer(&res)) + s.Data = h.Data + s.Len = h.Len / TimestampSizeBytes + s.Cap = h.Cap / TimestampSizeBytes + + return res +} + +// CastToBytes reinterprets the slice b to a slice of bytes. +func (timestampTraits) CastToBytes(b []Timestamp) []byte { + h := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + + var res []byte + s := (*reflect.SliceHeader)(unsafe.Pointer(&res)) + s.Data = h.Data + s.Len = h.Len * TimestampSizeBytes + s.Cap = h.Cap * TimestampSizeBytes + + return res +} + +// Copy copies src to dst. +func (timestampTraits) Copy(dst, src []Timestamp) { copy(dst, src) } + +// Time32 traits + +const ( + // Time32SizeBytes specifies the number of bytes required to store a single Time32 in memory + Time32SizeBytes = int(unsafe.Sizeof(Time32(0))) +) + +type time32Traits struct{} + +// BytesRequired returns the number of bytes required to store n elements in memory. +func (time32Traits) BytesRequired(n int) int { return Time32SizeBytes * n } + +// PutValue +func (time32Traits) PutValue(b []byte, v Time32) { + binary.LittleEndian.PutUint32(b, uint32(v)) +} + +// CastFromBytes reinterprets the slice b to a slice of type Time32. +// +// NOTE: len(b) must be a multiple of Time32SizeBytes. +func (time32Traits) CastFromBytes(b []byte) []Time32 { + h := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + + var res []Time32 + s := (*reflect.SliceHeader)(unsafe.Pointer(&res)) + s.Data = h.Data + s.Len = h.Len / Time32SizeBytes + s.Cap = h.Cap / Time32SizeBytes + + return res +} + +// CastToBytes reinterprets the slice b to a slice of bytes. +func (time32Traits) CastToBytes(b []Time32) []byte { + h := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + + var res []byte + s := (*reflect.SliceHeader)(unsafe.Pointer(&res)) + s.Data = h.Data + s.Len = h.Len * Time32SizeBytes + s.Cap = h.Cap * Time32SizeBytes + + return res +} + +// Copy copies src to dst. +func (time32Traits) Copy(dst, src []Time32) { copy(dst, src) } + +// Time64 traits + +const ( + // Time64SizeBytes specifies the number of bytes required to store a single Time64 in memory + Time64SizeBytes = int(unsafe.Sizeof(Time64(0))) +) + +type time64Traits struct{} + +// BytesRequired returns the number of bytes required to store n elements in memory. +func (time64Traits) BytesRequired(n int) int { return Time64SizeBytes * n } + +// PutValue +func (time64Traits) PutValue(b []byte, v Time64) { + binary.LittleEndian.PutUint64(b, uint64(v)) +} + +// CastFromBytes reinterprets the slice b to a slice of type Time64. +// +// NOTE: len(b) must be a multiple of Time64SizeBytes. +func (time64Traits) CastFromBytes(b []byte) []Time64 { + h := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + + var res []Time64 + s := (*reflect.SliceHeader)(unsafe.Pointer(&res)) + s.Data = h.Data + s.Len = h.Len / Time64SizeBytes + s.Cap = h.Cap / Time64SizeBytes + + return res +} + +// CastToBytes reinterprets the slice b to a slice of bytes. +func (time64Traits) CastToBytes(b []Time64) []byte { + h := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + + var res []byte + s := (*reflect.SliceHeader)(unsafe.Pointer(&res)) + s.Data = h.Data + s.Len = h.Len * Time64SizeBytes + s.Cap = h.Cap * Time64SizeBytes + + return res +} + +// Copy copies src to dst. +func (time64Traits) Copy(dst, src []Time64) { copy(dst, src) } + +// Date32 traits + +const ( + // Date32SizeBytes specifies the number of bytes required to store a single Date32 in memory + Date32SizeBytes = int(unsafe.Sizeof(Date32(0))) +) + +type date32Traits struct{} + +// BytesRequired returns the number of bytes required to store n elements in memory. +func (date32Traits) BytesRequired(n int) int { return Date32SizeBytes * n } + +// PutValue +func (date32Traits) PutValue(b []byte, v Date32) { + binary.LittleEndian.PutUint32(b, uint32(v)) +} + +// CastFromBytes reinterprets the slice b to a slice of type Date32. +// +// NOTE: len(b) must be a multiple of Date32SizeBytes. +func (date32Traits) CastFromBytes(b []byte) []Date32 { + h := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + + var res []Date32 + s := (*reflect.SliceHeader)(unsafe.Pointer(&res)) + s.Data = h.Data + s.Len = h.Len / Date32SizeBytes + s.Cap = h.Cap / Date32SizeBytes + + return res +} + +// CastToBytes reinterprets the slice b to a slice of bytes. +func (date32Traits) CastToBytes(b []Date32) []byte { + h := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + + var res []byte + s := (*reflect.SliceHeader)(unsafe.Pointer(&res)) + s.Data = h.Data + s.Len = h.Len * Date32SizeBytes + s.Cap = h.Cap * Date32SizeBytes + + return res +} + +// Copy copies src to dst. +func (date32Traits) Copy(dst, src []Date32) { copy(dst, src) } + +// Date64 traits + +const ( + // Date64SizeBytes specifies the number of bytes required to store a single Date64 in memory + Date64SizeBytes = int(unsafe.Sizeof(Date64(0))) +) + +type date64Traits struct{} + +// BytesRequired returns the number of bytes required to store n elements in memory. +func (date64Traits) BytesRequired(n int) int { return Date64SizeBytes * n } + +// PutValue +func (date64Traits) PutValue(b []byte, v Date64) { + binary.LittleEndian.PutUint64(b, uint64(v)) +} + +// CastFromBytes reinterprets the slice b to a slice of type Date64. +// +// NOTE: len(b) must be a multiple of Date64SizeBytes. +func (date64Traits) CastFromBytes(b []byte) []Date64 { + h := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + + var res []Date64 + s := (*reflect.SliceHeader)(unsafe.Pointer(&res)) + s.Data = h.Data + s.Len = h.Len / Date64SizeBytes + s.Cap = h.Cap / Date64SizeBytes + + return res +} + +// CastToBytes reinterprets the slice b to a slice of bytes. +func (date64Traits) CastToBytes(b []Date64) []byte { + h := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + + var res []byte + s := (*reflect.SliceHeader)(unsafe.Pointer(&res)) + s.Data = h.Data + s.Len = h.Len * Date64SizeBytes + s.Cap = h.Cap * Date64SizeBytes + + return res +} + +// Copy copies src to dst. +func (date64Traits) Copy(dst, src []Date64) { copy(dst, src) } + +// Duration traits + +const ( + // DurationSizeBytes specifies the number of bytes required to store a single Duration in memory + DurationSizeBytes = int(unsafe.Sizeof(Duration(0))) +) + +type durationTraits struct{} + +// BytesRequired returns the number of bytes required to store n elements in memory. +func (durationTraits) BytesRequired(n int) int { return DurationSizeBytes * n } + +// PutValue +func (durationTraits) PutValue(b []byte, v Duration) { + binary.LittleEndian.PutUint64(b, uint64(v)) +} + +// CastFromBytes reinterprets the slice b to a slice of type Duration. +// +// NOTE: len(b) must be a multiple of DurationSizeBytes. +func (durationTraits) CastFromBytes(b []byte) []Duration { + h := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + + var res []Duration + s := (*reflect.SliceHeader)(unsafe.Pointer(&res)) + s.Data = h.Data + s.Len = h.Len / DurationSizeBytes + s.Cap = h.Cap / DurationSizeBytes + + return res +} + +// CastToBytes reinterprets the slice b to a slice of bytes. +func (durationTraits) CastToBytes(b []Duration) []byte { + h := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + + var res []byte + s := (*reflect.SliceHeader)(unsafe.Pointer(&res)) + s.Data = h.Data + s.Len = h.Len * DurationSizeBytes + s.Cap = h.Cap * DurationSizeBytes + + return res +} + +// Copy copies src to dst. +func (durationTraits) Copy(dst, src []Duration) { copy(dst, src) } diff --git a/vendor/github.com/apache/arrow/go/arrow/type_traits_numeric.gen.go.tmpl b/vendor/github.com/apache/arrow/go/arrow/type_traits_numeric.gen.go.tmpl new file mode 100644 index 00000000000..c4a25ee3256 --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/type_traits_numeric.gen.go.tmpl @@ -0,0 +1,94 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package arrow // import "github.com/apache/arrow/go/arrow" + +import ( + "encoding/binary" + "math" + "reflect" + "unsafe" +) + +var ( +{{range .In}} + {{.Name}}Traits {{.name}}Traits +{{- end}} +) + +{{range .In}} +// {{.Name}} traits + +const ( + // {{.Name}}SizeBytes specifies the number of bytes required to store a single {{.Type}} in memory + {{.Name}}SizeBytes = int(unsafe.Sizeof({{.Type}}({{.Default}}))) +) + +type {{.name}}Traits struct{} + +// BytesRequired returns the number of bytes required to store n elements in memory. +func ({{.name}}Traits) BytesRequired(n int) int { return {{.Name}}SizeBytes * n } + +// PutValue +func ({{.name}}Traits) PutValue(b []byte, v {{.Type}}) { +{{- if eq .Type "float32" -}} + binary.LittleEndian.PutUint32(b, math.Float32bits(v)) +{{- else if eq .Type "float64" -}} + binary.LittleEndian.PutUint64(b, math.Float64bits(v)) +{{- else if eq .Size "1" -}} + b[0] = byte(v) +{{- else if eq .Size "2" -}} + binary.LittleEndian.PutUint16(b, uint16(v)) +{{- else if eq .Size "4" -}} + binary.LittleEndian.PutUint32(b, uint32(v)) +{{- else if eq .Size "8" -}} + binary.LittleEndian.PutUint64(b, uint64(v)) +{{- else -}} + panic("invalid type {{.Type}}") +{{end}} +} + +// CastFromBytes reinterprets the slice b to a slice of type {{.Type}}. +// +// NOTE: len(b) must be a multiple of {{.Name}}SizeBytes. +func ({{.name}}Traits) CastFromBytes(b []byte) []{{.Type}} { + h := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + + var res []{{.Type}} + s := (*reflect.SliceHeader)(unsafe.Pointer(&res)) + s.Data = h.Data + s.Len = h.Len/{{.Name}}SizeBytes + s.Cap = h.Cap/{{.Name}}SizeBytes + + return res +} + +// CastToBytes reinterprets the slice b to a slice of bytes. +func ({{.name}}Traits) CastToBytes(b []{{.Type}}) []byte { + h := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + + var res []byte + s := (*reflect.SliceHeader)(unsafe.Pointer(&res)) + s.Data = h.Data + s.Len = h.Len*{{.Name}}SizeBytes + s.Cap = h.Cap*{{.Name}}SizeBytes + + return res +} + +// Copy copies src to dst. +func ({{.name}}Traits) Copy(dst, src []{{.Type}}) { copy(dst, src) } +{{end}} diff --git a/vendor/github.com/apache/arrow/go/arrow/type_traits_numeric.gen_test.go.tmpl b/vendor/github.com/apache/arrow/go/arrow/type_traits_numeric.gen_test.go.tmpl new file mode 100644 index 00000000000..5a0e269b5d5 --- /dev/null +++ b/vendor/github.com/apache/arrow/go/arrow/type_traits_numeric.gen_test.go.tmpl @@ -0,0 +1,61 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package arrow_test + +import ( + "reflect" + "testing" + + "github.com/apache/arrow/go/arrow" +) + +{{- range .In}} + +func Test{{.Name}}Traits(t *testing.T) { + const N = 10 + b1 := arrow.{{.Name}}Traits.CastToBytes([]{{or .QualifiedType .Type}}{ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + }) + + b2 := make([]byte, arrow.{{.Name}}Traits.BytesRequired(N)) + for i := 0; i < N; i++ { + beg := i * arrow.{{.Name}}SizeBytes + end := (i + 1) * arrow.{{.Name}}SizeBytes + arrow.{{.Name}}Traits.PutValue(b2[beg:end], {{or .QualifiedType .Type}}(i)) + } + + if !reflect.DeepEqual(b1, b2) { + v1 := arrow.{{.Name}}Traits.CastFromBytes(b1) + v2 := arrow.{{.Name}}Traits.CastFromBytes(b2) + t.Fatalf("invalid values:\nb1=%v\nb2=%v\nv1=%v\nv2=%v\n", b1, b2, v1, v2) + } + + v1 := arrow.{{.Name}}Traits.CastFromBytes(b1) + for i, v := range v1 { + if got, want := v, {{or .QualifiedType .Type}}(i); got != want { + t.Fatalf("invalid value[%d]. got=%v, want=%v", i, got, want) + } + } + + v2 := make([]{{or .QualifiedType .Type}}, N) + arrow.{{.Name}}Traits.Copy(v2, v1) + + if !reflect.DeepEqual(v1, v2) { + t.Fatalf("invalid values:\nv1=%v\nv2=%v\n", v1, v2) + } +} +{{end}} diff --git a/vendor/github.com/google/flatbuffers/LICENSE.txt b/vendor/github.com/google/flatbuffers/LICENSE.txt new file mode 100644 index 00000000000..a4c5efd822f --- /dev/null +++ b/vendor/github.com/google/flatbuffers/LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2014 Google Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/vendor/github.com/google/flatbuffers/go/BUILD.bazel b/vendor/github.com/google/flatbuffers/go/BUILD.bazel new file mode 100644 index 00000000000..026e89c1930 --- /dev/null +++ b/vendor/github.com/google/flatbuffers/go/BUILD.bazel @@ -0,0 +1,17 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go", + srcs = [ + "builder.go", + "doc.go", + "encode.go", + "grpc.go", + "lib.go", + "sizes.go", + "struct.go", + "table.go", + ], + importpath = "github.com/google/flatbuffers/go", + visibility = ["//visibility:public"], +) diff --git a/vendor/github.com/google/flatbuffers/go/builder.go b/vendor/github.com/google/flatbuffers/go/builder.go new file mode 100644 index 00000000000..8d75700ce24 --- /dev/null +++ b/vendor/github.com/google/flatbuffers/go/builder.go @@ -0,0 +1,771 @@ +package flatbuffers + +// Builder is a state machine for creating FlatBuffer objects. +// Use a Builder to construct object(s) starting from leaf nodes. +// +// A Builder constructs byte buffers in a last-first manner for simplicity and +// performance. +type Builder struct { + // `Bytes` gives raw access to the buffer. Most users will want to use + // FinishedBytes() instead. + Bytes []byte + + minalign int + vtable []UOffsetT + objectEnd UOffsetT + vtables []UOffsetT + head UOffsetT + nested bool + finished bool +} + +const fileIdentifierLength = 4 + +// NewBuilder initializes a Builder of size `initial_size`. +// The internal buffer is grown as needed. +func NewBuilder(initialSize int) *Builder { + if initialSize <= 0 { + initialSize = 0 + } + + b := &Builder{} + b.Bytes = make([]byte, initialSize) + b.head = UOffsetT(initialSize) + b.minalign = 1 + b.vtables = make([]UOffsetT, 0, 16) // sensible default capacity + + return b +} + +// Reset truncates the underlying Builder buffer, facilitating alloc-free +// reuse of a Builder. It also resets bookkeeping data. +func (b *Builder) Reset() { + if b.Bytes != nil { + b.Bytes = b.Bytes[:cap(b.Bytes)] + } + + if b.vtables != nil { + b.vtables = b.vtables[:0] + } + + if b.vtable != nil { + b.vtable = b.vtable[:0] + } + + b.head = UOffsetT(len(b.Bytes)) + b.minalign = 1 + b.nested = false + b.finished = false +} + +// FinishedBytes returns a pointer to the written data in the byte buffer. +// Panics if the builder is not in a finished state (which is caused by calling +// `Finish()`). +func (b *Builder) FinishedBytes() []byte { + b.assertFinished() + return b.Bytes[b.Head():] +} + +// StartObject initializes bookkeeping for writing a new object. +func (b *Builder) StartObject(numfields int) { + b.assertNotNested() + b.nested = true + + // use 32-bit offsets so that arithmetic doesn't overflow. + if cap(b.vtable) < numfields || b.vtable == nil { + b.vtable = make([]UOffsetT, numfields) + } else { + b.vtable = b.vtable[:numfields] + for i := 0; i < len(b.vtable); i++ { + b.vtable[i] = 0 + } + } + + b.objectEnd = b.Offset() +} + +// WriteVtable serializes the vtable for the current object, if applicable. +// +// Before writing out the vtable, this checks pre-existing vtables for equality +// to this one. If an equal vtable is found, point the object to the existing +// vtable and return. +// +// Because vtable values are sensitive to alignment of object data, not all +// logically-equal vtables will be deduplicated. +// +// A vtable has the following format: +// +// +// * N, where N is the number of fields in +// the schema for this type. Includes deprecated fields. +// Thus, a vtable is made of 2 + N elements, each SizeVOffsetT bytes wide. +// +// An object has the following format: +// +// + +func (b *Builder) WriteVtable() (n UOffsetT) { + // Prepend a zero scalar to the object. Later in this function we'll + // write an offset here that points to the object's vtable: + b.PrependSOffsetT(0) + + objectOffset := b.Offset() + existingVtable := UOffsetT(0) + + // Trim vtable of trailing zeroes. + i := len(b.vtable) - 1 + for ; i >= 0 && b.vtable[i] == 0; i-- { + } + b.vtable = b.vtable[:i+1] + + // Search backwards through existing vtables, because similar vtables + // are likely to have been recently appended. See + // BenchmarkVtableDeduplication for a case in which this heuristic + // saves about 30% of the time used in writing objects with duplicate + // tables. + for i := len(b.vtables) - 1; i >= 0; i-- { + // Find the other vtable, which is associated with `i`: + vt2Offset := b.vtables[i] + vt2Start := len(b.Bytes) - int(vt2Offset) + vt2Len := GetVOffsetT(b.Bytes[vt2Start:]) + + metadata := VtableMetadataFields * SizeVOffsetT + vt2End := vt2Start + int(vt2Len) + vt2 := b.Bytes[vt2Start+metadata : vt2End] + + // Compare the other vtable to the one under consideration. + // If they are equal, store the offset and break: + if vtableEqual(b.vtable, objectOffset, vt2) { + existingVtable = vt2Offset + break + } + } + + if existingVtable == 0 { + // Did not find a vtable, so write this one to the buffer. + + // Write out the current vtable in reverse , because + // serialization occurs in last-first order: + for i := len(b.vtable) - 1; i >= 0; i-- { + var off UOffsetT + if b.vtable[i] != 0 { + // Forward reference to field; + // use 32bit number to assert no overflow: + off = objectOffset - b.vtable[i] + } + + b.PrependVOffsetT(VOffsetT(off)) + } + + // The two metadata fields are written last. + + // First, store the object bytesize: + objectSize := objectOffset - b.objectEnd + b.PrependVOffsetT(VOffsetT(objectSize)) + + // Second, store the vtable bytesize: + vBytes := (len(b.vtable) + VtableMetadataFields) * SizeVOffsetT + b.PrependVOffsetT(VOffsetT(vBytes)) + + // Next, write the offset to the new vtable in the + // already-allocated SOffsetT at the beginning of this object: + objectStart := SOffsetT(len(b.Bytes)) - SOffsetT(objectOffset) + WriteSOffsetT(b.Bytes[objectStart:], + SOffsetT(b.Offset())-SOffsetT(objectOffset)) + + // Finally, store this vtable in memory for future + // deduplication: + b.vtables = append(b.vtables, b.Offset()) + } else { + // Found a duplicate vtable. + + objectStart := SOffsetT(len(b.Bytes)) - SOffsetT(objectOffset) + b.head = UOffsetT(objectStart) + + // Write the offset to the found vtable in the + // already-allocated SOffsetT at the beginning of this object: + WriteSOffsetT(b.Bytes[b.head:], + SOffsetT(existingVtable)-SOffsetT(objectOffset)) + } + + b.vtable = b.vtable[:0] + return objectOffset +} + +// EndObject writes data necessary to finish object construction. +func (b *Builder) EndObject() UOffsetT { + b.assertNested() + n := b.WriteVtable() + b.nested = false + return n +} + +// Doubles the size of the byteslice, and copies the old data towards the +// end of the new byteslice (since we build the buffer backwards). +func (b *Builder) growByteBuffer() { + if (int64(len(b.Bytes)) & int64(0xC0000000)) != 0 { + panic("cannot grow buffer beyond 2 gigabytes") + } + newLen := len(b.Bytes) * 2 + if newLen == 0 { + newLen = 1 + } + + if cap(b.Bytes) >= newLen { + b.Bytes = b.Bytes[:newLen] + } else { + extension := make([]byte, newLen-len(b.Bytes)) + b.Bytes = append(b.Bytes, extension...) + } + + middle := newLen / 2 + copy(b.Bytes[middle:], b.Bytes[:middle]) +} + +// Head gives the start of useful data in the underlying byte buffer. +// Note: unlike other functions, this value is interpreted as from the left. +func (b *Builder) Head() UOffsetT { + return b.head +} + +// Offset relative to the end of the buffer. +func (b *Builder) Offset() UOffsetT { + return UOffsetT(len(b.Bytes)) - b.head +} + +// Pad places zeros at the current offset. +func (b *Builder) Pad(n int) { + for i := 0; i < n; i++ { + b.PlaceByte(0) + } +} + +// Prep prepares to write an element of `size` after `additional_bytes` +// have been written, e.g. if you write a string, you need to align such +// the int length field is aligned to SizeInt32, and the string data follows it +// directly. +// If all you need to do is align, `additionalBytes` will be 0. +func (b *Builder) Prep(size, additionalBytes int) { + // Track the biggest thing we've ever aligned to. + if size > b.minalign { + b.minalign = size + } + // Find the amount of alignment needed such that `size` is properly + // aligned after `additionalBytes`: + alignSize := (^(len(b.Bytes) - int(b.Head()) + additionalBytes)) + 1 + alignSize &= (size - 1) + + // Reallocate the buffer if needed: + for int(b.head) <= alignSize+size+additionalBytes { + oldBufSize := len(b.Bytes) + b.growByteBuffer() + b.head += UOffsetT(len(b.Bytes) - oldBufSize) + } + b.Pad(alignSize) +} + +// PrependSOffsetT prepends an SOffsetT, relative to where it will be written. +func (b *Builder) PrependSOffsetT(off SOffsetT) { + b.Prep(SizeSOffsetT, 0) // Ensure alignment is already done. + if !(UOffsetT(off) <= b.Offset()) { + panic("unreachable: off <= b.Offset()") + } + off2 := SOffsetT(b.Offset()) - off + SOffsetT(SizeSOffsetT) + b.PlaceSOffsetT(off2) +} + +// PrependUOffsetT prepends an UOffsetT, relative to where it will be written. +func (b *Builder) PrependUOffsetT(off UOffsetT) { + b.Prep(SizeUOffsetT, 0) // Ensure alignment is already done. + if !(off <= b.Offset()) { + panic("unreachable: off <= b.Offset()") + } + off2 := b.Offset() - off + UOffsetT(SizeUOffsetT) + b.PlaceUOffsetT(off2) +} + +// StartVector initializes bookkeeping for writing a new vector. +// +// A vector has the following format: +// +// +, where T is the type of elements of this vector. +func (b *Builder) StartVector(elemSize, numElems, alignment int) UOffsetT { + b.assertNotNested() + b.nested = true + b.Prep(SizeUint32, elemSize*numElems) + b.Prep(alignment, elemSize*numElems) // Just in case alignment > int. + return b.Offset() +} + +// EndVector writes data necessary to finish vector construction. +func (b *Builder) EndVector(vectorNumElems int) UOffsetT { + b.assertNested() + + // we already made space for this, so write without PrependUint32 + b.PlaceUOffsetT(UOffsetT(vectorNumElems)) + + b.nested = false + return b.Offset() +} + +// CreateString writes a null-terminated string as a vector. +func (b *Builder) CreateString(s string) UOffsetT { + b.assertNotNested() + b.nested = true + + b.Prep(int(SizeUOffsetT), (len(s)+1)*SizeByte) + b.PlaceByte(0) + + l := UOffsetT(len(s)) + + b.head -= l + copy(b.Bytes[b.head:b.head+l], s) + + return b.EndVector(len(s)) +} + +// CreateByteString writes a byte slice as a string (null-terminated). +func (b *Builder) CreateByteString(s []byte) UOffsetT { + b.assertNotNested() + b.nested = true + + b.Prep(int(SizeUOffsetT), (len(s)+1)*SizeByte) + b.PlaceByte(0) + + l := UOffsetT(len(s)) + + b.head -= l + copy(b.Bytes[b.head:b.head+l], s) + + return b.EndVector(len(s)) +} + +// CreateByteVector writes a ubyte vector +func (b *Builder) CreateByteVector(v []byte) UOffsetT { + b.assertNotNested() + b.nested = true + + b.Prep(int(SizeUOffsetT), len(v)*SizeByte) + + l := UOffsetT(len(v)) + + b.head -= l + copy(b.Bytes[b.head:b.head+l], v) + + return b.EndVector(len(v)) +} + +func (b *Builder) assertNested() { + // If you get this assert, you're in an object while trying to write + // data that belongs outside of an object. + // To fix this, write non-inline data (like vectors) before creating + // objects. + if !b.nested { + panic("Incorrect creation order: must be inside object.") + } +} + +func (b *Builder) assertNotNested() { + // If you hit this, you're trying to construct a Table/Vector/String + // during the construction of its parent table (between the MyTableBuilder + // and builder.Finish()). + // Move the creation of these sub-objects to above the MyTableBuilder to + // not get this assert. + // Ignoring this assert may appear to work in simple cases, but the reason + // it is here is that storing objects in-line may cause vtable offsets + // to not fit anymore. It also leads to vtable duplication. + if b.nested { + panic("Incorrect creation order: object must not be nested.") + } +} + +func (b *Builder) assertFinished() { + // If you get this assert, you're attempting to get access a buffer + // which hasn't been finished yet. Be sure to call builder.Finish() + // with your root table. + // If you really need to access an unfinished buffer, use the Bytes + // buffer directly. + if !b.finished { + panic("Incorrect use of FinishedBytes(): must call 'Finish' first.") + } +} + +// PrependBoolSlot prepends a bool onto the object at vtable slot `o`. +// If value `x` equals default `d`, then the slot will be set to zero and no +// other data will be written. +func (b *Builder) PrependBoolSlot(o int, x, d bool) { + val := byte(0) + if x { + val = 1 + } + def := byte(0) + if d { + def = 1 + } + b.PrependByteSlot(o, val, def) +} + +// PrependByteSlot prepends a byte onto the object at vtable slot `o`. +// If value `x` equals default `d`, then the slot will be set to zero and no +// other data will be written. +func (b *Builder) PrependByteSlot(o int, x, d byte) { + if x != d { + b.PrependByte(x) + b.Slot(o) + } +} + +// PrependUint8Slot prepends a uint8 onto the object at vtable slot `o`. +// If value `x` equals default `d`, then the slot will be set to zero and no +// other data will be written. +func (b *Builder) PrependUint8Slot(o int, x, d uint8) { + if x != d { + b.PrependUint8(x) + b.Slot(o) + } +} + +// PrependUint16Slot prepends a uint16 onto the object at vtable slot `o`. +// If value `x` equals default `d`, then the slot will be set to zero and no +// other data will be written. +func (b *Builder) PrependUint16Slot(o int, x, d uint16) { + if x != d { + b.PrependUint16(x) + b.Slot(o) + } +} + +// PrependUint32Slot prepends a uint32 onto the object at vtable slot `o`. +// If value `x` equals default `d`, then the slot will be set to zero and no +// other data will be written. +func (b *Builder) PrependUint32Slot(o int, x, d uint32) { + if x != d { + b.PrependUint32(x) + b.Slot(o) + } +} + +// PrependUint64Slot prepends a uint64 onto the object at vtable slot `o`. +// If value `x` equals default `d`, then the slot will be set to zero and no +// other data will be written. +func (b *Builder) PrependUint64Slot(o int, x, d uint64) { + if x != d { + b.PrependUint64(x) + b.Slot(o) + } +} + +// PrependInt8Slot prepends a int8 onto the object at vtable slot `o`. +// If value `x` equals default `d`, then the slot will be set to zero and no +// other data will be written. +func (b *Builder) PrependInt8Slot(o int, x, d int8) { + if x != d { + b.PrependInt8(x) + b.Slot(o) + } +} + +// PrependInt16Slot prepends a int16 onto the object at vtable slot `o`. +// If value `x` equals default `d`, then the slot will be set to zero and no +// other data will be written. +func (b *Builder) PrependInt16Slot(o int, x, d int16) { + if x != d { + b.PrependInt16(x) + b.Slot(o) + } +} + +// PrependInt32Slot prepends a int32 onto the object at vtable slot `o`. +// If value `x` equals default `d`, then the slot will be set to zero and no +// other data will be written. +func (b *Builder) PrependInt32Slot(o int, x, d int32) { + if x != d { + b.PrependInt32(x) + b.Slot(o) + } +} + +// PrependInt64Slot prepends a int64 onto the object at vtable slot `o`. +// If value `x` equals default `d`, then the slot will be set to zero and no +// other data will be written. +func (b *Builder) PrependInt64Slot(o int, x, d int64) { + if x != d { + b.PrependInt64(x) + b.Slot(o) + } +} + +// PrependFloat32Slot prepends a float32 onto the object at vtable slot `o`. +// If value `x` equals default `d`, then the slot will be set to zero and no +// other data will be written. +func (b *Builder) PrependFloat32Slot(o int, x, d float32) { + if x != d { + b.PrependFloat32(x) + b.Slot(o) + } +} + +// PrependFloat64Slot prepends a float64 onto the object at vtable slot `o`. +// If value `x` equals default `d`, then the slot will be set to zero and no +// other data will be written. +func (b *Builder) PrependFloat64Slot(o int, x, d float64) { + if x != d { + b.PrependFloat64(x) + b.Slot(o) + } +} + +// PrependUOffsetTSlot prepends an UOffsetT onto the object at vtable slot `o`. +// If value `x` equals default `d`, then the slot will be set to zero and no +// other data will be written. +func (b *Builder) PrependUOffsetTSlot(o int, x, d UOffsetT) { + if x != d { + b.PrependUOffsetT(x) + b.Slot(o) + } +} + +// PrependStructSlot prepends a struct onto the object at vtable slot `o`. +// Structs are stored inline, so nothing additional is being added. +// In generated code, `d` is always 0. +func (b *Builder) PrependStructSlot(voffset int, x, d UOffsetT) { + if x != d { + b.assertNested() + if x != b.Offset() { + panic("inline data write outside of object") + } + b.Slot(voffset) + } +} + +// Slot sets the vtable key `voffset` to the current location in the buffer. +func (b *Builder) Slot(slotnum int) { + b.vtable[slotnum] = UOffsetT(b.Offset()) +} + +// FinishWithFileIdentifier finalizes a buffer, pointing to the given `rootTable`. +// as well as applys a file identifier +func (b *Builder) FinishWithFileIdentifier(rootTable UOffsetT, fid []byte) { + if fid == nil || len(fid) != fileIdentifierLength { + panic("incorrect file identifier length") + } + // In order to add a file identifier to the flatbuffer message, we need + // to prepare an alignment and file identifier length + b.Prep(b.minalign, SizeInt32+fileIdentifierLength) + for i := fileIdentifierLength - 1; i >= 0; i-- { + // place the file identifier + b.PlaceByte(fid[i]) + } + // finish + b.Finish(rootTable) +} + +// Finish finalizes a buffer, pointing to the given `rootTable`. +func (b *Builder) Finish(rootTable UOffsetT) { + b.assertNotNested() + b.Prep(b.minalign, SizeUOffsetT) + b.PrependUOffsetT(rootTable) + b.finished = true +} + +// vtableEqual compares an unwritten vtable to a written vtable. +func vtableEqual(a []UOffsetT, objectStart UOffsetT, b []byte) bool { + if len(a)*SizeVOffsetT != len(b) { + return false + } + + for i := 0; i < len(a); i++ { + x := GetVOffsetT(b[i*SizeVOffsetT : (i+1)*SizeVOffsetT]) + + // Skip vtable entries that indicate a default value. + if x == 0 && a[i] == 0 { + continue + } + + y := SOffsetT(objectStart) - SOffsetT(a[i]) + if SOffsetT(x) != y { + return false + } + } + return true +} + +// PrependBool prepends a bool to the Builder buffer. +// Aligns and checks for space. +func (b *Builder) PrependBool(x bool) { + b.Prep(SizeBool, 0) + b.PlaceBool(x) +} + +// PrependUint8 prepends a uint8 to the Builder buffer. +// Aligns and checks for space. +func (b *Builder) PrependUint8(x uint8) { + b.Prep(SizeUint8, 0) + b.PlaceUint8(x) +} + +// PrependUint16 prepends a uint16 to the Builder buffer. +// Aligns and checks for space. +func (b *Builder) PrependUint16(x uint16) { + b.Prep(SizeUint16, 0) + b.PlaceUint16(x) +} + +// PrependUint32 prepends a uint32 to the Builder buffer. +// Aligns and checks for space. +func (b *Builder) PrependUint32(x uint32) { + b.Prep(SizeUint32, 0) + b.PlaceUint32(x) +} + +// PrependUint64 prepends a uint64 to the Builder buffer. +// Aligns and checks for space. +func (b *Builder) PrependUint64(x uint64) { + b.Prep(SizeUint64, 0) + b.PlaceUint64(x) +} + +// PrependInt8 prepends a int8 to the Builder buffer. +// Aligns and checks for space. +func (b *Builder) PrependInt8(x int8) { + b.Prep(SizeInt8, 0) + b.PlaceInt8(x) +} + +// PrependInt16 prepends a int16 to the Builder buffer. +// Aligns and checks for space. +func (b *Builder) PrependInt16(x int16) { + b.Prep(SizeInt16, 0) + b.PlaceInt16(x) +} + +// PrependInt32 prepends a int32 to the Builder buffer. +// Aligns and checks for space. +func (b *Builder) PrependInt32(x int32) { + b.Prep(SizeInt32, 0) + b.PlaceInt32(x) +} + +// PrependInt64 prepends a int64 to the Builder buffer. +// Aligns and checks for space. +func (b *Builder) PrependInt64(x int64) { + b.Prep(SizeInt64, 0) + b.PlaceInt64(x) +} + +// PrependFloat32 prepends a float32 to the Builder buffer. +// Aligns and checks for space. +func (b *Builder) PrependFloat32(x float32) { + b.Prep(SizeFloat32, 0) + b.PlaceFloat32(x) +} + +// PrependFloat64 prepends a float64 to the Builder buffer. +// Aligns and checks for space. +func (b *Builder) PrependFloat64(x float64) { + b.Prep(SizeFloat64, 0) + b.PlaceFloat64(x) +} + +// PrependByte prepends a byte to the Builder buffer. +// Aligns and checks for space. +func (b *Builder) PrependByte(x byte) { + b.Prep(SizeByte, 0) + b.PlaceByte(x) +} + +// PrependVOffsetT prepends a VOffsetT to the Builder buffer. +// Aligns and checks for space. +func (b *Builder) PrependVOffsetT(x VOffsetT) { + b.Prep(SizeVOffsetT, 0) + b.PlaceVOffsetT(x) +} + +// PlaceBool prepends a bool to the Builder, without checking for space. +func (b *Builder) PlaceBool(x bool) { + b.head -= UOffsetT(SizeBool) + WriteBool(b.Bytes[b.head:], x) +} + +// PlaceUint8 prepends a uint8 to the Builder, without checking for space. +func (b *Builder) PlaceUint8(x uint8) { + b.head -= UOffsetT(SizeUint8) + WriteUint8(b.Bytes[b.head:], x) +} + +// PlaceUint16 prepends a uint16 to the Builder, without checking for space. +func (b *Builder) PlaceUint16(x uint16) { + b.head -= UOffsetT(SizeUint16) + WriteUint16(b.Bytes[b.head:], x) +} + +// PlaceUint32 prepends a uint32 to the Builder, without checking for space. +func (b *Builder) PlaceUint32(x uint32) { + b.head -= UOffsetT(SizeUint32) + WriteUint32(b.Bytes[b.head:], x) +} + +// PlaceUint64 prepends a uint64 to the Builder, without checking for space. +func (b *Builder) PlaceUint64(x uint64) { + b.head -= UOffsetT(SizeUint64) + WriteUint64(b.Bytes[b.head:], x) +} + +// PlaceInt8 prepends a int8 to the Builder, without checking for space. +func (b *Builder) PlaceInt8(x int8) { + b.head -= UOffsetT(SizeInt8) + WriteInt8(b.Bytes[b.head:], x) +} + +// PlaceInt16 prepends a int16 to the Builder, without checking for space. +func (b *Builder) PlaceInt16(x int16) { + b.head -= UOffsetT(SizeInt16) + WriteInt16(b.Bytes[b.head:], x) +} + +// PlaceInt32 prepends a int32 to the Builder, without checking for space. +func (b *Builder) PlaceInt32(x int32) { + b.head -= UOffsetT(SizeInt32) + WriteInt32(b.Bytes[b.head:], x) +} + +// PlaceInt64 prepends a int64 to the Builder, without checking for space. +func (b *Builder) PlaceInt64(x int64) { + b.head -= UOffsetT(SizeInt64) + WriteInt64(b.Bytes[b.head:], x) +} + +// PlaceFloat32 prepends a float32 to the Builder, without checking for space. +func (b *Builder) PlaceFloat32(x float32) { + b.head -= UOffsetT(SizeFloat32) + WriteFloat32(b.Bytes[b.head:], x) +} + +// PlaceFloat64 prepends a float64 to the Builder, without checking for space. +func (b *Builder) PlaceFloat64(x float64) { + b.head -= UOffsetT(SizeFloat64) + WriteFloat64(b.Bytes[b.head:], x) +} + +// PlaceByte prepends a byte to the Builder, without checking for space. +func (b *Builder) PlaceByte(x byte) { + b.head -= UOffsetT(SizeByte) + WriteByte(b.Bytes[b.head:], x) +} + +// PlaceVOffsetT prepends a VOffsetT to the Builder, without checking for space. +func (b *Builder) PlaceVOffsetT(x VOffsetT) { + b.head -= UOffsetT(SizeVOffsetT) + WriteVOffsetT(b.Bytes[b.head:], x) +} + +// PlaceSOffsetT prepends a SOffsetT to the Builder, without checking for space. +func (b *Builder) PlaceSOffsetT(x SOffsetT) { + b.head -= UOffsetT(SizeSOffsetT) + WriteSOffsetT(b.Bytes[b.head:], x) +} + +// PlaceUOffsetT prepends a UOffsetT to the Builder, without checking for space. +func (b *Builder) PlaceUOffsetT(x UOffsetT) { + b.head -= UOffsetT(SizeUOffsetT) + WriteUOffsetT(b.Bytes[b.head:], x) +} diff --git a/vendor/github.com/google/flatbuffers/go/doc.go b/vendor/github.com/google/flatbuffers/go/doc.go new file mode 100644 index 00000000000..694edc763d8 --- /dev/null +++ b/vendor/github.com/google/flatbuffers/go/doc.go @@ -0,0 +1,3 @@ +// Package flatbuffers provides facilities to read and write flatbuffers +// objects. +package flatbuffers diff --git a/vendor/github.com/google/flatbuffers/go/encode.go b/vendor/github.com/google/flatbuffers/go/encode.go new file mode 100644 index 00000000000..72d4f3a1521 --- /dev/null +++ b/vendor/github.com/google/flatbuffers/go/encode.go @@ -0,0 +1,238 @@ +package flatbuffers + +import ( + "math" +) + +type ( + // A SOffsetT stores a signed offset into arbitrary data. + SOffsetT int32 + // A UOffsetT stores an unsigned offset into vector data. + UOffsetT uint32 + // A VOffsetT stores an unsigned offset in a vtable. + VOffsetT uint16 +) + +const ( + // VtableMetadataFields is the count of metadata fields in each vtable. + VtableMetadataFields = 2 +) + +// GetByte decodes a little-endian byte from a byte slice. +func GetByte(buf []byte) byte { + return byte(GetUint8(buf)) +} + +// GetBool decodes a little-endian bool from a byte slice. +func GetBool(buf []byte) bool { + return buf[0] == 1 +} + +// GetUint8 decodes a little-endian uint8 from a byte slice. +func GetUint8(buf []byte) (n uint8) { + n = uint8(buf[0]) + return +} + +// GetUint16 decodes a little-endian uint16 from a byte slice. +func GetUint16(buf []byte) (n uint16) { + _ = buf[1] // Force one bounds check. See: golang.org/issue/14808 + n |= uint16(buf[0]) + n |= uint16(buf[1]) << 8 + return +} + +// GetUint32 decodes a little-endian uint32 from a byte slice. +func GetUint32(buf []byte) (n uint32) { + _ = buf[3] // Force one bounds check. See: golang.org/issue/14808 + n |= uint32(buf[0]) + n |= uint32(buf[1]) << 8 + n |= uint32(buf[2]) << 16 + n |= uint32(buf[3]) << 24 + return +} + +// GetUint64 decodes a little-endian uint64 from a byte slice. +func GetUint64(buf []byte) (n uint64) { + _ = buf[7] // Force one bounds check. See: golang.org/issue/14808 + n |= uint64(buf[0]) + n |= uint64(buf[1]) << 8 + n |= uint64(buf[2]) << 16 + n |= uint64(buf[3]) << 24 + n |= uint64(buf[4]) << 32 + n |= uint64(buf[5]) << 40 + n |= uint64(buf[6]) << 48 + n |= uint64(buf[7]) << 56 + return +} + +// GetInt8 decodes a little-endian int8 from a byte slice. +func GetInt8(buf []byte) (n int8) { + n = int8(buf[0]) + return +} + +// GetInt16 decodes a little-endian int16 from a byte slice. +func GetInt16(buf []byte) (n int16) { + _ = buf[1] // Force one bounds check. See: golang.org/issue/14808 + n |= int16(buf[0]) + n |= int16(buf[1]) << 8 + return +} + +// GetInt32 decodes a little-endian int32 from a byte slice. +func GetInt32(buf []byte) (n int32) { + _ = buf[3] // Force one bounds check. See: golang.org/issue/14808 + n |= int32(buf[0]) + n |= int32(buf[1]) << 8 + n |= int32(buf[2]) << 16 + n |= int32(buf[3]) << 24 + return +} + +// GetInt64 decodes a little-endian int64 from a byte slice. +func GetInt64(buf []byte) (n int64) { + _ = buf[7] // Force one bounds check. See: golang.org/issue/14808 + n |= int64(buf[0]) + n |= int64(buf[1]) << 8 + n |= int64(buf[2]) << 16 + n |= int64(buf[3]) << 24 + n |= int64(buf[4]) << 32 + n |= int64(buf[5]) << 40 + n |= int64(buf[6]) << 48 + n |= int64(buf[7]) << 56 + return +} + +// GetFloat32 decodes a little-endian float32 from a byte slice. +func GetFloat32(buf []byte) float32 { + x := GetUint32(buf) + return math.Float32frombits(x) +} + +// GetFloat64 decodes a little-endian float64 from a byte slice. +func GetFloat64(buf []byte) float64 { + x := GetUint64(buf) + return math.Float64frombits(x) +} + +// GetUOffsetT decodes a little-endian UOffsetT from a byte slice. +func GetUOffsetT(buf []byte) UOffsetT { + return UOffsetT(GetInt32(buf)) +} + +// GetSOffsetT decodes a little-endian SOffsetT from a byte slice. +func GetSOffsetT(buf []byte) SOffsetT { + return SOffsetT(GetInt32(buf)) +} + +// GetVOffsetT decodes a little-endian VOffsetT from a byte slice. +func GetVOffsetT(buf []byte) VOffsetT { + return VOffsetT(GetUint16(buf)) +} + +// WriteByte encodes a little-endian uint8 into a byte slice. +func WriteByte(buf []byte, n byte) { + WriteUint8(buf, uint8(n)) +} + +// WriteBool encodes a little-endian bool into a byte slice. +func WriteBool(buf []byte, b bool) { + buf[0] = 0 + if b { + buf[0] = 1 + } +} + +// WriteUint8 encodes a little-endian uint8 into a byte slice. +func WriteUint8(buf []byte, n uint8) { + buf[0] = byte(n) +} + +// WriteUint16 encodes a little-endian uint16 into a byte slice. +func WriteUint16(buf []byte, n uint16) { + _ = buf[1] // Force one bounds check. See: golang.org/issue/14808 + buf[0] = byte(n) + buf[1] = byte(n >> 8) +} + +// WriteUint32 encodes a little-endian uint32 into a byte slice. +func WriteUint32(buf []byte, n uint32) { + _ = buf[3] // Force one bounds check. See: golang.org/issue/14808 + buf[0] = byte(n) + buf[1] = byte(n >> 8) + buf[2] = byte(n >> 16) + buf[3] = byte(n >> 24) +} + +// WriteUint64 encodes a little-endian uint64 into a byte slice. +func WriteUint64(buf []byte, n uint64) { + _ = buf[7] // Force one bounds check. See: golang.org/issue/14808 + buf[0] = byte(n) + buf[1] = byte(n >> 8) + buf[2] = byte(n >> 16) + buf[3] = byte(n >> 24) + buf[4] = byte(n >> 32) + buf[5] = byte(n >> 40) + buf[6] = byte(n >> 48) + buf[7] = byte(n >> 56) +} + +// WriteInt8 encodes a little-endian int8 into a byte slice. +func WriteInt8(buf []byte, n int8) { + buf[0] = byte(n) +} + +// WriteInt16 encodes a little-endian int16 into a byte slice. +func WriteInt16(buf []byte, n int16) { + _ = buf[1] // Force one bounds check. See: golang.org/issue/14808 + buf[0] = byte(n) + buf[1] = byte(n >> 8) +} + +// WriteInt32 encodes a little-endian int32 into a byte slice. +func WriteInt32(buf []byte, n int32) { + _ = buf[3] // Force one bounds check. See: golang.org/issue/14808 + buf[0] = byte(n) + buf[1] = byte(n >> 8) + buf[2] = byte(n >> 16) + buf[3] = byte(n >> 24) +} + +// WriteInt64 encodes a little-endian int64 into a byte slice. +func WriteInt64(buf []byte, n int64) { + _ = buf[7] // Force one bounds check. See: golang.org/issue/14808 + buf[0] = byte(n) + buf[1] = byte(n >> 8) + buf[2] = byte(n >> 16) + buf[3] = byte(n >> 24) + buf[4] = byte(n >> 32) + buf[5] = byte(n >> 40) + buf[6] = byte(n >> 48) + buf[7] = byte(n >> 56) +} + +// WriteFloat32 encodes a little-endian float32 into a byte slice. +func WriteFloat32(buf []byte, n float32) { + WriteUint32(buf, math.Float32bits(n)) +} + +// WriteFloat64 encodes a little-endian float64 into a byte slice. +func WriteFloat64(buf []byte, n float64) { + WriteUint64(buf, math.Float64bits(n)) +} + +// WriteVOffsetT encodes a little-endian VOffsetT into a byte slice. +func WriteVOffsetT(buf []byte, n VOffsetT) { + WriteUint16(buf, uint16(n)) +} + +// WriteSOffsetT encodes a little-endian SOffsetT into a byte slice. +func WriteSOffsetT(buf []byte, n SOffsetT) { + WriteInt32(buf, int32(n)) +} + +// WriteUOffsetT encodes a little-endian UOffsetT into a byte slice. +func WriteUOffsetT(buf []byte, n UOffsetT) { + WriteUint32(buf, uint32(n)) +} diff --git a/vendor/github.com/google/flatbuffers/go/grpc.go b/vendor/github.com/google/flatbuffers/go/grpc.go new file mode 100644 index 00000000000..e7dabd3c69a --- /dev/null +++ b/vendor/github.com/google/flatbuffers/go/grpc.go @@ -0,0 +1,23 @@ +package flatbuffers + +// Codec implements gRPC-go Codec which is used to encode and decode messages. +var Codec = "flatbuffers" + +type FlatbuffersCodec struct{} + +func (FlatbuffersCodec) Marshal(v interface{}) ([]byte, error) { + return v.(*Builder).FinishedBytes(), nil +} + +func (FlatbuffersCodec) Unmarshal(data []byte, v interface{}) error { + v.(flatbuffersInit).Init(data, GetUOffsetT(data)) + return nil +} + +func (FlatbuffersCodec) String() string { + return Codec +} + +type flatbuffersInit interface { + Init(data []byte, i UOffsetT) +} diff --git a/vendor/github.com/google/flatbuffers/go/lib.go b/vendor/github.com/google/flatbuffers/go/lib.go new file mode 100644 index 00000000000..adfce52efe5 --- /dev/null +++ b/vendor/github.com/google/flatbuffers/go/lib.go @@ -0,0 +1,13 @@ +package flatbuffers + +// FlatBuffer is the interface that represents a flatbuffer. +type FlatBuffer interface { + Table() Table + Init(buf []byte, i UOffsetT) +} + +// GetRootAs is a generic helper to initialize a FlatBuffer with the provided buffer bytes and its data offset. +func GetRootAs(buf []byte, offset UOffsetT, fb FlatBuffer) { + n := GetUOffsetT(buf[offset:]) + fb.Init(buf, n+offset) +} diff --git a/vendor/github.com/google/flatbuffers/go/sizes.go b/vendor/github.com/google/flatbuffers/go/sizes.go new file mode 100644 index 00000000000..ba221698455 --- /dev/null +++ b/vendor/github.com/google/flatbuffers/go/sizes.go @@ -0,0 +1,55 @@ +package flatbuffers + +import ( + "unsafe" +) + +const ( + // See http://golang.org/ref/spec#Numeric_types + + // SizeUint8 is the byte size of a uint8. + SizeUint8 = 1 + // SizeUint16 is the byte size of a uint16. + SizeUint16 = 2 + // SizeUint32 is the byte size of a uint32. + SizeUint32 = 4 + // SizeUint64 is the byte size of a uint64. + SizeUint64 = 8 + + // SizeInt8 is the byte size of a int8. + SizeInt8 = 1 + // SizeInt16 is the byte size of a int16. + SizeInt16 = 2 + // SizeInt32 is the byte size of a int32. + SizeInt32 = 4 + // SizeInt64 is the byte size of a int64. + SizeInt64 = 8 + + // SizeFloat32 is the byte size of a float32. + SizeFloat32 = 4 + // SizeFloat64 is the byte size of a float64. + SizeFloat64 = 8 + + // SizeByte is the byte size of a byte. + // The `byte` type is aliased (by Go definition) to uint8. + SizeByte = 1 + + // SizeBool is the byte size of a bool. + // The `bool` type is aliased (by flatbuffers convention) to uint8. + SizeBool = 1 + + // SizeSOffsetT is the byte size of an SOffsetT. + // The `SOffsetT` type is aliased (by flatbuffers convention) to int32. + SizeSOffsetT = 4 + // SizeUOffsetT is the byte size of an UOffsetT. + // The `UOffsetT` type is aliased (by flatbuffers convention) to uint32. + SizeUOffsetT = 4 + // SizeVOffsetT is the byte size of an VOffsetT. + // The `VOffsetT` type is aliased (by flatbuffers convention) to uint16. + SizeVOffsetT = 2 +) + +// byteSliceToString converts a []byte to string without a heap allocation. +func byteSliceToString(b []byte) string { + return *(*string)(unsafe.Pointer(&b)) +} diff --git a/vendor/github.com/google/flatbuffers/go/struct.go b/vendor/github.com/google/flatbuffers/go/struct.go new file mode 100644 index 00000000000..11258f715d4 --- /dev/null +++ b/vendor/github.com/google/flatbuffers/go/struct.go @@ -0,0 +1,8 @@ +package flatbuffers + +// Struct wraps a byte slice and provides read access to its data. +// +// Structs do not have a vtable. +type Struct struct { + Table +} diff --git a/vendor/github.com/google/flatbuffers/go/table.go b/vendor/github.com/google/flatbuffers/go/table.go new file mode 100644 index 00000000000..b273146fad4 --- /dev/null +++ b/vendor/github.com/google/flatbuffers/go/table.go @@ -0,0 +1,505 @@ +package flatbuffers + +// Table wraps a byte slice and provides read access to its data. +// +// The variable `Pos` indicates the root of the FlatBuffers object therein. +type Table struct { + Bytes []byte + Pos UOffsetT // Always < 1<<31. +} + +// Offset provides access into the Table's vtable. +// +// Fields which are deprecated are ignored by checking against the vtable's length. +func (t *Table) Offset(vtableOffset VOffsetT) VOffsetT { + vtable := UOffsetT(SOffsetT(t.Pos) - t.GetSOffsetT(t.Pos)) + if vtableOffset < t.GetVOffsetT(vtable) { + return t.GetVOffsetT(vtable + UOffsetT(vtableOffset)) + } + return 0 +} + +// Indirect retrieves the relative offset stored at `offset`. +func (t *Table) Indirect(off UOffsetT) UOffsetT { + return off + GetUOffsetT(t.Bytes[off:]) +} + +// String gets a string from data stored inside the flatbuffer. +func (t *Table) String(off UOffsetT) string { + b := t.ByteVector(off) + return byteSliceToString(b) +} + +// ByteVector gets a byte slice from data stored inside the flatbuffer. +func (t *Table) ByteVector(off UOffsetT) []byte { + off += GetUOffsetT(t.Bytes[off:]) + start := off + UOffsetT(SizeUOffsetT) + length := GetUOffsetT(t.Bytes[off:]) + return t.Bytes[start : start+length] +} + +// VectorLen retrieves the length of the vector whose offset is stored at +// "off" in this object. +func (t *Table) VectorLen(off UOffsetT) int { + off += t.Pos + off += GetUOffsetT(t.Bytes[off:]) + return int(GetUOffsetT(t.Bytes[off:])) +} + +// Vector retrieves the start of data of the vector whose offset is stored +// at "off" in this object. +func (t *Table) Vector(off UOffsetT) UOffsetT { + off += t.Pos + x := off + GetUOffsetT(t.Bytes[off:]) + // data starts after metadata containing the vector length + x += UOffsetT(SizeUOffsetT) + return x +} + +// Union initializes any Table-derived type to point to the union at the given +// offset. +func (t *Table) Union(t2 *Table, off UOffsetT) { + off += t.Pos + t2.Pos = off + t.GetUOffsetT(off) + t2.Bytes = t.Bytes +} + +// GetBool retrieves a bool at the given offset. +func (t *Table) GetBool(off UOffsetT) bool { + return GetBool(t.Bytes[off:]) +} + +// GetByte retrieves a byte at the given offset. +func (t *Table) GetByte(off UOffsetT) byte { + return GetByte(t.Bytes[off:]) +} + +// GetUint8 retrieves a uint8 at the given offset. +func (t *Table) GetUint8(off UOffsetT) uint8 { + return GetUint8(t.Bytes[off:]) +} + +// GetUint16 retrieves a uint16 at the given offset. +func (t *Table) GetUint16(off UOffsetT) uint16 { + return GetUint16(t.Bytes[off:]) +} + +// GetUint32 retrieves a uint32 at the given offset. +func (t *Table) GetUint32(off UOffsetT) uint32 { + return GetUint32(t.Bytes[off:]) +} + +// GetUint64 retrieves a uint64 at the given offset. +func (t *Table) GetUint64(off UOffsetT) uint64 { + return GetUint64(t.Bytes[off:]) +} + +// GetInt8 retrieves a int8 at the given offset. +func (t *Table) GetInt8(off UOffsetT) int8 { + return GetInt8(t.Bytes[off:]) +} + +// GetInt16 retrieves a int16 at the given offset. +func (t *Table) GetInt16(off UOffsetT) int16 { + return GetInt16(t.Bytes[off:]) +} + +// GetInt32 retrieves a int32 at the given offset. +func (t *Table) GetInt32(off UOffsetT) int32 { + return GetInt32(t.Bytes[off:]) +} + +// GetInt64 retrieves a int64 at the given offset. +func (t *Table) GetInt64(off UOffsetT) int64 { + return GetInt64(t.Bytes[off:]) +} + +// GetFloat32 retrieves a float32 at the given offset. +func (t *Table) GetFloat32(off UOffsetT) float32 { + return GetFloat32(t.Bytes[off:]) +} + +// GetFloat64 retrieves a float64 at the given offset. +func (t *Table) GetFloat64(off UOffsetT) float64 { + return GetFloat64(t.Bytes[off:]) +} + +// GetUOffsetT retrieves a UOffsetT at the given offset. +func (t *Table) GetUOffsetT(off UOffsetT) UOffsetT { + return GetUOffsetT(t.Bytes[off:]) +} + +// GetVOffsetT retrieves a VOffsetT at the given offset. +func (t *Table) GetVOffsetT(off UOffsetT) VOffsetT { + return GetVOffsetT(t.Bytes[off:]) +} + +// GetSOffsetT retrieves a SOffsetT at the given offset. +func (t *Table) GetSOffsetT(off UOffsetT) SOffsetT { + return GetSOffsetT(t.Bytes[off:]) +} + +// GetBoolSlot retrieves the bool that the given vtable location +// points to. If the vtable value is zero, the default value `d` +// will be returned. +func (t *Table) GetBoolSlot(slot VOffsetT, d bool) bool { + off := t.Offset(slot) + if off == 0 { + return d + } + + return t.GetBool(t.Pos + UOffsetT(off)) +} + +// GetByteSlot retrieves the byte that the given vtable location +// points to. If the vtable value is zero, the default value `d` +// will be returned. +func (t *Table) GetByteSlot(slot VOffsetT, d byte) byte { + off := t.Offset(slot) + if off == 0 { + return d + } + + return t.GetByte(t.Pos + UOffsetT(off)) +} + +// GetInt8Slot retrieves the int8 that the given vtable location +// points to. If the vtable value is zero, the default value `d` +// will be returned. +func (t *Table) GetInt8Slot(slot VOffsetT, d int8) int8 { + off := t.Offset(slot) + if off == 0 { + return d + } + + return t.GetInt8(t.Pos + UOffsetT(off)) +} + +// GetUint8Slot retrieves the uint8 that the given vtable location +// points to. If the vtable value is zero, the default value `d` +// will be returned. +func (t *Table) GetUint8Slot(slot VOffsetT, d uint8) uint8 { + off := t.Offset(slot) + if off == 0 { + return d + } + + return t.GetUint8(t.Pos + UOffsetT(off)) +} + +// GetInt16Slot retrieves the int16 that the given vtable location +// points to. If the vtable value is zero, the default value `d` +// will be returned. +func (t *Table) GetInt16Slot(slot VOffsetT, d int16) int16 { + off := t.Offset(slot) + if off == 0 { + return d + } + + return t.GetInt16(t.Pos + UOffsetT(off)) +} + +// GetUint16Slot retrieves the uint16 that the given vtable location +// points to. If the vtable value is zero, the default value `d` +// will be returned. +func (t *Table) GetUint16Slot(slot VOffsetT, d uint16) uint16 { + off := t.Offset(slot) + if off == 0 { + return d + } + + return t.GetUint16(t.Pos + UOffsetT(off)) +} + +// GetInt32Slot retrieves the int32 that the given vtable location +// points to. If the vtable value is zero, the default value `d` +// will be returned. +func (t *Table) GetInt32Slot(slot VOffsetT, d int32) int32 { + off := t.Offset(slot) + if off == 0 { + return d + } + + return t.GetInt32(t.Pos + UOffsetT(off)) +} + +// GetUint32Slot retrieves the uint32 that the given vtable location +// points to. If the vtable value is zero, the default value `d` +// will be returned. +func (t *Table) GetUint32Slot(slot VOffsetT, d uint32) uint32 { + off := t.Offset(slot) + if off == 0 { + return d + } + + return t.GetUint32(t.Pos + UOffsetT(off)) +} + +// GetInt64Slot retrieves the int64 that the given vtable location +// points to. If the vtable value is zero, the default value `d` +// will be returned. +func (t *Table) GetInt64Slot(slot VOffsetT, d int64) int64 { + off := t.Offset(slot) + if off == 0 { + return d + } + + return t.GetInt64(t.Pos + UOffsetT(off)) +} + +// GetUint64Slot retrieves the uint64 that the given vtable location +// points to. If the vtable value is zero, the default value `d` +// will be returned. +func (t *Table) GetUint64Slot(slot VOffsetT, d uint64) uint64 { + off := t.Offset(slot) + if off == 0 { + return d + } + + return t.GetUint64(t.Pos + UOffsetT(off)) +} + +// GetFloat32Slot retrieves the float32 that the given vtable location +// points to. If the vtable value is zero, the default value `d` +// will be returned. +func (t *Table) GetFloat32Slot(slot VOffsetT, d float32) float32 { + off := t.Offset(slot) + if off == 0 { + return d + } + + return t.GetFloat32(t.Pos + UOffsetT(off)) +} + +// GetFloat64Slot retrieves the float64 that the given vtable location +// points to. If the vtable value is zero, the default value `d` +// will be returned. +func (t *Table) GetFloat64Slot(slot VOffsetT, d float64) float64 { + off := t.Offset(slot) + if off == 0 { + return d + } + + return t.GetFloat64(t.Pos + UOffsetT(off)) +} + +// GetVOffsetTSlot retrieves the VOffsetT that the given vtable location +// points to. If the vtable value is zero, the default value `d` +// will be returned. +func (t *Table) GetVOffsetTSlot(slot VOffsetT, d VOffsetT) VOffsetT { + off := t.Offset(slot) + if off == 0 { + return d + } + return VOffsetT(off) +} + +// MutateBool updates a bool at the given offset. +func (t *Table) MutateBool(off UOffsetT, n bool) bool { + WriteBool(t.Bytes[off:], n) + return true +} + +// MutateByte updates a Byte at the given offset. +func (t *Table) MutateByte(off UOffsetT, n byte) bool { + WriteByte(t.Bytes[off:], n) + return true +} + +// MutateUint8 updates a Uint8 at the given offset. +func (t *Table) MutateUint8(off UOffsetT, n uint8) bool { + WriteUint8(t.Bytes[off:], n) + return true +} + +// MutateUint16 updates a Uint16 at the given offset. +func (t *Table) MutateUint16(off UOffsetT, n uint16) bool { + WriteUint16(t.Bytes[off:], n) + return true +} + +// MutateUint32 updates a Uint32 at the given offset. +func (t *Table) MutateUint32(off UOffsetT, n uint32) bool { + WriteUint32(t.Bytes[off:], n) + return true +} + +// MutateUint64 updates a Uint64 at the given offset. +func (t *Table) MutateUint64(off UOffsetT, n uint64) bool { + WriteUint64(t.Bytes[off:], n) + return true +} + +// MutateInt8 updates a Int8 at the given offset. +func (t *Table) MutateInt8(off UOffsetT, n int8) bool { + WriteInt8(t.Bytes[off:], n) + return true +} + +// MutateInt16 updates a Int16 at the given offset. +func (t *Table) MutateInt16(off UOffsetT, n int16) bool { + WriteInt16(t.Bytes[off:], n) + return true +} + +// MutateInt32 updates a Int32 at the given offset. +func (t *Table) MutateInt32(off UOffsetT, n int32) bool { + WriteInt32(t.Bytes[off:], n) + return true +} + +// MutateInt64 updates a Int64 at the given offset. +func (t *Table) MutateInt64(off UOffsetT, n int64) bool { + WriteInt64(t.Bytes[off:], n) + return true +} + +// MutateFloat32 updates a Float32 at the given offset. +func (t *Table) MutateFloat32(off UOffsetT, n float32) bool { + WriteFloat32(t.Bytes[off:], n) + return true +} + +// MutateFloat64 updates a Float64 at the given offset. +func (t *Table) MutateFloat64(off UOffsetT, n float64) bool { + WriteFloat64(t.Bytes[off:], n) + return true +} + +// MutateUOffsetT updates a UOffsetT at the given offset. +func (t *Table) MutateUOffsetT(off UOffsetT, n UOffsetT) bool { + WriteUOffsetT(t.Bytes[off:], n) + return true +} + +// MutateVOffsetT updates a VOffsetT at the given offset. +func (t *Table) MutateVOffsetT(off UOffsetT, n VOffsetT) bool { + WriteVOffsetT(t.Bytes[off:], n) + return true +} + +// MutateSOffsetT updates a SOffsetT at the given offset. +func (t *Table) MutateSOffsetT(off UOffsetT, n SOffsetT) bool { + WriteSOffsetT(t.Bytes[off:], n) + return true +} + +// MutateBoolSlot updates the bool at given vtable location +func (t *Table) MutateBoolSlot(slot VOffsetT, n bool) bool { + if off := t.Offset(slot); off != 0 { + t.MutateBool(t.Pos+UOffsetT(off), n) + return true + } + + return false +} + +// MutateByteSlot updates the byte at given vtable location +func (t *Table) MutateByteSlot(slot VOffsetT, n byte) bool { + if off := t.Offset(slot); off != 0 { + t.MutateByte(t.Pos+UOffsetT(off), n) + return true + } + + return false +} + +// MutateInt8Slot updates the int8 at given vtable location +func (t *Table) MutateInt8Slot(slot VOffsetT, n int8) bool { + if off := t.Offset(slot); off != 0 { + t.MutateInt8(t.Pos+UOffsetT(off), n) + return true + } + + return false +} + +// MutateUint8Slot updates the uint8 at given vtable location +func (t *Table) MutateUint8Slot(slot VOffsetT, n uint8) bool { + if off := t.Offset(slot); off != 0 { + t.MutateUint8(t.Pos+UOffsetT(off), n) + return true + } + + return false +} + +// MutateInt16Slot updates the int16 at given vtable location +func (t *Table) MutateInt16Slot(slot VOffsetT, n int16) bool { + if off := t.Offset(slot); off != 0 { + t.MutateInt16(t.Pos+UOffsetT(off), n) + return true + } + + return false +} + +// MutateUint16Slot updates the uint16 at given vtable location +func (t *Table) MutateUint16Slot(slot VOffsetT, n uint16) bool { + if off := t.Offset(slot); off != 0 { + t.MutateUint16(t.Pos+UOffsetT(off), n) + return true + } + + return false +} + +// MutateInt32Slot updates the int32 at given vtable location +func (t *Table) MutateInt32Slot(slot VOffsetT, n int32) bool { + if off := t.Offset(slot); off != 0 { + t.MutateInt32(t.Pos+UOffsetT(off), n) + return true + } + + return false +} + +// MutateUint32Slot updates the uint32 at given vtable location +func (t *Table) MutateUint32Slot(slot VOffsetT, n uint32) bool { + if off := t.Offset(slot); off != 0 { + t.MutateUint32(t.Pos+UOffsetT(off), n) + return true + } + + return false +} + +// MutateInt64Slot updates the int64 at given vtable location +func (t *Table) MutateInt64Slot(slot VOffsetT, n int64) bool { + if off := t.Offset(slot); off != 0 { + t.MutateInt64(t.Pos+UOffsetT(off), n) + return true + } + + return false +} + +// MutateUint64Slot updates the uint64 at given vtable location +func (t *Table) MutateUint64Slot(slot VOffsetT, n uint64) bool { + if off := t.Offset(slot); off != 0 { + t.MutateUint64(t.Pos+UOffsetT(off), n) + return true + } + + return false +} + +// MutateFloat32Slot updates the float32 at given vtable location +func (t *Table) MutateFloat32Slot(slot VOffsetT, n float32) bool { + if off := t.Offset(slot); off != 0 { + t.MutateFloat32(t.Pos+UOffsetT(off), n) + return true + } + + return false +} + +// MutateFloat64Slot updates the float64 at given vtable location +func (t *Table) MutateFloat64Slot(slot VOffsetT, n float64) bool { + if off := t.Offset(slot); off != 0 { + t.MutateFloat64(t.Pos+UOffsetT(off), n) + return true + } + + return false +} diff --git a/vendor/github.com/grafana/grafana-plugin-sdk-go/.gitignore b/vendor/github.com/grafana/grafana-plugin-sdk-go/.gitignore new file mode 100644 index 00000000000..d1543d6fdf9 --- /dev/null +++ b/vendor/github.com/grafana/grafana-plugin-sdk-go/.gitignore @@ -0,0 +1,2 @@ +node_modules +./js/lib diff --git a/vendor/github.com/grafana/grafana-plugin-sdk-go/Makefile b/vendor/github.com/grafana/grafana-plugin-sdk-go/Makefile new file mode 100644 index 00000000000..aef205a5c71 --- /dev/null +++ b/vendor/github.com/grafana/grafana-plugin-sdk-go/Makefile @@ -0,0 +1,14 @@ +SRC_DIR=./proto +DST_DIR=./genproto + +all: build + +${DST_DIR}/datasource/datasource.pb.go: ${SRC_DIR}/datasource.proto + protoc -I=${SRC_DIR} --go_out=plugins=grpc:${DST_DIR}/datasource/ ${SRC_DIR}/datasource.proto + +build-proto: ${DST_DIR}/datasource/datasource.pb.go + +build: build-proto + go build ./... + +.PHONY: all build build-proto diff --git a/vendor/github.com/grafana/grafana-plugin-sdk-go/README.md b/vendor/github.com/grafana/grafana-plugin-sdk-go/README.md new file mode 100644 index 00000000000..7bec479b0f2 --- /dev/null +++ b/vendor/github.com/grafana/grafana-plugin-sdk-go/README.md @@ -0,0 +1,51 @@ +# Grafana Plugin SDK for Go + +Develop Grafana backend plugins with this Go SDK. + +**Warning**: This SDK is currently in alpha and will likely have major breaking changes during early development. Please do not consider this SDK published until this warning has been removed. + +## Usage + +```go +package main + +import ( + "context" + "log" + "os" + + gf "github.com/grafana/grafana-plugin-sdk-go" +) + +const pluginID = "myorg-custom-datasource" + +type MyDataSource struct { + logger *log.Logger +} + +func (d *MyDataSource) Query(ctx context.Context, tr gf.TimeRange, ds gf.DataSourceInfo, queries []gf.Query) ([]gf.QueryResult, error) { + return []gf.QueryResult{}, nil +} + +func main() { + logger := log.New(os.Stderr, "", 0) + + srv := gf.NewServer() + + srv.HandleDataSource(pluginID, &MyDataSource{ + logger: logger, + }) + + if err := srv.Serve(); err != nil { + logger.Fatal(err) + } +} +``` + +## Developing + +Generate Go code for Protobuf definitions: + +``` +make build-proto +``` diff --git a/vendor/github.com/grafana/grafana-plugin-sdk-go/dataframe/README.md b/vendor/github.com/grafana/grafana-plugin-sdk-go/dataframe/README.md new file mode 100644 index 00000000000..acdeab2963e --- /dev/null +++ b/vendor/github.com/grafana/grafana-plugin-sdk-go/dataframe/README.md @@ -0,0 +1,13 @@ +# dataframe + +## Testing + +This package uses [golden files](https://medium.com/soon-london/testing-with-golden-files-in-go-7fccc71c43d3) to verify Arrow serialization. + +Add the `update` flag to `go test` to update the golden files: + +``` +go test -update +``` + +Make sure you check in any updated golden files. diff --git a/vendor/github.com/grafana/grafana-plugin-sdk-go/dataframe/arrow.go b/vendor/github.com/grafana/grafana-plugin-sdk-go/dataframe/arrow.go new file mode 100644 index 00000000000..cd4480c8581 --- /dev/null +++ b/vendor/github.com/grafana/grafana-plugin-sdk-go/dataframe/arrow.go @@ -0,0 +1,262 @@ +package dataframe + +import ( + "fmt" + "io" + "time" + + "github.com/apache/arrow/go/arrow" + "github.com/apache/arrow/go/arrow/array" + "github.com/apache/arrow/go/arrow/ipc" + "github.com/apache/arrow/go/arrow/memory" + "github.com/mattetti/filebuffer" +) + +// MarshalArrow converts the Frame to an arrow table and returns a byte +// representation of that table. +func MarshalArrow(f *Frame) ([]byte, error) { + arrowFields, err := buildArrowFields(f) + if err != nil { + return nil, err + } + + schema, err := buildArrowSchema(f, arrowFields) + if err != nil { + return nil, err + } + + columns, err := buildArrowColumns(f, arrowFields) + if err != nil { + return nil, err + } + defer func(cols []array.Column) { + for _, col := range cols { + col.Release() + } + }(columns) + + // Create a table from the schema and columns. + table := array.NewTable(schema, columns, -1) + defer table.Release() + + tableReader := array.NewTableReader(table, -1) + defer tableReader.Release() + + // Arrow tables with the Go API are written to files, so we create a fake + // file buffer that the FileWriter can write to. In the future, and with + // streaming, I think will likely be using the Arrow message type some how. + fb := filebuffer.New(nil) + + fw, err := ipc.NewFileWriter(fb, ipc.WithSchema(tableReader.Schema())) + if err != nil { + return nil, err + } + + for tableReader.Next() { + rec := tableReader.Record() + + if err := fw.Write(rec); err != nil { + rec.Release() + return nil, err + } + rec.Release() + } + + if err := fw.Close(); err != nil { + return nil, err + } + + return fb.Buff.Bytes(), nil +} + +// buildArrowFields builds Arrow field definitions from a DataFrame. +func buildArrowFields(f *Frame) ([]arrow.Field, error) { + arrowFields := make([]arrow.Field, len(f.Fields)) + + for i, field := range f.Fields { + t, err := fieldToArrow(field.Type) + if err != nil { + return nil, err + } + + fieldMeta := map[string]string{ + "name": field.Name, + "type": field.Type.String(), + } + + arrowFields[i] = arrow.Field{ + Name: field.Name, + Type: t, + Metadata: arrow.MetadataFrom(fieldMeta), + Nullable: true, + } + } + + return arrowFields, nil +} + +// buildArrowColumns builds Arrow columns from a DataFrame. +func buildArrowColumns(f *Frame, arrowFields []arrow.Field) ([]array.Column, error) { + pool := memory.NewGoAllocator() + columns := make([]array.Column, len(f.Fields)) + + for fieldIdx, field := range f.Fields { + switch field.Type { + case FieldTypeNumber: + columns[fieldIdx] = *buildFloatColumn(pool, arrowFields[fieldIdx], field.Vector.(*floatVector)) + case FieldTypeTime: + columns[fieldIdx] = *buildTimeColumn(pool, arrowFields[fieldIdx], field.Vector.(*timeVector)) + default: + return nil, fmt.Errorf("unsupported field type: %s", field.Type) + } + } + return columns, nil +} + +func buildFloatColumn(pool memory.Allocator, field arrow.Field, vec *floatVector) *array.Column { + builder := array.NewFloat64Builder(pool) + defer builder.Release() + + for _, v := range *vec { + if v == nil { + builder.AppendNull() + continue + } + builder.Append(*v) + } + + chunked := array.NewChunked(field.Type, []array.Interface{builder.NewArray()}) + defer chunked.Release() + + return array.NewColumn(field, chunked) +} + +func buildTimeColumn(pool memory.Allocator, field arrow.Field, vec *timeVector) *array.Column { + builder := array.NewTimestampBuilder(pool, &arrow.TimestampType{ + Unit: arrow.Nanosecond, + }) + defer builder.Release() + + for _, v := range *vec { + if v == nil { + builder.AppendNull() + continue + } + builder.Append(arrow.Timestamp((*v).UnixNano())) + } + + chunked := array.NewChunked(field.Type, []array.Interface{builder.NewArray()}) + defer chunked.Release() + + return array.NewColumn(field, chunked) +} + +// buildArrowSchema builds an Arrow schema for a DataFrame. +func buildArrowSchema(f *Frame, fs []arrow.Field) (*arrow.Schema, error) { + tableMetaMap := map[string]string{ + "name": f.Name, + "refId": f.RefID, + } + + if f.Labels != nil { + tableMetaMap["labels"] = f.Labels.String() + } + + tableMeta := arrow.MetadataFrom(tableMetaMap) + + return arrow.NewSchema(fs, &tableMeta), nil +} + +// fieldToArrow returns the corresponding Arrow primitive type to the fields' +// Vector primitives. +func fieldToArrow(f FieldType) (arrow.DataType, error) { + switch f { + case FieldTypeString: + return &arrow.StringType{}, nil + case FieldTypeNumber: + return &arrow.Float64Type{}, nil + case FieldTypeTime: + return &arrow.TimestampType{}, nil + default: + return nil, fmt.Errorf("unsupported type: %s", f) + } +} + +// UnMarshalArrow converts a byte representation of an arrow table to a Frame +func UnMarshalArrow(b []byte) (*Frame, error) { + fB := filebuffer.New(b) + fR, err := ipc.NewFileReader(fB) + defer fR.Close() + if err != nil { + return nil, err + } + schema := fR.Schema() + metaData := schema.Metadata() + frame := &Frame{} + getMDKey := func(key string) (string, bool) { + idx := metaData.FindKey(key) + if idx < 0 { + return "", false + } + return metaData.Values()[idx], true + } + frame.Name, _ = getMDKey("name") // No need to check ok, zero value ("") is returned + frame.RefID, _ = getMDKey("refId") + if labelsAsString, ok := getMDKey("labels"); ok { + frame.Labels, err = LabelsFromString(labelsAsString) + if err != nil { + return nil, err + } + } + for _, field := range schema.Fields() { + sdkField := &Field{ + Name: field.Name, + } + switch field.Type.ID() { + case arrow.FLOAT64: + sdkField.Type = FieldTypeNumber + sdkField.Vector = newVector(FieldTypeNumber, 0) + case arrow.TIMESTAMP: + sdkField.Type = FieldTypeTime + sdkField.Vector = newVector(FieldTypeTime, 0) + default: + return nil, fmt.Errorf("unsupported arrow type %s for conversion", field.Type) + } + frame.Fields = append(frame.Fields, sdkField) + } + + rIdx := 0 + for { + record, err := fR.Read() + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + for i := 0; i < len(frame.Fields); i++ { + col := record.Column(i) + switch col.DataType().ID() { + case arrow.FLOAT64: + v := array.NewFloat64Data(col.Data()) + for _, f := range v.Float64Values() { + vF := f + vec := append(*frame.Fields[i].Vector.(*floatVector), &vF) + frame.Fields[i].Vector = &vec + } + case arrow.TIMESTAMP: + v := array.NewTimestampData(col.Data()) + for _, ts := range v.TimestampValues() { + t := time.Unix(0, int64(ts)) // nanosecond assumption + vec := append(*frame.Fields[i].Vector.(*timeVector), &t) + frame.Fields[i].Vector = &vec + } + default: + return nil, fmt.Errorf("unsupported arrow type %s for conversion", col.DataType().ID()) + } + } + rIdx++ + } + + return frame, nil +} diff --git a/vendor/github.com/grafana/grafana-plugin-sdk-go/dataframe/bool_vector.go b/vendor/github.com/grafana/grafana-plugin-sdk-go/dataframe/bool_vector.go new file mode 100644 index 00000000000..dc563a9764a --- /dev/null +++ b/vendor/github.com/grafana/grafana-plugin-sdk-go/dataframe/bool_vector.go @@ -0,0 +1,12 @@ +package dataframe + +type boolVector []*bool + +func newBoolVector(l int) *boolVector { + v := make(boolVector, l) + return &v +} + +func (v *boolVector) Set(i int, val interface{}) { (*v)[i] = val.(*bool) } +func (v *boolVector) At(i int) interface{} { return (*v)[i] } +func (v *boolVector) Len() int { return len(*v) } diff --git a/vendor/github.com/grafana/grafana-plugin-sdk-go/dataframe/dataframe.go b/vendor/github.com/grafana/grafana-plugin-sdk-go/dataframe/dataframe.go new file mode 100644 index 00000000000..b6095503948 --- /dev/null +++ b/vendor/github.com/grafana/grafana-plugin-sdk-go/dataframe/dataframe.go @@ -0,0 +1,230 @@ +package dataframe + +import ( + "fmt" + "sort" + "strings" + "time" +) + +// FieldType is used to describe the type of data in a field. +type FieldType int + +// All valid field types. +const ( + FieldTypeOther FieldType = iota + FieldTypeTime + FieldTypeNumber + FieldTypeString + FieldTypeBoolean +) + +func (f FieldType) String() string { + switch f { + case FieldTypeOther: + return "other" + case FieldTypeNumber: + return "number" + case FieldTypeString: + return "string" + case FieldTypeBoolean: + return "boolean" + case FieldTypeTime: + return "time" + default: + return "unknown" + } +} + +// Field represents a column of data with a specific type. +type Field struct { + Name string + Type FieldType + Vector Vector +} + +type Fields []*Field + +func assertFieldType(got, want FieldType) { + if got != want { + panic(fmt.Sprintf("values doesn't match the specified field type")) + } +} + +// NewField returns a new instance of Field. +func NewField(name string, fieldType FieldType, values interface{}) *Field { + var vec Vector + + switch v := values.(type) { + case []*float64: + assertFieldType(fieldType, FieldTypeNumber) + vec = newVector(fieldType, len(v)) + for i := 0; i < len(v); i++ { + vec.Set(i, v[i]) + } + case []*int64: + assertFieldType(fieldType, FieldTypeNumber) + vec = newVector(fieldType, len(v)) + for i := 0; i < len(v); i++ { + vec.Set(i, v[i]) + } + case []*time.Time: + assertFieldType(fieldType, FieldTypeTime) + vec = newVector(fieldType, len(v)) + for i := 0; i < len(v); i++ { + vec.Set(i, v[i]) + } + case []*string: + assertFieldType(fieldType, FieldTypeString) + vec = newVector(fieldType, len(v)) + for i := 0; i < len(v); i++ { + vec.Set(i, v[i]) + } + case []*bool: + assertFieldType(fieldType, FieldTypeBoolean) + vec = newVector(fieldType, len(v)) + for i := 0; i < len(v); i++ { + vec.Set(i, v[i]) + } + case []float64: + assertFieldType(fieldType, FieldTypeNumber) + vec = newVector(fieldType, len(v)) + for i := 0; i < len(v); i++ { + vec.Set(i, &v[i]) + } + case []int64: + assertFieldType(fieldType, FieldTypeNumber) + vec = newVector(fieldType, len(v)) + for i := 0; i < len(v); i++ { + vec.Set(i, &v[i]) + } + case []time.Time: + assertFieldType(fieldType, FieldTypeTime) + vec = newVector(fieldType, len(v)) + for i := 0; i < len(v); i++ { + vec.Set(i, &v[i]) + } + case []string: + assertFieldType(fieldType, FieldTypeString) + vec = newVector(fieldType, len(v)) + for i := 0; i < len(v); i++ { + vec.Set(i, &v[i]) + } + case []bool: + assertFieldType(fieldType, FieldTypeBoolean) + vec = newVector(fieldType, len(v)) + for i := 0; i < len(v); i++ { + vec.Set(i, &v[i]) + } + } + + return &Field{ + Name: name, + Type: fieldType, + Vector: vec, + } +} + +// Len returns the number of elements in the field. +func (f *Field) Len() int { + return f.Vector.Len() +} + +// Labels are used to add metadata to an object. +type Labels map[string]string + +// Equals returns true if the argument has the same k=v pairs as the receiver. +func (l Labels) Equals(arg Labels) bool { + if len(l) != len(arg) { + return false + } + for k, v := range l { + if argVal, ok := arg[k]; !ok || argVal != v { + return false + } + } + return true +} + +// Contains returns true if all k=v pairs of the argument are in the receiver. +func (l Labels) Contains(arg Labels) bool { + if len(arg) > len(l) { + return false + } + for k, v := range arg { + if argVal, ok := l[k]; !ok || argVal != v { + return false + } + } + return true +} + +func (l Labels) String() string { + // Better structure, should be sorted, copy prom probably + keys := make([]string, len(l)) + i := 0 + for k := range l { + keys[i] = k + i++ + } + sort.Strings(keys) + + var sb strings.Builder + + i = 0 + for _, k := range keys { + sb.WriteString(k) + sb.WriteString("=") + sb.WriteString(l[k]) + if i != len(keys)-1 { + sb.WriteString(", ") + } + i++ + } + return sb.String() +} + +// LabelsFromString parses the output of Labels.String() into +// a Labels object. It probably has some flaws. +func LabelsFromString(s string) (Labels, error) { + labels := make(map[string]string) + if s == "" { + return labels, nil + } + + for _, rawKV := range strings.Split(s, ", ") { + kV := strings.SplitN(rawKV, "=", 2) + if len(kV) != 2 { + return nil, fmt.Errorf(`invalid label key=value pair "%v"`, rawKV) + } + labels[kV[0]] = kV[1] + } + + return labels, nil +} + +// Frame represents a columnar storage with optional labels. +type Frame struct { + Name string + Labels Labels + Fields []*Field + + RefID string +} + +// New returns a new instance of a Frame. +func New(name string, labels Labels, fields ...*Field) *Frame { + return &Frame{ + Name: name, + Labels: labels, + Fields: fields, + } +} + +// Rows returns the number of rows in the frame. +func (f *Frame) Rows() int { + if len(f.Fields) > 0 { + return f.Fields[0].Len() + } + return 0 +} diff --git a/vendor/github.com/grafana/grafana-plugin-sdk-go/dataframe/float_vector.go b/vendor/github.com/grafana/grafana-plugin-sdk-go/dataframe/float_vector.go new file mode 100644 index 00000000000..d7047dfa20e --- /dev/null +++ b/vendor/github.com/grafana/grafana-plugin-sdk-go/dataframe/float_vector.go @@ -0,0 +1,11 @@ +package dataframe + +type floatVector []*float64 + +func newFloatVector(l int) *floatVector { + v := make(floatVector, l) + return &v +} +func (v *floatVector) Set(i int, val interface{}) { (*v)[i] = val.(*float64) } +func (v *floatVector) At(i int) interface{} { return (*v)[i] } +func (v *floatVector) Len() int { return len(*v) } diff --git a/vendor/github.com/grafana/grafana-plugin-sdk-go/dataframe/string_vector.go b/vendor/github.com/grafana/grafana-plugin-sdk-go/dataframe/string_vector.go new file mode 100644 index 00000000000..c029ef2959d --- /dev/null +++ b/vendor/github.com/grafana/grafana-plugin-sdk-go/dataframe/string_vector.go @@ -0,0 +1,12 @@ +package dataframe + +type stringVector []*string + +func newStringVector(l int) *stringVector { + v := make(stringVector, l) + return &v +} + +func (v *stringVector) Set(i int, val interface{}) { (*v)[i] = val.(*string) } +func (v *stringVector) At(i int) interface{} { return (*v)[i] } +func (v *stringVector) Len() int { return len(*v) } diff --git a/vendor/github.com/grafana/grafana-plugin-sdk-go/dataframe/time_vector.go b/vendor/github.com/grafana/grafana-plugin-sdk-go/dataframe/time_vector.go new file mode 100644 index 00000000000..6ccdf364510 --- /dev/null +++ b/vendor/github.com/grafana/grafana-plugin-sdk-go/dataframe/time_vector.go @@ -0,0 +1,14 @@ +package dataframe + +import "time" + +type timeVector []*time.Time + +func newTimeVector(l int) *timeVector { + v := make(timeVector, l) + return &v +} + +func (v *timeVector) Set(i int, val interface{}) { (*v)[i] = val.(*time.Time) } +func (v *timeVector) At(i int) interface{} { return (*v)[i] } +func (v *timeVector) Len() int { return len(*v) } diff --git a/vendor/github.com/grafana/grafana-plugin-sdk-go/dataframe/vector.go b/vendor/github.com/grafana/grafana-plugin-sdk-go/dataframe/vector.go new file mode 100644 index 00000000000..38b2bc65aa3 --- /dev/null +++ b/vendor/github.com/grafana/grafana-plugin-sdk-go/dataframe/vector.go @@ -0,0 +1,23 @@ +package dataframe + +// Vector represents a collection of Elements. +type Vector interface { + Set(idx int, i interface{}) + At(i int) interface{} + Len() int +} + +func newVector(t FieldType, n int) Vector { + switch t { + case FieldTypeNumber: + return newFloatVector(n) + case FieldTypeTime: + return newTimeVector(n) + case FieldTypeString: + return newStringVector(n) + case FieldTypeBoolean: + return newBoolVector(n) + default: + return nil + } +} diff --git a/vendor/github.com/grafana/grafana-plugin-sdk-go/datasource.go b/vendor/github.com/grafana/grafana-plugin-sdk-go/datasource.go new file mode 100644 index 00000000000..a141d2d4e11 --- /dev/null +++ b/vendor/github.com/grafana/grafana-plugin-sdk-go/datasource.go @@ -0,0 +1,185 @@ +package grafana + +import ( + "context" + "encoding/json" + "fmt" + "time" + + "github.com/grafana/grafana-plugin-sdk-go/dataframe" + "github.com/grafana/grafana-plugin-sdk-go/genproto/datasource" + plugin "github.com/hashicorp/go-plugin" +) + +// TimeRange represents a time range for a query. +type TimeRange struct { + From time.Time + To time.Time +} + +// DataSourceInfo holds metadata for the queried data source. +type DataSourceInfo struct { + ID int64 + OrgID int64 + Name string + Type string + URL string + JSONData json.RawMessage +} + +// Query represents the query as sent from the frontend. +type Query struct { + RefID string + MaxDataPoints int64 + Interval time.Duration + ModelJSON json.RawMessage +} + +// QueryResult holds the results for a given query. +type QueryResult struct { + Error string + RefID string + MetaJSON string + DataFrames []*dataframe.Frame +} + +// DataSourceHandler handles data source queries. +type DataSourceHandler interface { + Query(ctx context.Context, tr TimeRange, ds DataSourceInfo, queries []Query, api GrafanaAPIHandler) ([]QueryResult, error) +} + +// datasourcePluginWrapper converts to and from protobuf types. +type datasourcePluginWrapper struct { + plugin.NetRPCUnsupportedPlugin + + handler DataSourceHandler +} + +func (p *datasourcePluginWrapper) Query(ctx context.Context, req *datasource.DatasourceRequest, api GrafanaAPI) (*datasource.DatasourceResponse, error) { + tr := TimeRange{ + From: time.Unix(0, req.TimeRange.FromEpochMs*int64(time.Millisecond)), + To: time.Unix(0, req.TimeRange.ToEpochMs*int64(time.Millisecond)), + } + + dsi := DataSourceInfo{ + ID: req.Datasource.Id, + OrgID: req.Datasource.OrgId, + Name: req.Datasource.Name, + Type: req.Datasource.Type, + URL: req.Datasource.Url, + JSONData: json.RawMessage(req.Datasource.JsonData), + } + + var queries []Query + for _, q := range req.Queries { + queries = append(queries, Query{ + RefID: q.RefId, + MaxDataPoints: q.MaxDataPoints, + Interval: time.Duration(q.IntervalMs) * time.Millisecond, + ModelJSON: []byte(q.ModelJson), + }) + } + + results, err := p.handler.Query(ctx, tr, dsi, queries, &grafanaAPIWrapper{api: api}) + if err != nil { + return nil, err + } + + if len(results) == 0 { + return &datasource.DatasourceResponse{ + Results: []*datasource.QueryResult{}, + }, nil + } + + var respResults []*datasource.QueryResult + + for _, res := range results { + encodedFrames := make([][]byte, len(res.DataFrames)) + for dfIdx, df := range res.DataFrames { + if len(df.Fields) == 0 { + continue + } + encodedFrames[dfIdx], err = dataframe.MarshalArrow(df) + if err != nil { + return nil, err + } + } + + queryResult := &datasource.QueryResult{ + Error: res.Error, + RefId: res.RefID, + MetaJson: res.MetaJSON, + Dataframes: encodedFrames, + } + + respResults = append(respResults, queryResult) + } + + return &datasource.DatasourceResponse{ + Results: respResults, + }, nil +} + +// DatasourceQueryResult holds the results for a given query. +type DatasourceQueryResult struct { + Error string + RefID string + MetaJSON string + DataFrames []*dataframe.Frame +} + +// GrafanaAPIHandler handles data source queries. +type GrafanaAPIHandler interface { + QueryDatasource(ctx context.Context, orgID int64, datasourceID int64, tr TimeRange, queries []Query) ([]DatasourceQueryResult, error) +} + +// grafanaAPIWrapper converts to and from Grafana types for calls from a datasource. +type grafanaAPIWrapper struct { + api GrafanaAPI +} + +func (w *grafanaAPIWrapper) QueryDatasource(ctx context.Context, orgID int64, datasourceID int64, tr TimeRange, queries []Query) ([]DatasourceQueryResult, error) { + rawQueries := make([]*datasource.Query, 0, len(queries)) + + for _, q := range queries { + rawQueries = append(rawQueries, &datasource.Query{ + RefId: q.RefID, + MaxDataPoints: q.MaxDataPoints, + IntervalMs: q.Interval.Milliseconds(), + ModelJson: string(q.ModelJSON), + }) + } + + rawResp, err := w.api.QueryDatasource(ctx, &datasource.QueryDatasourceRequest{ + OrgId: orgID, + DatasourceId: datasourceID, + TimeRange: &datasource.TimeRange{ + FromEpochMs: tr.From.UnixNano() / 1e6, + ToEpochMs: tr.To.UnixNano() / 1e6, + FromRaw: fmt.Sprintf("%v", tr.From.UnixNano()/1e6), + ToRaw: fmt.Sprintf("%v", tr.To.UnixNano()/1e6), + }, + Queries: rawQueries, + }) + if err != nil { + return nil, err + } + + results := make([]DatasourceQueryResult, len(rawResp.GetResults())) + + for resIdx, rawRes := range rawResp.GetResults() { + // TODO Error property etc + dfs := make([]*dataframe.Frame, len(rawRes.Dataframes)) + for dfIdx, b := range rawRes.Dataframes { + dfs[dfIdx], err = dataframe.UnMarshalArrow(b) + if err != nil { + return nil, err + } + } + results[resIdx] = DatasourceQueryResult{ + DataFrames: dfs, + } + } + + return results, nil +} diff --git a/vendor/github.com/grafana/grafana-plugin-sdk-go/genproto/datasource/datasource.pb.go b/vendor/github.com/grafana/grafana-plugin-sdk-go/genproto/datasource/datasource.pb.go new file mode 100644 index 00000000000..26772993843 --- /dev/null +++ b/vendor/github.com/grafana/grafana-plugin-sdk-go/genproto/datasource/datasource.pb.go @@ -0,0 +1,690 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// source: datasource.proto + +package datasource + +import proto "github.com/golang/protobuf/proto" +import fmt "fmt" +import math "math" + +import ( + context "golang.org/x/net/context" + grpc "google.golang.org/grpc" +) + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package + +type DatasourceRequest struct { + TimeRange *TimeRange `protobuf:"bytes,1,opt,name=timeRange" json:"timeRange,omitempty"` + Datasource *DatasourceInfo `protobuf:"bytes,2,opt,name=datasource" json:"datasource,omitempty"` + Queries []*Query `protobuf:"bytes,3,rep,name=queries" json:"queries,omitempty"` + RequestId uint32 `protobuf:"varint,4,opt,name=requestId" json:"requestId,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *DatasourceRequest) Reset() { *m = DatasourceRequest{} } +func (m *DatasourceRequest) String() string { return proto.CompactTextString(m) } +func (*DatasourceRequest) ProtoMessage() {} +func (*DatasourceRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_datasource_edbaf4ee6ed114da, []int{0} +} +func (m *DatasourceRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_DatasourceRequest.Unmarshal(m, b) +} +func (m *DatasourceRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_DatasourceRequest.Marshal(b, m, deterministic) +} +func (dst *DatasourceRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_DatasourceRequest.Merge(dst, src) +} +func (m *DatasourceRequest) XXX_Size() int { + return xxx_messageInfo_DatasourceRequest.Size(m) +} +func (m *DatasourceRequest) XXX_DiscardUnknown() { + xxx_messageInfo_DatasourceRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_DatasourceRequest proto.InternalMessageInfo + +func (m *DatasourceRequest) GetTimeRange() *TimeRange { + if m != nil { + return m.TimeRange + } + return nil +} + +func (m *DatasourceRequest) GetDatasource() *DatasourceInfo { + if m != nil { + return m.Datasource + } + return nil +} + +func (m *DatasourceRequest) GetQueries() []*Query { + if m != nil { + return m.Queries + } + return nil +} + +func (m *DatasourceRequest) GetRequestId() uint32 { + if m != nil { + return m.RequestId + } + return 0 +} + +type Query struct { + RefId string `protobuf:"bytes,1,opt,name=refId" json:"refId,omitempty"` + MaxDataPoints int64 `protobuf:"varint,2,opt,name=maxDataPoints" json:"maxDataPoints,omitempty"` + IntervalMs int64 `protobuf:"varint,3,opt,name=intervalMs" json:"intervalMs,omitempty"` + ModelJson string `protobuf:"bytes,4,opt,name=modelJson" json:"modelJson,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *Query) Reset() { *m = Query{} } +func (m *Query) String() string { return proto.CompactTextString(m) } +func (*Query) ProtoMessage() {} +func (*Query) Descriptor() ([]byte, []int) { + return fileDescriptor_datasource_edbaf4ee6ed114da, []int{1} +} +func (m *Query) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_Query.Unmarshal(m, b) +} +func (m *Query) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_Query.Marshal(b, m, deterministic) +} +func (dst *Query) XXX_Merge(src proto.Message) { + xxx_messageInfo_Query.Merge(dst, src) +} +func (m *Query) XXX_Size() int { + return xxx_messageInfo_Query.Size(m) +} +func (m *Query) XXX_DiscardUnknown() { + xxx_messageInfo_Query.DiscardUnknown(m) +} + +var xxx_messageInfo_Query proto.InternalMessageInfo + +func (m *Query) GetRefId() string { + if m != nil { + return m.RefId + } + return "" +} + +func (m *Query) GetMaxDataPoints() int64 { + if m != nil { + return m.MaxDataPoints + } + return 0 +} + +func (m *Query) GetIntervalMs() int64 { + if m != nil { + return m.IntervalMs + } + return 0 +} + +func (m *Query) GetModelJson() string { + if m != nil { + return m.ModelJson + } + return "" +} + +type TimeRange struct { + FromRaw string `protobuf:"bytes,1,opt,name=fromRaw" json:"fromRaw,omitempty"` + ToRaw string `protobuf:"bytes,2,opt,name=toRaw" json:"toRaw,omitempty"` + FromEpochMs int64 `protobuf:"varint,3,opt,name=fromEpochMs" json:"fromEpochMs,omitempty"` + ToEpochMs int64 `protobuf:"varint,4,opt,name=toEpochMs" json:"toEpochMs,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *TimeRange) Reset() { *m = TimeRange{} } +func (m *TimeRange) String() string { return proto.CompactTextString(m) } +func (*TimeRange) ProtoMessage() {} +func (*TimeRange) Descriptor() ([]byte, []int) { + return fileDescriptor_datasource_edbaf4ee6ed114da, []int{2} +} +func (m *TimeRange) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_TimeRange.Unmarshal(m, b) +} +func (m *TimeRange) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_TimeRange.Marshal(b, m, deterministic) +} +func (dst *TimeRange) XXX_Merge(src proto.Message) { + xxx_messageInfo_TimeRange.Merge(dst, src) +} +func (m *TimeRange) XXX_Size() int { + return xxx_messageInfo_TimeRange.Size(m) +} +func (m *TimeRange) XXX_DiscardUnknown() { + xxx_messageInfo_TimeRange.DiscardUnknown(m) +} + +var xxx_messageInfo_TimeRange proto.InternalMessageInfo + +func (m *TimeRange) GetFromRaw() string { + if m != nil { + return m.FromRaw + } + return "" +} + +func (m *TimeRange) GetToRaw() string { + if m != nil { + return m.ToRaw + } + return "" +} + +func (m *TimeRange) GetFromEpochMs() int64 { + if m != nil { + return m.FromEpochMs + } + return 0 +} + +func (m *TimeRange) GetToEpochMs() int64 { + if m != nil { + return m.ToEpochMs + } + return 0 +} + +type DatasourceResponse struct { + Results []*QueryResult `protobuf:"bytes,1,rep,name=results" json:"results,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *DatasourceResponse) Reset() { *m = DatasourceResponse{} } +func (m *DatasourceResponse) String() string { return proto.CompactTextString(m) } +func (*DatasourceResponse) ProtoMessage() {} +func (*DatasourceResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_datasource_edbaf4ee6ed114da, []int{3} +} +func (m *DatasourceResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_DatasourceResponse.Unmarshal(m, b) +} +func (m *DatasourceResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_DatasourceResponse.Marshal(b, m, deterministic) +} +func (dst *DatasourceResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_DatasourceResponse.Merge(dst, src) +} +func (m *DatasourceResponse) XXX_Size() int { + return xxx_messageInfo_DatasourceResponse.Size(m) +} +func (m *DatasourceResponse) XXX_DiscardUnknown() { + xxx_messageInfo_DatasourceResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_DatasourceResponse proto.InternalMessageInfo + +func (m *DatasourceResponse) GetResults() []*QueryResult { + if m != nil { + return m.Results + } + return nil +} + +type QueryResult struct { + Error string `protobuf:"bytes,1,opt,name=error" json:"error,omitempty"` + RefId string `protobuf:"bytes,2,opt,name=refId" json:"refId,omitempty"` + MetaJson string `protobuf:"bytes,3,opt,name=metaJson" json:"metaJson,omitempty"` + Dataframes [][]byte `protobuf:"bytes,4,rep,name=dataframes,proto3" json:"dataframes,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *QueryResult) Reset() { *m = QueryResult{} } +func (m *QueryResult) String() string { return proto.CompactTextString(m) } +func (*QueryResult) ProtoMessage() {} +func (*QueryResult) Descriptor() ([]byte, []int) { + return fileDescriptor_datasource_edbaf4ee6ed114da, []int{4} +} +func (m *QueryResult) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_QueryResult.Unmarshal(m, b) +} +func (m *QueryResult) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_QueryResult.Marshal(b, m, deterministic) +} +func (dst *QueryResult) XXX_Merge(src proto.Message) { + xxx_messageInfo_QueryResult.Merge(dst, src) +} +func (m *QueryResult) XXX_Size() int { + return xxx_messageInfo_QueryResult.Size(m) +} +func (m *QueryResult) XXX_DiscardUnknown() { + xxx_messageInfo_QueryResult.DiscardUnknown(m) +} + +var xxx_messageInfo_QueryResult proto.InternalMessageInfo + +func (m *QueryResult) GetError() string { + if m != nil { + return m.Error + } + return "" +} + +func (m *QueryResult) GetRefId() string { + if m != nil { + return m.RefId + } + return "" +} + +func (m *QueryResult) GetMetaJson() string { + if m != nil { + return m.MetaJson + } + return "" +} + +func (m *QueryResult) GetDataframes() [][]byte { + if m != nil { + return m.Dataframes + } + return nil +} + +type DatasourceInfo struct { + Id int64 `protobuf:"varint,1,opt,name=id" json:"id,omitempty"` + OrgId int64 `protobuf:"varint,2,opt,name=orgId" json:"orgId,omitempty"` + Name string `protobuf:"bytes,3,opt,name=name" json:"name,omitempty"` + Type string `protobuf:"bytes,4,opt,name=type" json:"type,omitempty"` + Url string `protobuf:"bytes,5,opt,name=url" json:"url,omitempty"` + JsonData string `protobuf:"bytes,6,opt,name=jsonData" json:"jsonData,omitempty"` + DecryptedSecureJsonData map[string]string `protobuf:"bytes,7,rep,name=decryptedSecureJsonData" json:"decryptedSecureJsonData,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *DatasourceInfo) Reset() { *m = DatasourceInfo{} } +func (m *DatasourceInfo) String() string { return proto.CompactTextString(m) } +func (*DatasourceInfo) ProtoMessage() {} +func (*DatasourceInfo) Descriptor() ([]byte, []int) { + return fileDescriptor_datasource_edbaf4ee6ed114da, []int{5} +} +func (m *DatasourceInfo) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_DatasourceInfo.Unmarshal(m, b) +} +func (m *DatasourceInfo) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_DatasourceInfo.Marshal(b, m, deterministic) +} +func (dst *DatasourceInfo) XXX_Merge(src proto.Message) { + xxx_messageInfo_DatasourceInfo.Merge(dst, src) +} +func (m *DatasourceInfo) XXX_Size() int { + return xxx_messageInfo_DatasourceInfo.Size(m) +} +func (m *DatasourceInfo) XXX_DiscardUnknown() { + xxx_messageInfo_DatasourceInfo.DiscardUnknown(m) +} + +var xxx_messageInfo_DatasourceInfo proto.InternalMessageInfo + +func (m *DatasourceInfo) GetId() int64 { + if m != nil { + return m.Id + } + return 0 +} + +func (m *DatasourceInfo) GetOrgId() int64 { + if m != nil { + return m.OrgId + } + return 0 +} + +func (m *DatasourceInfo) GetName() string { + if m != nil { + return m.Name + } + return "" +} + +func (m *DatasourceInfo) GetType() string { + if m != nil { + return m.Type + } + return "" +} + +func (m *DatasourceInfo) GetUrl() string { + if m != nil { + return m.Url + } + return "" +} + +func (m *DatasourceInfo) GetJsonData() string { + if m != nil { + return m.JsonData + } + return "" +} + +func (m *DatasourceInfo) GetDecryptedSecureJsonData() map[string]string { + if m != nil { + return m.DecryptedSecureJsonData + } + return nil +} + +type QueryDatasourceRequest struct { + TimeRange *TimeRange `protobuf:"bytes,1,opt,name=timeRange" json:"timeRange,omitempty"` + DatasourceId int64 `protobuf:"varint,2,opt,name=datasourceId" json:"datasourceId,omitempty"` + Queries []*Query `protobuf:"bytes,3,rep,name=queries" json:"queries,omitempty"` + OrgId int64 `protobuf:"varint,4,opt,name=orgId" json:"orgId,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *QueryDatasourceRequest) Reset() { *m = QueryDatasourceRequest{} } +func (m *QueryDatasourceRequest) String() string { return proto.CompactTextString(m) } +func (*QueryDatasourceRequest) ProtoMessage() {} +func (*QueryDatasourceRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_datasource_edbaf4ee6ed114da, []int{6} +} +func (m *QueryDatasourceRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_QueryDatasourceRequest.Unmarshal(m, b) +} +func (m *QueryDatasourceRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_QueryDatasourceRequest.Marshal(b, m, deterministic) +} +func (dst *QueryDatasourceRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_QueryDatasourceRequest.Merge(dst, src) +} +func (m *QueryDatasourceRequest) XXX_Size() int { + return xxx_messageInfo_QueryDatasourceRequest.Size(m) +} +func (m *QueryDatasourceRequest) XXX_DiscardUnknown() { + xxx_messageInfo_QueryDatasourceRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_QueryDatasourceRequest proto.InternalMessageInfo + +func (m *QueryDatasourceRequest) GetTimeRange() *TimeRange { + if m != nil { + return m.TimeRange + } + return nil +} + +func (m *QueryDatasourceRequest) GetDatasourceId() int64 { + if m != nil { + return m.DatasourceId + } + return 0 +} + +func (m *QueryDatasourceRequest) GetQueries() []*Query { + if m != nil { + return m.Queries + } + return nil +} + +func (m *QueryDatasourceRequest) GetOrgId() int64 { + if m != nil { + return m.OrgId + } + return 0 +} + +type QueryDatasourceResponse struct { + Results []*QueryResult `protobuf:"bytes,1,rep,name=results" json:"results,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *QueryDatasourceResponse) Reset() { *m = QueryDatasourceResponse{} } +func (m *QueryDatasourceResponse) String() string { return proto.CompactTextString(m) } +func (*QueryDatasourceResponse) ProtoMessage() {} +func (*QueryDatasourceResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_datasource_edbaf4ee6ed114da, []int{7} +} +func (m *QueryDatasourceResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_QueryDatasourceResponse.Unmarshal(m, b) +} +func (m *QueryDatasourceResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_QueryDatasourceResponse.Marshal(b, m, deterministic) +} +func (dst *QueryDatasourceResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_QueryDatasourceResponse.Merge(dst, src) +} +func (m *QueryDatasourceResponse) XXX_Size() int { + return xxx_messageInfo_QueryDatasourceResponse.Size(m) +} +func (m *QueryDatasourceResponse) XXX_DiscardUnknown() { + xxx_messageInfo_QueryDatasourceResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_QueryDatasourceResponse proto.InternalMessageInfo + +func (m *QueryDatasourceResponse) GetResults() []*QueryResult { + if m != nil { + return m.Results + } + return nil +} + +func init() { + proto.RegisterType((*DatasourceRequest)(nil), "modelsV2.DatasourceRequest") + proto.RegisterType((*Query)(nil), "modelsV2.Query") + proto.RegisterType((*TimeRange)(nil), "modelsV2.TimeRange") + proto.RegisterType((*DatasourceResponse)(nil), "modelsV2.DatasourceResponse") + proto.RegisterType((*QueryResult)(nil), "modelsV2.QueryResult") + proto.RegisterType((*DatasourceInfo)(nil), "modelsV2.DatasourceInfo") + proto.RegisterMapType((map[string]string)(nil), "modelsV2.DatasourceInfo.DecryptedSecureJsonDataEntry") + proto.RegisterType((*QueryDatasourceRequest)(nil), "modelsV2.QueryDatasourceRequest") + proto.RegisterType((*QueryDatasourceResponse)(nil), "modelsV2.QueryDatasourceResponse") +} + +// Reference imports to suppress errors if they are not otherwise used. +var _ context.Context +var _ grpc.ClientConn + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +const _ = grpc.SupportPackageIsVersion4 + +// DatasourcePluginClient is the client API for DatasourcePlugin service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. +type DatasourcePluginClient interface { + Query(ctx context.Context, in *DatasourceRequest, opts ...grpc.CallOption) (*DatasourceResponse, error) +} + +type datasourcePluginClient struct { + cc *grpc.ClientConn +} + +func NewDatasourcePluginClient(cc *grpc.ClientConn) DatasourcePluginClient { + return &datasourcePluginClient{cc} +} + +func (c *datasourcePluginClient) Query(ctx context.Context, in *DatasourceRequest, opts ...grpc.CallOption) (*DatasourceResponse, error) { + out := new(DatasourceResponse) + err := c.cc.Invoke(ctx, "/modelsV2.DatasourcePlugin/Query", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// Server API for DatasourcePlugin service + +type DatasourcePluginServer interface { + Query(context.Context, *DatasourceRequest) (*DatasourceResponse, error) +} + +func RegisterDatasourcePluginServer(s *grpc.Server, srv DatasourcePluginServer) { + s.RegisterService(&_DatasourcePlugin_serviceDesc, srv) +} + +func _DatasourcePlugin_Query_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(DatasourceRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(DatasourcePluginServer).Query(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/modelsV2.DatasourcePlugin/Query", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(DatasourcePluginServer).Query(ctx, req.(*DatasourceRequest)) + } + return interceptor(ctx, in, info, handler) +} + +var _DatasourcePlugin_serviceDesc = grpc.ServiceDesc{ + ServiceName: "modelsV2.DatasourcePlugin", + HandlerType: (*DatasourcePluginServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "Query", + Handler: _DatasourcePlugin_Query_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "datasource.proto", +} + +// GrafanaAPIClient is the client API for GrafanaAPI service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. +type GrafanaAPIClient interface { + QueryDatasource(ctx context.Context, in *QueryDatasourceRequest, opts ...grpc.CallOption) (*QueryDatasourceResponse, error) +} + +type grafanaAPIClient struct { + cc *grpc.ClientConn +} + +func NewGrafanaAPIClient(cc *grpc.ClientConn) GrafanaAPIClient { + return &grafanaAPIClient{cc} +} + +func (c *grafanaAPIClient) QueryDatasource(ctx context.Context, in *QueryDatasourceRequest, opts ...grpc.CallOption) (*QueryDatasourceResponse, error) { + out := new(QueryDatasourceResponse) + err := c.cc.Invoke(ctx, "/modelsV2.GrafanaAPI/QueryDatasource", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// Server API for GrafanaAPI service + +type GrafanaAPIServer interface { + QueryDatasource(context.Context, *QueryDatasourceRequest) (*QueryDatasourceResponse, error) +} + +func RegisterGrafanaAPIServer(s *grpc.Server, srv GrafanaAPIServer) { + s.RegisterService(&_GrafanaAPI_serviceDesc, srv) +} + +func _GrafanaAPI_QueryDatasource_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(QueryDatasourceRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(GrafanaAPIServer).QueryDatasource(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/modelsV2.GrafanaAPI/QueryDatasource", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(GrafanaAPIServer).QueryDatasource(ctx, req.(*QueryDatasourceRequest)) + } + return interceptor(ctx, in, info, handler) +} + +var _GrafanaAPI_serviceDesc = grpc.ServiceDesc{ + ServiceName: "modelsV2.GrafanaAPI", + HandlerType: (*GrafanaAPIServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "QueryDatasource", + Handler: _GrafanaAPI_QueryDatasource_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "datasource.proto", +} + +func init() { proto.RegisterFile("datasource.proto", fileDescriptor_datasource_edbaf4ee6ed114da) } + +var fileDescriptor_datasource_edbaf4ee6ed114da = []byte{ + // 599 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x94, 0x4f, 0x6f, 0xd3, 0x4c, + 0x10, 0xc6, 0x65, 0x3b, 0x69, 0x9a, 0x49, 0xda, 0xe6, 0xdd, 0x17, 0xa8, 0x15, 0x22, 0x14, 0x2c, + 0x0e, 0xe1, 0x12, 0x44, 0x10, 0x52, 0xc5, 0x8d, 0xaa, 0x11, 0x4a, 0x24, 0xa4, 0xb0, 0xa0, 0x1c, + 0xb8, 0x2d, 0xf1, 0x24, 0x18, 0x6c, 0x6f, 0xba, 0x5e, 0x17, 0x22, 0x4e, 0x7c, 0x23, 0xbe, 0x46, + 0xbf, 0x15, 0xda, 0xf5, 0x9f, 0x75, 0xd2, 0xb4, 0x12, 0x88, 0xdb, 0xce, 0x33, 0x33, 0x9e, 0x9f, + 0x9e, 0xf1, 0x2e, 0x74, 0x7c, 0x26, 0x59, 0xc2, 0x53, 0xb1, 0xc0, 0xe1, 0x5a, 0x70, 0xc9, 0xc9, + 0x61, 0xc4, 0x7d, 0x0c, 0x93, 0xf9, 0xc8, 0xbb, 0xb6, 0xe0, 0xbf, 0x8b, 0x32, 0x4d, 0xf1, 0x32, + 0xc5, 0x44, 0x92, 0xe7, 0xd0, 0x94, 0x41, 0x84, 0x94, 0xc5, 0x2b, 0x74, 0xad, 0xbe, 0x35, 0x68, + 0x8d, 0xfe, 0x1f, 0x16, 0x3d, 0xc3, 0x0f, 0x45, 0x8a, 0x9a, 0x2a, 0x72, 0x06, 0x60, 0xc6, 0xb8, + 0xb6, 0xee, 0x71, 0x4d, 0x8f, 0x99, 0x31, 0x89, 0x97, 0x9c, 0x56, 0x6a, 0xc9, 0x53, 0x68, 0x5c, + 0xa6, 0x28, 0x02, 0x4c, 0x5c, 0xa7, 0xef, 0x0c, 0x5a, 0xa3, 0x13, 0xd3, 0xf6, 0x2e, 0x45, 0xb1, + 0xa1, 0x45, 0x9e, 0xf4, 0xa0, 0x29, 0x32, 0xc4, 0x89, 0xef, 0xd6, 0xfa, 0xd6, 0xe0, 0x88, 0x1a, + 0xc1, 0xfb, 0x69, 0x41, 0x5d, 0x37, 0x90, 0x7b, 0x50, 0x17, 0xb8, 0x9c, 0xf8, 0x9a, 0xbd, 0x49, + 0xb3, 0x80, 0x3c, 0x81, 0xa3, 0x88, 0x7d, 0x57, 0x24, 0x33, 0x1e, 0xc4, 0x32, 0xd1, 0x94, 0x0e, + 0xdd, 0x16, 0xc9, 0x23, 0x80, 0x20, 0x96, 0x28, 0xae, 0x58, 0xf8, 0x56, 0x11, 0xa9, 0x92, 0x8a, + 0xa2, 0x18, 0x34, 0xde, 0x34, 0xe1, 0xb1, 0x66, 0x68, 0x52, 0x23, 0x78, 0x3f, 0xa0, 0x59, 0xda, + 0x43, 0x5c, 0x68, 0x2c, 0x05, 0x8f, 0x28, 0xfb, 0x96, 0x83, 0x14, 0xa1, 0x02, 0x94, 0x5c, 0xe9, + 0x76, 0x06, 0xa8, 0x03, 0xd2, 0x87, 0x96, 0x2a, 0x18, 0xaf, 0xf9, 0xe2, 0x73, 0x39, 0xbb, 0x2a, + 0xa9, 0xe1, 0x92, 0x17, 0xf9, 0x9a, 0xce, 0x1b, 0xc1, 0x1b, 0x03, 0xa9, 0xee, 0x32, 0x59, 0xf3, + 0x38, 0x41, 0xf2, 0x0c, 0x1a, 0x02, 0x93, 0x34, 0x94, 0x89, 0x6b, 0x69, 0x7f, 0xef, 0xef, 0xfa, + 0xab, 0xb3, 0xb4, 0xa8, 0xf2, 0x52, 0x68, 0x55, 0x74, 0xc5, 0x8a, 0x42, 0x70, 0x51, 0x98, 0xa9, + 0x03, 0x63, 0xb1, 0x5d, 0xb5, 0xb8, 0x0b, 0x87, 0x11, 0x4a, 0xa6, 0xbd, 0x71, 0x74, 0xa2, 0x8c, + 0x95, 0xb1, 0x6a, 0xeb, 0x4b, 0xc1, 0x22, 0x54, 0xf0, 0xce, 0xa0, 0x4d, 0x2b, 0x8a, 0x77, 0x6d, + 0xc3, 0xf1, 0xf6, 0x6f, 0x42, 0x8e, 0xc1, 0x0e, 0xb2, 0x25, 0x3a, 0xd4, 0x0e, 0x7c, 0x35, 0x94, + 0x8b, 0x55, 0x3e, 0xd4, 0xa1, 0x59, 0x40, 0x08, 0xd4, 0x62, 0x16, 0x61, 0x3e, 0x50, 0x9f, 0x95, + 0x26, 0x37, 0x6b, 0xcc, 0x17, 0xa4, 0xcf, 0xa4, 0x03, 0x4e, 0x2a, 0x42, 0xb7, 0xae, 0x25, 0x75, + 0x54, 0xb8, 0x5f, 0x12, 0x1e, 0xab, 0xa9, 0xee, 0x41, 0x86, 0x5b, 0xc4, 0x84, 0xc3, 0xa9, 0x8f, + 0x0b, 0xb1, 0x59, 0x4b, 0xf4, 0xdf, 0xe3, 0x22, 0x15, 0x38, 0x2d, 0x4a, 0x1b, 0xda, 0xc6, 0x97, + 0xb7, 0xfd, 0xdd, 0xc3, 0x8b, 0xfd, 0x7d, 0xe3, 0x58, 0x8a, 0x0d, 0xbd, 0xed, 0xab, 0xdd, 0x29, + 0xf4, 0xee, 0x6a, 0x54, 0xf8, 0x5f, 0x71, 0x93, 0x6f, 0x41, 0x1d, 0x95, 0x1d, 0x57, 0x2c, 0x4c, + 0xb1, 0xd8, 0x81, 0x0e, 0x5e, 0xd9, 0x67, 0x96, 0xf7, 0xcb, 0x82, 0x07, 0x7a, 0x87, 0xff, 0xe4, + 0x6e, 0x7b, 0xd0, 0x36, 0xf7, 0xb5, 0x74, 0x7f, 0x4b, 0xfb, 0x93, 0x5b, 0x5c, 0x6e, 0xb1, 0x56, + 0xd9, 0xa2, 0x37, 0x85, 0xd3, 0x1b, 0xc4, 0x7f, 0xf9, 0x07, 0x8f, 0xe6, 0xd0, 0x31, 0x9f, 0x99, + 0x85, 0xe9, 0x2a, 0x88, 0xc9, 0x79, 0xf1, 0x38, 0x3c, 0xdc, 0xb7, 0xb7, 0xdc, 0x9d, 0x6e, 0x6f, + 0x7f, 0x32, 0x03, 0x19, 0xf9, 0x00, 0x6f, 0x04, 0x5b, 0xb2, 0x98, 0xbd, 0x9e, 0x4d, 0xc8, 0x1c, + 0x4e, 0x76, 0x88, 0x49, 0x7f, 0x07, 0xec, 0xe6, 0x80, 0xc7, 0x77, 0x54, 0x64, 0x53, 0xce, 0xdb, + 0x1f, 0x2b, 0xcf, 0xe3, 0xa7, 0x03, 0xfd, 0x64, 0xbf, 0xf8, 0x1d, 0x00, 0x00, 0xff, 0xff, 0xc1, + 0xf0, 0xb5, 0x97, 0xc6, 0x05, 0x00, 0x00, +} diff --git a/vendor/github.com/grafana/grafana-plugin-sdk-go/go.mod b/vendor/github.com/grafana/grafana-plugin-sdk-go/go.mod new file mode 100644 index 00000000000..0aa0cb531c4 --- /dev/null +++ b/vendor/github.com/grafana/grafana-plugin-sdk-go/go.mod @@ -0,0 +1,14 @@ +module github.com/grafana/grafana-plugin-sdk-go + +go 1.12 + +require ( + github.com/apache/arrow/go/arrow v0.0.0-20190716210558-5f564424c71c + github.com/golang/protobuf v1.3.2 + github.com/hashicorp/go-hclog v0.0.0-20180709165350-ff2cf002a8dd + github.com/hashicorp/go-plugin v1.0.1 + github.com/mattetti/filebuffer v1.0.0 + github.com/stretchr/testify v1.3.0 + golang.org/x/net v0.0.0-20190311183353-d8887717615a + google.golang.org/grpc v1.23.1 +) diff --git a/vendor/github.com/grafana/grafana-plugin-sdk-go/go.sum b/vendor/github.com/grafana/grafana-plugin-sdk-go/go.sum new file mode 100644 index 00000000000..cf823544a5b --- /dev/null +++ b/vendor/github.com/grafana/grafana-plugin-sdk-go/go.sum @@ -0,0 +1,59 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/apache/arrow/go/arrow v0.0.0-20190716210558-5f564424c71c h1:iHUHzx3S1TU5xt+D7vLb0PAk3e+RfayF9IhR6+hyO/k= +github.com/apache/arrow/go/arrow v0.0.0-20190716210558-5f564424c71c/go.mod h1:VTxUBvSJ3s3eHAg65PNgrsn5BtqCRPdmyXh6rAfdxN0= +github.com/apache/arrow/go/arrow v0.0.0-20190926121000-5b4a08f3d2cf h1:yvUZ/QVWcn9Et/v+9lnKHqKySB3D4G4MkIr5UN2MvTk= +github.com/apache/arrow/go/arrow v0.0.0-20190926121000-5b4a08f3d2cf/go.mod h1:VTxUBvSJ3s3eHAg65PNgrsn5BtqCRPdmyXh6rAfdxN0= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/google/flatbuffers v1.11.0 h1:O7CEyB8Cb3/DmtxODGtLHcEvpr81Jm5qLg/hsHnxA2A= +github.com/google/flatbuffers v1.11.0/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/hashicorp/go-hclog v0.0.0-20180709165350-ff2cf002a8dd h1:rNuUHR+CvK1IS89MMtcF0EpcVMZtjKfPRp4MEmt/aTs= +github.com/hashicorp/go-hclog v0.0.0-20180709165350-ff2cf002a8dd/go.mod h1:9bjs9uLqI8l75knNv3lV1kA55veR+WUPSiKIWcQHudI= +github.com/hashicorp/go-plugin v1.0.1 h1:4OtAfUGbnKC6yS48p0CtMX2oFYtzFZVv6rok3cRWgnE= +github.com/hashicorp/go-plugin v1.0.1/go.mod h1:++UyYGoz3o5w9ZzAdZxtQKrWWP+iqPBn3cQptSMzBuY= +github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb h1:b5rjCoWHc7eqmAS4/qyk21ZsHyb6Mxv/jykxvNTkU4M= +github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= +github.com/mattetti/filebuffer v1.0.0 h1:ixTvQ0JjBTwWbdpDZ98lLrydo7KRi8xNRIi5RFszsbY= +github.com/mattetti/filebuffer v1.0.0/go.mod h1:X6nyAIge2JGVmuJt2MFCqmHrb/5IHiphfHtot0s5cnI= +github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77 h1:7GoSOOW2jpsfkntVKaS2rAr1TJqfcxotyaUcuxoZSzg= +github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/oklog/run v1.0.0 h1:Ru7dDtJNOyC66gQ5dQmaCa0qIsAUFY3sFpK1Xk8igrw= +github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= +github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.0/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190129075346-302c3dd5f1cc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8 h1:Nw54tB0rB7hY/N0NQvRW8DG4Yk3Q6T9cu9RcFQDu1tc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= +google.golang.org/grpc v1.23.1 h1:q4XQuHFC6I28BKZpo6IYyb3mNO+l7lSOxRuYTCiDfXk= +google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/vendor/github.com/grafana/grafana-plugin-sdk-go/grpc.go b/vendor/github.com/grafana/grafana-plugin-sdk-go/grpc.go new file mode 100644 index 00000000000..4d2a6c7e31c --- /dev/null +++ b/vendor/github.com/grafana/grafana-plugin-sdk-go/grpc.go @@ -0,0 +1,86 @@ +package grafana + +import ( + "context" + + "github.com/grafana/grafana-plugin-sdk-go/genproto/datasource" + "github.com/hashicorp/go-hclog" + "github.com/hashicorp/go-plugin" + "google.golang.org/grpc" +) + +type GRPCClient struct { + broker *plugin.GRPCBroker + client datasource.DatasourcePluginClient +} + +// DatasourcePlugin is the Grafana datasource interface. +type DatasourcePlugin interface { + Query(ctx context.Context, req *datasource.DatasourceRequest, api GrafanaAPI) (*datasource.DatasourceResponse, error) +} + +// GrafanaAPI is the Grafana API interface that allows a datasource plugin to callback and request additional information from Grafana. +type GrafanaAPI interface { + QueryDatasource(ctx context.Context, req *datasource.QueryDatasourceRequest) (*datasource.QueryDatasourceResponse, error) +} + +func (m *GRPCClient) Query(ctx context.Context, req *datasource.DatasourceRequest, api GrafanaAPI) (*datasource.DatasourceResponse, error) { + apiServer := &GRPCGrafanaAPIServer{Impl: api} + + var s *grpc.Server + serverFunc := func(opts []grpc.ServerOption) *grpc.Server { + s = grpc.NewServer(opts...) + datasource.RegisterGrafanaAPIServer(s, apiServer) + + return s + } + brokeId := m.broker.NextId() + go m.broker.AcceptAndServe(brokeId, serverFunc) + + req.RequestId = brokeId + res, err := m.client.Query(ctx, req) + + s.Stop() + return res, err +} + +type grpcServer struct { + broker *plugin.GRPCBroker + Impl datasourcePluginWrapper +} + +func (m *grpcServer) Query(ctx context.Context, req *datasource.DatasourceRequest) (*datasource.DatasourceResponse, error) { + conn, err := m.broker.Dial(req.RequestId) + if err != nil { + return nil, err + } + defer conn.Close() + + api := &GRPCGrafanaAPIClient{datasource.NewGrafanaAPIClient(conn)} + return m.Impl.Query(ctx, req, api) +} + +// GRPCGrafanaAPIClient is an implementation of GrafanaAPIClient that talks over RPC. +type GRPCGrafanaAPIClient struct{ client datasource.GrafanaAPIClient } + +func (m *GRPCGrafanaAPIClient) QueryDatasource(ctx context.Context, req *datasource.QueryDatasourceRequest) (*datasource.QueryDatasourceResponse, error) { + resp, err := m.client.QueryDatasource(ctx, req) + if err != nil { + hclog.Default().Info("grafana.QueryDatasource", "client", "start", "err", err) + return nil, err + } + return resp, err +} + +// GRPCGrafanaAPIServer is the gRPC server that GRPCGrafanaAPIClient talks to. +type GRPCGrafanaAPIServer struct { + Impl GrafanaAPI +} + +func (m *GRPCGrafanaAPIServer) QueryDatasource(ctx context.Context, req *datasource.QueryDatasourceRequest) (*datasource.QueryDatasourceResponse, error) { + resp, err := m.Impl.QueryDatasource(ctx, req) + if err != nil { + return nil, err + } + return resp, nil +} diff --git a/vendor/github.com/grafana/grafana-plugin-sdk-go/plugin.go b/vendor/github.com/grafana/grafana-plugin-sdk-go/plugin.go new file mode 100644 index 00000000000..1030a2fdf56 --- /dev/null +++ b/vendor/github.com/grafana/grafana-plugin-sdk-go/plugin.go @@ -0,0 +1,28 @@ +package grafana + +import ( + "context" + + "github.com/grafana/grafana-plugin-sdk-go/genproto/datasource" + plugin "github.com/hashicorp/go-plugin" + "google.golang.org/grpc" +) + +// DatasourcePluginImpl implements the plugin interface from github.com/hashicorp/go-plugin. +type DatasourcePluginImpl struct { + plugin.NetRPCUnsupportedPlugin + + Impl datasourcePluginWrapper +} + +func (p *DatasourcePluginImpl) GRPCServer(broker *plugin.GRPCBroker, s *grpc.Server) error { + datasource.RegisterDatasourcePluginServer(s, &grpcServer{ + Impl: p.Impl, + broker: broker, + }) + return nil +} + +func (p *DatasourcePluginImpl) GRPCClient(ctx context.Context, broker *plugin.GRPCBroker, c *grpc.ClientConn) (interface{}, error) { + return &GRPCClient{client: datasource.NewDatasourcePluginClient(c), broker: broker}, nil +} diff --git a/vendor/github.com/grafana/grafana-plugin-sdk-go/server.go b/vendor/github.com/grafana/grafana-plugin-sdk-go/server.go new file mode 100644 index 00000000000..89f65dc5c4e --- /dev/null +++ b/vendor/github.com/grafana/grafana-plugin-sdk-go/server.go @@ -0,0 +1,54 @@ +package grafana + +import ( + plugin "github.com/hashicorp/go-plugin" +) + +const ( + magicCookieKey = "grafana_plugin_type" + magicCookieValue = "datasource" +) + +// Server serves all registered data source handlers. +type Server struct { + datasources map[string]DataSourceHandler +} + +// NewServer returns a new instance of Server. +func NewServer() *Server { + return &Server{ + datasources: make(map[string]DataSourceHandler), + } +} + +// HandleDataSource registers a new data source. +// +// The plugin ID should be in the format --datasource. +func (g *Server) HandleDataSource(pluginID string, p DataSourceHandler) { + g.datasources[pluginID] = p +} + +// Serve starts serving the registered handlers over gRPC. +func (g *Server) Serve() error { + plugins := make(map[string]plugin.Plugin) + + for id, h := range g.datasources { + plugins[id] = &DatasourcePluginImpl{ + Impl: datasourcePluginWrapper{ + handler: h, + }, + } + } + + plugin.Serve(&plugin.ServeConfig{ + HandshakeConfig: plugin.HandshakeConfig{ + ProtocolVersion: 1, + MagicCookieKey: magicCookieKey, + MagicCookieValue: magicCookieValue, + }, + Plugins: plugins, + GRPCServer: plugin.DefaultGRPCServer, + }) + + return nil +} diff --git a/vendor/github.com/mattetti/filebuffer/.gitignore b/vendor/github.com/mattetti/filebuffer/.gitignore new file mode 100644 index 00000000000..daf913b1b34 --- /dev/null +++ b/vendor/github.com/mattetti/filebuffer/.gitignore @@ -0,0 +1,24 @@ +# Compiled Object files, Static and Dynamic libs (Shared Objects) +*.o +*.a +*.so + +# Folders +_obj +_test + +# Architecture specific extensions/prefixes +*.[568vq] +[568vq].out + +*.cgo1.go +*.cgo2.c +_cgo_defun.c +_cgo_gotypes.go +_cgo_export.* + +_testmain.go + +*.exe +*.test +*.prof diff --git a/vendor/github.com/mattetti/filebuffer/LICENSE b/vendor/github.com/mattetti/filebuffer/LICENSE new file mode 100644 index 00000000000..a5ee1224579 --- /dev/null +++ b/vendor/github.com/mattetti/filebuffer/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2016 Matt Aimonetti + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/vendor/github.com/mattetti/filebuffer/README.md b/vendor/github.com/mattetti/filebuffer/README.md new file mode 100644 index 00000000000..34d8b97e882 --- /dev/null +++ b/vendor/github.com/mattetti/filebuffer/README.md @@ -0,0 +1,10 @@ +# filebuffer + +filebuffer is a package implementing a few file-like interfaces such as `io.Reader`, `io.ReaderAt`, `io.Seeker` and more. +The implementation is backed by a byte buffer and the main purpose is to have in-memory alternative to using an io.File. + +More information can be found on the [GoDoc page](https://godoc.org/github.com/mattetti/filebuffer). + +[![GoDoc](http://godoc.org/github.com/mattetti/filebuffer?status.svg)](http://godoc.org/github.com/mattetti/filebuffer) + +[![Build Status](https://travis-ci.org/mattetti/filebuffer.png)](https://travis-ci.org/mattetti/filebuffer) diff --git a/vendor/github.com/mattetti/filebuffer/filebuffer.go b/vendor/github.com/mattetti/filebuffer/filebuffer.go new file mode 100644 index 00000000000..10e82242a30 --- /dev/null +++ b/vendor/github.com/mattetti/filebuffer/filebuffer.go @@ -0,0 +1,161 @@ +// filebuffer is a package implementing a few file like interfaces +// backed by a byte buffer. +// Implemented interfaces: +// +// * Reader +// * ReaderAt +// * Writer +// * Seeker +// * Closer +package filebuffer + +import ( + "bytes" + "errors" + "io" +) + +// Buffer implements interfaces implemented by files. +// The main purpose of this type is to have an in memory replacement for a +// file. +type Buffer struct { + // Buff is the backing buffer + Buff *bytes.Buffer + // Index indicates where in the buffer we are at + Index int64 + isClosed bool +} + +// New returns a new populated Buffer +func New(b []byte) *Buffer { + return &Buffer{Buff: bytes.NewBuffer(b)} +} + +// Bytes returns the bytes available until the end of the buffer. +func (f *Buffer) Bytes() []byte { + if f.isClosed || f.Index >= int64(f.Buff.Len()) { + return []byte{} + } + return f.Buff.Bytes()[f.Index:] +} + +// String implements the Stringer interface +func (f *Buffer) String() string { + return string(f.Buff.Bytes()[f.Index:]) +} + +// Read implements io.Reader https://golang.org/pkg/io/#Reader +// Read reads up to len(p) bytes into p. It returns the number of bytes read (0 <= n <= len(p)) +// and any error encountered. Even if Read returns n < len(p), it may use all of p as scratch +// space during the call. If some data is available but not len(p) bytes, Read conventionally +// returns what is available instead of waiting for more. + +// When Read encounters an error or end-of-file condition after successfully reading n > 0 bytes, +// it returns the number of bytes read. It may return the (non-nil) error from the same call or +// return the error (and n == 0) from a subsequent call. An instance of this general case is +// that a Reader returning a non-zero number of bytes at the end of the input stream may return +// either err == EOF or err == nil. The next Read should return 0, EOF. +func (f *Buffer) Read(b []byte) (n int, err error) { + if f.isClosed { + return 0, io.EOF + } + if len(b) == 0 { + return 0, nil + } + if f.Index >= int64(f.Buff.Len()) { + return 0, io.EOF + } + n, err = bytes.NewBuffer(f.Buff.Bytes()[f.Index:]).Read(b) + f.Index += int64(n) + + return n, err +} + +// ReadAt implements io.ReaderAt https://golang.org/pkg/io/#ReaderAt +// ReadAt reads len(p) bytes into p starting at offset off in the underlying input source. +// It returns the number of bytes read (0 <= n <= len(p)) and any error encountered. +// +// When ReadAt returns n < len(p), it returns a non-nil error explaining why more bytes were not returned. +// In this respect, ReadAt is stricter than Read. +// +// Even if ReadAt returns n < len(p), it may use all of p as scratch space during the call. +// If some data is available but not len(p) bytes, ReadAt blocks until either all the data is available or an error occurs. +// In this respect ReadAt is different from Read. +// +// If the n = len(p) bytes returned by ReadAt are at the end of the input source, +// ReadAt may return either err == EOF or err == nil. +// +// If ReadAt is reading from an input source with a seek offset, +// ReadAt should not affect nor be affected by the underlying seek offset. +// Clients of ReadAt can execute parallel ReadAt calls on the same input source. +func (f *Buffer) ReadAt(p []byte, off int64) (n int, err error) { + if f.isClosed { + return 0, io.EOF + } + if off < 0 { + return 0, errors.New("filebuffer.ReadAt: negative offset") + } + reqLen := len(p) + buffLen := int64(f.Buff.Len()) + if off >= buffLen { + return 0, io.EOF + } + + n = copy(p, f.Buff.Bytes()[off:]) + if n < reqLen { + err = io.EOF + } + return n, err +} + +// Write implements io.Writer https://golang.org/pkg/io/#Writer +// by appending the passed bytes to the buffer unless the buffer is closed or index negative. +func (f *Buffer) Write(p []byte) (n int, err error) { + if f.isClosed { + return 0, io.EOF + } + if f.Index < 0 { + return 0, io.EOF + } + // we might have rewinded, let's reset the buffer before appending to it + idx := int(f.Index) + buffLen := f.Buff.Len() + if idx != buffLen && idx <= buffLen { + f.Buff = bytes.NewBuffer(f.Bytes()[:f.Index]) + } + n, err = f.Buff.Write(p) + + f.Index += int64(n) + return n, err +} + +// Seek implements io.Seeker https://golang.org/pkg/io/#Seeker +func (f *Buffer) Seek(offset int64, whence int) (idx int64, err error) { + if f.isClosed { + return 0, io.EOF + } + + var abs int64 + switch whence { + case 0: + abs = offset + case 1: + abs = int64(f.Index) + offset + case 2: + abs = int64(f.Buff.Len()) + offset + default: + return 0, errors.New("filebuffer.Seek: invalid whence") + } + if abs < 0 { + return 0, errors.New("filebuffer.Seek: negative position") + } + f.Index = abs + return abs, nil +} + +// Close implements io.Closer https://golang.org/pkg/io/#Closer +// It closes the buffer, rendering it unusable for I/O. It returns an error, if any. +func (f *Buffer) Close() error { + f.isClosed = true + return nil +} diff --git a/vendor/github.com/mattetti/filebuffer/go.mod b/vendor/github.com/mattetti/filebuffer/go.mod new file mode 100644 index 00000000000..ff8d8475c77 --- /dev/null +++ b/vendor/github.com/mattetti/filebuffer/go.mod @@ -0,0 +1,3 @@ +module github.com/mattetti/filebuffer + +go 1.12 diff --git a/vendor/google.golang.org/grpc/CONTRIBUTING.md b/vendor/google.golang.org/grpc/CONTRIBUTING.md index 0863eb26b60..6e69b28c270 100644 --- a/vendor/google.golang.org/grpc/CONTRIBUTING.md +++ b/vendor/google.golang.org/grpc/CONTRIBUTING.md @@ -11,26 +11,50 @@ In order to protect both you and ourselves, you will need to sign the ## Guidelines for Pull Requests How to get your contributions merged smoothly and quickly. - -- Create **small PRs** that are narrowly focused on **addressing a single concern**. We often times receive PRs that are trying to fix several things at a time, but only one fix is considered acceptable, nothing gets merged and both author's & review's time is wasted. Create more PRs to address different concerns and everyone will be happy. - -- For speculative changes, consider opening an issue and discussing it first. If you are suggesting a behavioral or API change, consider starting with a [gRFC proposal](https://github.com/grpc/proposal). - -- Provide a good **PR description** as a record of **what** change is being made and **why** it was made. Link to a github issue if it exists. - -- Don't fix code style and formatting unless you are already changing that line to address an issue. PRs with irrelevant changes won't be merged. If you do want to fix formatting or style, do that in a separate PR. - -- Unless your PR is trivial, you should expect there will be reviewer comments that you'll need to address before merging. We expect you to be reasonably responsive to those comments, otherwise the PR will be closed after 2-3 weeks of inactivity. - -- Maintain **clean commit history** and use **meaningful commit messages**. PRs with messy commit history are difficult to review and won't be merged. Use `rebase -i upstream/master` to curate your commit history and/or to bring in latest changes from master (but avoid rebasing in the middle of a code review). - -- Keep your PR up to date with upstream/master (if there are merge conflicts, we can't really merge your change). - -- **All tests need to be passing** before your change can be merged. We recommend you **run tests locally** before creating your PR to catch breakages early on. + +- Create **small PRs** that are narrowly focused on **addressing a single + concern**. We often times receive PRs that are trying to fix several things at + a time, but only one fix is considered acceptable, nothing gets merged and + both author's & review's time is wasted. Create more PRs to address different + concerns and everyone will be happy. + +- The grpc package should only depend on standard Go packages and a small number + of exceptions. If your contribution introduces new dependencies which are NOT + in the [list](https://godoc.org/google.golang.org/grpc?imports), you need a + discussion with gRPC-Go authors and consultants. + +- For speculative changes, consider opening an issue and discussing it first. If + you are suggesting a behavioral or API change, consider starting with a [gRFC + proposal](https://github.com/grpc/proposal). + +- Provide a good **PR description** as a record of **what** change is being made + and **why** it was made. Link to a github issue if it exists. + +- Don't fix code style and formatting unless you are already changing that line + to address an issue. PRs with irrelevant changes won't be merged. If you do + want to fix formatting or style, do that in a separate PR. + +- Unless your PR is trivial, you should expect there will be reviewer comments + that you'll need to address before merging. We expect you to be reasonably + responsive to those comments, otherwise the PR will be closed after 2-3 weeks + of inactivity. + +- Maintain **clean commit history** and use **meaningful commit messages**. PRs + with messy commit history are difficult to review and won't be merged. Use + `rebase -i upstream/master` to curate your commit history and/or to bring in + latest changes from master (but avoid rebasing in the middle of a code + review). + +- Keep your PR up to date with upstream/master (if there are merge conflicts, we + can't really merge your change). + +- **All tests need to be passing** before your change can be merged. We + recommend you **run tests locally** before creating your PR to catch breakages + early on. - `make all` to test everything, OR - `make vet` to catch vet errors - `make test` to run the tests - `make testrace` to run tests in race mode + - optional `make testappengine` to run tests with appengine - Exceptions to the rules can be made if there's a compelling reason for doing so. - diff --git a/vendor/google.golang.org/grpc/Makefile b/vendor/google.golang.org/grpc/Makefile index 50454530f4f..db982aabde6 100644 --- a/vendor/google.golang.org/grpc/Makefile +++ b/vendor/google.golang.org/grpc/Makefile @@ -1,23 +1,14 @@ all: vet test testrace -deps: - go get -d -v google.golang.org/grpc/... - -updatedeps: - go get -d -v -u -f google.golang.org/grpc/... - -testdeps: - go get -d -v -t google.golang.org/grpc/... - -testgaedeps: - goapp get -d -v -t -tags 'appengine appenginevm' google.golang.org/grpc/... - -updatetestdeps: - go get -d -v -t -u -f google.golang.org/grpc/... - build: deps go build google.golang.org/grpc/... +clean: + go clean -i google.golang.org/grpc/... + +deps: + go get -d -v google.golang.org/grpc/... + proto: @ if ! which protoc > /dev/null; then \ echo "error: protoc not installed" >&2; \ @@ -25,31 +16,45 @@ proto: fi go generate google.golang.org/grpc/... -vet: - ./vet.sh - test: testdeps - go test -cpu 1,4 -timeout 5m google.golang.org/grpc/... + go test -cpu 1,4 -timeout 7m google.golang.org/grpc/... + +testappengine: testappenginedeps + goapp test -cpu 1,4 -timeout 7m google.golang.org/grpc/... + +testappenginedeps: + goapp get -d -v -t -tags 'appengine appenginevm' google.golang.org/grpc/... + +testdeps: + go get -d -v -t google.golang.org/grpc/... testrace: testdeps go test -race -cpu 1,4 -timeout 7m google.golang.org/grpc/... -testappengine: testgaedeps - goapp test -cpu 1,4 -timeout 5m google.golang.org/grpc/... +updatedeps: + go get -d -v -u -f google.golang.org/grpc/... -clean: - go clean -i google.golang.org/grpc/... +updatetestdeps: + go get -d -v -t -u -f google.golang.org/grpc/... + +vet: vetdeps + ./vet.sh + +vetdeps: + ./vet.sh -install .PHONY: \ all \ - deps \ - updatedeps \ - testdeps \ - testgaedeps \ - updatetestdeps \ build \ + clean \ + deps \ proto \ - vet \ test \ + testappengine \ + testappenginedeps \ + testdeps \ testrace \ - clean + updatedeps \ + updatetestdeps \ + vet \ + vetdeps diff --git a/vendor/google.golang.org/grpc/README.md b/vendor/google.golang.org/grpc/README.md index 789adfd6536..afbc43db510 100644 --- a/vendor/google.golang.org/grpc/README.md +++ b/vendor/google.golang.org/grpc/README.md @@ -1,45 +1,121 @@ # gRPC-Go -[![Build Status](https://travis-ci.org/grpc/grpc-go.svg)](https://travis-ci.org/grpc/grpc-go) [![GoDoc](https://godoc.org/google.golang.org/grpc?status.svg)](https://godoc.org/google.golang.org/grpc) [![GoReportCard](https://goreportcard.com/badge/grpc/grpc-go)](https://goreportcard.com/report/github.com/grpc/grpc-go) +[![Build Status](https://travis-ci.org/grpc/grpc-go.svg)](https://travis-ci.org/grpc/grpc-go) +[![GoDoc](https://godoc.org/google.golang.org/grpc?status.svg)](https://godoc.org/google.golang.org/grpc) +[![GoReportCard](https://goreportcard.com/badge/grpc/grpc-go)](https://goreportcard.com/report/github.com/grpc/grpc-go) -The Go implementation of [gRPC](https://grpc.io/): A high performance, open source, general RPC framework that puts mobile and HTTP/2 first. For more information see the [gRPC Quick Start: Go](https://grpc.io/docs/quickstart/go.html) guide. +The Go implementation of [gRPC](https://grpc.io/): A high performance, open +source, general RPC framework that puts mobile and HTTP/2 first. For more +information see the [gRPC Quick Start: +Go](https://grpc.io/docs/quickstart/go.html) guide. Installation ------------ -To install this package, you need to install Go and setup your Go workspace on your computer. The simplest way to install the library is to run: +To install this package, you need to install Go and setup your Go workspace on +your computer. The simplest way to install the library is to run: ``` $ go get -u google.golang.org/grpc ``` +With Go module support (Go 1.11+), simply `import "google.golang.org/grpc"` in +your source code and `go [build|run|test]` will automatically download the +necessary dependencies ([Go modules +ref](https://github.com/golang/go/wiki/Modules)). + +If you are trying to access grpc-go from within China, please see the +[FAQ](#FAQ) below. + Prerequisites ------------- - -This requires Go 1.6 or later. Go 1.7 will be required soon. - -Constraints ------------ -The grpc package should only depend on standard Go packages and a small number of exceptions. If your contribution introduces new dependencies which are NOT in the [list](http://godoc.org/google.golang.org/grpc?imports), you need a discussion with gRPC-Go authors and consultants. +gRPC-Go requires Go 1.9 or later. Documentation ------------- -See [API documentation](https://godoc.org/google.golang.org/grpc) for package and API descriptions and find examples in the [examples directory](examples/). +- See [godoc](https://godoc.org/google.golang.org/grpc) for package and API + descriptions. +- Documentation on specific topics can be found in the [Documentation + directory](Documentation/). +- Examples can be found in the [examples directory](examples/). Performance ----------- -See the current benchmarks for some of the languages supported in [this dashboard](https://performance-dot-grpc-testing.appspot.com/explore?dashboard=5652536396611584&widget=490377658&container=1286539696). +Performance benchmark data for grpc-go and other languages is maintained in +[this +dashboard](https://performance-dot-grpc-testing.appspot.com/explore?dashboard=5652536396611584&widget=490377658&container=1286539696). Status ------ -General Availability [Google Cloud Platform Launch Stages](https://cloud.google.com/terms/launch-stages). +General Availability [Google Cloud Platform Launch +Stages](https://cloud.google.com/terms/launch-stages). FAQ --- +#### I/O Timeout Errors + +The `golang.org` domain may be blocked from some countries. `go get` usually +produces an error like the following when this happens: + +``` +$ go get -u google.golang.org/grpc +package google.golang.org/grpc: unrecognized import path "google.golang.org/grpc" (https fetch: Get https://google.golang.org/grpc?go-get=1: dial tcp 216.239.37.1:443: i/o timeout) +``` + +To build Go code, there are several options: + +- Set up a VPN and access google.golang.org through that. + +- Without Go module support: `git clone` the repo manually: + + ``` + git clone https://github.com/grpc/grpc-go.git $GOPATH/src/google.golang.org/grpc + ``` + + You will need to do the same for all of grpc's dependencies in `golang.org`, + e.g. `golang.org/x/net`. + +- With Go module support: it is possible to use the `replace` feature of `go + mod` to create aliases for golang.org packages. In your project's directory: + + ``` + go mod edit -replace=google.golang.org/grpc=github.com/grpc/grpc-go@latest + go mod tidy + go mod vendor + go build -mod=vendor + ``` + + Again, this will need to be done for all transitive dependencies hosted on + golang.org as well. Please refer to [this + issue](https://github.com/golang/go/issues/28652) in the golang repo regarding + this concern. + #### Compiling error, undefined: grpc.SupportPackageIsVersion Please update proto package, gRPC package and rebuild the proto files: - `go get -u github.com/golang/protobuf/{proto,protoc-gen-go}` - `go get -u google.golang.org/grpc` - `protoc --go_out=plugins=grpc:. *.proto` + +#### How to turn on logging + +The default logger is controlled by the environment variables. Turn everything +on by setting: + +``` +GRPC_GO_LOG_VERBOSITY_LEVEL=99 GRPC_GO_LOG_SEVERITY_LEVEL=info +``` + +#### The RPC failed with error `"code = Unavailable desc = transport is closing"` + +This error means the connection the RPC is using was closed, and there are many +possible reasons, including: + 1. mis-configured transport credentials, connection failed on handshaking + 1. bytes disrupted, possibly by a proxy in between + 1. server shutdown + +It can be tricky to debug this because the error happens on the client side but +the root cause of the connection being closed is on the server side. Turn on +logging on __both client and server__, and see if there are any transport +errors. diff --git a/vendor/google.golang.org/grpc/backoff.go b/vendor/google.golang.org/grpc/backoff.go index fa31565fd28..97c6e2568f4 100644 --- a/vendor/google.golang.org/grpc/backoff.go +++ b/vendor/google.golang.org/grpc/backoff.go @@ -17,7 +17,7 @@ */ // See internal/backoff package for the backoff implementation. This file is -// kept for the exported types and API backward compatility. +// kept for the exported types and API backward compatibility. package grpc diff --git a/vendor/google.golang.org/grpc/balancer.go b/vendor/google.golang.org/grpc/balancer.go index e1730166cde..a8eb0f47609 100644 --- a/vendor/google.golang.org/grpc/balancer.go +++ b/vendor/google.golang.org/grpc/balancer.go @@ -19,11 +19,10 @@ package grpc import ( - "fmt" + "context" "net" "sync" - "golang.org/x/net/context" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials" "google.golang.org/grpc/grpclog" @@ -44,7 +43,7 @@ type Address struct { // BalancerConfig specifies the configurations for Balancer. // -// Deprecated: please use package balancer. +// Deprecated: please use package balancer. May be removed in a future 1.x release. type BalancerConfig struct { // DialCreds is the transport credential the Balancer implementation can // use to dial to a remote load balancer server. The Balancer implementations @@ -58,7 +57,7 @@ type BalancerConfig struct { // BalancerGetOptions configures a Get call. // -// Deprecated: please use package balancer. +// Deprecated: please use package balancer. May be removed in a future 1.x release. type BalancerGetOptions struct { // BlockingWait specifies whether Get should block when there is no // connected address. @@ -67,7 +66,7 @@ type BalancerGetOptions struct { // Balancer chooses network addresses for RPCs. // -// Deprecated: please use package balancer. +// Deprecated: please use package balancer. May be removed in a future 1.x release. type Balancer interface { // Start does the initialization work to bootstrap a Balancer. For example, // this function may start the name resolution and watch the updates. It will @@ -118,30 +117,10 @@ type Balancer interface { Close() error } -// downErr implements net.Error. It is constructed by gRPC internals and passed to the down -// call of Balancer. -type downErr struct { - timeout bool - temporary bool - desc string -} - -func (e downErr) Error() string { return e.desc } -func (e downErr) Timeout() bool { return e.timeout } -func (e downErr) Temporary() bool { return e.temporary } - -func downErrorf(timeout, temporary bool, format string, a ...interface{}) downErr { - return downErr{ - timeout: timeout, - temporary: temporary, - desc: fmt.Sprintf(format, a...), - } -} - // RoundRobin returns a Balancer that selects addresses round-robin. It uses r to watch // the name resolution updates and updates the addresses available correspondingly. // -// Deprecated: please use package balancer/roundrobin. +// Deprecated: please use package balancer/roundrobin. May be removed in a future 1.x release. func RoundRobin(r naming.Resolver) Balancer { return &roundRobin{r: r} } @@ -410,7 +389,3 @@ func (rr *roundRobin) Close() error { type pickFirst struct { *roundRobin } - -func pickFirstBalancerV1(r naming.Resolver) Balancer { - return &pickFirst{&roundRobin{r: r}} -} diff --git a/vendor/google.golang.org/grpc/balancer/balancer.go b/vendor/google.golang.org/grpc/balancer/balancer.go index 069feb1e723..c266f4ec102 100644 --- a/vendor/google.golang.org/grpc/balancer/balancer.go +++ b/vendor/google.golang.org/grpc/balancer/balancer.go @@ -21,14 +21,18 @@ package balancer import ( + "context" + "encoding/json" "errors" "net" "strings" - "golang.org/x/net/context" "google.golang.org/grpc/connectivity" "google.golang.org/grpc/credentials" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/metadata" "google.golang.org/grpc/resolver" + "google.golang.org/grpc/serviceconfig" ) var ( @@ -37,7 +41,10 @@ var ( ) // Register registers the balancer builder to the balancer map. b.Name -// (lowercased) will be used as the name registered with this builder. +// (lowercased) will be used as the name registered with this builder. If the +// Builder implements ConfigParser, ParseConfig will be called when new service +// configs are received by the resolver, and the result will be provided to the +// Balancer in UpdateClientConnState. // // NOTE: this function must only be called during initialization time (i.e. in // an init() function), and is not thread-safe. If multiple Balancers are @@ -46,8 +53,20 @@ func Register(b Builder) { m[strings.ToLower(b.Name())] = b } +// unregisterForTesting deletes the balancer with the given name from the +// balancer map. +// +// This function is not thread-safe. +func unregisterForTesting(name string) { + delete(m, name) +} + +func init() { + internal.BalancerUnregister = unregisterForTesting +} + // Get returns the resolver builder registered with the given name. -// Note that the compare is done in a case-insenstive fashion. +// Note that the compare is done in a case-insensitive fashion. // If no builder is register with the name, nil will be returned. func Get(name string) Builder { if b, ok := m[strings.ToLower(name)]; ok { @@ -88,7 +107,15 @@ type SubConn interface { } // NewSubConnOptions contains options to create new SubConn. -type NewSubConnOptions struct{} +type NewSubConnOptions struct { + // CredsBundle is the credentials bundle that will be used in the created + // SubConn. If it's nil, the original creds from grpc DialOptions will be + // used. + CredsBundle credentials.Bundle + // HealthCheckEnabled indicates whether health check service should be + // enabled on this SubConn + HealthCheckEnabled bool +} // ClientConn represents a gRPC ClientConn. // @@ -105,7 +132,7 @@ type ClientConn interface { // The SubConn will be shutdown. RemoveSubConn(SubConn) - // UpdateBalancerState is called by balancer to nofity gRPC that some internal + // UpdateBalancerState is called by balancer to notify gRPC that some internal // state in balancer has changed. // // gRPC will update the connectivity state of the ClientConn, and will call pick @@ -116,6 +143,8 @@ type ClientConn interface { ResolveNow(resolver.ResolveNowOption) // Target returns the dial target for this ClientConn. + // + // Deprecated: Use the Target field in the BuildOptions instead. Target() string } @@ -125,12 +154,18 @@ type BuildOptions struct { // use to dial to a remote load balancer server. The Balancer implementations // can ignore this if it does not need to talk to another party securely. DialCreds credentials.TransportCredentials + // CredsBundle is the credentials bundle that the Balancer can use. + CredsBundle credentials.Bundle // Dialer is the custom dialer the Balancer implementation can use to dial // to a remote load balancer server. The Balancer implementations // can ignore this if it doesn't need to talk to remote balancer. Dialer func(context.Context, string) (net.Conn, error) // ChannelzParentID is the entity parent's channelz unique identification number. ChannelzParentID int64 + // Target contains the parsed address info of the dial target. It is the same resolver.Target as + // passed to the resolver. + // See the documentation for the resolver.Target type for details about what it contains. + Target resolver.Target } // Builder creates a balancer. @@ -142,6 +177,14 @@ type Builder interface { Name() string } +// ConfigParser parses load balancer configs. +type ConfigParser interface { + // ParseConfig parses the JSON load balancer config provided into an + // internal form or returns an error if the config is invalid. For future + // compatibility reasons, unknown fields in the config should be ignored. + ParseConfig(LoadBalancingConfigJSON json.RawMessage) (serviceconfig.LoadBalancingConfig, error) +} + // PickOptions contains addition information for the Pick operation. type PickOptions struct { // FullMethodName is the method name that NewClientStream() is called @@ -153,10 +196,17 @@ type PickOptions struct { type DoneInfo struct { // Err is the rpc error the RPC finished with. It could be nil. Err error + // Trailer contains the metadata from the RPC's trailer, if present. + Trailer metadata.MD // BytesSent indicates if any bytes have been sent to the server. BytesSent bool // BytesReceived indicates if any byte has been received from the server. BytesReceived bool + // ServerLoad is the load received from server. It's usually sent as part of + // trailing metadata. + // + // The only supported type now is *orca_v1.LoadReport. + ServerLoad interface{} } var ( @@ -186,8 +236,10 @@ type Picker interface { // // If a SubConn is returned: // - If it is READY, gRPC will send the RPC on it; - // - If it is not ready, or becomes not ready after it's returned, gRPC will block - // until UpdateBalancerState() is called and will call pick on the new picker. + // - If it is not ready, or becomes not ready after it's returned, gRPC will + // block until UpdateBalancerState() is called and will call pick on the + // new picker. The done function returned from Pick(), if not nil, will be + // called with nil error, no bytes sent and no bytes received. // // If the returned error is not nil: // - If the error is ErrNoSubConnAvailable, gRPC will block until UpdateBalancerState() @@ -198,9 +250,10 @@ type Picker interface { // - Else (error is other non-nil error): // - The RPC will fail with unavailable error. // - // The returned done() function will be called once the rpc has finished, with the - // final status of that RPC. - // done may be nil if balancer doesn't care about the RPC status. + // The returned done() function will be called once the rpc has finished, + // with the final status of that RPC. If the SubConn returned is not a + // valid SubConn type, done may not be called. done may be nil if balancer + // doesn't care about the RPC status. Pick(ctx context.Context, opts PickOptions) (conn SubConn, done func(DoneInfo), err error) } @@ -219,18 +272,55 @@ type Balancer interface { // that back to gRPC. // Balancer should also generate and update Pickers when its internal state has // been changed by the new state. + // + // Deprecated: if V2Balancer is implemented by the Balancer, + // UpdateSubConnState will be called instead. HandleSubConnStateChange(sc SubConn, state connectivity.State) // HandleResolvedAddrs is called by gRPC to send updated resolved addresses to // balancers. // Balancer can create new SubConn or remove SubConn with the addresses. // An empty address slice and a non-nil error will be passed if the resolver returns // non-nil error to gRPC. + // + // Deprecated: if V2Balancer is implemented by the Balancer, + // UpdateClientConnState will be called instead. HandleResolvedAddrs([]resolver.Address, error) // Close closes the balancer. The balancer is not required to call // ClientConn.RemoveSubConn for its existing SubConns. Close() } +// SubConnState describes the state of a SubConn. +type SubConnState struct { + ConnectivityState connectivity.State + // TODO: add last connection error +} + +// ClientConnState describes the state of a ClientConn relevant to the +// balancer. +type ClientConnState struct { + ResolverState resolver.State + // The parsed load balancing configuration returned by the builder's + // ParseConfig method, if implemented. + BalancerConfig serviceconfig.LoadBalancingConfig +} + +// V2Balancer is defined for documentation purposes. If a Balancer also +// implements V2Balancer, its UpdateClientConnState method will be called +// instead of HandleResolvedAddrs and its UpdateSubConnState will be called +// instead of HandleSubConnStateChange. +type V2Balancer interface { + // UpdateClientConnState is called by gRPC when the state of the ClientConn + // changes. + UpdateClientConnState(ClientConnState) + // UpdateSubConnState is called by gRPC when the state of a SubConn + // changes. + UpdateSubConnState(SubConn, SubConnState) + // Close closes the balancer. The balancer is not required to call + // ClientConn.RemoveSubConn for its existing SubConns. + Close() +} + // ConnectivityStateEvaluator takes the connectivity states of multiple SubConns // and returns one aggregated connectivity state. // diff --git a/vendor/google.golang.org/grpc/balancer/base/balancer.go b/vendor/google.golang.org/grpc/balancer/base/balancer.go index 23d13511bb2..1af88f0a3f1 100644 --- a/vendor/google.golang.org/grpc/balancer/base/balancer.go +++ b/vendor/google.golang.org/grpc/balancer/base/balancer.go @@ -19,7 +19,8 @@ package base import ( - "golang.org/x/net/context" + "context" + "google.golang.org/grpc/balancer" "google.golang.org/grpc/connectivity" "google.golang.org/grpc/grpclog" @@ -29,6 +30,7 @@ import ( type baseBuilder struct { name string pickerBuilder PickerBuilder + config Config } func (bb *baseBuilder) Build(cc balancer.ClientConn, opt balancer.BuildOptions) balancer.Balancer { @@ -38,11 +40,12 @@ func (bb *baseBuilder) Build(cc balancer.ClientConn, opt balancer.BuildOptions) subConns: make(map[resolver.Address]balancer.SubConn), scStates: make(map[balancer.SubConn]connectivity.State), - csEvltr: &connectivityStateEvaluator{}, + csEvltr: &balancer.ConnectivityStateEvaluator{}, // Initialize picker to a picker that always return // ErrNoSubConnAvailable, because when state of a SubConn changes, we // may call UpdateBalancerState with this picker. picker: NewErrPicker(balancer.ErrNoSubConnAvailable), + config: bb.config, } } @@ -54,27 +57,32 @@ type baseBalancer struct { cc balancer.ClientConn pickerBuilder PickerBuilder - csEvltr *connectivityStateEvaluator + csEvltr *balancer.ConnectivityStateEvaluator state connectivity.State subConns map[resolver.Address]balancer.SubConn scStates map[balancer.SubConn]connectivity.State picker balancer.Picker + config Config } func (b *baseBalancer) HandleResolvedAddrs(addrs []resolver.Address, err error) { - if err != nil { - grpclog.Infof("base.baseBalancer: HandleResolvedAddrs called with error %v", err) - return + panic("not implemented") +} + +func (b *baseBalancer) UpdateClientConnState(s balancer.ClientConnState) { + // TODO: handle s.ResolverState.Err (log if not nil) once implemented. + // TODO: handle s.ResolverState.ServiceConfig? + if grpclog.V(2) { + grpclog.Infoln("base.baseBalancer: got new ClientConn state: ", s) } - grpclog.Infoln("base.baseBalancer: got new resolved addresses: ", addrs) // addrsSet is the set converted from addrs, it's used for quick lookup of an address. addrsSet := make(map[resolver.Address]struct{}) - for _, a := range addrs { + for _, a := range s.ResolverState.Addresses { addrsSet[a] = struct{}{} if _, ok := b.subConns[a]; !ok { // a is a new address (not existing in b.subConns). - sc, err := b.cc.NewSubConn([]resolver.Address{a}, balancer.NewSubConnOptions{}) + sc, err := b.cc.NewSubConn([]resolver.Address{a}, balancer.NewSubConnOptions{HealthCheckEnabled: b.config.HealthCheck}) if err != nil { grpclog.Warningf("base.baseBalancer: failed to create new SubConn: %v", err) continue @@ -116,10 +124,19 @@ func (b *baseBalancer) regeneratePicker() { } func (b *baseBalancer) HandleSubConnStateChange(sc balancer.SubConn, s connectivity.State) { - grpclog.Infof("base.baseBalancer: handle SubConn state change: %p, %v", sc, s) + panic("not implemented") +} + +func (b *baseBalancer) UpdateSubConnState(sc balancer.SubConn, state balancer.SubConnState) { + s := state.ConnectivityState + if grpclog.V(2) { + grpclog.Infof("base.baseBalancer: handle SubConn state change: %p, %v", sc, s) + } oldS, ok := b.scStates[sc] if !ok { - grpclog.Infof("base.baseBalancer: got state changes for an unknown SubConn: %p, %v", sc, s) + if grpclog.V(2) { + grpclog.Infof("base.baseBalancer: got state changes for an unknown SubConn: %p, %v", sc, s) + } return } b.scStates[sc] = s @@ -133,7 +150,7 @@ func (b *baseBalancer) HandleSubConnStateChange(sc balancer.SubConn, s connectiv } oldAggrState := b.state - b.state = b.csEvltr.recordTransition(oldS, s) + b.state = b.csEvltr.RecordTransition(oldS, s) // Regenerate picker when one of the following happens: // - this sc became ready from not-ready @@ -165,44 +182,3 @@ type errPicker struct { func (p *errPicker) Pick(ctx context.Context, opts balancer.PickOptions) (balancer.SubConn, func(balancer.DoneInfo), error) { return nil, nil, p.err } - -// connectivityStateEvaluator gets updated by addrConns when their -// states transition, based on which it evaluates the state of -// ClientConn. -type connectivityStateEvaluator struct { - numReady uint64 // Number of addrConns in ready state. - numConnecting uint64 // Number of addrConns in connecting state. - numTransientFailure uint64 // Number of addrConns in transientFailure. -} - -// recordTransition records state change happening in every subConn and based on -// that it evaluates what aggregated state should be. -// It can only transition between Ready, Connecting and TransientFailure. Other states, -// Idle and Shutdown are transitioned into by ClientConn; in the beginning of the connection -// before any subConn is created ClientConn is in idle state. In the end when ClientConn -// closes it is in Shutdown state. -// -// recordTransition should only be called synchronously from the same goroutine. -func (cse *connectivityStateEvaluator) recordTransition(oldState, newState connectivity.State) connectivity.State { - // Update counters. - for idx, state := range []connectivity.State{oldState, newState} { - updateVal := 2*uint64(idx) - 1 // -1 for oldState and +1 for new. - switch state { - case connectivity.Ready: - cse.numReady += updateVal - case connectivity.Connecting: - cse.numConnecting += updateVal - case connectivity.TransientFailure: - cse.numTransientFailure += updateVal - } - } - - // Evaluate. - if cse.numReady > 0 { - return connectivity.Ready - } - if cse.numConnecting > 0 { - return connectivity.Connecting - } - return connectivity.TransientFailure -} diff --git a/vendor/google.golang.org/grpc/balancer/base/base.go b/vendor/google.golang.org/grpc/balancer/base/base.go index 012ace2f2f7..34b1f2994a7 100644 --- a/vendor/google.golang.org/grpc/balancer/base/base.go +++ b/vendor/google.golang.org/grpc/balancer/base/base.go @@ -45,8 +45,20 @@ type PickerBuilder interface { // NewBalancerBuilder returns a balancer builder. The balancers // built by this builder will use the picker builder to build pickers. func NewBalancerBuilder(name string, pb PickerBuilder) balancer.Builder { + return NewBalancerBuilderWithConfig(name, pb, Config{}) +} + +// Config contains the config info about the base balancer builder. +type Config struct { + // HealthCheck indicates whether health checking should be enabled for this specific balancer. + HealthCheck bool +} + +// NewBalancerBuilderWithConfig returns a base balancer builder configured by the provided config. +func NewBalancerBuilderWithConfig(name string, pb PickerBuilder, config Config) balancer.Builder { return &baseBuilder{ name: name, pickerBuilder: pb, + config: config, } } diff --git a/vendor/google.golang.org/grpc/balancer/roundrobin/roundrobin.go b/vendor/google.golang.org/grpc/balancer/roundrobin/roundrobin.go index 2eda0a1c210..29f7a4ddd68 100644 --- a/vendor/google.golang.org/grpc/balancer/roundrobin/roundrobin.go +++ b/vendor/google.golang.org/grpc/balancer/roundrobin/roundrobin.go @@ -22,12 +22,13 @@ package roundrobin import ( + "context" "sync" - "golang.org/x/net/context" "google.golang.org/grpc/balancer" "google.golang.org/grpc/balancer/base" "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/internal/grpcrand" "google.golang.org/grpc/resolver" ) @@ -36,7 +37,7 @@ const Name = "round_robin" // newBuilder creates a new roundrobin balancer builder. func newBuilder() balancer.Builder { - return base.NewBalancerBuilder(Name, &rrPickerBuilder{}) + return base.NewBalancerBuilderWithConfig(Name, &rrPickerBuilder{}, base.Config{HealthCheck: true}) } func init() { @@ -47,12 +48,19 @@ type rrPickerBuilder struct{} func (*rrPickerBuilder) Build(readySCs map[resolver.Address]balancer.SubConn) balancer.Picker { grpclog.Infof("roundrobinPicker: newPicker called with readySCs: %v", readySCs) + if len(readySCs) == 0 { + return base.NewErrPicker(balancer.ErrNoSubConnAvailable) + } var scs []balancer.SubConn for _, sc := range readySCs { scs = append(scs, sc) } return &rrPicker{ subConns: scs, + // Start at a random index, as the same RR balancer rebuilds a new + // picker when SubConn states change, and we don't want to apply excess + // load to the first server in the list. + next: grpcrand.Intn(len(scs)), } } @@ -67,10 +75,6 @@ type rrPicker struct { } func (p *rrPicker) Pick(ctx context.Context, opts balancer.PickOptions) (balancer.SubConn, func(balancer.DoneInfo), error) { - if len(p.subConns) <= 0 { - return nil, nil, balancer.ErrNoSubConnAvailable - } - p.mu.Lock() sc := p.subConns[p.next] p.next = (p.next + 1) % len(p.subConns) diff --git a/vendor/google.golang.org/grpc/balancer_conn_wrappers.go b/vendor/google.golang.org/grpc/balancer_conn_wrappers.go index c23f81706fb..8df4095ca95 100644 --- a/vendor/google.golang.org/grpc/balancer_conn_wrappers.go +++ b/vendor/google.golang.org/grpc/balancer_conn_wrappers.go @@ -82,20 +82,13 @@ func (b *scStateUpdateBuffer) get() <-chan *scStateUpdate { return b.c } -// resolverUpdate contains the new resolved addresses or error if there's -// any. -type resolverUpdate struct { - addrs []resolver.Address - err error -} - // ccBalancerWrapper is a wrapper on top of cc for balancers. // It implements balancer.ClientConn interface. type ccBalancerWrapper struct { cc *ClientConn balancer balancer.Balancer stateChangeQueue *scStateUpdateBuffer - resolverUpdateCh chan *resolverUpdate + ccUpdateCh chan *balancer.ClientConnState done chan struct{} mu sync.Mutex @@ -106,7 +99,7 @@ func newCCBalancerWrapper(cc *ClientConn, b balancer.Builder, bopts balancer.Bui ccb := &ccBalancerWrapper{ cc: cc, stateChangeQueue: newSCStateUpdateBuffer(), - resolverUpdateCh: make(chan *resolverUpdate, 1), + ccUpdateCh: make(chan *balancer.ClientConnState, 1), done: make(chan struct{}), subConns: make(map[*acBalancerWrapper]struct{}), } @@ -128,15 +121,23 @@ func (ccb *ccBalancerWrapper) watcher() { return default: } - ccb.balancer.HandleSubConnStateChange(t.sc, t.state) - case t := <-ccb.resolverUpdateCh: + if ub, ok := ccb.balancer.(balancer.V2Balancer); ok { + ub.UpdateSubConnState(t.sc, balancer.SubConnState{ConnectivityState: t.state}) + } else { + ccb.balancer.HandleSubConnStateChange(t.sc, t.state) + } + case s := <-ccb.ccUpdateCh: select { case <-ccb.done: ccb.balancer.Close() return default: } - ccb.balancer.HandleResolvedAddrs(t.addrs, t.err) + if ub, ok := ccb.balancer.(balancer.V2Balancer); ok { + ub.UpdateClientConnState(*s) + } else { + ccb.balancer.HandleResolvedAddrs(s.ResolverState.Addresses, nil) + } case <-ccb.done: } @@ -150,9 +151,11 @@ func (ccb *ccBalancerWrapper) watcher() { for acbw := range scs { ccb.cc.removeAddrConn(acbw.getAddrConn(), errConnDrain) } + ccb.UpdateBalancerState(connectivity.Connecting, nil) return default: } + ccb.cc.firstResolveEvent.Fire() } } @@ -177,15 +180,24 @@ func (ccb *ccBalancerWrapper) handleSubConnStateChange(sc balancer.SubConn, s co }) } -func (ccb *ccBalancerWrapper) handleResolvedAddrs(addrs []resolver.Address, err error) { +func (ccb *ccBalancerWrapper) updateClientConnState(ccs *balancer.ClientConnState) { + if ccb.cc.curBalancerName != grpclbName { + // Filter any grpclb addresses since we don't have the grpclb balancer. + s := &ccs.ResolverState + for i := 0; i < len(s.Addresses); { + if s.Addresses[i].Type == resolver.GRPCLB { + copy(s.Addresses[i:], s.Addresses[i+1:]) + s.Addresses = s.Addresses[:len(s.Addresses)-1] + continue + } + i++ + } + } select { - case <-ccb.resolverUpdateCh: + case <-ccb.ccUpdateCh: default: } - ccb.resolverUpdateCh <- &resolverUpdate{ - addrs: addrs, - err: err, - } + ccb.ccUpdateCh <- ccs } func (ccb *ccBalancerWrapper) NewSubConn(addrs []resolver.Address, opts balancer.NewSubConnOptions) (balancer.SubConn, error) { @@ -197,7 +209,7 @@ func (ccb *ccBalancerWrapper) NewSubConn(addrs []resolver.Address, opts balancer if ccb.subConns == nil { return nil, fmt.Errorf("grpc: ClientConn balancer wrapper was closed") } - ac, err := ccb.cc.newAddrConn(addrs) + ac, err := ccb.cc.newAddrConn(addrs, opts) if err != nil { return nil, err } @@ -229,8 +241,13 @@ func (ccb *ccBalancerWrapper) UpdateBalancerState(s connectivity.State, p balanc if ccb.subConns == nil { return } - ccb.cc.csMgr.updateState(s) + // Update picker before updating state. Even though the ordering here does + // not matter, it can lead to multiple calls of Pick in the common start-up + // case where we wait for ready and then perform an RPC. If the picker is + // updated later, we could call the "connecting" picker when the state is + // updated, and then call the "ready" picker after the picker gets updated. ccb.cc.blockingpicker.updatePicker(p) + ccb.cc.csMgr.updateState(s) } func (ccb *ccBalancerWrapper) ResolveNow(o resolver.ResolveNowOption) { @@ -257,6 +274,7 @@ func (acbw *acBalancerWrapper) UpdateAddresses(addrs []resolver.Address) { } if !acbw.ac.tryUpdateAddrs(addrs) { cc := acbw.ac.cc + opts := acbw.ac.scopts acbw.ac.mu.Lock() // Set old ac.acbw to nil so the Shutdown state update will be ignored // by balancer. @@ -272,7 +290,7 @@ func (acbw *acBalancerWrapper) UpdateAddresses(addrs []resolver.Address) { return } - ac, err := cc.newAddrConn(addrs) + ac, err := cc.newAddrConn(addrs, opts) if err != nil { grpclog.Warningf("acBalancerWrapper: UpdateAddresses: failed to newAddrConn: %v", err) return diff --git a/vendor/google.golang.org/grpc/balancer_v1_wrapper.go b/vendor/google.golang.org/grpc/balancer_v1_wrapper.go index e0ce32cfb6d..66e9a44ac4d 100644 --- a/vendor/google.golang.org/grpc/balancer_v1_wrapper.go +++ b/vendor/google.golang.org/grpc/balancer_v1_wrapper.go @@ -19,16 +19,13 @@ package grpc import ( - "strings" + "context" "sync" - "golang.org/x/net/context" "google.golang.org/grpc/balancer" - "google.golang.org/grpc/codes" "google.golang.org/grpc/connectivity" "google.golang.org/grpc/grpclog" "google.golang.org/grpc/resolver" - "google.golang.org/grpc/status" ) type balancerWrapperBuilder struct { @@ -36,13 +33,7 @@ type balancerWrapperBuilder struct { } func (bwb *balancerWrapperBuilder) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Balancer { - targetAddr := cc.Target() - targetSplitted := strings.Split(targetAddr, ":///") - if len(targetSplitted) >= 2 { - targetAddr = targetSplitted[1] - } - - bwb.b.Start(targetAddr, BalancerConfig{ + bwb.b.Start(opts.Target.Endpoint, BalancerConfig{ DialCreds: opts.DialCreds, Dialer: opts.Dialer, }) @@ -51,7 +42,7 @@ func (bwb *balancerWrapperBuilder) Build(cc balancer.ClientConn, opts balancer.B balancer: bwb.b, pickfirst: pickfirst, cc: cc, - targetAddr: targetAddr, + targetAddr: opts.Target.Endpoint, startCh: make(chan struct{}), conns: make(map[resolver.Address]balancer.SubConn), connSt: make(map[balancer.SubConn]*scState), @@ -122,7 +113,7 @@ func (bw *balancerWrapper) lbWatcher() { } for addrs := range notifyCh { - grpclog.Infof("balancerWrapper: got update addr from Notify: %v\n", addrs) + grpclog.Infof("balancerWrapper: got update addr from Notify: %v", addrs) if bw.pickfirst { var ( oldA resolver.Address @@ -283,9 +274,8 @@ func (bw *balancerWrapper) Close() { } // The picker is the balancerWrapper itself. -// Pick should never return ErrNoSubConnAvailable. // It either blocks or returns error, consistent with v1 balancer Get(). -func (bw *balancerWrapper) Pick(ctx context.Context, opts balancer.PickOptions) (balancer.SubConn, func(balancer.DoneInfo), error) { +func (bw *balancerWrapper) Pick(ctx context.Context, opts balancer.PickOptions) (sc balancer.SubConn, done func(balancer.DoneInfo), err error) { failfast := true // Default failfast is true. if ss, ok := rpcInfoFromContext(ctx); ok { failfast = ss.failfast @@ -294,35 +284,51 @@ func (bw *balancerWrapper) Pick(ctx context.Context, opts balancer.PickOptions) if err != nil { return nil, nil, err } - var done func(balancer.DoneInfo) if p != nil { - done = func(i balancer.DoneInfo) { p() } + done = func(balancer.DoneInfo) { p() } + defer func() { + if err != nil { + p() + } + }() } - var sc balancer.SubConn + bw.mu.Lock() defer bw.mu.Unlock() if bw.pickfirst { // Get the first sc in conns. - for _, sc = range bw.conns { - break - } - } else { - var ok bool - sc, ok = bw.conns[resolver.Address{ - Addr: a.Addr, - Type: resolver.Backend, - ServerName: "", - Metadata: a.Metadata, - }] - if !ok && failfast { - return nil, nil, status.Errorf(codes.Unavailable, "there is no connection available") - } - if s, ok := bw.connSt[sc]; failfast && (!ok || s.s != connectivity.Ready) { - // If the returned sc is not ready and RPC is failfast, - // return error, and this RPC will fail. - return nil, nil, status.Errorf(codes.Unavailable, "there is no connection available") + for _, sc := range bw.conns { + return sc, done, nil } + return nil, nil, balancer.ErrNoSubConnAvailable + } + sc, ok1 := bw.conns[resolver.Address{ + Addr: a.Addr, + Type: resolver.Backend, + ServerName: "", + Metadata: a.Metadata, + }] + s, ok2 := bw.connSt[sc] + if !ok1 || !ok2 { + // This can only happen due to a race where Get() returned an address + // that was subsequently removed by Notify. In this case we should + // retry always. + return nil, nil, balancer.ErrNoSubConnAvailable + } + switch s.s { + case connectivity.Ready, connectivity.Idle: + return sc, done, nil + case connectivity.Shutdown, connectivity.TransientFailure: + // If the returned sc has been shut down or is in transient failure, + // return error, and this RPC will fail or wait for another picker (if + // non-failfast). + return nil, nil, balancer.ErrTransientFailure + default: + // For other states (connecting or unknown), the v1 balancer would + // traditionally wait until ready and then issue the RPC. Returning + // ErrNoSubConnAvailable will be a slight improvement in that it will + // allow the balancer to choose another address in case others are + // connected. + return nil, nil, balancer.ErrNoSubConnAvailable } - - return sc, done, nil } diff --git a/vendor/google.golang.org/grpc/binarylog/grpc_binarylog_v1/binarylog.pb.go b/vendor/google.golang.org/grpc/binarylog/grpc_binarylog_v1/binarylog.pb.go new file mode 100644 index 00000000000..f393bb66187 --- /dev/null +++ b/vendor/google.golang.org/grpc/binarylog/grpc_binarylog_v1/binarylog.pb.go @@ -0,0 +1,900 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// source: grpc/binarylog/grpc_binarylog_v1/binarylog.proto + +package grpc_binarylog_v1 // import "google.golang.org/grpc/binarylog/grpc_binarylog_v1" + +import proto "github.com/golang/protobuf/proto" +import fmt "fmt" +import math "math" +import duration "github.com/golang/protobuf/ptypes/duration" +import timestamp "github.com/golang/protobuf/ptypes/timestamp" + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package + +// Enumerates the type of event +// Note the terminology is different from the RPC semantics +// definition, but the same meaning is expressed here. +type GrpcLogEntry_EventType int32 + +const ( + GrpcLogEntry_EVENT_TYPE_UNKNOWN GrpcLogEntry_EventType = 0 + // Header sent from client to server + GrpcLogEntry_EVENT_TYPE_CLIENT_HEADER GrpcLogEntry_EventType = 1 + // Header sent from server to client + GrpcLogEntry_EVENT_TYPE_SERVER_HEADER GrpcLogEntry_EventType = 2 + // Message sent from client to server + GrpcLogEntry_EVENT_TYPE_CLIENT_MESSAGE GrpcLogEntry_EventType = 3 + // Message sent from server to client + GrpcLogEntry_EVENT_TYPE_SERVER_MESSAGE GrpcLogEntry_EventType = 4 + // A signal that client is done sending + GrpcLogEntry_EVENT_TYPE_CLIENT_HALF_CLOSE GrpcLogEntry_EventType = 5 + // Trailer indicates the end of the RPC. + // On client side, this event means a trailer was either received + // from the network or the gRPC library locally generated a status + // to inform the application about a failure. + // On server side, this event means the server application requested + // to send a trailer. Note: EVENT_TYPE_CANCEL may still arrive after + // this due to races on server side. + GrpcLogEntry_EVENT_TYPE_SERVER_TRAILER GrpcLogEntry_EventType = 6 + // A signal that the RPC is cancelled. On client side, this + // indicates the client application requests a cancellation. + // On server side, this indicates that cancellation was detected. + // Note: This marks the end of the RPC. Events may arrive after + // this due to races. For example, on client side a trailer + // may arrive even though the application requested to cancel the RPC. + GrpcLogEntry_EVENT_TYPE_CANCEL GrpcLogEntry_EventType = 7 +) + +var GrpcLogEntry_EventType_name = map[int32]string{ + 0: "EVENT_TYPE_UNKNOWN", + 1: "EVENT_TYPE_CLIENT_HEADER", + 2: "EVENT_TYPE_SERVER_HEADER", + 3: "EVENT_TYPE_CLIENT_MESSAGE", + 4: "EVENT_TYPE_SERVER_MESSAGE", + 5: "EVENT_TYPE_CLIENT_HALF_CLOSE", + 6: "EVENT_TYPE_SERVER_TRAILER", + 7: "EVENT_TYPE_CANCEL", +} +var GrpcLogEntry_EventType_value = map[string]int32{ + "EVENT_TYPE_UNKNOWN": 0, + "EVENT_TYPE_CLIENT_HEADER": 1, + "EVENT_TYPE_SERVER_HEADER": 2, + "EVENT_TYPE_CLIENT_MESSAGE": 3, + "EVENT_TYPE_SERVER_MESSAGE": 4, + "EVENT_TYPE_CLIENT_HALF_CLOSE": 5, + "EVENT_TYPE_SERVER_TRAILER": 6, + "EVENT_TYPE_CANCEL": 7, +} + +func (x GrpcLogEntry_EventType) String() string { + return proto.EnumName(GrpcLogEntry_EventType_name, int32(x)) +} +func (GrpcLogEntry_EventType) EnumDescriptor() ([]byte, []int) { + return fileDescriptor_binarylog_264c8c9c551ce911, []int{0, 0} +} + +// Enumerates the entity that generates the log entry +type GrpcLogEntry_Logger int32 + +const ( + GrpcLogEntry_LOGGER_UNKNOWN GrpcLogEntry_Logger = 0 + GrpcLogEntry_LOGGER_CLIENT GrpcLogEntry_Logger = 1 + GrpcLogEntry_LOGGER_SERVER GrpcLogEntry_Logger = 2 +) + +var GrpcLogEntry_Logger_name = map[int32]string{ + 0: "LOGGER_UNKNOWN", + 1: "LOGGER_CLIENT", + 2: "LOGGER_SERVER", +} +var GrpcLogEntry_Logger_value = map[string]int32{ + "LOGGER_UNKNOWN": 0, + "LOGGER_CLIENT": 1, + "LOGGER_SERVER": 2, +} + +func (x GrpcLogEntry_Logger) String() string { + return proto.EnumName(GrpcLogEntry_Logger_name, int32(x)) +} +func (GrpcLogEntry_Logger) EnumDescriptor() ([]byte, []int) { + return fileDescriptor_binarylog_264c8c9c551ce911, []int{0, 1} +} + +type Address_Type int32 + +const ( + Address_TYPE_UNKNOWN Address_Type = 0 + // address is in 1.2.3.4 form + Address_TYPE_IPV4 Address_Type = 1 + // address is in IPv6 canonical form (RFC5952 section 4) + // The scope is NOT included in the address string. + Address_TYPE_IPV6 Address_Type = 2 + // address is UDS string + Address_TYPE_UNIX Address_Type = 3 +) + +var Address_Type_name = map[int32]string{ + 0: "TYPE_UNKNOWN", + 1: "TYPE_IPV4", + 2: "TYPE_IPV6", + 3: "TYPE_UNIX", +} +var Address_Type_value = map[string]int32{ + "TYPE_UNKNOWN": 0, + "TYPE_IPV4": 1, + "TYPE_IPV6": 2, + "TYPE_UNIX": 3, +} + +func (x Address_Type) String() string { + return proto.EnumName(Address_Type_name, int32(x)) +} +func (Address_Type) EnumDescriptor() ([]byte, []int) { + return fileDescriptor_binarylog_264c8c9c551ce911, []int{7, 0} +} + +// Log entry we store in binary logs +type GrpcLogEntry struct { + // The timestamp of the binary log message + Timestamp *timestamp.Timestamp `protobuf:"bytes,1,opt,name=timestamp,proto3" json:"timestamp,omitempty"` + // Uniquely identifies a call. The value must not be 0 in order to disambiguate + // from an unset value. + // Each call may have several log entries, they will all have the same call_id. + // Nothing is guaranteed about their value other than they are unique across + // different RPCs in the same gRPC process. + CallId uint64 `protobuf:"varint,2,opt,name=call_id,json=callId,proto3" json:"call_id,omitempty"` + // The entry sequence id for this call. The first GrpcLogEntry has a + // value of 1, to disambiguate from an unset value. The purpose of + // this field is to detect missing entries in environments where + // durability or ordering is not guaranteed. + SequenceIdWithinCall uint64 `protobuf:"varint,3,opt,name=sequence_id_within_call,json=sequenceIdWithinCall,proto3" json:"sequence_id_within_call,omitempty"` + Type GrpcLogEntry_EventType `protobuf:"varint,4,opt,name=type,proto3,enum=grpc.binarylog.v1.GrpcLogEntry_EventType" json:"type,omitempty"` + Logger GrpcLogEntry_Logger `protobuf:"varint,5,opt,name=logger,proto3,enum=grpc.binarylog.v1.GrpcLogEntry_Logger" json:"logger,omitempty"` + // The logger uses one of the following fields to record the payload, + // according to the type of the log entry. + // + // Types that are valid to be assigned to Payload: + // *GrpcLogEntry_ClientHeader + // *GrpcLogEntry_ServerHeader + // *GrpcLogEntry_Message + // *GrpcLogEntry_Trailer + Payload isGrpcLogEntry_Payload `protobuf_oneof:"payload"` + // true if payload does not represent the full message or metadata. + PayloadTruncated bool `protobuf:"varint,10,opt,name=payload_truncated,json=payloadTruncated,proto3" json:"payload_truncated,omitempty"` + // Peer address information, will only be recorded on the first + // incoming event. On client side, peer is logged on + // EVENT_TYPE_SERVER_HEADER normally or EVENT_TYPE_SERVER_TRAILER in + // the case of trailers-only. On server side, peer is always + // logged on EVENT_TYPE_CLIENT_HEADER. + Peer *Address `protobuf:"bytes,11,opt,name=peer,proto3" json:"peer,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *GrpcLogEntry) Reset() { *m = GrpcLogEntry{} } +func (m *GrpcLogEntry) String() string { return proto.CompactTextString(m) } +func (*GrpcLogEntry) ProtoMessage() {} +func (*GrpcLogEntry) Descriptor() ([]byte, []int) { + return fileDescriptor_binarylog_264c8c9c551ce911, []int{0} +} +func (m *GrpcLogEntry) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_GrpcLogEntry.Unmarshal(m, b) +} +func (m *GrpcLogEntry) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_GrpcLogEntry.Marshal(b, m, deterministic) +} +func (dst *GrpcLogEntry) XXX_Merge(src proto.Message) { + xxx_messageInfo_GrpcLogEntry.Merge(dst, src) +} +func (m *GrpcLogEntry) XXX_Size() int { + return xxx_messageInfo_GrpcLogEntry.Size(m) +} +func (m *GrpcLogEntry) XXX_DiscardUnknown() { + xxx_messageInfo_GrpcLogEntry.DiscardUnknown(m) +} + +var xxx_messageInfo_GrpcLogEntry proto.InternalMessageInfo + +func (m *GrpcLogEntry) GetTimestamp() *timestamp.Timestamp { + if m != nil { + return m.Timestamp + } + return nil +} + +func (m *GrpcLogEntry) GetCallId() uint64 { + if m != nil { + return m.CallId + } + return 0 +} + +func (m *GrpcLogEntry) GetSequenceIdWithinCall() uint64 { + if m != nil { + return m.SequenceIdWithinCall + } + return 0 +} + +func (m *GrpcLogEntry) GetType() GrpcLogEntry_EventType { + if m != nil { + return m.Type + } + return GrpcLogEntry_EVENT_TYPE_UNKNOWN +} + +func (m *GrpcLogEntry) GetLogger() GrpcLogEntry_Logger { + if m != nil { + return m.Logger + } + return GrpcLogEntry_LOGGER_UNKNOWN +} + +type isGrpcLogEntry_Payload interface { + isGrpcLogEntry_Payload() +} + +type GrpcLogEntry_ClientHeader struct { + ClientHeader *ClientHeader `protobuf:"bytes,6,opt,name=client_header,json=clientHeader,proto3,oneof"` +} + +type GrpcLogEntry_ServerHeader struct { + ServerHeader *ServerHeader `protobuf:"bytes,7,opt,name=server_header,json=serverHeader,proto3,oneof"` +} + +type GrpcLogEntry_Message struct { + Message *Message `protobuf:"bytes,8,opt,name=message,proto3,oneof"` +} + +type GrpcLogEntry_Trailer struct { + Trailer *Trailer `protobuf:"bytes,9,opt,name=trailer,proto3,oneof"` +} + +func (*GrpcLogEntry_ClientHeader) isGrpcLogEntry_Payload() {} + +func (*GrpcLogEntry_ServerHeader) isGrpcLogEntry_Payload() {} + +func (*GrpcLogEntry_Message) isGrpcLogEntry_Payload() {} + +func (*GrpcLogEntry_Trailer) isGrpcLogEntry_Payload() {} + +func (m *GrpcLogEntry) GetPayload() isGrpcLogEntry_Payload { + if m != nil { + return m.Payload + } + return nil +} + +func (m *GrpcLogEntry) GetClientHeader() *ClientHeader { + if x, ok := m.GetPayload().(*GrpcLogEntry_ClientHeader); ok { + return x.ClientHeader + } + return nil +} + +func (m *GrpcLogEntry) GetServerHeader() *ServerHeader { + if x, ok := m.GetPayload().(*GrpcLogEntry_ServerHeader); ok { + return x.ServerHeader + } + return nil +} + +func (m *GrpcLogEntry) GetMessage() *Message { + if x, ok := m.GetPayload().(*GrpcLogEntry_Message); ok { + return x.Message + } + return nil +} + +func (m *GrpcLogEntry) GetTrailer() *Trailer { + if x, ok := m.GetPayload().(*GrpcLogEntry_Trailer); ok { + return x.Trailer + } + return nil +} + +func (m *GrpcLogEntry) GetPayloadTruncated() bool { + if m != nil { + return m.PayloadTruncated + } + return false +} + +func (m *GrpcLogEntry) GetPeer() *Address { + if m != nil { + return m.Peer + } + return nil +} + +// XXX_OneofFuncs is for the internal use of the proto package. +func (*GrpcLogEntry) XXX_OneofFuncs() (func(msg proto.Message, b *proto.Buffer) error, func(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error), func(msg proto.Message) (n int), []interface{}) { + return _GrpcLogEntry_OneofMarshaler, _GrpcLogEntry_OneofUnmarshaler, _GrpcLogEntry_OneofSizer, []interface{}{ + (*GrpcLogEntry_ClientHeader)(nil), + (*GrpcLogEntry_ServerHeader)(nil), + (*GrpcLogEntry_Message)(nil), + (*GrpcLogEntry_Trailer)(nil), + } +} + +func _GrpcLogEntry_OneofMarshaler(msg proto.Message, b *proto.Buffer) error { + m := msg.(*GrpcLogEntry) + // payload + switch x := m.Payload.(type) { + case *GrpcLogEntry_ClientHeader: + b.EncodeVarint(6<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.ClientHeader); err != nil { + return err + } + case *GrpcLogEntry_ServerHeader: + b.EncodeVarint(7<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.ServerHeader); err != nil { + return err + } + case *GrpcLogEntry_Message: + b.EncodeVarint(8<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.Message); err != nil { + return err + } + case *GrpcLogEntry_Trailer: + b.EncodeVarint(9<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.Trailer); err != nil { + return err + } + case nil: + default: + return fmt.Errorf("GrpcLogEntry.Payload has unexpected type %T", x) + } + return nil +} + +func _GrpcLogEntry_OneofUnmarshaler(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error) { + m := msg.(*GrpcLogEntry) + switch tag { + case 6: // payload.client_header + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(ClientHeader) + err := b.DecodeMessage(msg) + m.Payload = &GrpcLogEntry_ClientHeader{msg} + return true, err + case 7: // payload.server_header + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(ServerHeader) + err := b.DecodeMessage(msg) + m.Payload = &GrpcLogEntry_ServerHeader{msg} + return true, err + case 8: // payload.message + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(Message) + err := b.DecodeMessage(msg) + m.Payload = &GrpcLogEntry_Message{msg} + return true, err + case 9: // payload.trailer + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(Trailer) + err := b.DecodeMessage(msg) + m.Payload = &GrpcLogEntry_Trailer{msg} + return true, err + default: + return false, nil + } +} + +func _GrpcLogEntry_OneofSizer(msg proto.Message) (n int) { + m := msg.(*GrpcLogEntry) + // payload + switch x := m.Payload.(type) { + case *GrpcLogEntry_ClientHeader: + s := proto.Size(x.ClientHeader) + n += 1 // tag and wire + n += proto.SizeVarint(uint64(s)) + n += s + case *GrpcLogEntry_ServerHeader: + s := proto.Size(x.ServerHeader) + n += 1 // tag and wire + n += proto.SizeVarint(uint64(s)) + n += s + case *GrpcLogEntry_Message: + s := proto.Size(x.Message) + n += 1 // tag and wire + n += proto.SizeVarint(uint64(s)) + n += s + case *GrpcLogEntry_Trailer: + s := proto.Size(x.Trailer) + n += 1 // tag and wire + n += proto.SizeVarint(uint64(s)) + n += s + case nil: + default: + panic(fmt.Sprintf("proto: unexpected type %T in oneof", x)) + } + return n +} + +type ClientHeader struct { + // This contains only the metadata from the application. + Metadata *Metadata `protobuf:"bytes,1,opt,name=metadata,proto3" json:"metadata,omitempty"` + // The name of the RPC method, which looks something like: + // // + // Note the leading "/" character. + MethodName string `protobuf:"bytes,2,opt,name=method_name,json=methodName,proto3" json:"method_name,omitempty"` + // A single process may be used to run multiple virtual + // servers with different identities. + // The authority is the name of such a server identitiy. + // It is typically a portion of the URI in the form of + // or : . + Authority string `protobuf:"bytes,3,opt,name=authority,proto3" json:"authority,omitempty"` + // the RPC timeout + Timeout *duration.Duration `protobuf:"bytes,4,opt,name=timeout,proto3" json:"timeout,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ClientHeader) Reset() { *m = ClientHeader{} } +func (m *ClientHeader) String() string { return proto.CompactTextString(m) } +func (*ClientHeader) ProtoMessage() {} +func (*ClientHeader) Descriptor() ([]byte, []int) { + return fileDescriptor_binarylog_264c8c9c551ce911, []int{1} +} +func (m *ClientHeader) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_ClientHeader.Unmarshal(m, b) +} +func (m *ClientHeader) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_ClientHeader.Marshal(b, m, deterministic) +} +func (dst *ClientHeader) XXX_Merge(src proto.Message) { + xxx_messageInfo_ClientHeader.Merge(dst, src) +} +func (m *ClientHeader) XXX_Size() int { + return xxx_messageInfo_ClientHeader.Size(m) +} +func (m *ClientHeader) XXX_DiscardUnknown() { + xxx_messageInfo_ClientHeader.DiscardUnknown(m) +} + +var xxx_messageInfo_ClientHeader proto.InternalMessageInfo + +func (m *ClientHeader) GetMetadata() *Metadata { + if m != nil { + return m.Metadata + } + return nil +} + +func (m *ClientHeader) GetMethodName() string { + if m != nil { + return m.MethodName + } + return "" +} + +func (m *ClientHeader) GetAuthority() string { + if m != nil { + return m.Authority + } + return "" +} + +func (m *ClientHeader) GetTimeout() *duration.Duration { + if m != nil { + return m.Timeout + } + return nil +} + +type ServerHeader struct { + // This contains only the metadata from the application. + Metadata *Metadata `protobuf:"bytes,1,opt,name=metadata,proto3" json:"metadata,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ServerHeader) Reset() { *m = ServerHeader{} } +func (m *ServerHeader) String() string { return proto.CompactTextString(m) } +func (*ServerHeader) ProtoMessage() {} +func (*ServerHeader) Descriptor() ([]byte, []int) { + return fileDescriptor_binarylog_264c8c9c551ce911, []int{2} +} +func (m *ServerHeader) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_ServerHeader.Unmarshal(m, b) +} +func (m *ServerHeader) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_ServerHeader.Marshal(b, m, deterministic) +} +func (dst *ServerHeader) XXX_Merge(src proto.Message) { + xxx_messageInfo_ServerHeader.Merge(dst, src) +} +func (m *ServerHeader) XXX_Size() int { + return xxx_messageInfo_ServerHeader.Size(m) +} +func (m *ServerHeader) XXX_DiscardUnknown() { + xxx_messageInfo_ServerHeader.DiscardUnknown(m) +} + +var xxx_messageInfo_ServerHeader proto.InternalMessageInfo + +func (m *ServerHeader) GetMetadata() *Metadata { + if m != nil { + return m.Metadata + } + return nil +} + +type Trailer struct { + // This contains only the metadata from the application. + Metadata *Metadata `protobuf:"bytes,1,opt,name=metadata,proto3" json:"metadata,omitempty"` + // The gRPC status code. + StatusCode uint32 `protobuf:"varint,2,opt,name=status_code,json=statusCode,proto3" json:"status_code,omitempty"` + // An original status message before any transport specific + // encoding. + StatusMessage string `protobuf:"bytes,3,opt,name=status_message,json=statusMessage,proto3" json:"status_message,omitempty"` + // The value of the 'grpc-status-details-bin' metadata key. If + // present, this is always an encoded 'google.rpc.Status' message. + StatusDetails []byte `protobuf:"bytes,4,opt,name=status_details,json=statusDetails,proto3" json:"status_details,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *Trailer) Reset() { *m = Trailer{} } +func (m *Trailer) String() string { return proto.CompactTextString(m) } +func (*Trailer) ProtoMessage() {} +func (*Trailer) Descriptor() ([]byte, []int) { + return fileDescriptor_binarylog_264c8c9c551ce911, []int{3} +} +func (m *Trailer) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_Trailer.Unmarshal(m, b) +} +func (m *Trailer) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_Trailer.Marshal(b, m, deterministic) +} +func (dst *Trailer) XXX_Merge(src proto.Message) { + xxx_messageInfo_Trailer.Merge(dst, src) +} +func (m *Trailer) XXX_Size() int { + return xxx_messageInfo_Trailer.Size(m) +} +func (m *Trailer) XXX_DiscardUnknown() { + xxx_messageInfo_Trailer.DiscardUnknown(m) +} + +var xxx_messageInfo_Trailer proto.InternalMessageInfo + +func (m *Trailer) GetMetadata() *Metadata { + if m != nil { + return m.Metadata + } + return nil +} + +func (m *Trailer) GetStatusCode() uint32 { + if m != nil { + return m.StatusCode + } + return 0 +} + +func (m *Trailer) GetStatusMessage() string { + if m != nil { + return m.StatusMessage + } + return "" +} + +func (m *Trailer) GetStatusDetails() []byte { + if m != nil { + return m.StatusDetails + } + return nil +} + +// Message payload, used by CLIENT_MESSAGE and SERVER_MESSAGE +type Message struct { + // Length of the message. It may not be the same as the length of the + // data field, as the logging payload can be truncated or omitted. + Length uint32 `protobuf:"varint,1,opt,name=length,proto3" json:"length,omitempty"` + // May be truncated or omitted. + Data []byte `protobuf:"bytes,2,opt,name=data,proto3" json:"data,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *Message) Reset() { *m = Message{} } +func (m *Message) String() string { return proto.CompactTextString(m) } +func (*Message) ProtoMessage() {} +func (*Message) Descriptor() ([]byte, []int) { + return fileDescriptor_binarylog_264c8c9c551ce911, []int{4} +} +func (m *Message) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_Message.Unmarshal(m, b) +} +func (m *Message) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_Message.Marshal(b, m, deterministic) +} +func (dst *Message) XXX_Merge(src proto.Message) { + xxx_messageInfo_Message.Merge(dst, src) +} +func (m *Message) XXX_Size() int { + return xxx_messageInfo_Message.Size(m) +} +func (m *Message) XXX_DiscardUnknown() { + xxx_messageInfo_Message.DiscardUnknown(m) +} + +var xxx_messageInfo_Message proto.InternalMessageInfo + +func (m *Message) GetLength() uint32 { + if m != nil { + return m.Length + } + return 0 +} + +func (m *Message) GetData() []byte { + if m != nil { + return m.Data + } + return nil +} + +// A list of metadata pairs, used in the payload of client header, +// server header, and server trailer. +// Implementations may omit some entries to honor the header limits +// of GRPC_BINARY_LOG_CONFIG. +// +// Header keys added by gRPC are omitted. To be more specific, +// implementations will not log the following entries, and this is +// not to be treated as a truncation: +// - entries handled by grpc that are not user visible, such as those +// that begin with 'grpc-' (with exception of grpc-trace-bin) +// or keys like 'lb-token' +// - transport specific entries, including but not limited to: +// ':path', ':authority', 'content-encoding', 'user-agent', 'te', etc +// - entries added for call credentials +// +// Implementations must always log grpc-trace-bin if it is present. +// Practically speaking it will only be visible on server side because +// grpc-trace-bin is managed by low level client side mechanisms +// inaccessible from the application level. On server side, the +// header is just a normal metadata key. +// The pair will not count towards the size limit. +type Metadata struct { + Entry []*MetadataEntry `protobuf:"bytes,1,rep,name=entry,proto3" json:"entry,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *Metadata) Reset() { *m = Metadata{} } +func (m *Metadata) String() string { return proto.CompactTextString(m) } +func (*Metadata) ProtoMessage() {} +func (*Metadata) Descriptor() ([]byte, []int) { + return fileDescriptor_binarylog_264c8c9c551ce911, []int{5} +} +func (m *Metadata) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_Metadata.Unmarshal(m, b) +} +func (m *Metadata) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_Metadata.Marshal(b, m, deterministic) +} +func (dst *Metadata) XXX_Merge(src proto.Message) { + xxx_messageInfo_Metadata.Merge(dst, src) +} +func (m *Metadata) XXX_Size() int { + return xxx_messageInfo_Metadata.Size(m) +} +func (m *Metadata) XXX_DiscardUnknown() { + xxx_messageInfo_Metadata.DiscardUnknown(m) +} + +var xxx_messageInfo_Metadata proto.InternalMessageInfo + +func (m *Metadata) GetEntry() []*MetadataEntry { + if m != nil { + return m.Entry + } + return nil +} + +// A metadata key value pair +type MetadataEntry struct { + Key string `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` + Value []byte `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *MetadataEntry) Reset() { *m = MetadataEntry{} } +func (m *MetadataEntry) String() string { return proto.CompactTextString(m) } +func (*MetadataEntry) ProtoMessage() {} +func (*MetadataEntry) Descriptor() ([]byte, []int) { + return fileDescriptor_binarylog_264c8c9c551ce911, []int{6} +} +func (m *MetadataEntry) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_MetadataEntry.Unmarshal(m, b) +} +func (m *MetadataEntry) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_MetadataEntry.Marshal(b, m, deterministic) +} +func (dst *MetadataEntry) XXX_Merge(src proto.Message) { + xxx_messageInfo_MetadataEntry.Merge(dst, src) +} +func (m *MetadataEntry) XXX_Size() int { + return xxx_messageInfo_MetadataEntry.Size(m) +} +func (m *MetadataEntry) XXX_DiscardUnknown() { + xxx_messageInfo_MetadataEntry.DiscardUnknown(m) +} + +var xxx_messageInfo_MetadataEntry proto.InternalMessageInfo + +func (m *MetadataEntry) GetKey() string { + if m != nil { + return m.Key + } + return "" +} + +func (m *MetadataEntry) GetValue() []byte { + if m != nil { + return m.Value + } + return nil +} + +// Address information +type Address struct { + Type Address_Type `protobuf:"varint,1,opt,name=type,proto3,enum=grpc.binarylog.v1.Address_Type" json:"type,omitempty"` + Address string `protobuf:"bytes,2,opt,name=address,proto3" json:"address,omitempty"` + // only for TYPE_IPV4 and TYPE_IPV6 + IpPort uint32 `protobuf:"varint,3,opt,name=ip_port,json=ipPort,proto3" json:"ip_port,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *Address) Reset() { *m = Address{} } +func (m *Address) String() string { return proto.CompactTextString(m) } +func (*Address) ProtoMessage() {} +func (*Address) Descriptor() ([]byte, []int) { + return fileDescriptor_binarylog_264c8c9c551ce911, []int{7} +} +func (m *Address) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_Address.Unmarshal(m, b) +} +func (m *Address) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_Address.Marshal(b, m, deterministic) +} +func (dst *Address) XXX_Merge(src proto.Message) { + xxx_messageInfo_Address.Merge(dst, src) +} +func (m *Address) XXX_Size() int { + return xxx_messageInfo_Address.Size(m) +} +func (m *Address) XXX_DiscardUnknown() { + xxx_messageInfo_Address.DiscardUnknown(m) +} + +var xxx_messageInfo_Address proto.InternalMessageInfo + +func (m *Address) GetType() Address_Type { + if m != nil { + return m.Type + } + return Address_TYPE_UNKNOWN +} + +func (m *Address) GetAddress() string { + if m != nil { + return m.Address + } + return "" +} + +func (m *Address) GetIpPort() uint32 { + if m != nil { + return m.IpPort + } + return 0 +} + +func init() { + proto.RegisterType((*GrpcLogEntry)(nil), "grpc.binarylog.v1.GrpcLogEntry") + proto.RegisterType((*ClientHeader)(nil), "grpc.binarylog.v1.ClientHeader") + proto.RegisterType((*ServerHeader)(nil), "grpc.binarylog.v1.ServerHeader") + proto.RegisterType((*Trailer)(nil), "grpc.binarylog.v1.Trailer") + proto.RegisterType((*Message)(nil), "grpc.binarylog.v1.Message") + proto.RegisterType((*Metadata)(nil), "grpc.binarylog.v1.Metadata") + proto.RegisterType((*MetadataEntry)(nil), "grpc.binarylog.v1.MetadataEntry") + proto.RegisterType((*Address)(nil), "grpc.binarylog.v1.Address") + proto.RegisterEnum("grpc.binarylog.v1.GrpcLogEntry_EventType", GrpcLogEntry_EventType_name, GrpcLogEntry_EventType_value) + proto.RegisterEnum("grpc.binarylog.v1.GrpcLogEntry_Logger", GrpcLogEntry_Logger_name, GrpcLogEntry_Logger_value) + proto.RegisterEnum("grpc.binarylog.v1.Address_Type", Address_Type_name, Address_Type_value) +} + +func init() { + proto.RegisterFile("grpc/binarylog/grpc_binarylog_v1/binarylog.proto", fileDescriptor_binarylog_264c8c9c551ce911) +} + +var fileDescriptor_binarylog_264c8c9c551ce911 = []byte{ + // 900 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x55, 0x51, 0x6f, 0xe3, 0x44, + 0x10, 0x3e, 0x37, 0x69, 0xdc, 0x4c, 0x92, 0xca, 0x5d, 0x95, 0x3b, 0x5f, 0x29, 0x34, 0xb2, 0x04, + 0x0a, 0x42, 0x72, 0xb9, 0x94, 0xeb, 0xf1, 0x02, 0x52, 0x92, 0xfa, 0xd2, 0x88, 0x5c, 0x1a, 0x6d, + 0x72, 0x3d, 0x40, 0x48, 0xd6, 0x36, 0x5e, 0x1c, 0x0b, 0xc7, 0x6b, 0xd6, 0x9b, 0xa0, 0xfc, 0x2c, + 0xde, 0x90, 0xee, 0x77, 0xf1, 0x8e, 0xbc, 0x6b, 0x27, 0xa6, 0x69, 0x0f, 0x09, 0xde, 0x3c, 0xdf, + 0x7c, 0xf3, 0xcd, 0xee, 0x78, 0x66, 0x16, 0xbe, 0xf2, 0x79, 0x3c, 0x3b, 0xbf, 0x0b, 0x22, 0xc2, + 0xd7, 0x21, 0xf3, 0xcf, 0x53, 0xd3, 0xdd, 0x98, 0xee, 0xea, 0xc5, 0xd6, 0x67, 0xc7, 0x9c, 0x09, + 0x86, 0x8e, 0x52, 0x8a, 0xbd, 0x45, 0x57, 0x2f, 0x4e, 0x3e, 0xf5, 0x19, 0xf3, 0x43, 0x7a, 0x2e, + 0x09, 0x77, 0xcb, 0x5f, 0xce, 0xbd, 0x25, 0x27, 0x22, 0x60, 0x91, 0x0a, 0x39, 0x39, 0xbb, 0xef, + 0x17, 0xc1, 0x82, 0x26, 0x82, 0x2c, 0x62, 0x45, 0xb0, 0xde, 0xeb, 0x50, 0xef, 0xf3, 0x78, 0x36, + 0x64, 0xbe, 0x13, 0x09, 0xbe, 0x46, 0xdf, 0x40, 0x75, 0xc3, 0x31, 0xb5, 0xa6, 0xd6, 0xaa, 0xb5, + 0x4f, 0x6c, 0xa5, 0x62, 0xe7, 0x2a, 0xf6, 0x34, 0x67, 0xe0, 0x2d, 0x19, 0x3d, 0x03, 0x7d, 0x46, + 0xc2, 0xd0, 0x0d, 0x3c, 0x73, 0xaf, 0xa9, 0xb5, 0xca, 0xb8, 0x92, 0x9a, 0x03, 0x0f, 0xbd, 0x84, + 0x67, 0x09, 0xfd, 0x6d, 0x49, 0xa3, 0x19, 0x75, 0x03, 0xcf, 0xfd, 0x3d, 0x10, 0xf3, 0x20, 0x72, + 0x53, 0xa7, 0x59, 0x92, 0xc4, 0xe3, 0xdc, 0x3d, 0xf0, 0xde, 0x49, 0x67, 0x8f, 0x84, 0x21, 0xfa, + 0x16, 0xca, 0x62, 0x1d, 0x53, 0xb3, 0xdc, 0xd4, 0x5a, 0x87, 0xed, 0x2f, 0xec, 0x9d, 0xdb, 0xdb, + 0xc5, 0x83, 0xdb, 0xce, 0x8a, 0x46, 0x62, 0xba, 0x8e, 0x29, 0x96, 0x61, 0xe8, 0x3b, 0xa8, 0x84, + 0xcc, 0xf7, 0x29, 0x37, 0xf7, 0xa5, 0xc0, 0xe7, 0xff, 0x26, 0x30, 0x94, 0x6c, 0x9c, 0x45, 0xa1, + 0xd7, 0xd0, 0x98, 0x85, 0x01, 0x8d, 0x84, 0x3b, 0xa7, 0xc4, 0xa3, 0xdc, 0xac, 0xc8, 0x62, 0x9c, + 0x3d, 0x20, 0xd3, 0x93, 0xbc, 0x6b, 0x49, 0xbb, 0x7e, 0x82, 0xeb, 0xb3, 0x82, 0x9d, 0xea, 0x24, + 0x94, 0xaf, 0x28, 0xcf, 0x75, 0xf4, 0x47, 0x75, 0x26, 0x92, 0xb7, 0xd5, 0x49, 0x0a, 0x36, 0xba, + 0x04, 0x7d, 0x41, 0x93, 0x84, 0xf8, 0xd4, 0x3c, 0xc8, 0x7f, 0xcb, 0x8e, 0xc2, 0x1b, 0xc5, 0xb8, + 0x7e, 0x82, 0x73, 0x72, 0x1a, 0x27, 0x38, 0x09, 0x42, 0xca, 0xcd, 0xea, 0xa3, 0x71, 0x53, 0xc5, + 0x48, 0xe3, 0x32, 0x32, 0xfa, 0x12, 0x8e, 0x62, 0xb2, 0x0e, 0x19, 0xf1, 0x5c, 0xc1, 0x97, 0xd1, + 0x8c, 0x08, 0xea, 0x99, 0xd0, 0xd4, 0x5a, 0x07, 0xd8, 0xc8, 0x1c, 0xd3, 0x1c, 0x47, 0x36, 0x94, + 0x63, 0x4a, 0xb9, 0x59, 0x7b, 0x34, 0x43, 0xc7, 0xf3, 0x38, 0x4d, 0x12, 0x2c, 0x79, 0xd6, 0x5f, + 0x1a, 0x54, 0x37, 0x3f, 0x0c, 0x3d, 0x05, 0xe4, 0xdc, 0x3a, 0xa3, 0xa9, 0x3b, 0xfd, 0x71, 0xec, + 0xb8, 0x6f, 0x47, 0xdf, 0x8f, 0x6e, 0xde, 0x8d, 0x8c, 0x27, 0xe8, 0x14, 0xcc, 0x02, 0xde, 0x1b, + 0x0e, 0xd2, 0xef, 0x6b, 0xa7, 0x73, 0xe5, 0x60, 0x43, 0xbb, 0xe7, 0x9d, 0x38, 0xf8, 0xd6, 0xc1, + 0xb9, 0x77, 0x0f, 0x7d, 0x02, 0xcf, 0x77, 0x63, 0xdf, 0x38, 0x93, 0x49, 0xa7, 0xef, 0x18, 0xa5, + 0x7b, 0xee, 0x2c, 0x38, 0x77, 0x97, 0x51, 0x13, 0x4e, 0x1f, 0xc8, 0xdc, 0x19, 0xbe, 0x76, 0x7b, + 0xc3, 0x9b, 0x89, 0x63, 0xec, 0x3f, 0x2c, 0x30, 0xc5, 0x9d, 0xc1, 0xd0, 0xc1, 0x46, 0x05, 0x7d, + 0x04, 0x47, 0x45, 0x81, 0xce, 0xa8, 0xe7, 0x0c, 0x0d, 0xdd, 0xea, 0x42, 0x45, 0xb5, 0x19, 0x42, + 0x70, 0x38, 0xbc, 0xe9, 0xf7, 0x1d, 0x5c, 0xb8, 0xef, 0x11, 0x34, 0x32, 0x4c, 0x65, 0x34, 0xb4, + 0x02, 0xa4, 0x52, 0x18, 0x7b, 0xdd, 0x2a, 0xe8, 0x59, 0xfd, 0xad, 0xf7, 0x1a, 0xd4, 0x8b, 0xcd, + 0x87, 0x5e, 0xc1, 0xc1, 0x82, 0x0a, 0xe2, 0x11, 0x41, 0xb2, 0xe1, 0xfd, 0xf8, 0xc1, 0x2e, 0x51, + 0x14, 0xbc, 0x21, 0xa3, 0x33, 0xa8, 0x2d, 0xa8, 0x98, 0x33, 0xcf, 0x8d, 0xc8, 0x82, 0xca, 0x01, + 0xae, 0x62, 0x50, 0xd0, 0x88, 0x2c, 0x28, 0x3a, 0x85, 0x2a, 0x59, 0x8a, 0x39, 0xe3, 0x81, 0x58, + 0xcb, 0xb1, 0xad, 0xe2, 0x2d, 0x80, 0x2e, 0x40, 0x4f, 0x17, 0x01, 0x5b, 0x0a, 0x39, 0xae, 0xb5, + 0xf6, 0xf3, 0x9d, 0x9d, 0x71, 0x95, 0x6d, 0x26, 0x9c, 0x33, 0xad, 0x3e, 0xd4, 0x8b, 0x1d, 0xff, + 0x9f, 0x0f, 0x6f, 0xfd, 0xa1, 0x81, 0x9e, 0x75, 0xf0, 0xff, 0xaa, 0x40, 0x22, 0x88, 0x58, 0x26, + 0xee, 0x8c, 0x79, 0xaa, 0x02, 0x0d, 0x0c, 0x0a, 0xea, 0x31, 0x8f, 0xa2, 0xcf, 0xe0, 0x30, 0x23, + 0xe4, 0x73, 0xa8, 0xca, 0xd0, 0x50, 0x68, 0x36, 0x7a, 0x05, 0x9a, 0x47, 0x05, 0x09, 0xc2, 0x44, + 0x56, 0xa4, 0x9e, 0xd3, 0xae, 0x14, 0x68, 0xbd, 0x04, 0x3d, 0x8f, 0x78, 0x0a, 0x95, 0x90, 0x46, + 0xbe, 0x98, 0xcb, 0x03, 0x37, 0x70, 0x66, 0x21, 0x04, 0x65, 0x79, 0x8d, 0x3d, 0x19, 0x2f, 0xbf, + 0xad, 0x2e, 0x1c, 0xe4, 0x67, 0x47, 0x97, 0xb0, 0x4f, 0xd3, 0xcd, 0x65, 0x6a, 0xcd, 0x52, 0xab, + 0xd6, 0x6e, 0x7e, 0xe0, 0x9e, 0x72, 0xc3, 0x61, 0x45, 0xb7, 0x5e, 0x41, 0xe3, 0x1f, 0x38, 0x32, + 0xa0, 0xf4, 0x2b, 0x5d, 0xcb, 0xec, 0x55, 0x9c, 0x7e, 0xa2, 0x63, 0xd8, 0x5f, 0x91, 0x70, 0x49, + 0xb3, 0xdc, 0xca, 0xb0, 0xfe, 0xd4, 0x40, 0xcf, 0xe6, 0x18, 0x5d, 0x64, 0xdb, 0x59, 0x93, 0xcb, + 0xf5, 0xec, 0xf1, 0x89, 0xb7, 0x0b, 0x3b, 0xd9, 0x04, 0x9d, 0x28, 0x34, 0xeb, 0xb0, 0xdc, 0x4c, + 0x1f, 0x8f, 0x20, 0x76, 0x63, 0xc6, 0x85, 0xac, 0x6a, 0x03, 0x57, 0x82, 0x78, 0xcc, 0xb8, 0xb0, + 0x1c, 0x28, 0xcb, 0x1d, 0x61, 0x40, 0xfd, 0xde, 0x76, 0x68, 0x40, 0x55, 0x22, 0x83, 0xf1, 0xed, + 0xd7, 0x86, 0x56, 0x34, 0x2f, 0x8d, 0xbd, 0x8d, 0xf9, 0x76, 0x34, 0xf8, 0xc1, 0x28, 0x75, 0x7f, + 0x86, 0xe3, 0x80, 0xed, 0x1e, 0xb2, 0x7b, 0xd8, 0x95, 0xd6, 0x90, 0xf9, 0xe3, 0xb4, 0x51, 0xc7, + 0xda, 0x4f, 0xed, 0xac, 0x71, 0x7d, 0x16, 0x92, 0xc8, 0xb7, 0x19, 0x57, 0x4f, 0xf3, 0x87, 0x5e, + 0xea, 0xbb, 0x8a, 0xec, 0xf2, 0x8b, 0xbf, 0x03, 0x00, 0x00, 0xff, 0xff, 0xe7, 0xf6, 0x4b, 0x50, + 0xd4, 0x07, 0x00, 0x00, +} diff --git a/vendor/google.golang.org/grpc/call.go b/vendor/google.golang.org/grpc/call.go index 180d79d0656..9e20e4d385f 100644 --- a/vendor/google.golang.org/grpc/call.go +++ b/vendor/google.golang.org/grpc/call.go @@ -19,7 +19,7 @@ package grpc import ( - "golang.org/x/net/context" + "context" ) // Invoke sends the RPC request on the wire and returns after response is @@ -40,7 +40,7 @@ func (cc *ClientConn) Invoke(ctx context.Context, method string, args, reply int func combine(o1 []CallOption, o2 []CallOption) []CallOption { // we don't use append because o1 could have extra capacity whose // elements would be overwritten, which could cause inadvertent - // sharing (and race connditions) between concurrent calls + // sharing (and race conditions) between concurrent calls if len(o1) == 0 { return o2 } else if len(o2) == 0 { diff --git a/vendor/google.golang.org/grpc/clientconn.go b/vendor/google.golang.org/grpc/clientconn.go index 9b035e8f563..a7643df7d29 100644 --- a/vendor/google.golang.org/grpc/clientconn.go +++ b/vendor/google.golang.org/grpc/clientconn.go @@ -19,6 +19,7 @@ package grpc import ( + "context" "errors" "fmt" "math" @@ -29,8 +30,6 @@ import ( "sync/atomic" "time" - "golang.org/x/net/context" - "golang.org/x/net/trace" "google.golang.org/grpc/balancer" _ "google.golang.org/grpc/balancer/roundrobin" // To register roundrobin. "google.golang.org/grpc/codes" @@ -39,11 +38,13 @@ import ( "google.golang.org/grpc/grpclog" "google.golang.org/grpc/internal/backoff" "google.golang.org/grpc/internal/channelz" + "google.golang.org/grpc/internal/grpcsync" "google.golang.org/grpc/internal/transport" "google.golang.org/grpc/keepalive" "google.golang.org/grpc/resolver" _ "google.golang.org/grpc/resolver/dns" // To register dns resolver. _ "google.golang.org/grpc/resolver/passthrough" // To register passthrough resolver. + "google.golang.org/grpc/serviceconfig" "google.golang.org/grpc/status" ) @@ -65,15 +66,11 @@ var ( errConnDrain = errors.New("grpc: the connection is drained") // errConnClosing indicates that the connection is closing. errConnClosing = errors.New("grpc: the connection is closing") - // errConnUnavailable indicates that the connection is unavailable. - errConnUnavailable = errors.New("grpc: the connection is unavailable") // errBalancerClosed indicates that the balancer is closed. errBalancerClosed = errors.New("grpc: balancer is closed") - // We use an accessor so that minConnectTimeout can be - // atomically read and updated while testing. - getMinConnectTimeout = func() time.Duration { - return minConnectTimeout - } + // invalidDefaultServiceConfigErrPrefix is used to prefix the json parsing error for the default + // service config. + invalidDefaultServiceConfigErrPrefix = "grpc: the provided default service config is invalid" ) // The following errors are returned from Dial and DialContext @@ -82,15 +79,16 @@ var ( // being set for ClientConn. Users should either set one or explicitly // call WithInsecure DialOption to disable security. errNoTransportSecurity = errors.New("grpc: no transport security set (use grpc.WithInsecure() explicitly or set credentials)") + // errTransportCredsAndBundle indicates that creds bundle is used together + // with other individual Transport Credentials. + errTransportCredsAndBundle = errors.New("grpc: credentials.Bundle may not be used with individual TransportCredentials") // errTransportCredentialsMissing indicates that users want to transmit security - // information (e.g., oauth2 token) which requires secure connection on an insecure + // information (e.g., OAuth2 token) which requires secure connection on an insecure // connection. errTransportCredentialsMissing = errors.New("grpc: the credentials require transport level security (use grpc.WithTransportCredentials() to set)") // errCredentialsConflict indicates that grpc.WithTransportCredentials() // and grpc.WithInsecure() are both called for a connection. errCredentialsConflict = errors.New("grpc: transport credentials are set for an insecure connection (grpc.WithTransportCredentials() and grpc.WithInsecure() are both called)") - // errNetworkIO indicates that the connection is down due to some network I/O error. - errNetworkIO = errors.New("grpc: failed with network I/O error") ) const ( @@ -101,12 +99,6 @@ const ( defaultReadBufSize = 32 * 1024 ) -// RegisterChannelz turns on channelz service. -// This is an EXPERIMENTAL API. -func RegisterChannelz() { - channelz.TurnOn() -} - // Dial creates a client connection to the given target. func Dial(target string, opts ...DialOption) (*ClientConn, error) { return DialContext(context.Background(), target, opts...) @@ -130,11 +122,13 @@ func Dial(target string, opts ...DialOption) (*ClientConn, error) { // e.g. to use dns resolver, a "dns:///" prefix should be applied to the target. func DialContext(ctx context.Context, target string, opts ...DialOption) (conn *ClientConn, err error) { cc := &ClientConn{ - target: target, - csMgr: &connectivityStateManager{}, - conns: make(map[*addrConn]struct{}), - dopts: defaultDialOptions(), - blockingpicker: newPickerWrapper(), + target: target, + csMgr: &connectivityStateManager{}, + conns: make(map[*addrConn]struct{}), + dopts: defaultDialOptions(), + blockingpicker: newPickerWrapper(), + czData: new(channelzData), + firstResolveEvent: grpcsync.NewEvent(), } cc.retryThrottler.Store((*retryThrottler)(nil)) cc.ctx, cc.cancel = context.WithCancel(context.Background()) @@ -143,20 +137,45 @@ func DialContext(ctx context.Context, target string, opts ...DialOption) (conn * opt.apply(&cc.dopts) } + chainUnaryClientInterceptors(cc) + chainStreamClientInterceptors(cc) + + defer func() { + if err != nil { + cc.Close() + } + }() + if channelz.IsOn() { if cc.dopts.channelzParentID != 0 { - cc.channelzID = channelz.RegisterChannel(cc, cc.dopts.channelzParentID, target) + cc.channelzID = channelz.RegisterChannel(&channelzChannel{cc}, cc.dopts.channelzParentID, target) + channelz.AddTraceEvent(cc.channelzID, &channelz.TraceEventDesc{ + Desc: "Channel Created", + Severity: channelz.CtINFO, + Parent: &channelz.TraceEventDesc{ + Desc: fmt.Sprintf("Nested Channel(id:%d) created", cc.channelzID), + Severity: channelz.CtINFO, + }, + }) } else { - cc.channelzID = channelz.RegisterChannel(cc, 0, target) + cc.channelzID = channelz.RegisterChannel(&channelzChannel{cc}, 0, target) + channelz.AddTraceEvent(cc.channelzID, &channelz.TraceEventDesc{ + Desc: "Channel Created", + Severity: channelz.CtINFO, + }) } + cc.csMgr.channelzID = cc.channelzID } if !cc.dopts.insecure { - if cc.dopts.copts.TransportCredentials == nil { + if cc.dopts.copts.TransportCredentials == nil && cc.dopts.copts.CredsBundle == nil { return nil, errNoTransportSecurity } + if cc.dopts.copts.TransportCredentials != nil && cc.dopts.copts.CredsBundle != nil { + return nil, errTransportCredsAndBundle + } } else { - if cc.dopts.copts.TransportCredentials != nil { + if cc.dopts.copts.TransportCredentials != nil || cc.dopts.copts.CredsBundle != nil { return nil, errCredentialsConflict } for _, cd := range cc.dopts.copts.PerRPCCredentials { @@ -166,13 +185,20 @@ func DialContext(ctx context.Context, target string, opts ...DialOption) (conn * } } + if cc.dopts.defaultServiceConfigRawJSON != nil { + sc, err := parseServiceConfig(*cc.dopts.defaultServiceConfigRawJSON) + if err != nil { + return nil, fmt.Errorf("%s: %v", invalidDefaultServiceConfigErrPrefix, err) + } + cc.dopts.defaultServiceConfig = sc + } cc.mkp = cc.dopts.copts.KeepaliveParams if cc.dopts.copts.Dialer == nil { cc.dopts.copts.Dialer = newProxyDialer( func(ctx context.Context, addr string) (net.Conn, error) { network, addr := parseDialTarget(addr) - return dialContext(ctx, network, addr) + return (&net.Dialer{}).DialContext(ctx, network, addr) }, ) } @@ -188,17 +214,12 @@ func DialContext(ctx context.Context, target string, opts ...DialOption) (conn * ctx, cancel = context.WithTimeout(ctx, cc.dopts.timeout) defer cancel() } - defer func() { select { case <-ctx.Done(): conn, err = nil, ctx.Err() default: } - - if err != nil { - cc.Close() - } }() scSet := false @@ -207,7 +228,7 @@ func DialContext(ctx context.Context, target string, opts ...DialOption) (conn * select { case sc, ok := <-cc.dopts.scChan: if ok { - cc.sc = sc + cc.sc = &sc scSet = true } default: @@ -224,9 +245,9 @@ func DialContext(ctx context.Context, target string, opts ...DialOption) (conn * grpclog.Infof("parsed scheme: %q", cc.parsedTarget.Scheme) cc.dopts.resolverBuilder = resolver.Get(cc.parsedTarget.Scheme) if cc.dopts.resolverBuilder == nil { - // If resolver builder is still nil, the parse target's scheme is + // If resolver builder is still nil, the parsed target's scheme is // not registered. Fallback to default resolver and set Endpoint to - // the original unparsed target. + // the original target. grpclog.Infof("scheme %q not registered, fallback to default scheme", cc.parsedTarget.Scheme) cc.parsedTarget = resolver.Target{ Scheme: resolver.GetDefaultScheme(), @@ -253,7 +274,7 @@ func DialContext(ctx context.Context, target string, opts ...DialOption) (conn * select { case sc, ok := <-cc.dopts.scChan: if ok { - cc.sc = sc + cc.sc = &sc } case <-ctx.Done(): return nil, ctx.Err() @@ -269,30 +290,36 @@ func DialContext(ctx context.Context, target string, opts ...DialOption) (conn * } cc.balancerBuildOpts = balancer.BuildOptions{ DialCreds: credsClone, + CredsBundle: cc.dopts.copts.CredsBundle, Dialer: cc.dopts.copts.Dialer, ChannelzParentID: cc.channelzID, + Target: cc.parsedTarget, } // Build the resolver. - cc.resolverWrapper, err = newCCResolverWrapper(cc) + rWrapper, err := newCCResolverWrapper(cc) if err != nil { return nil, fmt.Errorf("failed to build resolver: %v", err) } - // Start the resolver wrapper goroutine after resolverWrapper is created. - // - // If the goroutine is started before resolverWrapper is ready, the - // following may happen: The goroutine sends updates to cc. cc forwards - // those to balancer. Balancer creates new addrConn. addrConn fails to - // connect, and calls resolveNow(). resolveNow() tries to use the non-ready - // resolverWrapper. - cc.resolverWrapper.start() + cc.mu.Lock() + cc.resolverWrapper = rWrapper + cc.mu.Unlock() // A blocking dial blocks until the clientConn is ready. if cc.dopts.block { for { s := cc.GetState() if s == connectivity.Ready { break + } else if cc.dopts.copts.FailOnNonTempDialError && s == connectivity.TransientFailure { + if err = cc.blockingpicker.connectionError(); err != nil { + terr, ok := err.(interface { + Temporary() bool + }) + if ok && !terr.Temporary() { + return nil, err + } + } } if !cc.WaitForStateChange(ctx, s) { // ctx got timeout or canceled. @@ -304,12 +331,75 @@ func DialContext(ctx context.Context, target string, opts ...DialOption) (conn * return cc, nil } +// chainUnaryClientInterceptors chains all unary client interceptors into one. +func chainUnaryClientInterceptors(cc *ClientConn) { + interceptors := cc.dopts.chainUnaryInts + // Prepend dopts.unaryInt to the chaining interceptors if it exists, since unaryInt will + // be executed before any other chained interceptors. + if cc.dopts.unaryInt != nil { + interceptors = append([]UnaryClientInterceptor{cc.dopts.unaryInt}, interceptors...) + } + var chainedInt UnaryClientInterceptor + if len(interceptors) == 0 { + chainedInt = nil + } else if len(interceptors) == 1 { + chainedInt = interceptors[0] + } else { + chainedInt = func(ctx context.Context, method string, req, reply interface{}, cc *ClientConn, invoker UnaryInvoker, opts ...CallOption) error { + return interceptors[0](ctx, method, req, reply, cc, getChainUnaryInvoker(interceptors, 0, invoker), opts...) + } + } + cc.dopts.unaryInt = chainedInt +} + +// getChainUnaryInvoker recursively generate the chained unary invoker. +func getChainUnaryInvoker(interceptors []UnaryClientInterceptor, curr int, finalInvoker UnaryInvoker) UnaryInvoker { + if curr == len(interceptors)-1 { + return finalInvoker + } + return func(ctx context.Context, method string, req, reply interface{}, cc *ClientConn, opts ...CallOption) error { + return interceptors[curr+1](ctx, method, req, reply, cc, getChainUnaryInvoker(interceptors, curr+1, finalInvoker), opts...) + } +} + +// chainStreamClientInterceptors chains all stream client interceptors into one. +func chainStreamClientInterceptors(cc *ClientConn) { + interceptors := cc.dopts.chainStreamInts + // Prepend dopts.streamInt to the chaining interceptors if it exists, since streamInt will + // be executed before any other chained interceptors. + if cc.dopts.streamInt != nil { + interceptors = append([]StreamClientInterceptor{cc.dopts.streamInt}, interceptors...) + } + var chainedInt StreamClientInterceptor + if len(interceptors) == 0 { + chainedInt = nil + } else if len(interceptors) == 1 { + chainedInt = interceptors[0] + } else { + chainedInt = func(ctx context.Context, desc *StreamDesc, cc *ClientConn, method string, streamer Streamer, opts ...CallOption) (ClientStream, error) { + return interceptors[0](ctx, desc, cc, method, getChainStreamer(interceptors, 0, streamer), opts...) + } + } + cc.dopts.streamInt = chainedInt +} + +// getChainStreamer recursively generate the chained client stream constructor. +func getChainStreamer(interceptors []StreamClientInterceptor, curr int, finalStreamer Streamer) Streamer { + if curr == len(interceptors)-1 { + return finalStreamer + } + return func(ctx context.Context, desc *StreamDesc, cc *ClientConn, method string, opts ...CallOption) (ClientStream, error) { + return interceptors[curr+1](ctx, desc, cc, method, getChainStreamer(interceptors, curr+1, finalStreamer), opts...) + } +} + // connectivityStateManager keeps the connectivity.State of ClientConn. // This struct will eventually be exported so the balancers can access it. type connectivityStateManager struct { mu sync.Mutex state connectivity.State notifyChan chan struct{} + channelzID int64 } // updateState updates the connectivity.State of ClientConn. @@ -325,6 +415,12 @@ func (csm *connectivityStateManager) updateState(state connectivity.State) { return } csm.state = state + if channelz.IsOn() { + channelz.AddTraceEvent(csm.channelzID, &channelz.TraceEventDesc{ + Desc: fmt.Sprintf("Channel Connectivity change to %v", state), + Severity: channelz.CtINFO, + }) + } if csm.notifyChan != nil { // There are other goroutines waiting on this channel. close(csm.notifyChan) @@ -359,27 +455,22 @@ type ClientConn struct { csMgr *connectivityStateManager balancerBuildOpts balancer.BuildOptions - resolverWrapper *ccResolverWrapper blockingpicker *pickerWrapper - mu sync.RWMutex - sc ServiceConfig - scRaw string - conns map[*addrConn]struct{} + mu sync.RWMutex + resolverWrapper *ccResolverWrapper + sc *ServiceConfig + conns map[*addrConn]struct{} // Keepalive parameter can be updated if a GoAway is received. mkp keepalive.ClientParameters curBalancerName string - preBalancerName string // previous balancer name. - curAddresses []resolver.Address balancerWrapper *ccBalancerWrapper retryThrottler atomic.Value - channelzID int64 // channelz unique identification number - czmu sync.RWMutex - callsStarted int64 - callsSucceeded int64 - callsFailed int64 - lastCallStartedTime time.Time + firstResolveEvent *grpcsync.Event + + channelzID int64 // channelz unique identification number + czData *channelzData } // WaitForStateChange waits until the connectivity.State of ClientConn changes from sourceState or @@ -413,9 +504,8 @@ func (cc *ClientConn) scWatcher() { } cc.mu.Lock() // TODO: load balance policy runtime change is ignored. - // We may revist this decision in the future. - cc.sc = sc - cc.scRaw = "" + // We may revisit this decision in the future. + cc.sc = &sc cc.mu.Unlock() case <-cc.ctx.Done(): return @@ -423,47 +513,64 @@ func (cc *ClientConn) scWatcher() { } } -func (cc *ClientConn) handleResolvedAddrs(addrs []resolver.Address, err error) { +// waitForResolvedAddrs blocks until the resolver has provided addresses or the +// context expires. Returns nil unless the context expires first; otherwise +// returns a status error based on the context. +func (cc *ClientConn) waitForResolvedAddrs(ctx context.Context) error { + // This is on the RPC path, so we use a fast path to avoid the + // more-expensive "select" below after the resolver has returned once. + if cc.firstResolveEvent.HasFired() { + return nil + } + select { + case <-cc.firstResolveEvent.Done(): + return nil + case <-ctx.Done(): + return status.FromContextError(ctx.Err()).Err() + case <-cc.ctx.Done(): + return ErrClientConnClosing + } +} + +func (cc *ClientConn) updateResolverState(s resolver.State) error { cc.mu.Lock() defer cc.mu.Unlock() + // Check if the ClientConn is already closed. Some fields (e.g. + // balancerWrapper) are set to nil when closing the ClientConn, and could + // cause nil pointer panic if we don't have this check. if cc.conns == nil { - // cc was closed. - return + return nil } - if reflect.DeepEqual(cc.curAddresses, addrs) { - return + if cc.dopts.disableServiceConfig || s.ServiceConfig == nil { + if cc.dopts.defaultServiceConfig != nil && cc.sc == nil { + cc.applyServiceConfig(cc.dopts.defaultServiceConfig) + } + } else if sc, ok := s.ServiceConfig.(*ServiceConfig); ok { + cc.applyServiceConfig(sc) } - cc.curAddresses = addrs - + var balCfg serviceconfig.LoadBalancingConfig if cc.dopts.balancerBuilder == nil { // Only look at balancer types and switch balancer if balancer dial // option is not set. - var isGRPCLB bool - for _, a := range addrs { - if a.Type == resolver.GRPCLB { - isGRPCLB = true - break - } - } var newBalancerName string - if isGRPCLB { - newBalancerName = grpclbName + if cc.sc != nil && cc.sc.lbConfig != nil { + newBalancerName = cc.sc.lbConfig.name + balCfg = cc.sc.lbConfig.cfg } else { - // Address list doesn't contain grpclb address. Try to pick a - // non-grpclb balancer. - newBalancerName = cc.curBalancerName - // If current balancer is grpclb, switch to the previous one. - if newBalancerName == grpclbName { - newBalancerName = cc.preBalancerName + var isGRPCLB bool + for _, a := range s.Addresses { + if a.Type == resolver.GRPCLB { + isGRPCLB = true + break + } } - // The following could be true in two cases: - // - the first time handling resolved addresses - // (curBalancerName="") - // - the first time handling non-grpclb addresses - // (curBalancerName="grpclb", preBalancerName="") - if newBalancerName == "" { + if isGRPCLB { + newBalancerName = grpclbName + } else if cc.sc != nil && cc.sc.LB != nil { + newBalancerName = *cc.sc.LB + } else { newBalancerName = PickFirstBalancerName } } @@ -471,10 +578,12 @@ func (cc *ClientConn) handleResolvedAddrs(addrs []resolver.Address, err error) { } else if cc.balancerWrapper == nil { // Balancer dial option was set, and this is the first time handling // resolved addresses. Build a balancer with dopts.balancerBuilder. + cc.curBalancerName = cc.dopts.balancerBuilder.Name() cc.balancerWrapper = newCCBalancerWrapper(cc, cc.dopts.balancerBuilder, cc.balancerBuildOpts) } - cc.balancerWrapper.handleResolvedAddrs(addrs, nil) + cc.balancerWrapper.updateClientConnState(&balancer.ClientConnState{ResolverState: s, BalancerConfig: balCfg}) + return nil } // switchBalancer starts the switching from current balancer to the balancer @@ -486,11 +595,7 @@ func (cc *ClientConn) handleResolvedAddrs(addrs []resolver.Address, err error) { // // Caller must hold cc.mu. func (cc *ClientConn) switchBalancer(name string) { - if cc.conns == nil { - return - } - - if strings.ToLower(cc.curBalancerName) == strings.ToLower(name) { + if strings.EqualFold(cc.curBalancerName, name) { return } @@ -499,18 +604,29 @@ func (cc *ClientConn) switchBalancer(name string) { grpclog.Infoln("ignoring balancer switching: Balancer DialOption used instead") return } - // TODO(bar switching) change this to two steps: drain and close. - // Keep track of sc in wrapper. if cc.balancerWrapper != nil { cc.balancerWrapper.close() } builder := balancer.Get(name) + if channelz.IsOn() { + if builder == nil { + channelz.AddTraceEvent(cc.channelzID, &channelz.TraceEventDesc{ + Desc: fmt.Sprintf("Channel switches to new LB policy %q due to fallback from invalid balancer name", PickFirstBalancerName), + Severity: channelz.CtWarning, + }) + } else { + channelz.AddTraceEvent(cc.channelzID, &channelz.TraceEventDesc{ + Desc: fmt.Sprintf("Channel switches to new LB policy %q", name), + Severity: channelz.CtINFO, + }) + } + } if builder == nil { grpclog.Infof("failed to get balancer builder for: %v, using pick_first instead", name) builder = newPickfirstBuilder() } - cc.preBalancerName = cc.curBalancerName + cc.curBalancerName = builder.Name() cc.balancerWrapper = newCCBalancerWrapper(cc, builder, cc.balancerBuildOpts) } @@ -530,11 +646,14 @@ func (cc *ClientConn) handleSubConnStateChange(sc balancer.SubConn, s connectivi // newAddrConn creates an addrConn for addrs and adds it to cc.conns. // // Caller needs to make sure len(addrs) > 0. -func (cc *ClientConn) newAddrConn(addrs []resolver.Address) (*addrConn, error) { +func (cc *ClientConn) newAddrConn(addrs []resolver.Address, opts balancer.NewSubConnOptions) (*addrConn, error) { ac := &addrConn{ - cc: cc, - addrs: addrs, - dopts: cc.dopts, + cc: cc, + addrs: addrs, + scopts: opts, + dopts: cc.dopts, + czData: new(channelzData), + resetBackoff: make(chan struct{}), } ac.ctx, ac.cancel = context.WithCancel(cc.ctx) // Track ac in cc. This needs to be done before any getTransport(...) is called. @@ -545,6 +664,14 @@ func (cc *ClientConn) newAddrConn(addrs []resolver.Address) (*addrConn, error) { } if channelz.IsOn() { ac.channelzID = channelz.RegisterSubChannel(ac, cc.channelzID, "") + channelz.AddTraceEvent(ac.channelzID, &channelz.TraceEventDesc{ + Desc: "Subchannel Created", + Severity: channelz.CtINFO, + Parent: &channelz.TraceEventDesc{ + Desc: fmt.Sprintf("Subchannel(id:%d) created", ac.channelzID), + Severity: channelz.CtINFO, + }, + }) } cc.conns[ac] = struct{}{} cc.mu.Unlock() @@ -564,19 +691,14 @@ func (cc *ClientConn) removeAddrConn(ac *addrConn, err error) { ac.tearDown(err) } -// ChannelzMetric returns ChannelInternalMetric of current ClientConn. -// This is an EXPERIMENTAL API. -func (cc *ClientConn) ChannelzMetric() *channelz.ChannelInternalMetric { - state := cc.GetState() - cc.czmu.RLock() - defer cc.czmu.RUnlock() +func (cc *ClientConn) channelzMetric() *channelz.ChannelInternalMetric { return &channelz.ChannelInternalMetric{ - State: state, + State: cc.GetState(), Target: cc.target, - CallsStarted: cc.callsStarted, - CallsSucceeded: cc.callsSucceeded, - CallsFailed: cc.callsFailed, - LastCallStartedTimestamp: cc.lastCallStartedTime, + CallsStarted: atomic.LoadInt64(&cc.czData.callsStarted), + CallsSucceeded: atomic.LoadInt64(&cc.czData.callsSucceeded), + CallsFailed: atomic.LoadInt64(&cc.czData.callsFailed), + LastCallStartedTimestamp: time.Unix(0, atomic.LoadInt64(&cc.czData.lastCallStartedTime)), } } @@ -587,30 +709,21 @@ func (cc *ClientConn) Target() string { } func (cc *ClientConn) incrCallsStarted() { - cc.czmu.Lock() - cc.callsStarted++ - // TODO(yuxuanli): will make this a time.Time pointer improve performance? - cc.lastCallStartedTime = time.Now() - cc.czmu.Unlock() + atomic.AddInt64(&cc.czData.callsStarted, 1) + atomic.StoreInt64(&cc.czData.lastCallStartedTime, time.Now().UnixNano()) } func (cc *ClientConn) incrCallsSucceeded() { - cc.czmu.Lock() - cc.callsSucceeded++ - cc.czmu.Unlock() + atomic.AddInt64(&cc.czData.callsSucceeded, 1) } func (cc *ClientConn) incrCallsFailed() { - cc.czmu.Lock() - cc.callsFailed++ - cc.czmu.Unlock() + atomic.AddInt64(&cc.czData.callsFailed, 1) } -// connect starts to creating transport and also starts the transport monitor -// goroutine for this ac. +// connect starts creating a transport. // It does nothing if the ac is not IDLE. // TODO(bar) Move this to the addrConn section. -// This was part of resetAddrConn, keep it here to make the diff look clean. func (ac *addrConn) connect() error { ac.mu.Lock() if ac.state == connectivity.Shutdown { @@ -621,28 +734,28 @@ func (ac *addrConn) connect() error { ac.mu.Unlock() return nil } - ac.state = connectivity.Connecting - ac.cc.handleSubConnStateChange(ac.acbw, ac.state) + // Update connectivity state within the lock to prevent subsequent or + // concurrent calls from resetting the transport more than once. + ac.updateConnectivityState(connectivity.Connecting) ac.mu.Unlock() // Start a goroutine connecting to the server asynchronously. - go func() { - if err := ac.resetTransport(); err != nil { - grpclog.Warningf("Failed to dial %s: %v; please retry.", ac.addrs[0].Addr, err) - if err != errConnClosing { - // Keep this ac in cc.conns, to get the reason it's torn down. - ac.tearDown(err) - } - return - } - ac.transportMonitor() - }() + go ac.resetTransport() return nil } // tryUpdateAddrs tries to update ac.addrs with the new addresses list. // -// It checks whether current connected address of ac is in the new addrs list. +// If ac is Connecting, it returns false. The caller should tear down the ac and +// create a new one. Note that the backoff will be reset when this happens. +// +// If ac is TransientFailure, it updates ac.addrs and returns true. The updated +// addresses will be picked up by retry in the next iteration after backoff. +// +// If ac is Shutdown or Idle, it updates ac.addrs and returns true. +// +// If ac is Ready, it checks whether current connected address of ac is in the +// new addrs list. // - If true, it updates ac.addrs and returns true. The ac will keep using // the existing connection. // - If false, it does nothing and returns false. @@ -650,11 +763,18 @@ func (ac *addrConn) tryUpdateAddrs(addrs []resolver.Address) bool { ac.mu.Lock() defer ac.mu.Unlock() grpclog.Infof("addrConn: tryUpdateAddrs curAddr: %v, addrs: %v", ac.curAddr, addrs) - if ac.state == connectivity.Shutdown { + if ac.state == connectivity.Shutdown || + ac.state == connectivity.TransientFailure || + ac.state == connectivity.Idle { ac.addrs = addrs return true } + if ac.state == connectivity.Connecting { + return false + } + + // ac.state is Ready, try to find the connected address. var curAddrFound bool for _, a := range addrs { if reflect.DeepEqual(ac.curAddr, a) { @@ -665,7 +785,6 @@ func (ac *addrConn) tryUpdateAddrs(addrs []resolver.Address) bool { grpclog.Infof("addrConn: tryUpdateAddrs curAddrFound: %v", curAddrFound) if curAddrFound { ac.addrs = addrs - ac.reconnectIdx = 0 // Start reconnecting from beginning in the new list. } return curAddrFound @@ -682,6 +801,9 @@ func (cc *ClientConn) GetMethodConfig(method string) MethodConfig { // TODO: Avoid the locking here. cc.mu.RLock() defer cc.mu.RUnlock() + if cc.sc == nil { + return MethodConfig{} + } m, ok := cc.sc.Methods[method] if !ok { i := strings.LastIndex(method, "/") @@ -690,6 +812,15 @@ func (cc *ClientConn) GetMethodConfig(method string) MethodConfig { return m } +func (cc *ClientConn) healthCheckConfig() *healthCheckConfig { + cc.mu.RLock() + defer cc.mu.RUnlock() + if cc.sc == nil { + return nil + } + return cc.sc.healthCheckConfig +} + func (cc *ClientConn) getTransport(ctx context.Context, failfast bool, method string) (transport.ClientTransport, func(balancer.DoneInfo), error) { t, done, err := cc.blockingpicker.pick(ctx, failfast, balancer.PickOptions{ FullMethodName: method, @@ -700,47 +831,25 @@ func (cc *ClientConn) getTransport(ctx context.Context, failfast bool, method st return t, done, nil } -// handleServiceConfig parses the service config string in JSON format to Go native -// struct ServiceConfig, and store both the struct and the JSON string in ClientConn. -func (cc *ClientConn) handleServiceConfig(js string) error { - if cc.dopts.disableServiceConfig { - return nil +func (cc *ClientConn) applyServiceConfig(sc *ServiceConfig) error { + if sc == nil { + // should never reach here. + return fmt.Errorf("got nil pointer for service config") } - sc, err := parseServiceConfig(js) - if err != nil { - return err - } - cc.mu.Lock() - cc.scRaw = js cc.sc = sc - if sc.retryThrottling != nil { + if cc.sc.retryThrottling != nil { newThrottler := &retryThrottler{ - tokens: sc.retryThrottling.MaxTokens, - max: sc.retryThrottling.MaxTokens, - thresh: sc.retryThrottling.MaxTokens / 2, - ratio: sc.retryThrottling.TokenRatio, + tokens: cc.sc.retryThrottling.MaxTokens, + max: cc.sc.retryThrottling.MaxTokens, + thresh: cc.sc.retryThrottling.MaxTokens / 2, + ratio: cc.sc.retryThrottling.TokenRatio, } cc.retryThrottler.Store(newThrottler) } else { cc.retryThrottler.Store((*retryThrottler)(nil)) } - if sc.LB != nil && *sc.LB != grpclbName { // "grpclb" is not a valid balancer option in service config. - if cc.curBalancerName == grpclbName { - // If current balancer is grpclb, there's at least one grpclb - // balancer address in the resolved list. Don't switch the balancer, - // but change the previous balancer name, so if a new resolved - // address list doesn't contain grpclb address, balancer will be - // switched to *sc.LB. - cc.preBalancerName = *sc.LB - } else { - cc.switchBalancer(*sc.LB) - cc.balancerWrapper.handleResolvedAddrs(cc.curAddresses, nil) - } - } - - cc.mu.Unlock() return nil } @@ -754,6 +863,24 @@ func (cc *ClientConn) resolveNow(o resolver.ResolveNowOption) { go r.resolveNow(o) } +// ResetConnectBackoff wakes up all subchannels in transient failure and causes +// them to attempt another connection immediately. It also resets the backoff +// times used for subsequent attempts regardless of the current state. +// +// In general, this function should not be used. Typical service or network +// outages result in a reasonable client reconnection strategy by default. +// However, if a previously unavailable network becomes available, this may be +// used to trigger an immediate reconnect. +// +// This API is EXPERIMENTAL. +func (cc *ClientConn) ResetConnectBackoff() { + cc.mu.Lock() + defer cc.mu.Unlock() + for ac := range cc.conns { + ac.resetConnectBackoff() + } +} + // Close tears down the ClientConn and all underlying connections. func (cc *ClientConn) Close() error { defer cc.cancel() @@ -786,6 +913,19 @@ func (cc *ClientConn) Close() error { ac.tearDown(ErrClientConnClosing) } if channelz.IsOn() { + ted := &channelz.TraceEventDesc{ + Desc: "Channel Deleted", + Severity: channelz.CtINFO, + } + if cc.dopts.channelzParentID != 0 { + ted.Parent = &channelz.TraceEventDesc{ + Desc: fmt.Sprintf("Nested channel(id:%d) deleted", cc.channelzID), + Severity: channelz.CtINFO, + } + } + channelz.AddTraceEvent(cc.channelzID, ted) + // TraceEvent needs to be called before RemoveEntry, as TraceEvent may add trace reference to + // the entity being deleted, and thus prevent it from being deleted right away. channelz.RemoveEntry(cc.channelzID) } return nil @@ -797,37 +937,45 @@ type addrConn struct { cancel context.CancelFunc cc *ClientConn - addrs []resolver.Address dopts dialOptions - events trace.EventLog acbw balancer.SubConn + scopts balancer.NewSubConnOptions - mu sync.Mutex - curAddr resolver.Address - reconnectIdx int // The index in addrs list to start reconnecting from. - state connectivity.State - // ready is closed and becomes nil when a new transport is up or failed - // due to timeout. - ready chan struct{} - transport transport.ClientTransport + // transport is set when there's a viable transport (note: ac state may not be READY as LB channel + // health checking may require server to report healthy to set ac to READY), and is reset + // to nil when the current transport should no longer be used to create a stream (e.g. after GoAway + // is received, transport is closed, ac has been torn down). + transport transport.ClientTransport // The current transport. - // The reason this addrConn is torn down. - tearDownErr error + mu sync.Mutex + curAddr resolver.Address // The current address. + addrs []resolver.Address // All addresses that the resolver resolved to. - connectRetryNum int - // backoffDeadline is the time until which resetTransport needs to - // wait before increasing connectRetryNum count. - backoffDeadline time.Time - // connectDeadline is the time by which all connection - // negotiations must complete. - connectDeadline time.Time + // Use updateConnectivityState for updating addrConn's connectivity state. + state connectivity.State - channelzID int64 // channelz unique identification number - czmu sync.RWMutex - callsStarted int64 - callsSucceeded int64 - callsFailed int64 - lastCallStartedTime time.Time + backoffIdx int // Needs to be stateful for resetConnectBackoff. + resetBackoff chan struct{} + + channelzID int64 // channelz unique identification number. + czData *channelzData +} + +// Note: this requires a lock on ac.mu. +func (ac *addrConn) updateConnectivityState(s connectivity.State) { + if ac.state == s { + return + } + + updateMsg := fmt.Sprintf("Subchannel Connectivity change to %v", s) + ac.state = s + if channelz.IsOn() { + channelz.AddTraceEvent(ac.channelzID, &channelz.TraceEventDesc{ + Desc: updateMsg, + Severity: channelz.CtINFO, + }) + } + ac.cc.handleSubConnStateChange(ac.acbw, s) } // adjustParams updates parameters used to create transports upon @@ -844,289 +992,307 @@ func (ac *addrConn) adjustParams(r transport.GoAwayReason) { } } -// printf records an event in ac's event log, unless ac has been closed. -// REQUIRES ac.mu is held. -func (ac *addrConn) printf(format string, a ...interface{}) { - if ac.events != nil { - ac.events.Printf(format, a...) - } -} - -// errorf records an error in ac's event log, unless ac has been closed. -// REQUIRES ac.mu is held. -func (ac *addrConn) errorf(format string, a ...interface{}) { - if ac.events != nil { - ac.events.Errorf(format, a...) - } -} - -// resetTransport recreates a transport to the address for ac. The old -// transport will close itself on error or when the clientconn is closed. -// The created transport must receive initial settings frame from the server. -// In case that doesn't happen, transportMonitor will kill the newly created -// transport after connectDeadline has expired. -// In case there was an error on the transport before the settings frame was -// received, resetTransport resumes connecting to backends after the one that -// was previously connected to. In case end of the list is reached, resetTransport -// backs off until the original deadline. -// If the DialOption WithWaitForHandshake was set, resetTrasport returns -// successfully only after server settings are received. -// -// TODO(bar) make sure all state transitions are valid. -func (ac *addrConn) resetTransport() error { - ac.mu.Lock() - if ac.state == connectivity.Shutdown { - ac.mu.Unlock() - return errConnClosing - } - if ac.ready != nil { - close(ac.ready) - ac.ready = nil - } - ac.transport = nil - ridx := ac.reconnectIdx - ac.mu.Unlock() - ac.cc.mu.RLock() - ac.dopts.copts.KeepaliveParams = ac.cc.mkp - ac.cc.mu.RUnlock() - var backoffDeadline, connectDeadline time.Time - for connectRetryNum := 0; ; connectRetryNum++ { - ac.mu.Lock() - if ac.backoffDeadline.IsZero() { - // This means either a successful HTTP2 connection was established - // or this is the first time this addrConn is trying to establish a - // connection. - backoffFor := ac.dopts.bs.Backoff(connectRetryNum) // time.Duration. - // This will be the duration that dial gets to finish. - dialDuration := getMinConnectTimeout() - if backoffFor > dialDuration { - // Give dial more time as we keep failing to connect. - dialDuration = backoffFor - } - start := time.Now() - backoffDeadline = start.Add(backoffFor) - connectDeadline = start.Add(dialDuration) - ridx = 0 // Start connecting from the beginning. - } else { - // Continue trying to connect with the same deadlines. - connectRetryNum = ac.connectRetryNum - backoffDeadline = ac.backoffDeadline - connectDeadline = ac.connectDeadline - ac.backoffDeadline = time.Time{} - ac.connectDeadline = time.Time{} - ac.connectRetryNum = 0 +func (ac *addrConn) resetTransport() { + for i := 0; ; i++ { + if i > 0 { + ac.cc.resolveNow(resolver.ResolveNowOption{}) } + + ac.mu.Lock() if ac.state == connectivity.Shutdown { ac.mu.Unlock() - return errConnClosing + return } - ac.printf("connecting") - if ac.state != connectivity.Connecting { - ac.state = connectivity.Connecting - ac.cc.handleSubConnStateChange(ac.acbw, ac.state) - } - // copy ac.addrs in case of race - addrsIter := make([]resolver.Address, len(ac.addrs)) - copy(addrsIter, ac.addrs) - copts := ac.dopts.copts - ac.mu.Unlock() - connected, err := ac.createTransport(connectRetryNum, ridx, backoffDeadline, connectDeadline, addrsIter, copts) - if err != nil { - return err - } - if connected { - return nil - } - } -} -// createTransport creates a connection to one of the backends in addrs. -// It returns true if a connection was established. -func (ac *addrConn) createTransport(connectRetryNum, ridx int, backoffDeadline, connectDeadline time.Time, addrs []resolver.Address, copts transport.ConnectOptions) (bool, error) { - for i := ridx; i < len(addrs); i++ { - addr := addrs[i] - target := transport.TargetInfo{ - Addr: addr.Addr, - Metadata: addr.Metadata, - Authority: ac.cc.authority, + addrs := ac.addrs + backoffFor := ac.dopts.bs.Backoff(ac.backoffIdx) + // This will be the duration that dial gets to finish. + dialDuration := minConnectTimeout + if ac.dopts.minConnectTimeout != nil { + dialDuration = ac.dopts.minConnectTimeout() } - done := make(chan struct{}) - onPrefaceReceipt := func() { - ac.mu.Lock() - close(done) - if !ac.backoffDeadline.IsZero() { - // If we haven't already started reconnecting to - // other backends. - // Note, this can happen when writer notices an error - // and triggers resetTransport while at the same time - // reader receives the preface and invokes this closure. - ac.backoffDeadline = time.Time{} - ac.connectDeadline = time.Time{} - ac.connectRetryNum = 0 - } - ac.mu.Unlock() + + if dialDuration < backoffFor { + // Give dial more time as we keep failing to connect. + dialDuration = backoffFor } - // Do not cancel in the success path because of - // this issue in Go1.6: https://github.com/golang/go/issues/15078. - connectCtx, cancel := context.WithDeadline(ac.ctx, connectDeadline) - if channelz.IsOn() { - copts.ChannelzParentID = ac.channelzID - } - newTr, err := transport.NewClientTransport(connectCtx, ac.cc.ctx, target, copts, onPrefaceReceipt) + // We can potentially spend all the time trying the first address, and + // if the server accepts the connection and then hangs, the following + // addresses will never be tried. + // + // The spec doesn't mention what should be done for multiple addresses. + // https://github.com/grpc/grpc/blob/master/doc/connection-backoff.md#proposed-backoff-algorithm + connectDeadline := time.Now().Add(dialDuration) + + ac.updateConnectivityState(connectivity.Connecting) + ac.transport = nil + ac.mu.Unlock() + + newTr, addr, reconnect, err := ac.tryAllAddrs(addrs, connectDeadline) if err != nil { - cancel() - ac.cc.blockingpicker.updateConnectionError(err) + // After exhausting all addresses, the addrConn enters + // TRANSIENT_FAILURE. ac.mu.Lock() if ac.state == connectivity.Shutdown { - // ac.tearDown(...) has been invoked. ac.mu.Unlock() - return false, errConnClosing + return } + ac.updateConnectivityState(connectivity.TransientFailure) + + // Backoff. + b := ac.resetBackoff ac.mu.Unlock() - grpclog.Warningf("grpc: addrConn.createTransport failed to connect to %v. Err :%v. Reconnecting...", addr, err) + + timer := time.NewTimer(backoffFor) + select { + case <-timer.C: + ac.mu.Lock() + ac.backoffIdx++ + ac.mu.Unlock() + case <-b: + timer.Stop() + case <-ac.ctx.Done(): + timer.Stop() + return + } continue } - if ac.dopts.waitForHandshake { - select { - case <-done: - case <-connectCtx.Done(): - // Didn't receive server preface, must kill this new transport now. - grpclog.Warningf("grpc: addrConn.createTransport failed to receive server preface before deadline.") - newTr.Close() - continue - case <-ac.ctx.Done(): - } - } + ac.mu.Lock() if ac.state == connectivity.Shutdown { ac.mu.Unlock() - // ac.tearDonn(...) has been invoked. newTr.Close() - return false, errConnClosing + return } - ac.printf("ready") - ac.state = connectivity.Ready - ac.cc.handleSubConnStateChange(ac.acbw, ac.state) - ac.transport = newTr ac.curAddr = addr - if ac.ready != nil { - close(ac.ready) - ac.ready = nil - } - select { - case <-done: - // If the server has responded back with preface already, - // don't set the reconnect parameters. - default: - ac.connectRetryNum = connectRetryNum - ac.backoffDeadline = backoffDeadline - ac.connectDeadline = connectDeadline - ac.reconnectIdx = i + 1 // Start reconnecting from the next backend in the list. - } + ac.transport = newTr + ac.backoffIdx = 0 + + hctx, hcancel := context.WithCancel(ac.ctx) + ac.startHealthCheck(hctx) ac.mu.Unlock() - return true, nil + + // Block until the created transport is down. And when this happens, + // we restart from the top of the addr list. + <-reconnect.Done() + hcancel() + // restart connecting - the top of the loop will set state to + // CONNECTING. This is against the current connectivity semantics doc, + // however it allows for graceful behavior for RPCs not yet dispatched + // - unfortunate timing would otherwise lead to the RPC failing even + // though the TRANSIENT_FAILURE state (called for by the doc) would be + // instantaneous. + // + // Ideally we should transition to Idle here and block until there is + // RPC activity that leads to the balancer requesting a reconnect of + // the associated SubConn. } - ac.mu.Lock() - if ac.state == connectivity.Shutdown { - ac.mu.Unlock() - return false, errConnClosing - } - ac.state = connectivity.TransientFailure - ac.cc.handleSubConnStateChange(ac.acbw, ac.state) - ac.cc.resolveNow(resolver.ResolveNowOption{}) - if ac.ready != nil { - close(ac.ready) - ac.ready = nil - } - ac.mu.Unlock() - timer := time.NewTimer(backoffDeadline.Sub(time.Now())) - select { - case <-timer.C: - case <-ac.ctx.Done(): - timer.Stop() - return false, ac.ctx.Err() - } - return false, nil } -// Run in a goroutine to track the error in transport and create the -// new transport if an error happens. It returns when the channel is closing. -func (ac *addrConn) transportMonitor() { - for { - var timer *time.Timer - var cdeadline <-chan time.Time - ac.mu.Lock() - t := ac.transport - if !ac.connectDeadline.IsZero() { - timer = time.NewTimer(ac.connectDeadline.Sub(time.Now())) - cdeadline = timer.C - } - ac.mu.Unlock() - // Block until we receive a goaway or an error occurs. - select { - case <-t.GoAway(): - done := t.Error() - cleanup := t.Close - // Since this transport will be orphaned (won't have a transportMonitor) - // we need to launch a goroutine to keep track of clientConn.Close() - // happening since it might not be noticed by any other goroutine for a while. - go func() { - <-done - cleanup() - }() - case <-t.Error(): - // In case this is triggered because clientConn.Close() - // was called, we want to immeditately close the transport - // since no other goroutine might notice it for a while. - t.Close() - case <-cdeadline: - ac.mu.Lock() - // This implies that client received server preface. - if ac.backoffDeadline.IsZero() { - ac.mu.Unlock() - continue - } - ac.mu.Unlock() - timer = nil - // No server preface received until deadline. - // Kill the connection. - grpclog.Warningf("grpc: addrConn.transportMonitor didn't get server preface after waiting. Closing the new transport now.") - t.Close() - } - if timer != nil { - timer.Stop() - } - // If a GoAway happened, regardless of error, adjust our keepalive - // parameters as appropriate. - select { - case <-t.GoAway(): - ac.adjustParams(t.GetGoAwayReason()) - default: - } +// tryAllAddrs tries to creates a connection to the addresses, and stop when at the +// first successful one. It returns the transport, the address and a Event in +// the successful case. The Event fires when the returned transport disconnects. +func (ac *addrConn) tryAllAddrs(addrs []resolver.Address, connectDeadline time.Time) (transport.ClientTransport, resolver.Address, *grpcsync.Event, error) { + for _, addr := range addrs { ac.mu.Lock() if ac.state == connectivity.Shutdown { ac.mu.Unlock() - return + return nil, resolver.Address{}, nil, errConnClosing + } + + ac.cc.mu.RLock() + ac.dopts.copts.KeepaliveParams = ac.cc.mkp + ac.cc.mu.RUnlock() + + copts := ac.dopts.copts + if ac.scopts.CredsBundle != nil { + copts.CredsBundle = ac.scopts.CredsBundle } - // Set connectivity state to TransientFailure before calling - // resetTransport. Transition READY->CONNECTING is not valid. - ac.state = connectivity.TransientFailure - ac.cc.handleSubConnStateChange(ac.acbw, ac.state) - ac.cc.resolveNow(resolver.ResolveNowOption{}) - ac.curAddr = resolver.Address{} ac.mu.Unlock() - if err := ac.resetTransport(); err != nil { - ac.mu.Lock() - ac.printf("transport exiting: %v", err) - ac.mu.Unlock() - grpclog.Warningf("grpc: addrConn.transportMonitor exits due to: %v", err) - if err != errConnClosing { - // Keep this ac in cc.conns, to get the reason it's torn down. - ac.tearDown(err) + + if channelz.IsOn() { + channelz.AddTraceEvent(ac.channelzID, &channelz.TraceEventDesc{ + Desc: fmt.Sprintf("Subchannel picks a new address %q to connect", addr.Addr), + Severity: channelz.CtINFO, + }) + } + + newTr, reconnect, err := ac.createTransport(addr, copts, connectDeadline) + if err == nil { + return newTr, addr, reconnect, nil + } + ac.cc.blockingpicker.updateConnectionError(err) + } + + // Couldn't connect to any address. + return nil, resolver.Address{}, nil, fmt.Errorf("couldn't connect to any address") +} + +// createTransport creates a connection to addr. It returns the transport and a +// Event in the successful case. The Event fires when the returned transport +// disconnects. +func (ac *addrConn) createTransport(addr resolver.Address, copts transport.ConnectOptions, connectDeadline time.Time) (transport.ClientTransport, *grpcsync.Event, error) { + prefaceReceived := make(chan struct{}) + onCloseCalled := make(chan struct{}) + reconnect := grpcsync.NewEvent() + + target := transport.TargetInfo{ + Addr: addr.Addr, + Metadata: addr.Metadata, + Authority: ac.cc.authority, + } + + once := sync.Once{} + onGoAway := func(r transport.GoAwayReason) { + ac.mu.Lock() + ac.adjustParams(r) + once.Do(func() { + if ac.state == connectivity.Ready { + // Prevent this SubConn from being used for new RPCs by setting its + // state to Connecting. + // + // TODO: this should be Idle when grpc-go properly supports it. + ac.updateConnectivityState(connectivity.Connecting) } + }) + ac.mu.Unlock() + reconnect.Fire() + } + + onClose := func() { + ac.mu.Lock() + once.Do(func() { + if ac.state == connectivity.Ready { + // Prevent this SubConn from being used for new RPCs by setting its + // state to Connecting. + // + // TODO: this should be Idle when grpc-go properly supports it. + ac.updateConnectivityState(connectivity.Connecting) + } + }) + ac.mu.Unlock() + close(onCloseCalled) + reconnect.Fire() + } + + onPrefaceReceipt := func() { + close(prefaceReceived) + } + + connectCtx, cancel := context.WithDeadline(ac.ctx, connectDeadline) + defer cancel() + if channelz.IsOn() { + copts.ChannelzParentID = ac.channelzID + } + + newTr, err := transport.NewClientTransport(connectCtx, ac.cc.ctx, target, copts, onPrefaceReceipt, onGoAway, onClose) + if err != nil { + // newTr is either nil, or closed. + grpclog.Warningf("grpc: addrConn.createTransport failed to connect to %v. Err :%v. Reconnecting...", addr, err) + return nil, nil, err + } + + select { + case <-time.After(connectDeadline.Sub(time.Now())): + // We didn't get the preface in time. + newTr.Close() + grpclog.Warningf("grpc: addrConn.createTransport failed to connect to %v: didn't receive server preface in time. Reconnecting...", addr) + return nil, nil, errors.New("timed out waiting for server handshake") + case <-prefaceReceived: + // We got the preface - huzzah! things are good. + case <-onCloseCalled: + // The transport has already closed - noop. + return nil, nil, errors.New("connection closed") + // TODO(deklerk) this should bail on ac.ctx.Done(). Add a test and fix. + } + return newTr, reconnect, nil +} + +// startHealthCheck starts the health checking stream (RPC) to watch the health +// stats of this connection if health checking is requested and configured. +// +// LB channel health checking is enabled when all requirements below are met: +// 1. it is not disabled by the user with the WithDisableHealthCheck DialOption +// 2. internal.HealthCheckFunc is set by importing the grpc/healthcheck package +// 3. a service config with non-empty healthCheckConfig field is provided +// 4. the load balancer requests it +// +// It sets addrConn to READY if the health checking stream is not started. +// +// Caller must hold ac.mu. +func (ac *addrConn) startHealthCheck(ctx context.Context) { + var healthcheckManagingState bool + defer func() { + if !healthcheckManagingState { + ac.updateConnectivityState(connectivity.Ready) + } + }() + + if ac.cc.dopts.disableHealthCheck { + return + } + healthCheckConfig := ac.cc.healthCheckConfig() + if healthCheckConfig == nil { + return + } + if !ac.scopts.HealthCheckEnabled { + return + } + healthCheckFunc := ac.cc.dopts.healthCheckFunc + if healthCheckFunc == nil { + // The health package is not imported to set health check function. + // + // TODO: add a link to the health check doc in the error message. + grpclog.Error("Health check is requested but health check function is not set.") + return + } + + healthcheckManagingState = true + + // Set up the health check helper functions. + currentTr := ac.transport + newStream := func(method string) (interface{}, error) { + ac.mu.Lock() + if ac.transport != currentTr { + ac.mu.Unlock() + return nil, status.Error(codes.Canceled, "the provided transport is no longer valid to use") + } + ac.mu.Unlock() + return newNonRetryClientStream(ctx, &StreamDesc{ServerStreams: true}, method, currentTr, ac) + } + setConnectivityState := func(s connectivity.State) { + ac.mu.Lock() + defer ac.mu.Unlock() + if ac.transport != currentTr { return } + ac.updateConnectivityState(s) } + // Start the health checking stream. + go func() { + err := ac.cc.dopts.healthCheckFunc(ctx, newStream, setConnectivityState, healthCheckConfig.ServiceName) + if err != nil { + if status.Code(err) == codes.Unimplemented { + if channelz.IsOn() { + channelz.AddTraceEvent(ac.channelzID, &channelz.TraceEventDesc{ + Desc: "Subchannel health check is unimplemented at server side, thus health check is disabled", + Severity: channelz.CtError, + }) + } + grpclog.Error("Subchannel health check is unimplemented at server side, thus health check is disabled") + } else { + grpclog.Errorf("HealthCheckFunc exits with unexpected error %v", err) + } + } + }() +} + +func (ac *addrConn) resetConnectBackoff() { + ac.mu.Lock() + close(ac.resetBackoff) + ac.backoffIdx = 0 + ac.resetBackoff = make(chan struct{}) + ac.mu.Unlock() } // getReadyTransport returns the transport if ac's state is READY. @@ -1134,7 +1300,7 @@ func (ac *addrConn) transportMonitor() { // If ac's state is IDLE, it will trigger ac to connect. func (ac *addrConn) getReadyTransport() (transport.ClientTransport, bool) { ac.mu.Lock() - if ac.state == connectivity.Ready { + if ac.state == connectivity.Ready && ac.transport != nil { t := ac.transport ac.mu.Unlock() return t, true @@ -1157,34 +1323,42 @@ func (ac *addrConn) getReadyTransport() (transport.ClientTransport, bool) { // tight loop. // tearDown doesn't remove ac from ac.cc.conns. func (ac *addrConn) tearDown(err error) { - ac.cancel() ac.mu.Lock() - defer ac.mu.Unlock() if ac.state == connectivity.Shutdown { + ac.mu.Unlock() return } + curTr := ac.transport + ac.transport = nil + // We have to set the state to Shutdown before anything else to prevent races + // between setting the state and logic that waits on context cancelation / etc. + ac.updateConnectivityState(connectivity.Shutdown) + ac.cancel() ac.curAddr = resolver.Address{} - if err == errConnDrain && ac.transport != nil { + if err == errConnDrain && curTr != nil { // GracefulClose(...) may be executed multiple times when // i) receiving multiple GoAway frames from the server; or // ii) there are concurrent name resolver/Balancer triggered // address removal and GoAway. - ac.transport.GracefulClose() - } - ac.state = connectivity.Shutdown - ac.tearDownErr = err - ac.cc.handleSubConnStateChange(ac.acbw, ac.state) - if ac.events != nil { - ac.events.Finish() - ac.events = nil - } - if ac.ready != nil { - close(ac.ready) - ac.ready = nil + // We have to unlock and re-lock here because GracefulClose => Close => onClose, which requires locking ac.mu. + ac.mu.Unlock() + curTr.GracefulClose() + ac.mu.Lock() } if channelz.IsOn() { + channelz.AddTraceEvent(ac.channelzID, &channelz.TraceEventDesc{ + Desc: "Subchannel Deleted", + Severity: channelz.CtINFO, + Parent: &channelz.TraceEventDesc{ + Desc: fmt.Sprintf("Subchanel(id:%d) deleted", ac.channelzID), + Severity: channelz.CtINFO, + }, + }) + // TraceEvent needs to be called before RemoveEntry, as TraceEvent may add trace reference to + // the entity beng deleted, and thus prevent it from being deleted right away. channelz.RemoveEntry(ac.channelzID) } + ac.mu.Unlock() } func (ac *addrConn) getState() connectivity.State { @@ -1197,36 +1371,27 @@ func (ac *addrConn) ChannelzMetric() *channelz.ChannelInternalMetric { ac.mu.Lock() addr := ac.curAddr.Addr ac.mu.Unlock() - state := ac.getState() - ac.czmu.RLock() - defer ac.czmu.RUnlock() return &channelz.ChannelInternalMetric{ - State: state, + State: ac.getState(), Target: addr, - CallsStarted: ac.callsStarted, - CallsSucceeded: ac.callsSucceeded, - CallsFailed: ac.callsFailed, - LastCallStartedTimestamp: ac.lastCallStartedTime, + CallsStarted: atomic.LoadInt64(&ac.czData.callsStarted), + CallsSucceeded: atomic.LoadInt64(&ac.czData.callsSucceeded), + CallsFailed: atomic.LoadInt64(&ac.czData.callsFailed), + LastCallStartedTimestamp: time.Unix(0, atomic.LoadInt64(&ac.czData.lastCallStartedTime)), } } func (ac *addrConn) incrCallsStarted() { - ac.czmu.Lock() - ac.callsStarted++ - ac.lastCallStartedTime = time.Now() - ac.czmu.Unlock() + atomic.AddInt64(&ac.czData.callsStarted, 1) + atomic.StoreInt64(&ac.czData.lastCallStartedTime, time.Now().UnixNano()) } func (ac *addrConn) incrCallsSucceeded() { - ac.czmu.Lock() - ac.callsSucceeded++ - ac.czmu.Unlock() + atomic.AddInt64(&ac.czData.callsSucceeded, 1) } func (ac *addrConn) incrCallsFailed() { - ac.czmu.Lock() - ac.callsFailed++ - ac.czmu.Unlock() + atomic.AddInt64(&ac.czData.callsFailed, 1) } type retryThrottler struct { @@ -1266,6 +1431,14 @@ func (rt *retryThrottler) successfulRPC() { } } +type channelzChannel struct { + cc *ClientConn +} + +func (c *channelzChannel) ChannelzMetric() *channelz.ChannelInternalMetric { + return c.cc.channelzMetric() +} + // ErrClientConnTimeout indicates that the ClientConn cannot establish the // underlying connections within the specified timeout. // diff --git a/vendor/google.golang.org/grpc/codes/codes.go b/vendor/google.golang.org/grpc/codes/codes.go index d9b9d5782e0..02738839dd9 100644 --- a/vendor/google.golang.org/grpc/codes/codes.go +++ b/vendor/google.golang.org/grpc/codes/codes.go @@ -132,7 +132,8 @@ const ( // Unavailable indicates the service is currently unavailable. // This is a most likely a transient condition and may be corrected - // by retrying with a backoff. + // by retrying with a backoff. Note that it is not always safe to retry + // non-idempotent operations. // // See litmus test above for deciding between FailedPrecondition, // Aborted, and Unavailable. diff --git a/vendor/google.golang.org/grpc/connectivity/connectivity.go b/vendor/google.golang.org/grpc/connectivity/connectivity.go index 568ef5dc68b..34ec36fbf6d 100644 --- a/vendor/google.golang.org/grpc/connectivity/connectivity.go +++ b/vendor/google.golang.org/grpc/connectivity/connectivity.go @@ -22,7 +22,8 @@ package connectivity import ( - "golang.org/x/net/context" + "context" + "google.golang.org/grpc/grpclog" ) @@ -51,7 +52,7 @@ func (s State) String() string { const ( // Idle indicates the ClientConn is idle. Idle State = iota - // Connecting indicates the ClienConn is connecting. + // Connecting indicates the ClientConn is connecting. Connecting // Ready indicates the ClientConn is ready for work. Ready diff --git a/vendor/google.golang.org/grpc/credentials/credentials.go b/vendor/google.golang.org/grpc/credentials/credentials.go index 1dae57ab188..8ea3d4a1dc2 100644 --- a/vendor/google.golang.org/grpc/credentials/credentials.go +++ b/vendor/google.golang.org/grpc/credentials/credentials.go @@ -23,6 +23,7 @@ package credentials // import "google.golang.org/grpc/credentials" import ( + "context" "crypto/tls" "crypto/x509" "errors" @@ -32,12 +33,9 @@ import ( "strings" "github.com/golang/protobuf/proto" - "golang.org/x/net/context" + "google.golang.org/grpc/credentials/internal" ) -// alpnProtoStr are the specified application level protocols for gRPC. -var alpnProtoStr = []string{"h2"} - // PerRPCCredentials defines the common interface for the credentials which need to // attach security information to every RPC (e.g., oauth2). type PerRPCCredentials interface { @@ -108,6 +106,25 @@ type TransportCredentials interface { OverrideServerName(string) error } +// Bundle is a combination of TransportCredentials and PerRPCCredentials. +// +// It also contains a mode switching method, so it can be used as a combination +// of different credential policies. +// +// Bundle cannot be used together with individual TransportCredentials. +// PerRPCCredentials from Bundle will be appended to other PerRPCCredentials. +// +// This API is experimental. +type Bundle interface { + TransportCredentials() TransportCredentials + PerRPCCredentials() PerRPCCredentials + // NewWithMode should make a copy of Bundle, and switch mode. Modifying the + // existing Bundle may cause races. + // + // NewWithMode returns nil if the requested mode is not supported. + NewWithMode(mode string) (Bundle, error) +} + // TLSInfo contains the auth information for a TLS authenticated connection. // It implements the AuthInfo interface. type TLSInfo struct { @@ -119,8 +136,8 @@ func (t TLSInfo) AuthType() string { return "tls" } -// GetChannelzSecurityValue returns security info requested by channelz. -func (t TLSInfo) GetChannelzSecurityValue() ChannelzSecurityValue { +// GetSecurityValue returns security info requested by channelz. +func (t TLSInfo) GetSecurityValue() ChannelzSecurityValue { v := &TLSChannelzSecurityValue{ StandardName: cipherSuiteLookup[t.State.CipherSuite], } @@ -168,7 +185,7 @@ func (c *tlsCreds) ClientHandshake(ctx context.Context, authority string, rawCon case <-ctx.Done(): return nil, nil, ctx.Err() } - return tlsConn{Conn: conn, rawConn: rawConn}, TLSInfo{conn.ConnectionState()}, nil + return internal.WrapSyscallConn(rawConn, conn), TLSInfo{conn.ConnectionState()}, nil } func (c *tlsCreds) ServerHandshake(rawConn net.Conn) (net.Conn, AuthInfo, error) { @@ -176,7 +193,7 @@ func (c *tlsCreds) ServerHandshake(rawConn net.Conn) (net.Conn, AuthInfo, error) if err := conn.Handshake(); err != nil { return nil, nil, err } - return tlsConn{Conn: conn, rawConn: rawConn}, TLSInfo{conn.ConnectionState()}, nil + return internal.WrapSyscallConn(rawConn, conn), TLSInfo{conn.ConnectionState()}, nil } func (c *tlsCreds) Clone() TransportCredentials { @@ -188,10 +205,23 @@ func (c *tlsCreds) OverrideServerName(serverNameOverride string) error { return nil } +const alpnProtoStrH2 = "h2" + +func appendH2ToNextProtos(ps []string) []string { + for _, p := range ps { + if p == alpnProtoStrH2 { + return ps + } + } + ret := make([]string, 0, len(ps)+1) + ret = append(ret, ps...) + return append(ret, alpnProtoStrH2) +} + // NewTLS uses c to construct a TransportCredentials based on TLS. func NewTLS(c *tls.Config) TransportCredentials { tc := &tlsCreds{cloneTLSConfig(c)} - tc.config.NextProtos = alpnProtoStr + tc.config.NextProtos = appendH2ToNextProtos(tc.config.NextProtos) return tc } @@ -248,29 +278,22 @@ type ChannelzSecurityValue interface { // TLSChannelzSecurityValue defines the struct that TLS protocol should return // from GetSecurityValue(), containing security info like cipher and certificate used. type TLSChannelzSecurityValue struct { + ChannelzSecurityValue StandardName string LocalCertificate []byte RemoteCertificate []byte } -func (*TLSChannelzSecurityValue) isChannelzSecurityValue() {} - // OtherChannelzSecurityValue defines the struct that non-TLS protocol should return // from GetSecurityValue(), which contains protocol specific security info. Note // the Value field will be sent to users of channelz requesting channel info, and // thus sensitive info should better be avoided. type OtherChannelzSecurityValue struct { + ChannelzSecurityValue Name string Value proto.Message } -func (*OtherChannelzSecurityValue) isChannelzSecurityValue() {} - -type tlsConn struct { - *tls.Conn - rawConn net.Conn -} - var cipherSuiteLookup = map[uint16]string{ tls.TLS_RSA_WITH_RC4_128_SHA: "TLS_RSA_WITH_RC4_128_SHA", tls.TLS_RSA_WITH_3DES_EDE_CBC_SHA: "TLS_RSA_WITH_3DES_EDE_CBC_SHA", @@ -290,4 +313,24 @@ var cipherSuiteLookup = map[uint16]string{ tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384: "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384", tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384: "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384", tls.TLS_FALLBACK_SCSV: "TLS_FALLBACK_SCSV", + tls.TLS_RSA_WITH_AES_128_CBC_SHA256: "TLS_RSA_WITH_AES_128_CBC_SHA256", + tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256: "TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256", + tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256: "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256", + tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305: "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305", + tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305: "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305", +} + +// cloneTLSConfig returns a shallow clone of the exported +// fields of cfg, ignoring the unexported sync.Once, which +// contains a mutex and must not be copied. +// +// If cfg is nil, a new zero tls.Config is returned. +// +// TODO: inline this function if possible. +func cloneTLSConfig(cfg *tls.Config) *tls.Config { + if cfg == nil { + return &tls.Config{} + } + + return cfg.Clone() } diff --git a/vendor/google.golang.org/grpc/credentials/go16.go b/vendor/google.golang.org/grpc/credentials/go16.go deleted file mode 100644 index d6bbcc9fdd9..00000000000 --- a/vendor/google.golang.org/grpc/credentials/go16.go +++ /dev/null @@ -1,57 +0,0 @@ -// +build !go1.7 - -/* - * - * Copyright 2016 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package credentials - -import ( - "crypto/tls" -) - -// cloneTLSConfig returns a shallow clone of the exported -// fields of cfg, ignoring the unexported sync.Once, which -// contains a mutex and must not be copied. -// -// If cfg is nil, a new zero tls.Config is returned. -func cloneTLSConfig(cfg *tls.Config) *tls.Config { - if cfg == nil { - return &tls.Config{} - } - return &tls.Config{ - Rand: cfg.Rand, - Time: cfg.Time, - Certificates: cfg.Certificates, - NameToCertificate: cfg.NameToCertificate, - GetCertificate: cfg.GetCertificate, - RootCAs: cfg.RootCAs, - NextProtos: cfg.NextProtos, - ServerName: cfg.ServerName, - ClientAuth: cfg.ClientAuth, - ClientCAs: cfg.ClientCAs, - InsecureSkipVerify: cfg.InsecureSkipVerify, - CipherSuites: cfg.CipherSuites, - PreferServerCipherSuites: cfg.PreferServerCipherSuites, - SessionTicketsDisabled: cfg.SessionTicketsDisabled, - SessionTicketKey: cfg.SessionTicketKey, - ClientSessionCache: cfg.ClientSessionCache, - MinVersion: cfg.MinVersion, - MaxVersion: cfg.MaxVersion, - CurvePreferences: cfg.CurvePreferences, - } -} diff --git a/vendor/google.golang.org/grpc/credentials/go17.go b/vendor/google.golang.org/grpc/credentials/go17.go deleted file mode 100644 index fbd50000200..00000000000 --- a/vendor/google.golang.org/grpc/credentials/go17.go +++ /dev/null @@ -1,59 +0,0 @@ -// +build go1.7,!go1.8 - -/* - * - * Copyright 2016 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package credentials - -import ( - "crypto/tls" -) - -// cloneTLSConfig returns a shallow clone of the exported -// fields of cfg, ignoring the unexported sync.Once, which -// contains a mutex and must not be copied. -// -// If cfg is nil, a new zero tls.Config is returned. -func cloneTLSConfig(cfg *tls.Config) *tls.Config { - if cfg == nil { - return &tls.Config{} - } - return &tls.Config{ - Rand: cfg.Rand, - Time: cfg.Time, - Certificates: cfg.Certificates, - NameToCertificate: cfg.NameToCertificate, - GetCertificate: cfg.GetCertificate, - RootCAs: cfg.RootCAs, - NextProtos: cfg.NextProtos, - ServerName: cfg.ServerName, - ClientAuth: cfg.ClientAuth, - ClientCAs: cfg.ClientCAs, - InsecureSkipVerify: cfg.InsecureSkipVerify, - CipherSuites: cfg.CipherSuites, - PreferServerCipherSuites: cfg.PreferServerCipherSuites, - SessionTicketsDisabled: cfg.SessionTicketsDisabled, - SessionTicketKey: cfg.SessionTicketKey, - ClientSessionCache: cfg.ClientSessionCache, - MinVersion: cfg.MinVersion, - MaxVersion: cfg.MaxVersion, - CurvePreferences: cfg.CurvePreferences, - DynamicRecordSizingDisabled: cfg.DynamicRecordSizingDisabled, - Renegotiation: cfg.Renegotiation, - } -} diff --git a/vendor/google.golang.org/grpc/credentials/go18.go b/vendor/google.golang.org/grpc/credentials/go18.go deleted file mode 100644 index db30d46cc57..00000000000 --- a/vendor/google.golang.org/grpc/credentials/go18.go +++ /dev/null @@ -1,46 +0,0 @@ -// +build go1.8 - -/* - * - * Copyright 2017 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package credentials - -import ( - "crypto/tls" -) - -func init() { - cipherSuiteLookup[tls.TLS_RSA_WITH_AES_128_CBC_SHA256] = "TLS_RSA_WITH_AES_128_CBC_SHA256" - cipherSuiteLookup[tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256] = "TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256" - cipherSuiteLookup[tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256] = "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256" - cipherSuiteLookup[tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305] = "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305" - cipherSuiteLookup[tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305] = "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305" -} - -// cloneTLSConfig returns a shallow clone of the exported -// fields of cfg, ignoring the unexported sync.Once, which -// contains a mutex and must not be copied. -// -// If cfg is nil, a new zero tls.Config is returned. -func cloneTLSConfig(cfg *tls.Config) *tls.Config { - if cfg == nil { - return &tls.Config{} - } - - return cfg.Clone() -} diff --git a/vendor/google.golang.org/grpc/credentials/internal/syscallconn.go b/vendor/google.golang.org/grpc/credentials/internal/syscallconn.go new file mode 100644 index 00000000000..2f4472becc7 --- /dev/null +++ b/vendor/google.golang.org/grpc/credentials/internal/syscallconn.go @@ -0,0 +1,61 @@ +// +build !appengine + +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package internal contains credentials-internal code. +package internal + +import ( + "net" + "syscall" +) + +type sysConn = syscall.Conn + +// syscallConn keeps reference of rawConn to support syscall.Conn for channelz. +// SyscallConn() (the method in interface syscall.Conn) is explicitly +// implemented on this type, +// +// Interface syscall.Conn is implemented by most net.Conn implementations (e.g. +// TCPConn, UnixConn), but is not part of net.Conn interface. So wrapper conns +// that embed net.Conn don't implement syscall.Conn. (Side note: tls.Conn +// doesn't embed net.Conn, so even if syscall.Conn is part of net.Conn, it won't +// help here). +type syscallConn struct { + net.Conn + // sysConn is a type alias of syscall.Conn. It's necessary because the name + // `Conn` collides with `net.Conn`. + sysConn +} + +// WrapSyscallConn tries to wrap rawConn and newConn into a net.Conn that +// implements syscall.Conn. rawConn will be used to support syscall, and newConn +// will be used for read/write. +// +// This function returns newConn if rawConn doesn't implement syscall.Conn. +func WrapSyscallConn(rawConn, newConn net.Conn) net.Conn { + sysConn, ok := rawConn.(syscall.Conn) + if !ok { + return newConn + } + return &syscallConn{ + Conn: newConn, + sysConn: sysConn, + } +} diff --git a/vendor/google.golang.org/grpc/naming/go18.go b/vendor/google.golang.org/grpc/credentials/internal/syscallconn_appengine.go similarity index 73% rename from vendor/google.golang.org/grpc/naming/go18.go rename to vendor/google.golang.org/grpc/credentials/internal/syscallconn_appengine.go index b5a0f842748..d4346e9eabe 100644 --- a/vendor/google.golang.org/grpc/naming/go18.go +++ b/vendor/google.golang.org/grpc/credentials/internal/syscallconn_appengine.go @@ -1,8 +1,8 @@ -// +build go1.8 +// +build appengine /* * - * Copyright 2017 gRPC authors. + * Copyright 2018 gRPC authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,11 +18,13 @@ * */ -package naming +package internal -import "net" - -var ( - lookupHost = net.DefaultResolver.LookupHost - lookupSRV = net.DefaultResolver.LookupSRV +import ( + "net" ) + +// WrapSyscallConn returns newConn on appengine. +func WrapSyscallConn(rawConn, newConn net.Conn) net.Conn { + return newConn +} diff --git a/vendor/google.golang.org/grpc/credentials/go19.go b/vendor/google.golang.org/grpc/credentials/tls13.go similarity index 60% rename from vendor/google.golang.org/grpc/credentials/go19.go rename to vendor/google.golang.org/grpc/credentials/tls13.go index 2a4ca1a57d6..ccbf35b3312 100644 --- a/vendor/google.golang.org/grpc/credentials/go19.go +++ b/vendor/google.golang.org/grpc/credentials/tls13.go @@ -1,8 +1,8 @@ -// +build go1.9,!appengine +// +build go1.12 /* * - * Copyright 2018 gRPC authors. + * Copyright 2019 gRPC authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,16 +20,11 @@ package credentials -import ( - "errors" - "syscall" -) +import "crypto/tls" -// implements the syscall.Conn interface -func (c tlsConn) SyscallConn() (syscall.RawConn, error) { - conn, ok := c.rawConn.(syscall.Conn) - if !ok { - return nil, errors.New("RawConn does not implement syscall.Conn") - } - return conn.SyscallConn() +// This init function adds cipher suite constants only defined in Go 1.12. +func init() { + cipherSuiteLookup[tls.TLS_AES_128_GCM_SHA256] = "TLS_AES_128_GCM_SHA256" + cipherSuiteLookup[tls.TLS_AES_256_GCM_SHA384] = "TLS_AES_256_GCM_SHA384" + cipherSuiteLookup[tls.TLS_CHACHA20_POLY1305_SHA256] = "TLS_CHACHA20_POLY1305_SHA256" } diff --git a/vendor/google.golang.org/grpc/dialoptions.go b/vendor/google.golang.org/grpc/dialoptions.go index 20accf1a192..e8f34d0d6ea 100644 --- a/vendor/google.golang.org/grpc/dialoptions.go +++ b/vendor/google.golang.org/grpc/dialoptions.go @@ -19,13 +19,14 @@ package grpc import ( + "context" "fmt" "net" "time" - "golang.org/x/net/context" "google.golang.org/grpc/balancer" "google.golang.org/grpc/credentials" + "google.golang.org/grpc/grpclog" "google.golang.org/grpc/internal" "google.golang.org/grpc/internal/backoff" "google.golang.org/grpc/internal/envconfig" @@ -38,8 +39,12 @@ import ( // dialOptions configure a Dial call. dialOptions are set by the DialOption // values passed to Dial. type dialOptions struct { - unaryInt UnaryClientInterceptor - streamInt StreamClientInterceptor + unaryInt UnaryClientInterceptor + streamInt StreamClientInterceptor + + chainUnaryInts []UnaryClientInterceptor + chainStreamInts []StreamClientInterceptor + cp Compressor dc Decompressor bs backoff.Strategy @@ -54,11 +59,15 @@ type dialOptions struct { // balancer, and also by WithBalancerName dial option. balancerBuilder balancer.Builder // This is to support grpclb. - resolverBuilder resolver.Builder - waitForHandshake bool - channelzParentID int64 - disableServiceConfig bool - disableRetry bool + resolverBuilder resolver.Builder + channelzParentID int64 + disableServiceConfig bool + disableRetry bool + disableHealthCheck bool + healthCheckFunc internal.HealthChecker + minConnectTimeout func() time.Duration + defaultServiceConfig *ServiceConfig // defaultServiceConfig is parsed from defaultServiceConfigRawJSON. + defaultServiceConfigRawJSON *string } // DialOption configures how we set up the connection. @@ -90,14 +99,6 @@ func newFuncDialOption(f func(*dialOptions)) *funcDialOption { } } -// WithWaitForHandshake blocks until the initial settings frame is received from -// the server before assigning RPCs to the connection. Experimental API. -func WithWaitForHandshake() DialOption { - return newFuncDialOption(func(o *dialOptions) { - o.waitForHandshake = true - }) -} - // WithWriteBufferSize determines how much data can be batched before doing a // write on the wire. The corresponding memory allocation for this buffer will // be twice the size to keep syscalls low. The default value for this buffer is @@ -143,7 +144,8 @@ func WithInitialConnWindowSize(s int32) DialOption { // WithMaxMsgSize returns a DialOption which sets the maximum message size the // client can receive. // -// Deprecated: use WithDefaultCallOptions(MaxCallRecvMsgSize(s)) instead. +// Deprecated: use WithDefaultCallOptions(MaxCallRecvMsgSize(s)) instead. Will +// be supported throughout 1.x. func WithMaxMsgSize(s int) DialOption { return WithDefaultCallOptions(MaxCallRecvMsgSize(s)) } @@ -159,7 +161,8 @@ func WithDefaultCallOptions(cos ...CallOption) DialOption { // WithCodec returns a DialOption which sets a codec for message marshaling and // unmarshaling. // -// Deprecated: use WithDefaultCallOptions(CallCustomCodec(c)) instead. +// Deprecated: use WithDefaultCallOptions(ForceCodec(_)) instead. Will be +// supported throughout 1.x. func WithCodec(c Codec) DialOption { return WithDefaultCallOptions(CallCustomCodec(c)) } @@ -168,7 +171,7 @@ func WithCodec(c Codec) DialOption { // message compression. It has lower priority than the compressor set by the // UseCompressor CallOption. // -// Deprecated: use UseCompressor instead. +// Deprecated: use UseCompressor instead. Will be supported throughout 1.x. func WithCompressor(cp Compressor) DialOption { return newFuncDialOption(func(o *dialOptions) { o.cp = cp @@ -183,7 +186,8 @@ func WithCompressor(cp Compressor) DialOption { // message. If no compressor is registered for the encoding, an Unimplemented // status error will be returned. // -// Deprecated: use encoding.RegisterCompressor instead. +// Deprecated: use encoding.RegisterCompressor instead. Will be supported +// throughout 1.x. func WithDecompressor(dc Decompressor) DialOption { return newFuncDialOption(func(o *dialOptions) { o.dc = dc @@ -194,7 +198,7 @@ func WithDecompressor(dc Decompressor) DialOption { // Name resolver will be ignored if this DialOption is specified. // // Deprecated: use the new balancer APIs in balancer package and -// WithBalancerName. +// WithBalancerName. Will be removed in a future 1.x release. func WithBalancer(b Balancer) DialOption { return newFuncDialOption(func(o *dialOptions) { o.balancerBuilder = &balancerWrapperBuilder{ @@ -210,7 +214,8 @@ func WithBalancer(b Balancer) DialOption { // The balancer cannot be overridden by balancer option specified by service // config. // -// This is an EXPERIMENTAL API. +// Deprecated: use WithDefaultServiceConfig and WithDisableServiceConfig +// instead. Will be removed in a future 1.x release. func WithBalancerName(balancerName string) DialOption { builder := balancer.Get(balancerName) if builder == nil { @@ -231,9 +236,10 @@ func withResolverBuilder(b resolver.Builder) DialOption { // WithServiceConfig returns a DialOption which has a channel to read the // service configuration. // -// Deprecated: service config should be received through name resolver, as -// specified here. -// https://github.com/grpc/grpc/blob/master/doc/service_config.md +// Deprecated: service config should be received through name resolver or via +// WithDefaultServiceConfig, as specified at +// https://github.com/grpc/grpc/blob/master/doc/service_config.md. Will be +// removed in a future 1.x release. func WithServiceConfig(c <-chan ServiceConfig) DialOption { return newFuncDialOption(func(o *dialOptions) { o.scChan = c @@ -286,7 +292,8 @@ func WithInsecure() DialOption { } // WithTransportCredentials returns a DialOption which configures a connection -// level security credentials (e.g., TLS/SSL). +// level security credentials (e.g., TLS/SSL). This should not be used together +// with WithCredentialsBundle. func WithTransportCredentials(creds credentials.TransportCredentials) DialOption { return newFuncDialOption(func(o *dialOptions) { o.copts.TransportCredentials = creds @@ -301,36 +308,55 @@ func WithPerRPCCredentials(creds credentials.PerRPCCredentials) DialOption { }) } +// WithCredentialsBundle returns a DialOption to set a credentials bundle for +// the ClientConn.WithCreds. This should not be used together with +// WithTransportCredentials. +// +// This API is experimental. +func WithCredentialsBundle(b credentials.Bundle) DialOption { + return newFuncDialOption(func(o *dialOptions) { + o.copts.CredsBundle = b + }) +} + // WithTimeout returns a DialOption that configures a timeout for dialing a // ClientConn initially. This is valid if and only if WithBlock() is present. // -// Deprecated: use DialContext and context.WithTimeout instead. +// Deprecated: use DialContext and context.WithTimeout instead. Will be +// supported throughout 1.x. func WithTimeout(d time.Duration) DialOption { return newFuncDialOption(func(o *dialOptions) { o.timeout = d }) } -func withContextDialer(f func(context.Context, string) (net.Conn, error)) DialOption { +// WithContextDialer returns a DialOption that sets a dialer to create +// connections. If FailOnNonTempDialError() is set to true, and an error is +// returned by f, gRPC checks the error's Temporary() method to decide if it +// should try to reconnect to the network address. +func WithContextDialer(f func(context.Context, string) (net.Conn, error)) DialOption { return newFuncDialOption(func(o *dialOptions) { o.copts.Dialer = f }) } func init() { - internal.WithContextDialer = withContextDialer internal.WithResolverBuilder = withResolverBuilder + internal.WithHealthCheckFunc = withHealthCheckFunc } // WithDialer returns a DialOption that specifies a function to use for dialing // network addresses. If FailOnNonTempDialError() is set to true, and an error // is returned by f, gRPC checks the error's Temporary() method to decide if it // should try to reconnect to the network address. +// +// Deprecated: use WithContextDialer instead. Will be supported throughout +// 1.x. func WithDialer(f func(string, time.Duration) (net.Conn, error)) DialOption { - return withContextDialer( + return WithContextDialer( func(ctx context.Context, addr string) (net.Conn, error) { if deadline, ok := ctx.Deadline(); ok { - return f(addr, deadline.Sub(time.Now())) + return f(addr, time.Until(deadline)) } return f(addr, 0) }) @@ -349,6 +375,9 @@ func WithStatsHandler(h stats.Handler) DialOption { // error, gRPC will fail the connection to the network address and won't try to // reconnect. The default value of FailOnNonTempDialError is false. // +// FailOnNonTempDialError only affects the initial dial, and does not do +// anything useful unless you are also using WithBlock(). +// // This is an EXPERIMENTAL API. func FailOnNonTempDialError(f bool) DialOption { return newFuncDialOption(func(o *dialOptions) { @@ -367,6 +396,10 @@ func WithUserAgent(s string) DialOption { // WithKeepaliveParams returns a DialOption that specifies keepalive parameters // for the client transport. func WithKeepaliveParams(kp keepalive.ClientParameters) DialOption { + if kp.Time < internal.KeepaliveMinPingTime { + grpclog.Warningf("Adjusting keepalive ping interval to minimum period of %v", internal.KeepaliveMinPingTime) + kp.Time = internal.KeepaliveMinPingTime + } return newFuncDialOption(func(o *dialOptions) { o.copts.KeepaliveParams = kp }) @@ -380,6 +413,17 @@ func WithUnaryInterceptor(f UnaryClientInterceptor) DialOption { }) } +// WithChainUnaryInterceptor returns a DialOption that specifies the chained +// interceptor for unary RPCs. The first interceptor will be the outer most, +// while the last interceptor will be the inner most wrapper around the real call. +// All interceptors added by this method will be chained, and the interceptor +// defined by WithUnaryInterceptor will always be prepended to the chain. +func WithChainUnaryInterceptor(interceptors ...UnaryClientInterceptor) DialOption { + return newFuncDialOption(func(o *dialOptions) { + o.chainUnaryInts = append(o.chainUnaryInts, interceptors...) + }) +} + // WithStreamInterceptor returns a DialOption that specifies the interceptor for // streaming RPCs. func WithStreamInterceptor(f StreamClientInterceptor) DialOption { @@ -388,6 +432,17 @@ func WithStreamInterceptor(f StreamClientInterceptor) DialOption { }) } +// WithChainStreamInterceptor returns a DialOption that specifies the chained +// interceptor for unary RPCs. The first interceptor will be the outer most, +// while the last interceptor will be the inner most wrapper around the real call. +// All interceptors added by this method will be chained, and the interceptor +// defined by WithStreamInterceptor will always be prepended to the chain. +func WithChainStreamInterceptor(interceptors ...StreamClientInterceptor) DialOption { + return newFuncDialOption(func(o *dialOptions) { + o.chainStreamInts = append(o.chainStreamInts, interceptors...) + }) +} + // WithAuthority returns a DialOption that specifies the value to be used as the // :authority pseudo-header. This value only works with WithInsecure and has no // effect if TransportCredentials are present. @@ -406,15 +461,32 @@ func WithChannelzParentID(id int64) DialOption { }) } -// WithDisableServiceConfig returns a DialOption that causes grpc to ignore any +// WithDisableServiceConfig returns a DialOption that causes gRPC to ignore any // service config provided by the resolver and provides a hint to the resolver // to not fetch service configs. +// +// Note that this dial option only disables service config from resolver. If +// default service config is provided, gRPC will use the default service config. func WithDisableServiceConfig() DialOption { return newFuncDialOption(func(o *dialOptions) { o.disableServiceConfig = true }) } +// WithDefaultServiceConfig returns a DialOption that configures the default +// service config, which will be used in cases where: +// +// 1. WithDisableServiceConfig is also used. +// 2. Resolver does not return a service config or if the resolver returns an +// invalid service config. +// +// This API is EXPERIMENTAL. +func WithDefaultServiceConfig(s string) DialOption { + return newFuncDialOption(func(o *dialOptions) { + o.defaultServiceConfigRawJSON = &s + }) +} + // WithDisableRetry returns a DialOption that disables retries, even if the // service config enables them. This does not impact transparent retries, which // will happen automatically if no data is written to the wire or if the RPC is @@ -439,12 +511,44 @@ func WithMaxHeaderListSize(s uint32) DialOption { }) } +// WithDisableHealthCheck disables the LB channel health checking for all +// SubConns of this ClientConn. +// +// This API is EXPERIMENTAL. +func WithDisableHealthCheck() DialOption { + return newFuncDialOption(func(o *dialOptions) { + o.disableHealthCheck = true + }) +} + +// withHealthCheckFunc replaces the default health check function with the +// provided one. It makes tests easier to change the health check function. +// +// For testing purpose only. +func withHealthCheckFunc(f internal.HealthChecker) DialOption { + return newFuncDialOption(func(o *dialOptions) { + o.healthCheckFunc = f + }) +} + func defaultDialOptions() dialOptions { return dialOptions{ - disableRetry: !envconfig.Retry, + disableRetry: !envconfig.Retry, + healthCheckFunc: internal.HealthCheckFunc, copts: transport.ConnectOptions{ WriteBufferSize: defaultWriteBufSize, ReadBufferSize: defaultReadBufSize, }, } } + +// withGetMinConnectDeadline specifies the function that clientconn uses to +// get minConnectDeadline. This can be used to make connection attempts happen +// faster/slower. +// +// For testing purpose only. +func withMinConnectDeadline(f func() time.Duration) DialOption { + return newFuncDialOption(func(o *dialOptions) { + o.minConnectTimeout = f + }) +} diff --git a/vendor/google.golang.org/grpc/encoding/encoding.go b/vendor/google.golang.org/grpc/encoding/encoding.go index ade8b7cec73..30a75da99d5 100644 --- a/vendor/google.golang.org/grpc/encoding/encoding.go +++ b/vendor/google.golang.org/grpc/encoding/encoding.go @@ -102,10 +102,10 @@ func RegisterCodec(codec Codec) { if codec == nil { panic("cannot register a nil Codec") } - contentSubtype := strings.ToLower(codec.Name()) - if contentSubtype == "" { - panic("cannot register Codec with empty string result for String()") + if codec.Name() == "" { + panic("cannot register Codec with empty string result for Name()") } + contentSubtype := strings.ToLower(codec.Name()) registeredCodecs[contentSubtype] = codec } diff --git a/vendor/google.golang.org/grpc/go.mod b/vendor/google.golang.org/grpc/go.mod new file mode 100644 index 00000000000..c1a8340c5ba --- /dev/null +++ b/vendor/google.golang.org/grpc/go.mod @@ -0,0 +1,19 @@ +module google.golang.org/grpc + +require ( + cloud.google.com/go v0.26.0 // indirect + github.com/BurntSushi/toml v0.3.1 // indirect + github.com/client9/misspell v0.3.4 + github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b + github.com/golang/mock v1.1.1 + github.com/golang/protobuf v1.2.0 + github.com/google/go-cmp v0.2.0 + golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3 + golang.org/x/net v0.0.0-20190311183353-d8887717615a + golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be + golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a + golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135 + google.golang.org/appengine v1.1.0 // indirect + google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8 + honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc +) diff --git a/vendor/google.golang.org/grpc/go.sum b/vendor/google.golang.org/grpc/go.sum new file mode 100644 index 00000000000..741677d2e81 --- /dev/null +++ b/vendor/google.golang.org/grpc/go.sum @@ -0,0 +1,37 @@ +cloud.google.com/go v0.26.0 h1:e0WKqKTd5BnrG8aKH3J3h+QvEIQtSUcf2n5UZ5ZgLtQ= +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/client9/misspell v0.3.4 h1:ta993UF76GwbvJcIo3Y68y/M3WxlpEHPWIGDkJYwzJI= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/mock v1.1.1 h1:G5FRp8JnTd7RQH5kemVNlMeyXQAztQ3mOWV95KxsXH8= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3 h1:XQyxROzUlZH+WIQwySDgnISgOivlhjIEwaQaJEJrrN0= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be h1:vEDujvNQGv4jgYKudGeI/+DAX4Jffq6hpD55MmoEvKs= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd h1:/e+gpKk9r3dJobndpTytxS2gOy6m5uvpg+ISQoEcusQ= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135 h1:5Beo0mZN8dRzgrMMkDp0jc8YXQKx9DiJ2k1dkvGsn5A= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +google.golang.org/appengine v1.1.0 h1:igQkv0AAhEIvTEpD5LIpAfav2eeVO9HBTjvKHVJPRSs= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8 h1:Nw54tB0rB7hY/N0NQvRW8DG4Yk3Q6T9cu9RcFQDu1tc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc h1:/hemPrYIhOhy8zYrNj+069zDB68us2sMGsfkFJO0iZs= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/vendor/google.golang.org/grpc/go16.go b/vendor/google.golang.org/grpc/go16.go deleted file mode 100644 index b1db21af6f4..00000000000 --- a/vendor/google.golang.org/grpc/go16.go +++ /dev/null @@ -1,71 +0,0 @@ -// +build go1.6,!go1.7 - -/* - * - * Copyright 2016 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package grpc - -import ( - "fmt" - "io" - "net" - "net/http" - - "golang.org/x/net/context" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/internal/transport" - "google.golang.org/grpc/status" -) - -// dialContext connects to the address on the named network. -func dialContext(ctx context.Context, network, address string) (net.Conn, error) { - return (&net.Dialer{Cancel: ctx.Done()}).Dial(network, address) -} - -func sendHTTPRequest(ctx context.Context, req *http.Request, conn net.Conn) error { - req.Cancel = ctx.Done() - if err := req.Write(conn); err != nil { - return fmt.Errorf("failed to write the HTTP request: %v", err) - } - return nil -} - -// toRPCErr converts an error into an error from the status package. -func toRPCErr(err error) error { - if err == nil || err == io.EOF { - return err - } - if err == io.ErrUnexpectedEOF { - return status.Error(codes.Internal, err.Error()) - } - if _, ok := status.FromError(err); ok { - return err - } - switch e := err.(type) { - case transport.ConnectionError: - return status.Error(codes.Unavailable, e.Desc) - default: - switch err { - case context.DeadlineExceeded: - return status.Error(codes.DeadlineExceeded, err.Error()) - case context.Canceled: - return status.Error(codes.Canceled, err.Error()) - } - } - return status.Error(codes.Unknown, err.Error()) -} diff --git a/vendor/google.golang.org/grpc/go17.go b/vendor/google.golang.org/grpc/go17.go deleted file mode 100644 index 71a72e8fe15..00000000000 --- a/vendor/google.golang.org/grpc/go17.go +++ /dev/null @@ -1,72 +0,0 @@ -// +build go1.7 - -/* - * - * Copyright 2016 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package grpc - -import ( - "context" - "fmt" - "io" - "net" - "net/http" - - netctx "golang.org/x/net/context" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/internal/transport" - "google.golang.org/grpc/status" -) - -// dialContext connects to the address on the named network. -func dialContext(ctx context.Context, network, address string) (net.Conn, error) { - return (&net.Dialer{}).DialContext(ctx, network, address) -} - -func sendHTTPRequest(ctx context.Context, req *http.Request, conn net.Conn) error { - req = req.WithContext(ctx) - if err := req.Write(conn); err != nil { - return fmt.Errorf("failed to write the HTTP request: %v", err) - } - return nil -} - -// toRPCErr converts an error into an error from the status package. -func toRPCErr(err error) error { - if err == nil || err == io.EOF { - return err - } - if err == io.ErrUnexpectedEOF { - return status.Error(codes.Internal, err.Error()) - } - if _, ok := status.FromError(err); ok { - return err - } - switch e := err.(type) { - case transport.ConnectionError: - return status.Error(codes.Unavailable, e.Desc) - default: - switch err { - case context.DeadlineExceeded, netctx.DeadlineExceeded: - return status.Error(codes.DeadlineExceeded, err.Error()) - case context.Canceled, netctx.Canceled: - return status.Error(codes.Canceled, err.Error()) - } - } - return status.Error(codes.Unknown, err.Error()) -} diff --git a/vendor/google.golang.org/grpc/grpclog/grpclog.go b/vendor/google.golang.org/grpc/grpclog/grpclog.go index 1fabb11e1ba..51bb9457cda 100644 --- a/vendor/google.golang.org/grpc/grpclog/grpclog.go +++ b/vendor/google.golang.org/grpc/grpclog/grpclog.go @@ -18,7 +18,7 @@ // Package grpclog defines logging for grpc. // -// All logs in transport package only go to verbose level 2. +// All logs in transport and grpclb packages only go to verbose level 2. // All logs in other packages in grpc are logged in spite of the verbosity level. // // In the default logger, diff --git a/vendor/google.golang.org/grpc/health/client.go b/vendor/google.golang.org/grpc/health/client.go new file mode 100644 index 00000000000..b43746e616c --- /dev/null +++ b/vendor/google.golang.org/grpc/health/client.go @@ -0,0 +1,117 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package health + +import ( + "context" + "fmt" + "io" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" + healthpb "google.golang.org/grpc/health/grpc_health_v1" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/backoff" + "google.golang.org/grpc/status" +) + +const maxDelay = 120 * time.Second + +var backoffStrategy = backoff.Exponential{MaxDelay: maxDelay} +var backoffFunc = func(ctx context.Context, retries int) bool { + d := backoffStrategy.Backoff(retries) + timer := time.NewTimer(d) + select { + case <-timer.C: + return true + case <-ctx.Done(): + timer.Stop() + return false + } +} + +func init() { + internal.HealthCheckFunc = clientHealthCheck +} + +const healthCheckMethod = "/grpc.health.v1.Health/Watch" + +// This function implements the protocol defined at: +// https://github.com/grpc/grpc/blob/master/doc/health-checking.md +func clientHealthCheck(ctx context.Context, newStream func(string) (interface{}, error), setConnectivityState func(connectivity.State), service string) error { + tryCnt := 0 + +retryConnection: + for { + // Backs off if the connection has failed in some way without receiving a message in the previous retry. + if tryCnt > 0 && !backoffFunc(ctx, tryCnt-1) { + return nil + } + tryCnt++ + + if ctx.Err() != nil { + return nil + } + setConnectivityState(connectivity.Connecting) + rawS, err := newStream(healthCheckMethod) + if err != nil { + continue retryConnection + } + + s, ok := rawS.(grpc.ClientStream) + // Ideally, this should never happen. But if it happens, the server is marked as healthy for LBing purposes. + if !ok { + setConnectivityState(connectivity.Ready) + return fmt.Errorf("newStream returned %v (type %T); want grpc.ClientStream", rawS, rawS) + } + + if err = s.SendMsg(&healthpb.HealthCheckRequest{Service: service}); err != nil && err != io.EOF { + // Stream should have been closed, so we can safely continue to create a new stream. + continue retryConnection + } + s.CloseSend() + + resp := new(healthpb.HealthCheckResponse) + for { + err = s.RecvMsg(resp) + + // Reports healthy for the LBing purposes if health check is not implemented in the server. + if status.Code(err) == codes.Unimplemented { + setConnectivityState(connectivity.Ready) + return err + } + + // Reports unhealthy if server's Watch method gives an error other than UNIMPLEMENTED. + if err != nil { + setConnectivityState(connectivity.TransientFailure) + continue retryConnection + } + + // As a message has been received, removes the need for backoff for the next retry by reseting the try count. + tryCnt = 0 + if resp.Status == healthpb.HealthCheckResponse_SERVING { + setConnectivityState(connectivity.Ready) + } else { + setConnectivityState(connectivity.TransientFailure) + } + } + } +} diff --git a/vendor/google.golang.org/grpc/health/grpc_health_v1/health.pb.go b/vendor/google.golang.org/grpc/health/grpc_health_v1/health.pb.go index a1fda2801b8..c2f2c7729d0 100644 --- a/vendor/google.golang.org/grpc/health/grpc_health_v1/health.pb.go +++ b/vendor/google.golang.org/grpc/health/grpc_health_v1/health.pb.go @@ -26,27 +26,30 @@ const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package type HealthCheckResponse_ServingStatus int32 const ( - HealthCheckResponse_UNKNOWN HealthCheckResponse_ServingStatus = 0 - HealthCheckResponse_SERVING HealthCheckResponse_ServingStatus = 1 - HealthCheckResponse_NOT_SERVING HealthCheckResponse_ServingStatus = 2 + HealthCheckResponse_UNKNOWN HealthCheckResponse_ServingStatus = 0 + HealthCheckResponse_SERVING HealthCheckResponse_ServingStatus = 1 + HealthCheckResponse_NOT_SERVING HealthCheckResponse_ServingStatus = 2 + HealthCheckResponse_SERVICE_UNKNOWN HealthCheckResponse_ServingStatus = 3 ) var HealthCheckResponse_ServingStatus_name = map[int32]string{ 0: "UNKNOWN", 1: "SERVING", 2: "NOT_SERVING", + 3: "SERVICE_UNKNOWN", } var HealthCheckResponse_ServingStatus_value = map[string]int32{ - "UNKNOWN": 0, - "SERVING": 1, - "NOT_SERVING": 2, + "UNKNOWN": 0, + "SERVING": 1, + "NOT_SERVING": 2, + "SERVICE_UNKNOWN": 3, } func (x HealthCheckResponse_ServingStatus) String() string { return proto.EnumName(HealthCheckResponse_ServingStatus_name, int32(x)) } func (HealthCheckResponse_ServingStatus) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_health_85731b6c49265086, []int{1, 0} + return fileDescriptor_health_6b1a06aa67f91efd, []int{1, 0} } type HealthCheckRequest struct { @@ -60,7 +63,7 @@ func (m *HealthCheckRequest) Reset() { *m = HealthCheckRequest{} } func (m *HealthCheckRequest) String() string { return proto.CompactTextString(m) } func (*HealthCheckRequest) ProtoMessage() {} func (*HealthCheckRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_health_85731b6c49265086, []int{0} + return fileDescriptor_health_6b1a06aa67f91efd, []int{0} } func (m *HealthCheckRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_HealthCheckRequest.Unmarshal(m, b) @@ -98,7 +101,7 @@ func (m *HealthCheckResponse) Reset() { *m = HealthCheckResponse{} } func (m *HealthCheckResponse) String() string { return proto.CompactTextString(m) } func (*HealthCheckResponse) ProtoMessage() {} func (*HealthCheckResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_health_85731b6c49265086, []int{1} + return fileDescriptor_health_6b1a06aa67f91efd, []int{1} } func (m *HealthCheckResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_HealthCheckResponse.Unmarshal(m, b) @@ -143,7 +146,25 @@ const _ = grpc.SupportPackageIsVersion4 // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. type HealthClient interface { + // If the requested service is unknown, the call will fail with status + // NOT_FOUND. Check(ctx context.Context, in *HealthCheckRequest, opts ...grpc.CallOption) (*HealthCheckResponse, error) + // Performs a watch for the serving status of the requested service. + // The server will immediately send back a message indicating the current + // serving status. It will then subsequently send a new message whenever + // the service's serving status changes. + // + // If the requested service is unknown when the call is received, the + // server will send a message setting the serving status to + // SERVICE_UNKNOWN but will *not* terminate the call. If at some + // future point, the serving status of the service becomes known, the + // server will send a new message with the service's serving status. + // + // If the call terminates with status UNIMPLEMENTED, then clients + // should assume this method is not supported and should not retry the + // call. If the call terminates with any other status (including OK), + // clients should retry the call with appropriate exponential backoff. + Watch(ctx context.Context, in *HealthCheckRequest, opts ...grpc.CallOption) (Health_WatchClient, error) } type healthClient struct { @@ -163,9 +184,59 @@ func (c *healthClient) Check(ctx context.Context, in *HealthCheckRequest, opts . return out, nil } +func (c *healthClient) Watch(ctx context.Context, in *HealthCheckRequest, opts ...grpc.CallOption) (Health_WatchClient, error) { + stream, err := c.cc.NewStream(ctx, &_Health_serviceDesc.Streams[0], "/grpc.health.v1.Health/Watch", opts...) + if err != nil { + return nil, err + } + x := &healthWatchClient{stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil +} + +type Health_WatchClient interface { + Recv() (*HealthCheckResponse, error) + grpc.ClientStream +} + +type healthWatchClient struct { + grpc.ClientStream +} + +func (x *healthWatchClient) Recv() (*HealthCheckResponse, error) { + m := new(HealthCheckResponse) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + // HealthServer is the server API for Health service. type HealthServer interface { + // If the requested service is unknown, the call will fail with status + // NOT_FOUND. Check(context.Context, *HealthCheckRequest) (*HealthCheckResponse, error) + // Performs a watch for the serving status of the requested service. + // The server will immediately send back a message indicating the current + // serving status. It will then subsequently send a new message whenever + // the service's serving status changes. + // + // If the requested service is unknown when the call is received, the + // server will send a message setting the serving status to + // SERVICE_UNKNOWN but will *not* terminate the call. If at some + // future point, the serving status of the service becomes known, the + // server will send a new message with the service's serving status. + // + // If the call terminates with status UNIMPLEMENTED, then clients + // should assume this method is not supported and should not retry the + // call. If the call terminates with any other status (including OK), + // clients should retry the call with appropriate exponential backoff. + Watch(*HealthCheckRequest, Health_WatchServer) error } func RegisterHealthServer(s *grpc.Server, srv HealthServer) { @@ -190,6 +261,27 @@ func _Health_Check_Handler(srv interface{}, ctx context.Context, dec func(interf return interceptor(ctx, in, info, handler) } +func _Health_Watch_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(HealthCheckRequest) + if err := stream.RecvMsg(m); err != nil { + return err + } + return srv.(HealthServer).Watch(m, &healthWatchServer{stream}) +} + +type Health_WatchServer interface { + Send(*HealthCheckResponse) error + grpc.ServerStream +} + +type healthWatchServer struct { + grpc.ServerStream +} + +func (x *healthWatchServer) Send(m *HealthCheckResponse) error { + return x.ServerStream.SendMsg(m) +} + var _Health_serviceDesc = grpc.ServiceDesc{ ServiceName: "grpc.health.v1.Health", HandlerType: (*HealthServer)(nil), @@ -199,29 +291,37 @@ var _Health_serviceDesc = grpc.ServiceDesc{ Handler: _Health_Check_Handler, }, }, - Streams: []grpc.StreamDesc{}, + Streams: []grpc.StreamDesc{ + { + StreamName: "Watch", + Handler: _Health_Watch_Handler, + ServerStreams: true, + }, + }, Metadata: "grpc/health/v1/health.proto", } -func init() { proto.RegisterFile("grpc/health/v1/health.proto", fileDescriptor_health_85731b6c49265086) } +func init() { proto.RegisterFile("grpc/health/v1/health.proto", fileDescriptor_health_6b1a06aa67f91efd) } -var fileDescriptor_health_85731b6c49265086 = []byte{ - // 271 bytes of a gzipped FileDescriptorProto +var fileDescriptor_health_6b1a06aa67f91efd = []byte{ + // 297 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x92, 0x4e, 0x2f, 0x2a, 0x48, 0xd6, 0xcf, 0x48, 0x4d, 0xcc, 0x29, 0xc9, 0xd0, 0x2f, 0x33, 0x84, 0xb2, 0xf4, 0x0a, 0x8a, 0xf2, 0x4b, 0xf2, 0x85, 0xf8, 0x40, 0x92, 0x7a, 0x50, 0xa1, 0x32, 0x43, 0x25, 0x3d, 0x2e, 0x21, 0x0f, 0x30, 0xc7, 0x39, 0x23, 0x35, 0x39, 0x3b, 0x28, 0xb5, 0xb0, 0x34, 0xb5, 0xb8, 0x44, 0x48, 0x82, 0x8b, 0xbd, 0x38, 0xb5, 0xa8, 0x2c, 0x33, 0x39, 0x55, 0x82, 0x51, 0x81, 0x51, 0x83, 0x33, 0x08, - 0xc6, 0x55, 0x9a, 0xc3, 0xc8, 0x25, 0x8c, 0xa2, 0xa1, 0xb8, 0x20, 0x3f, 0xaf, 0x38, 0x55, 0xc8, + 0xc6, 0x55, 0xda, 0xc8, 0xc8, 0x25, 0x8c, 0xa2, 0xa1, 0xb8, 0x20, 0x3f, 0xaf, 0x38, 0x55, 0xc8, 0x93, 0x8b, 0xad, 0xb8, 0x24, 0xb1, 0xa4, 0xb4, 0x18, 0xac, 0x81, 0xcf, 0xc8, 0x50, 0x0f, 0xd5, 0x22, 0x3d, 0x2c, 0x9a, 0xf4, 0x82, 0x41, 0x86, 0xe6, 0xa5, 0x07, 0x83, 0x35, 0x06, 0x41, 0x0d, - 0x50, 0xb2, 0xe2, 0xe2, 0x45, 0x91, 0x10, 0xe2, 0xe6, 0x62, 0x0f, 0xf5, 0xf3, 0xf6, 0xf3, 0x0f, + 0x50, 0xf2, 0xe7, 0xe2, 0x45, 0x91, 0x10, 0xe2, 0xe6, 0x62, 0x0f, 0xf5, 0xf3, 0xf6, 0xf3, 0x0f, 0xf7, 0x13, 0x60, 0x00, 0x71, 0x82, 0x5d, 0x83, 0xc2, 0x3c, 0xfd, 0xdc, 0x05, 0x18, 0x85, 0xf8, - 0xb9, 0xb8, 0xfd, 0xfc, 0x43, 0xe2, 0x61, 0x02, 0x4c, 0x46, 0x51, 0x5c, 0x6c, 0x10, 0x8b, 0x84, - 0x02, 0xb8, 0x58, 0xc1, 0x96, 0x09, 0x29, 0xe1, 0x75, 0x09, 0xd8, 0xbf, 0x52, 0xca, 0x44, 0xb8, - 0xd6, 0x29, 0x91, 0x4b, 0x30, 0x33, 0x1f, 0x4d, 0xa1, 0x13, 0x37, 0x44, 0x65, 0x00, 0x28, 0x70, - 0x03, 0x18, 0xa3, 0x74, 0xd2, 0xf3, 0xf3, 0xd3, 0x73, 0x52, 0xf5, 0xd2, 0xf3, 0x73, 0x12, 0xf3, - 0xd2, 0xf5, 0xf2, 0x8b, 0xd2, 0xf5, 0x91, 0x63, 0x03, 0xc4, 0x8e, 0x87, 0xb0, 0xe3, 0xcb, 0x0c, - 0x57, 0x31, 0xf1, 0xb9, 0x83, 0x4c, 0x83, 0x18, 0xa1, 0x17, 0x66, 0x98, 0xc4, 0x06, 0x8e, 0x24, - 0x63, 0x40, 0x00, 0x00, 0x00, 0xff, 0xff, 0xec, 0x66, 0x81, 0xcb, 0xc3, 0x01, 0x00, 0x00, + 0xb9, 0xb8, 0xfd, 0xfc, 0x43, 0xe2, 0x61, 0x02, 0x4c, 0x42, 0xc2, 0x5c, 0xfc, 0x60, 0x8e, 0xb3, + 0x6b, 0x3c, 0x4c, 0x0b, 0xb3, 0xd1, 0x3a, 0x46, 0x2e, 0x36, 0x88, 0xf5, 0x42, 0x01, 0x5c, 0xac, + 0x60, 0x27, 0x08, 0x29, 0xe1, 0x75, 0x1f, 0x38, 0x14, 0xa4, 0x94, 0x89, 0xf0, 0x83, 0x50, 0x10, + 0x17, 0x6b, 0x78, 0x62, 0x49, 0x72, 0x06, 0xd5, 0x4c, 0x34, 0x60, 0x74, 0x4a, 0xe4, 0x12, 0xcc, + 0xcc, 0x47, 0x53, 0xea, 0xc4, 0x0d, 0x51, 0x1b, 0x00, 0x8a, 0xc6, 0x00, 0xc6, 0x28, 0x9d, 0xf4, + 0xfc, 0xfc, 0xf4, 0x9c, 0x54, 0xbd, 0xf4, 0xfc, 0x9c, 0xc4, 0xbc, 0x74, 0xbd, 0xfc, 0xa2, 0x74, + 0x7d, 0xe4, 0x78, 0x07, 0xb1, 0xe3, 0x21, 0xec, 0xf8, 0x32, 0xc3, 0x55, 0x4c, 0x7c, 0xee, 0x20, + 0xd3, 0x20, 0x46, 0xe8, 0x85, 0x19, 0x26, 0xb1, 0x81, 0x93, 0x83, 0x31, 0x20, 0x00, 0x00, 0xff, + 0xff, 0x12, 0x7d, 0x96, 0xcb, 0x2d, 0x02, 0x00, 0x00, } diff --git a/vendor/google.golang.org/grpc/health/health.go b/vendor/google.golang.org/grpc/health/health.go deleted file mode 100644 index c2588867e5a..00000000000 --- a/vendor/google.golang.org/grpc/health/health.go +++ /dev/null @@ -1,72 +0,0 @@ -/* - * - * Copyright 2017 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -//go:generate ./regenerate.sh - -// Package health provides some utility functions to health-check a server. The implementation -// is based on protobuf. Users need to write their own implementations if other IDLs are used. -package health - -import ( - "sync" - - "golang.org/x/net/context" - "google.golang.org/grpc/codes" - healthpb "google.golang.org/grpc/health/grpc_health_v1" - "google.golang.org/grpc/status" -) - -// Server implements `service Health`. -type Server struct { - mu sync.Mutex - // statusMap stores the serving status of the services this Server monitors. - statusMap map[string]healthpb.HealthCheckResponse_ServingStatus -} - -// NewServer returns a new Server. -func NewServer() *Server { - return &Server{ - statusMap: make(map[string]healthpb.HealthCheckResponse_ServingStatus), - } -} - -// Check implements `service Health`. -func (s *Server) Check(ctx context.Context, in *healthpb.HealthCheckRequest) (*healthpb.HealthCheckResponse, error) { - s.mu.Lock() - defer s.mu.Unlock() - if in.Service == "" { - // check the server overall health status. - return &healthpb.HealthCheckResponse{ - Status: healthpb.HealthCheckResponse_SERVING, - }, nil - } - if status, ok := s.statusMap[in.Service]; ok { - return &healthpb.HealthCheckResponse{ - Status: status, - }, nil - } - return nil, status.Error(codes.NotFound, "unknown service") -} - -// SetServingStatus is called when need to reset the serving status of a service -// or insert a new service entry into the statusMap. -func (s *Server) SetServingStatus(service string, status healthpb.HealthCheckResponse_ServingStatus) { - s.mu.Lock() - s.statusMap[service] = status - s.mu.Unlock() -} diff --git a/vendor/google.golang.org/grpc/health/server.go b/vendor/google.golang.org/grpc/health/server.go new file mode 100644 index 00000000000..c79f9d2ab80 --- /dev/null +++ b/vendor/google.golang.org/grpc/health/server.go @@ -0,0 +1,165 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +//go:generate ./regenerate.sh + +// Package health provides a service that exposes server's health and it must be +// imported to enable support for client-side health checks. +package health + +import ( + "context" + "sync" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/grpclog" + healthgrpc "google.golang.org/grpc/health/grpc_health_v1" + healthpb "google.golang.org/grpc/health/grpc_health_v1" + "google.golang.org/grpc/status" +) + +// Server implements `service Health`. +type Server struct { + mu sync.Mutex + // If shutdown is true, it's expected all serving status is NOT_SERVING, and + // will stay in NOT_SERVING. + shutdown bool + // statusMap stores the serving status of the services this Server monitors. + statusMap map[string]healthpb.HealthCheckResponse_ServingStatus + updates map[string]map[healthgrpc.Health_WatchServer]chan healthpb.HealthCheckResponse_ServingStatus +} + +// NewServer returns a new Server. +func NewServer() *Server { + return &Server{ + statusMap: map[string]healthpb.HealthCheckResponse_ServingStatus{"": healthpb.HealthCheckResponse_SERVING}, + updates: make(map[string]map[healthgrpc.Health_WatchServer]chan healthpb.HealthCheckResponse_ServingStatus), + } +} + +// Check implements `service Health`. +func (s *Server) Check(ctx context.Context, in *healthpb.HealthCheckRequest) (*healthpb.HealthCheckResponse, error) { + s.mu.Lock() + defer s.mu.Unlock() + if servingStatus, ok := s.statusMap[in.Service]; ok { + return &healthpb.HealthCheckResponse{ + Status: servingStatus, + }, nil + } + return nil, status.Error(codes.NotFound, "unknown service") +} + +// Watch implements `service Health`. +func (s *Server) Watch(in *healthpb.HealthCheckRequest, stream healthgrpc.Health_WatchServer) error { + service := in.Service + // update channel is used for getting service status updates. + update := make(chan healthpb.HealthCheckResponse_ServingStatus, 1) + s.mu.Lock() + // Puts the initial status to the channel. + if servingStatus, ok := s.statusMap[service]; ok { + update <- servingStatus + } else { + update <- healthpb.HealthCheckResponse_SERVICE_UNKNOWN + } + + // Registers the update channel to the correct place in the updates map. + if _, ok := s.updates[service]; !ok { + s.updates[service] = make(map[healthgrpc.Health_WatchServer]chan healthpb.HealthCheckResponse_ServingStatus) + } + s.updates[service][stream] = update + defer func() { + s.mu.Lock() + delete(s.updates[service], stream) + s.mu.Unlock() + }() + s.mu.Unlock() + + var lastSentStatus healthpb.HealthCheckResponse_ServingStatus = -1 + for { + select { + // Status updated. Sends the up-to-date status to the client. + case servingStatus := <-update: + if lastSentStatus == servingStatus { + continue + } + lastSentStatus = servingStatus + err := stream.Send(&healthpb.HealthCheckResponse{Status: servingStatus}) + if err != nil { + return status.Error(codes.Canceled, "Stream has ended.") + } + // Context done. Removes the update channel from the updates map. + case <-stream.Context().Done(): + return status.Error(codes.Canceled, "Stream has ended.") + } + } +} + +// SetServingStatus is called when need to reset the serving status of a service +// or insert a new service entry into the statusMap. +func (s *Server) SetServingStatus(service string, servingStatus healthpb.HealthCheckResponse_ServingStatus) { + s.mu.Lock() + defer s.mu.Unlock() + if s.shutdown { + grpclog.Infof("health: status changing for %s to %v is ignored because health service is shutdown", service, servingStatus) + return + } + + s.setServingStatusLocked(service, servingStatus) +} + +func (s *Server) setServingStatusLocked(service string, servingStatus healthpb.HealthCheckResponse_ServingStatus) { + s.statusMap[service] = servingStatus + for _, update := range s.updates[service] { + // Clears previous updates, that are not sent to the client, from the channel. + // This can happen if the client is not reading and the server gets flow control limited. + select { + case <-update: + default: + } + // Puts the most recent update to the channel. + update <- servingStatus + } +} + +// Shutdown sets all serving status to NOT_SERVING, and configures the server to +// ignore all future status changes. +// +// This changes serving status for all services. To set status for a perticular +// services, call SetServingStatus(). +func (s *Server) Shutdown() { + s.mu.Lock() + defer s.mu.Unlock() + s.shutdown = true + for service := range s.statusMap { + s.setServingStatusLocked(service, healthpb.HealthCheckResponse_NOT_SERVING) + } +} + +// Resume sets all serving status to SERVING, and configures the server to +// accept all future status changes. +// +// This changes serving status for all services. To set status for a perticular +// services, call SetServingStatus(). +func (s *Server) Resume() { + s.mu.Lock() + defer s.mu.Unlock() + s.shutdown = false + for service := range s.statusMap { + s.setServingStatusLocked(service, healthpb.HealthCheckResponse_SERVING) + } +} diff --git a/vendor/google.golang.org/grpc/install_gae.sh b/vendor/google.golang.org/grpc/install_gae.sh index d4236f3b80e..7c7bcada504 100644 --- a/vendor/google.golang.org/grpc/install_gae.sh +++ b/vendor/google.golang.org/grpc/install_gae.sh @@ -1,6 +1,6 @@ #!/bin/bash TMP=$(mktemp -d /tmp/sdk.XXX) \ -&& curl -o $TMP.zip "https://storage.googleapis.com/appengine-sdks/featured/go_appengine_sdk_linux_amd64-1.9.64.zip" \ +&& curl -o $TMP.zip "https://storage.googleapis.com/appengine-sdks/featured/go_appengine_sdk_linux_amd64-1.9.68.zip" \ && unzip -q $TMP.zip -d $TMP \ && export PATH="$PATH:$TMP/go_appengine" diff --git a/vendor/google.golang.org/grpc/interceptor.go b/vendor/google.golang.org/grpc/interceptor.go index 1f6ef678035..8b7350022ad 100644 --- a/vendor/google.golang.org/grpc/interceptor.go +++ b/vendor/google.golang.org/grpc/interceptor.go @@ -19,7 +19,7 @@ package grpc import ( - "golang.org/x/net/context" + "context" ) // UnaryInvoker is called by UnaryClientInterceptor to complete RPCs. diff --git a/vendor/google.golang.org/grpc/internal/balancerload/load.go b/vendor/google.golang.org/grpc/internal/balancerload/load.go new file mode 100644 index 00000000000..3a905d96657 --- /dev/null +++ b/vendor/google.golang.org/grpc/internal/balancerload/load.go @@ -0,0 +1,46 @@ +/* + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package balancerload defines APIs to parse server loads in trailers. The +// parsed loads are sent to balancers in DoneInfo. +package balancerload + +import ( + "google.golang.org/grpc/metadata" +) + +// Parser converts loads from metadata into a concrete type. +type Parser interface { + // Parse parses loads from metadata. + Parse(md metadata.MD) interface{} +} + +var parser Parser + +// SetParser sets the load parser. +// +// Not mutex-protected, should be called before any gRPC functions. +func SetParser(lr Parser) { + parser = lr +} + +// Parse calls parser.Read(). +func Parse(md metadata.MD) interface{} { + if parser == nil { + return nil + } + return parser.Parse(md) +} diff --git a/vendor/google.golang.org/grpc/internal/binarylog/binarylog.go b/vendor/google.golang.org/grpc/internal/binarylog/binarylog.go new file mode 100644 index 00000000000..fee6aecd08f --- /dev/null +++ b/vendor/google.golang.org/grpc/internal/binarylog/binarylog.go @@ -0,0 +1,167 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package binarylog implementation binary logging as defined in +// https://github.com/grpc/proposal/blob/master/A16-binary-logging.md. +package binarylog + +import ( + "fmt" + "os" + + "google.golang.org/grpc/grpclog" +) + +// Logger is the global binary logger. It can be used to get binary logger for +// each method. +type Logger interface { + getMethodLogger(methodName string) *MethodLogger +} + +// binLogger is the global binary logger for the binary. One of this should be +// built at init time from the configuration (environment varialbe or flags). +// +// It is used to get a methodLogger for each individual method. +var binLogger Logger + +// SetLogger sets the binarg logger. +// +// Only call this at init time. +func SetLogger(l Logger) { + binLogger = l +} + +// GetMethodLogger returns the methodLogger for the given methodName. +// +// methodName should be in the format of "/service/method". +// +// Each methodLogger returned by this method is a new instance. This is to +// generate sequence id within the call. +func GetMethodLogger(methodName string) *MethodLogger { + if binLogger == nil { + return nil + } + return binLogger.getMethodLogger(methodName) +} + +func init() { + const envStr = "GRPC_BINARY_LOG_FILTER" + configStr := os.Getenv(envStr) + binLogger = NewLoggerFromConfigString(configStr) +} + +type methodLoggerConfig struct { + // Max length of header and message. + hdr, msg uint64 +} + +type logger struct { + all *methodLoggerConfig + services map[string]*methodLoggerConfig + methods map[string]*methodLoggerConfig + + blacklist map[string]struct{} +} + +// newEmptyLogger creates an empty logger. The map fields need to be filled in +// using the set* functions. +func newEmptyLogger() *logger { + return &logger{} +} + +// Set method logger for "*". +func (l *logger) setDefaultMethodLogger(ml *methodLoggerConfig) error { + if l.all != nil { + return fmt.Errorf("conflicting global rules found") + } + l.all = ml + return nil +} + +// Set method logger for "service/*". +// +// New methodLogger with same service overrides the old one. +func (l *logger) setServiceMethodLogger(service string, ml *methodLoggerConfig) error { + if _, ok := l.services[service]; ok { + return fmt.Errorf("conflicting rules for service %v found", service) + } + if l.services == nil { + l.services = make(map[string]*methodLoggerConfig) + } + l.services[service] = ml + return nil +} + +// Set method logger for "service/method". +// +// New methodLogger with same method overrides the old one. +func (l *logger) setMethodMethodLogger(method string, ml *methodLoggerConfig) error { + if _, ok := l.blacklist[method]; ok { + return fmt.Errorf("conflicting rules for method %v found", method) + } + if _, ok := l.methods[method]; ok { + return fmt.Errorf("conflicting rules for method %v found", method) + } + if l.methods == nil { + l.methods = make(map[string]*methodLoggerConfig) + } + l.methods[method] = ml + return nil +} + +// Set blacklist method for "-service/method". +func (l *logger) setBlacklist(method string) error { + if _, ok := l.blacklist[method]; ok { + return fmt.Errorf("conflicting rules for method %v found", method) + } + if _, ok := l.methods[method]; ok { + return fmt.Errorf("conflicting rules for method %v found", method) + } + if l.blacklist == nil { + l.blacklist = make(map[string]struct{}) + } + l.blacklist[method] = struct{}{} + return nil +} + +// getMethodLogger returns the methodLogger for the given methodName. +// +// methodName should be in the format of "/service/method". +// +// Each methodLogger returned by this method is a new instance. This is to +// generate sequence id within the call. +func (l *logger) getMethodLogger(methodName string) *MethodLogger { + s, m, err := parseMethodName(methodName) + if err != nil { + grpclog.Infof("binarylogging: failed to parse %q: %v", methodName, err) + return nil + } + if ml, ok := l.methods[s+"/"+m]; ok { + return newMethodLogger(ml.hdr, ml.msg) + } + if _, ok := l.blacklist[s+"/"+m]; ok { + return nil + } + if ml, ok := l.services[s]; ok { + return newMethodLogger(ml.hdr, ml.msg) + } + if l.all == nil { + return nil + } + return newMethodLogger(l.all.hdr, l.all.msg) +} diff --git a/vendor/google.golang.org/grpc/internal/binarylog/binarylog_testutil.go b/vendor/google.golang.org/grpc/internal/binarylog/binarylog_testutil.go new file mode 100644 index 00000000000..1ee00a39ac7 --- /dev/null +++ b/vendor/google.golang.org/grpc/internal/binarylog/binarylog_testutil.go @@ -0,0 +1,42 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// This file contains exported variables/functions that are exported for testing +// only. +// +// An ideal way for this would be to put those in a *_test.go but in binarylog +// package. But this doesn't work with staticcheck with go module. Error was: +// "MdToMetadataProto not declared by package binarylog". This could be caused +// by the way staticcheck looks for files for a certain package, which doesn't +// support *_test.go files. +// +// Move those to binary_test.go when staticcheck is fixed. + +package binarylog + +var ( + // AllLogger is a logger that logs all headers/messages for all RPCs. It's + // for testing only. + AllLogger = NewLoggerFromConfigString("*") + // MdToMetadataProto converts metadata to a binary logging proto message. + // It's for testing only. + MdToMetadataProto = mdToMetadataProto + // AddrToProto converts an address to a binary logging proto message. It's + // for testing only. + AddrToProto = addrToProto +) diff --git a/vendor/google.golang.org/grpc/internal/binarylog/env_config.go b/vendor/google.golang.org/grpc/internal/binarylog/env_config.go new file mode 100644 index 00000000000..4cc2525df7f --- /dev/null +++ b/vendor/google.golang.org/grpc/internal/binarylog/env_config.go @@ -0,0 +1,210 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package binarylog + +import ( + "errors" + "fmt" + "regexp" + "strconv" + "strings" + + "google.golang.org/grpc/grpclog" +) + +// NewLoggerFromConfigString reads the string and build a logger. It can be used +// to build a new logger and assign it to binarylog.Logger. +// +// Example filter config strings: +// - "" Nothing will be logged +// - "*" All headers and messages will be fully logged. +// - "*{h}" Only headers will be logged. +// - "*{m:256}" Only the first 256 bytes of each message will be logged. +// - "Foo/*" Logs every method in service Foo +// - "Foo/*,-Foo/Bar" Logs every method in service Foo except method /Foo/Bar +// - "Foo/*,Foo/Bar{m:256}" Logs the first 256 bytes of each message in method +// /Foo/Bar, logs all headers and messages in every other method in service +// Foo. +// +// If two configs exist for one certain method or service, the one specified +// later overrides the privous config. +func NewLoggerFromConfigString(s string) Logger { + if s == "" { + return nil + } + l := newEmptyLogger() + methods := strings.Split(s, ",") + for _, method := range methods { + if err := l.fillMethodLoggerWithConfigString(method); err != nil { + grpclog.Warningf("failed to parse binary log config: %v", err) + return nil + } + } + return l +} + +// fillMethodLoggerWithConfigString parses config, creates methodLogger and adds +// it to the right map in the logger. +func (l *logger) fillMethodLoggerWithConfigString(config string) error { + // "" is invalid. + if config == "" { + return errors.New("empty string is not a valid method binary logging config") + } + + // "-service/method", blacklist, no * or {} allowed. + if config[0] == '-' { + s, m, suffix, err := parseMethodConfigAndSuffix(config[1:]) + if err != nil { + return fmt.Errorf("invalid config: %q, %v", config, err) + } + if m == "*" { + return fmt.Errorf("invalid config: %q, %v", config, "* not allowd in blacklist config") + } + if suffix != "" { + return fmt.Errorf("invalid config: %q, %v", config, "header/message limit not allowed in blacklist config") + } + if err := l.setBlacklist(s + "/" + m); err != nil { + return fmt.Errorf("invalid config: %v", err) + } + return nil + } + + // "*{h:256;m:256}" + if config[0] == '*' { + hdr, msg, err := parseHeaderMessageLengthConfig(config[1:]) + if err != nil { + return fmt.Errorf("invalid config: %q, %v", config, err) + } + if err := l.setDefaultMethodLogger(&methodLoggerConfig{hdr: hdr, msg: msg}); err != nil { + return fmt.Errorf("invalid config: %v", err) + } + return nil + } + + s, m, suffix, err := parseMethodConfigAndSuffix(config) + if err != nil { + return fmt.Errorf("invalid config: %q, %v", config, err) + } + hdr, msg, err := parseHeaderMessageLengthConfig(suffix) + if err != nil { + return fmt.Errorf("invalid header/message length config: %q, %v", suffix, err) + } + if m == "*" { + if err := l.setServiceMethodLogger(s, &methodLoggerConfig{hdr: hdr, msg: msg}); err != nil { + return fmt.Errorf("invalid config: %v", err) + } + } else { + if err := l.setMethodMethodLogger(s+"/"+m, &methodLoggerConfig{hdr: hdr, msg: msg}); err != nil { + return fmt.Errorf("invalid config: %v", err) + } + } + return nil +} + +const ( + // TODO: this const is only used by env_config now. But could be useful for + // other config. Move to binarylog.go if necessary. + maxUInt = ^uint64(0) + + // For "p.s/m" plus any suffix. Suffix will be parsed again. See test for + // expected output. + longMethodConfigRegexpStr = `^([\w./]+)/((?:\w+)|[*])(.+)?$` + + // For suffix from above, "{h:123,m:123}". See test for expected output. + optionalLengthRegexpStr = `(?::(\d+))?` // Optional ":123". + headerConfigRegexpStr = `^{h` + optionalLengthRegexpStr + `}$` + messageConfigRegexpStr = `^{m` + optionalLengthRegexpStr + `}$` + headerMessageConfigRegexpStr = `^{h` + optionalLengthRegexpStr + `;m` + optionalLengthRegexpStr + `}$` +) + +var ( + longMethodConfigRegexp = regexp.MustCompile(longMethodConfigRegexpStr) + headerConfigRegexp = regexp.MustCompile(headerConfigRegexpStr) + messageConfigRegexp = regexp.MustCompile(messageConfigRegexpStr) + headerMessageConfigRegexp = regexp.MustCompile(headerMessageConfigRegexpStr) +) + +// Turn "service/method{h;m}" into "service", "method", "{h;m}". +func parseMethodConfigAndSuffix(c string) (service, method, suffix string, _ error) { + // Regexp result: + // + // in: "p.s/m{h:123,m:123}", + // out: []string{"p.s/m{h:123,m:123}", "p.s", "m", "{h:123,m:123}"}, + match := longMethodConfigRegexp.FindStringSubmatch(c) + if match == nil { + return "", "", "", fmt.Errorf("%q contains invalid substring", c) + } + service = match[1] + method = match[2] + suffix = match[3] + return +} + +// Turn "{h:123;m:345}" into 123, 345. +// +// Return maxUInt if length is unspecified. +func parseHeaderMessageLengthConfig(c string) (hdrLenStr, msgLenStr uint64, err error) { + if c == "" { + return maxUInt, maxUInt, nil + } + // Header config only. + if match := headerConfigRegexp.FindStringSubmatch(c); match != nil { + if s := match[1]; s != "" { + hdrLenStr, err = strconv.ParseUint(s, 10, 64) + if err != nil { + return 0, 0, fmt.Errorf("failed to convert %q to uint", s) + } + return hdrLenStr, 0, nil + } + return maxUInt, 0, nil + } + + // Message config only. + if match := messageConfigRegexp.FindStringSubmatch(c); match != nil { + if s := match[1]; s != "" { + msgLenStr, err = strconv.ParseUint(s, 10, 64) + if err != nil { + return 0, 0, fmt.Errorf("failed to convert %q to uint", s) + } + return 0, msgLenStr, nil + } + return 0, maxUInt, nil + } + + // Header and message config both. + if match := headerMessageConfigRegexp.FindStringSubmatch(c); match != nil { + // Both hdr and msg are specified, but one or two of them might be empty. + hdrLenStr = maxUInt + msgLenStr = maxUInt + if s := match[1]; s != "" { + hdrLenStr, err = strconv.ParseUint(s, 10, 64) + if err != nil { + return 0, 0, fmt.Errorf("failed to convert %q to uint", s) + } + } + if s := match[2]; s != "" { + msgLenStr, err = strconv.ParseUint(s, 10, 64) + if err != nil { + return 0, 0, fmt.Errorf("failed to convert %q to uint", s) + } + } + return hdrLenStr, msgLenStr, nil + } + return 0, 0, fmt.Errorf("%q contains invalid substring", c) +} diff --git a/vendor/google.golang.org/grpc/internal/binarylog/method_logger.go b/vendor/google.golang.org/grpc/internal/binarylog/method_logger.go new file mode 100644 index 00000000000..160f6e8616f --- /dev/null +++ b/vendor/google.golang.org/grpc/internal/binarylog/method_logger.go @@ -0,0 +1,423 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package binarylog + +import ( + "net" + "strings" + "sync/atomic" + "time" + + "github.com/golang/protobuf/proto" + "github.com/golang/protobuf/ptypes" + pb "google.golang.org/grpc/binarylog/grpc_binarylog_v1" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" +) + +type callIDGenerator struct { + id uint64 +} + +func (g *callIDGenerator) next() uint64 { + id := atomic.AddUint64(&g.id, 1) + return id +} + +// reset is for testing only, and doesn't need to be thread safe. +func (g *callIDGenerator) reset() { + g.id = 0 +} + +var idGen callIDGenerator + +// MethodLogger is the sub-logger for each method. +type MethodLogger struct { + headerMaxLen, messageMaxLen uint64 + + callID uint64 + idWithinCallGen *callIDGenerator + + sink Sink // TODO(blog): make this plugable. +} + +func newMethodLogger(h, m uint64) *MethodLogger { + return &MethodLogger{ + headerMaxLen: h, + messageMaxLen: m, + + callID: idGen.next(), + idWithinCallGen: &callIDGenerator{}, + + sink: defaultSink, // TODO(blog): make it plugable. + } +} + +// Log creates a proto binary log entry, and logs it to the sink. +func (ml *MethodLogger) Log(c LogEntryConfig) { + m := c.toProto() + timestamp, _ := ptypes.TimestampProto(time.Now()) + m.Timestamp = timestamp + m.CallId = ml.callID + m.SequenceIdWithinCall = ml.idWithinCallGen.next() + + switch pay := m.Payload.(type) { + case *pb.GrpcLogEntry_ClientHeader: + m.PayloadTruncated = ml.truncateMetadata(pay.ClientHeader.GetMetadata()) + case *pb.GrpcLogEntry_ServerHeader: + m.PayloadTruncated = ml.truncateMetadata(pay.ServerHeader.GetMetadata()) + case *pb.GrpcLogEntry_Message: + m.PayloadTruncated = ml.truncateMessage(pay.Message) + } + + ml.sink.Write(m) +} + +func (ml *MethodLogger) truncateMetadata(mdPb *pb.Metadata) (truncated bool) { + if ml.headerMaxLen == maxUInt { + return false + } + var ( + bytesLimit = ml.headerMaxLen + index int + ) + // At the end of the loop, index will be the first entry where the total + // size is greater than the limit: + // + // len(entry[:index]) <= ml.hdr && len(entry[:index+1]) > ml.hdr. + for ; index < len(mdPb.Entry); index++ { + entry := mdPb.Entry[index] + if entry.Key == "grpc-trace-bin" { + // "grpc-trace-bin" is a special key. It's kept in the log entry, + // but not counted towards the size limit. + continue + } + currentEntryLen := uint64(len(entry.Value)) + if currentEntryLen > bytesLimit { + break + } + bytesLimit -= currentEntryLen + } + truncated = index < len(mdPb.Entry) + mdPb.Entry = mdPb.Entry[:index] + return truncated +} + +func (ml *MethodLogger) truncateMessage(msgPb *pb.Message) (truncated bool) { + if ml.messageMaxLen == maxUInt { + return false + } + if ml.messageMaxLen >= uint64(len(msgPb.Data)) { + return false + } + msgPb.Data = msgPb.Data[:ml.messageMaxLen] + return true +} + +// LogEntryConfig represents the configuration for binary log entry. +type LogEntryConfig interface { + toProto() *pb.GrpcLogEntry +} + +// ClientHeader configs the binary log entry to be a ClientHeader entry. +type ClientHeader struct { + OnClientSide bool + Header metadata.MD + MethodName string + Authority string + Timeout time.Duration + // PeerAddr is required only when it's on server side. + PeerAddr net.Addr +} + +func (c *ClientHeader) toProto() *pb.GrpcLogEntry { + // This function doesn't need to set all the fields (e.g. seq ID). The Log + // function will set the fields when necessary. + clientHeader := &pb.ClientHeader{ + Metadata: mdToMetadataProto(c.Header), + MethodName: c.MethodName, + Authority: c.Authority, + } + if c.Timeout > 0 { + clientHeader.Timeout = ptypes.DurationProto(c.Timeout) + } + ret := &pb.GrpcLogEntry{ + Type: pb.GrpcLogEntry_EVENT_TYPE_CLIENT_HEADER, + Payload: &pb.GrpcLogEntry_ClientHeader{ + ClientHeader: clientHeader, + }, + } + if c.OnClientSide { + ret.Logger = pb.GrpcLogEntry_LOGGER_CLIENT + } else { + ret.Logger = pb.GrpcLogEntry_LOGGER_SERVER + } + if c.PeerAddr != nil { + ret.Peer = addrToProto(c.PeerAddr) + } + return ret +} + +// ServerHeader configs the binary log entry to be a ServerHeader entry. +type ServerHeader struct { + OnClientSide bool + Header metadata.MD + // PeerAddr is required only when it's on client side. + PeerAddr net.Addr +} + +func (c *ServerHeader) toProto() *pb.GrpcLogEntry { + ret := &pb.GrpcLogEntry{ + Type: pb.GrpcLogEntry_EVENT_TYPE_SERVER_HEADER, + Payload: &pb.GrpcLogEntry_ServerHeader{ + ServerHeader: &pb.ServerHeader{ + Metadata: mdToMetadataProto(c.Header), + }, + }, + } + if c.OnClientSide { + ret.Logger = pb.GrpcLogEntry_LOGGER_CLIENT + } else { + ret.Logger = pb.GrpcLogEntry_LOGGER_SERVER + } + if c.PeerAddr != nil { + ret.Peer = addrToProto(c.PeerAddr) + } + return ret +} + +// ClientMessage configs the binary log entry to be a ClientMessage entry. +type ClientMessage struct { + OnClientSide bool + // Message can be a proto.Message or []byte. Other messages formats are not + // supported. + Message interface{} +} + +func (c *ClientMessage) toProto() *pb.GrpcLogEntry { + var ( + data []byte + err error + ) + if m, ok := c.Message.(proto.Message); ok { + data, err = proto.Marshal(m) + if err != nil { + grpclog.Infof("binarylogging: failed to marshal proto message: %v", err) + } + } else if b, ok := c.Message.([]byte); ok { + data = b + } else { + grpclog.Infof("binarylogging: message to log is neither proto.message nor []byte") + } + ret := &pb.GrpcLogEntry{ + Type: pb.GrpcLogEntry_EVENT_TYPE_CLIENT_MESSAGE, + Payload: &pb.GrpcLogEntry_Message{ + Message: &pb.Message{ + Length: uint32(len(data)), + Data: data, + }, + }, + } + if c.OnClientSide { + ret.Logger = pb.GrpcLogEntry_LOGGER_CLIENT + } else { + ret.Logger = pb.GrpcLogEntry_LOGGER_SERVER + } + return ret +} + +// ServerMessage configs the binary log entry to be a ServerMessage entry. +type ServerMessage struct { + OnClientSide bool + // Message can be a proto.Message or []byte. Other messages formats are not + // supported. + Message interface{} +} + +func (c *ServerMessage) toProto() *pb.GrpcLogEntry { + var ( + data []byte + err error + ) + if m, ok := c.Message.(proto.Message); ok { + data, err = proto.Marshal(m) + if err != nil { + grpclog.Infof("binarylogging: failed to marshal proto message: %v", err) + } + } else if b, ok := c.Message.([]byte); ok { + data = b + } else { + grpclog.Infof("binarylogging: message to log is neither proto.message nor []byte") + } + ret := &pb.GrpcLogEntry{ + Type: pb.GrpcLogEntry_EVENT_TYPE_SERVER_MESSAGE, + Payload: &pb.GrpcLogEntry_Message{ + Message: &pb.Message{ + Length: uint32(len(data)), + Data: data, + }, + }, + } + if c.OnClientSide { + ret.Logger = pb.GrpcLogEntry_LOGGER_CLIENT + } else { + ret.Logger = pb.GrpcLogEntry_LOGGER_SERVER + } + return ret +} + +// ClientHalfClose configs the binary log entry to be a ClientHalfClose entry. +type ClientHalfClose struct { + OnClientSide bool +} + +func (c *ClientHalfClose) toProto() *pb.GrpcLogEntry { + ret := &pb.GrpcLogEntry{ + Type: pb.GrpcLogEntry_EVENT_TYPE_CLIENT_HALF_CLOSE, + Payload: nil, // No payload here. + } + if c.OnClientSide { + ret.Logger = pb.GrpcLogEntry_LOGGER_CLIENT + } else { + ret.Logger = pb.GrpcLogEntry_LOGGER_SERVER + } + return ret +} + +// ServerTrailer configs the binary log entry to be a ServerTrailer entry. +type ServerTrailer struct { + OnClientSide bool + Trailer metadata.MD + // Err is the status error. + Err error + // PeerAddr is required only when it's on client side and the RPC is trailer + // only. + PeerAddr net.Addr +} + +func (c *ServerTrailer) toProto() *pb.GrpcLogEntry { + st, ok := status.FromError(c.Err) + if !ok { + grpclog.Info("binarylogging: error in trailer is not a status error") + } + var ( + detailsBytes []byte + err error + ) + stProto := st.Proto() + if stProto != nil && len(stProto.Details) != 0 { + detailsBytes, err = proto.Marshal(stProto) + if err != nil { + grpclog.Infof("binarylogging: failed to marshal status proto: %v", err) + } + } + ret := &pb.GrpcLogEntry{ + Type: pb.GrpcLogEntry_EVENT_TYPE_SERVER_TRAILER, + Payload: &pb.GrpcLogEntry_Trailer{ + Trailer: &pb.Trailer{ + Metadata: mdToMetadataProto(c.Trailer), + StatusCode: uint32(st.Code()), + StatusMessage: st.Message(), + StatusDetails: detailsBytes, + }, + }, + } + if c.OnClientSide { + ret.Logger = pb.GrpcLogEntry_LOGGER_CLIENT + } else { + ret.Logger = pb.GrpcLogEntry_LOGGER_SERVER + } + if c.PeerAddr != nil { + ret.Peer = addrToProto(c.PeerAddr) + } + return ret +} + +// Cancel configs the binary log entry to be a Cancel entry. +type Cancel struct { + OnClientSide bool +} + +func (c *Cancel) toProto() *pb.GrpcLogEntry { + ret := &pb.GrpcLogEntry{ + Type: pb.GrpcLogEntry_EVENT_TYPE_CANCEL, + Payload: nil, + } + if c.OnClientSide { + ret.Logger = pb.GrpcLogEntry_LOGGER_CLIENT + } else { + ret.Logger = pb.GrpcLogEntry_LOGGER_SERVER + } + return ret +} + +// metadataKeyOmit returns whether the metadata entry with this key should be +// omitted. +func metadataKeyOmit(key string) bool { + switch key { + case "lb-token", ":path", ":authority", "content-encoding", "content-type", "user-agent", "te": + return true + case "grpc-trace-bin": // grpc-trace-bin is special because it's visiable to users. + return false + } + return strings.HasPrefix(key, "grpc-") +} + +func mdToMetadataProto(md metadata.MD) *pb.Metadata { + ret := &pb.Metadata{} + for k, vv := range md { + if metadataKeyOmit(k) { + continue + } + for _, v := range vv { + ret.Entry = append(ret.Entry, + &pb.MetadataEntry{ + Key: k, + Value: []byte(v), + }, + ) + } + } + return ret +} + +func addrToProto(addr net.Addr) *pb.Address { + ret := &pb.Address{} + switch a := addr.(type) { + case *net.TCPAddr: + if a.IP.To4() != nil { + ret.Type = pb.Address_TYPE_IPV4 + } else if a.IP.To16() != nil { + ret.Type = pb.Address_TYPE_IPV6 + } else { + ret.Type = pb.Address_TYPE_UNKNOWN + // Do not set address and port fields. + break + } + ret.Address = a.IP.String() + ret.IpPort = uint32(a.Port) + case *net.UnixAddr: + ret.Type = pb.Address_TYPE_UNIX + ret.Address = a.String() + default: + ret.Type = pb.Address_TYPE_UNKNOWN + } + return ret +} diff --git a/vendor/google.golang.org/grpc/internal/binarylog/regenerate.sh b/vendor/google.golang.org/grpc/internal/binarylog/regenerate.sh new file mode 100644 index 00000000000..113d40cbe16 --- /dev/null +++ b/vendor/google.golang.org/grpc/internal/binarylog/regenerate.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# Copyright 2018 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eux -o pipefail + +TMP=$(mktemp -d) + +function finish { + rm -rf "$TMP" +} +trap finish EXIT + +pushd "$TMP" +mkdir -p grpc/binarylog/grpc_binarylog_v1 +curl https://raw.githubusercontent.com/grpc/grpc-proto/master/grpc/binlog/v1/binarylog.proto > grpc/binarylog/grpc_binarylog_v1/binarylog.proto + +protoc --go_out=plugins=grpc,paths=source_relative:. -I. grpc/binarylog/grpc_binarylog_v1/*.proto +popd +rm -f ./grpc_binarylog_v1/*.pb.go +cp "$TMP"/grpc/binarylog/grpc_binarylog_v1/*.pb.go ../../binarylog/grpc_binarylog_v1/ + diff --git a/vendor/google.golang.org/grpc/internal/binarylog/sink.go b/vendor/google.golang.org/grpc/internal/binarylog/sink.go new file mode 100644 index 00000000000..20d044f0fd7 --- /dev/null +++ b/vendor/google.golang.org/grpc/internal/binarylog/sink.go @@ -0,0 +1,162 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package binarylog + +import ( + "bufio" + "encoding/binary" + "fmt" + "io" + "io/ioutil" + "sync" + "time" + + "github.com/golang/protobuf/proto" + pb "google.golang.org/grpc/binarylog/grpc_binarylog_v1" + "google.golang.org/grpc/grpclog" +) + +var ( + defaultSink Sink = &noopSink{} // TODO(blog): change this default (file in /tmp). +) + +// SetDefaultSink sets the sink where binary logs will be written to. +// +// Not thread safe. Only set during initialization. +func SetDefaultSink(s Sink) { + if defaultSink != nil { + defaultSink.Close() + } + defaultSink = s +} + +// Sink writes log entry into the binary log sink. +type Sink interface { + // Write will be called to write the log entry into the sink. + // + // It should be thread-safe so it can be called in parallel. + Write(*pb.GrpcLogEntry) error + // Close will be called when the Sink is replaced by a new Sink. + Close() error +} + +type noopSink struct{} + +func (ns *noopSink) Write(*pb.GrpcLogEntry) error { return nil } +func (ns *noopSink) Close() error { return nil } + +// newWriterSink creates a binary log sink with the given writer. +// +// Write() marshalls the proto message and writes it to the given writer. Each +// message is prefixed with a 4 byte big endian unsigned integer as the length. +// +// No buffer is done, Close() doesn't try to close the writer. +func newWriterSink(w io.Writer) *writerSink { + return &writerSink{out: w} +} + +type writerSink struct { + out io.Writer +} + +func (ws *writerSink) Write(e *pb.GrpcLogEntry) error { + b, err := proto.Marshal(e) + if err != nil { + grpclog.Infof("binary logging: failed to marshal proto message: %v", err) + } + hdr := make([]byte, 4) + binary.BigEndian.PutUint32(hdr, uint32(len(b))) + if _, err := ws.out.Write(hdr); err != nil { + return err + } + if _, err := ws.out.Write(b); err != nil { + return err + } + return nil +} + +func (ws *writerSink) Close() error { return nil } + +type bufWriteCloserSink struct { + mu sync.Mutex + closer io.Closer + out *writerSink // out is built on buf. + buf *bufio.Writer // buf is kept for flush. + + writeStartOnce sync.Once + writeTicker *time.Ticker +} + +func (fs *bufWriteCloserSink) Write(e *pb.GrpcLogEntry) error { + // Start the write loop when Write is called. + fs.writeStartOnce.Do(fs.startFlushGoroutine) + fs.mu.Lock() + if err := fs.out.Write(e); err != nil { + fs.mu.Unlock() + return err + } + fs.mu.Unlock() + return nil +} + +const ( + bufFlushDuration = 60 * time.Second +) + +func (fs *bufWriteCloserSink) startFlushGoroutine() { + fs.writeTicker = time.NewTicker(bufFlushDuration) + go func() { + for range fs.writeTicker.C { + fs.mu.Lock() + fs.buf.Flush() + fs.mu.Unlock() + } + }() +} + +func (fs *bufWriteCloserSink) Close() error { + if fs.writeTicker != nil { + fs.writeTicker.Stop() + } + fs.mu.Lock() + fs.buf.Flush() + fs.closer.Close() + fs.out.Close() + fs.mu.Unlock() + return nil +} + +func newBufWriteCloserSink(o io.WriteCloser) Sink { + bufW := bufio.NewWriter(o) + return &bufWriteCloserSink{ + closer: o, + out: newWriterSink(bufW), + buf: bufW, + } +} + +// NewTempFileSink creates a temp file and returns a Sink that writes to this +// file. +func NewTempFileSink() (Sink, error) { + tempFile, err := ioutil.TempFile("/tmp", "grpcgo_binarylog_*.txt") + if err != nil { + return nil, fmt.Errorf("failed to create temp file: %v", err) + } + return newBufWriteCloserSink(tempFile), nil +} diff --git a/vendor/google.golang.org/grpc/status/go16.go b/vendor/google.golang.org/grpc/internal/binarylog/util.go similarity index 51% rename from vendor/google.golang.org/grpc/status/go16.go rename to vendor/google.golang.org/grpc/internal/binarylog/util.go index e59b53e82be..15dc7803d8b 100644 --- a/vendor/google.golang.org/grpc/status/go16.go +++ b/vendor/google.golang.org/grpc/internal/binarylog/util.go @@ -1,5 +1,3 @@ -// +build go1.6,!go1.7 - /* * * Copyright 2018 gRPC authors. @@ -18,25 +16,26 @@ * */ -package status +package binarylog import ( - "golang.org/x/net/context" - "google.golang.org/grpc/codes" + "errors" + "strings" ) -// FromContextError converts a context error into a Status. It returns a -// Status with codes.OK if err is nil, or a Status with codes.Unknown if err is -// non-nil and not a context error. -func FromContextError(err error) *Status { - switch err { - case nil: - return New(codes.OK, "") - case context.DeadlineExceeded: - return New(codes.DeadlineExceeded, err.Error()) - case context.Canceled: - return New(codes.Canceled, err.Error()) - default: - return New(codes.Unknown, err.Error()) +// parseMethodName splits service and method from the input. It expects format +// "/service/method". +// +// TODO: move to internal/grpcutil. +func parseMethodName(methodName string) (service, method string, _ error) { + if !strings.HasPrefix(methodName, "/") { + return "", "", errors.New("invalid method name: should start with /") } + methodName = methodName[1:] + + pos := strings.LastIndex(methodName, "/") + if pos < 0 { + return "", "", errors.New("invalid method name: suffix /method is missing") + } + return methodName[:pos], methodName[pos+1:], nil } diff --git a/vendor/google.golang.org/grpc/internal/channelz/funcs.go b/vendor/google.golang.org/grpc/internal/channelz/funcs.go index 586a0336b47..f0744f9937e 100644 --- a/vendor/google.golang.org/grpc/internal/channelz/funcs.go +++ b/vendor/google.golang.org/grpc/internal/channelz/funcs.go @@ -24,19 +24,26 @@ package channelz import ( + "fmt" "sort" "sync" "sync/atomic" + "time" "google.golang.org/grpc/grpclog" ) +const ( + defaultMaxTraceEntry int32 = 30 +) + var ( db dbWrapper idGen idGenerator // EntryPerPage defines the number of channelz entries to be shown on a web page. - EntryPerPage = 50 - curState int32 + EntryPerPage = int64(50) + curState int32 + maxTraceEntry = defaultMaxTraceEntry ) // TurnOn turns on channelz data collection. @@ -52,6 +59,22 @@ func IsOn() bool { return atomic.CompareAndSwapInt32(&curState, 1, 1) } +// SetMaxTraceEntry sets maximum number of trace entry per entity (i.e. channel/subchannel). +// Setting it to 0 will disable channel tracing. +func SetMaxTraceEntry(i int32) { + atomic.StoreInt32(&maxTraceEntry, i) +} + +// ResetMaxTraceEntryToDefault resets the maximum number of trace entry per entity to default. +func ResetMaxTraceEntryToDefault() { + atomic.StoreInt32(&maxTraceEntry, defaultMaxTraceEntry) +} + +func getMaxTraceEntry() int { + i := atomic.LoadInt32(&maxTraceEntry) + return int(i) +} + // dbWarpper wraps around a reference to internal channelz data storage, and // provide synchronized functionality to set and get the reference. type dbWrapper struct { @@ -73,9 +96,14 @@ func (d *dbWrapper) get() *channelMap { // NewChannelzStorage initializes channelz data storage and id generator. // +// This function returns a cleanup function to wait for all channelz state to be reset by the +// grpc goroutines when those entities get closed. By using this cleanup function, we make sure tests +// don't mess up each other, i.e. lingering goroutine from previous test doing entity removal happen +// to remove some entity just register by the new test, since the id space is the same. +// // Note: This function is exported for testing purpose only. User should not call // it in most cases. -func NewChannelzStorage() { +func NewChannelzStorage() (cleanup func() error) { db.set(&channelMap{ topLevelChannels: make(map[int64]struct{}), channels: make(map[int64]*channel), @@ -85,26 +113,48 @@ func NewChannelzStorage() { subChannels: make(map[int64]*subChannel), }) idGen.reset() + return func() error { + var err error + cm := db.get() + if cm == nil { + return nil + } + for i := 0; i < 1000; i++ { + cm.mu.Lock() + if len(cm.topLevelChannels) == 0 && len(cm.servers) == 0 && len(cm.channels) == 0 && len(cm.subChannels) == 0 && len(cm.listenSockets) == 0 && len(cm.normalSockets) == 0 { + cm.mu.Unlock() + // all things stored in the channelz map have been cleared. + return nil + } + cm.mu.Unlock() + time.Sleep(10 * time.Millisecond) + } + + cm.mu.Lock() + err = fmt.Errorf("after 10s the channelz map has not been cleaned up yet, topchannels: %d, servers: %d, channels: %d, subchannels: %d, listen sockets: %d, normal sockets: %d", len(cm.topLevelChannels), len(cm.servers), len(cm.channels), len(cm.subChannels), len(cm.listenSockets), len(cm.normalSockets)) + cm.mu.Unlock() + return err + } } // GetTopChannels returns a slice of top channel's ChannelMetric, along with a // boolean indicating whether there's more top channels to be queried for. // // The arg id specifies that only top channel with id at or above it will be included -// in the result. The returned slice is up to a length of EntryPerPage, and is -// sorted in ascending id order. -func GetTopChannels(id int64) ([]*ChannelMetric, bool) { - return db.get().GetTopChannels(id) +// in the result. The returned slice is up to a length of the arg maxResults or +// EntryPerPage if maxResults is zero, and is sorted in ascending id order. +func GetTopChannels(id int64, maxResults int64) ([]*ChannelMetric, bool) { + return db.get().GetTopChannels(id, maxResults) } // GetServers returns a slice of server's ServerMetric, along with a // boolean indicating whether there's more servers to be queried for. // // The arg id specifies that only server with id at or above it will be included -// in the result. The returned slice is up to a length of EntryPerPage, and is -// sorted in ascending id order. -func GetServers(id int64) ([]*ServerMetric, bool) { - return db.get().GetServers(id) +// in the result. The returned slice is up to a length of the arg maxResults or +// EntryPerPage if maxResults is zero, and is sorted in ascending id order. +func GetServers(id int64, maxResults int64) ([]*ServerMetric, bool) { + return db.get().GetServers(id, maxResults) } // GetServerSockets returns a slice of server's (identified by id) normal socket's @@ -112,10 +162,10 @@ func GetServers(id int64) ([]*ServerMetric, bool) { // be queried for. // // The arg startID specifies that only sockets with id at or above it will be -// included in the result. The returned slice is up to a length of EntryPerPage, -// and is sorted in ascending id order. -func GetServerSockets(id int64, startID int64) ([]*SocketMetric, bool) { - return db.get().GetServerSockets(id, startID) +// included in the result. The returned slice is up to a length of the arg maxResults +// or EntryPerPage if maxResults is zero, and is sorted in ascending id order. +func GetServerSockets(id int64, startID int64, maxResults int64) ([]*SocketMetric, bool) { + return db.get().GetServerSockets(id, startID, maxResults) } // GetChannel returns the ChannelMetric for the channel (identified by id). @@ -133,6 +183,11 @@ func GetSocket(id int64) *SocketMetric { return db.get().GetSocket(id) } +// GetServer returns the ServerMetric for the server (identified by id). +func GetServer(id int64) *ServerMetric { + return db.get().GetServer(id) +} + // RegisterChannel registers the given channel c in channelz database with ref // as its reference name, and add it to the child list of its parent (identified // by pid). pid = 0 means no parent. It returns the unique channelz tracking id @@ -146,6 +201,7 @@ func RegisterChannel(c Channel, pid int64, ref string) int64 { nestedChans: make(map[int64]string), id: id, pid: pid, + trace: &channelTrace{createdTime: time.Now(), events: make([]*TraceEvent, 0, getMaxTraceEntry())}, } if pid == 0 { db.get().addChannel(id, cn, true, pid, ref) @@ -170,6 +226,7 @@ func RegisterSubChannel(c Channel, pid int64, ref string) int64 { sockets: make(map[int64]string), id: id, pid: pid, + trace: &channelTrace{createdTime: time.Now(), events: make([]*TraceEvent, 0, getMaxTraceEntry())}, } db.get().addSubChannel(id, sc, pid, ref) return id @@ -226,6 +283,24 @@ func RemoveEntry(id int64) { db.get().removeEntry(id) } +// TraceEventDesc is what the caller of AddTraceEvent should provide to describe the event to be added +// to the channel trace. +// The Parent field is optional. It is used for event that will be recorded in the entity's parent +// trace also. +type TraceEventDesc struct { + Desc string + Severity Severity + Parent *TraceEventDesc +} + +// AddTraceEvent adds trace related to the entity with specified id, using the provided TraceEventDesc. +func AddTraceEvent(id int64, desc *TraceEventDesc) { + if getMaxTraceEntry() == 0 { + return + } + db.get().traceEvent(id, desc) +} + // channelMap is the storage data structure for channelz. // Methods of channelMap can be divided in two two categories with respect to locking. // 1. Methods acquire the global lock. @@ -251,6 +326,7 @@ func (c *channelMap) addServer(id int64, s *server) { func (c *channelMap) addChannel(id int64, cn *channel, isTopChannel bool, pid int64, ref string) { c.mu.Lock() cn.cm = c + cn.trace.cm = c c.channels[id] = cn if isTopChannel { c.topLevelChannels[id] = struct{}{} @@ -263,6 +339,7 @@ func (c *channelMap) addChannel(id int64, cn *channel, isTopChannel bool, pid in func (c *channelMap) addSubChannel(id int64, sc *subChannel, pid int64, ref string) { c.mu.Lock() sc.cm = c + sc.trace.cm = c c.subChannels[id] = sc c.findEntry(pid).addChild(id, sc) c.mu.Unlock() @@ -284,16 +361,25 @@ func (c *channelMap) addNormalSocket(id int64, ns *normalSocket, pid int64, ref c.mu.Unlock() } -// removeEntry triggers the removal of an entry, which may not indeed delete the -// entry, if it has to wait on the deletion of its children, or may lead to a chain -// of entry deletion. For example, deleting the last socket of a gracefully shutting -// down server will lead to the server being also deleted. +// removeEntry triggers the removal of an entry, which may not indeed delete the entry, if it has to +// wait on the deletion of its children and until no other entity's channel trace references it. +// It may lead to a chain of entry deletion. For example, deleting the last socket of a gracefully +// shutting down server will lead to the server being also deleted. func (c *channelMap) removeEntry(id int64) { c.mu.Lock() c.findEntry(id).triggerDelete() c.mu.Unlock() } +// c.mu must be held by the caller +func (c *channelMap) decrTraceRefCount(id int64) { + e := c.findEntry(id) + if v, ok := e.(tracedChannel); ok { + v.decrTraceRefCount() + e.deleteSelfIfReady() + } +} + // c.mu must be held by the caller. func (c *channelMap) findEntry(id int64) entry { var v entry @@ -347,6 +433,39 @@ func (c *channelMap) deleteEntry(id int64) { } } +func (c *channelMap) traceEvent(id int64, desc *TraceEventDesc) { + c.mu.Lock() + child := c.findEntry(id) + childTC, ok := child.(tracedChannel) + if !ok { + c.mu.Unlock() + return + } + childTC.getChannelTrace().append(&TraceEvent{Desc: desc.Desc, Severity: desc.Severity, Timestamp: time.Now()}) + if desc.Parent != nil { + parent := c.findEntry(child.getParentID()) + var chanType RefChannelType + switch child.(type) { + case *channel: + chanType = RefChannel + case *subChannel: + chanType = RefSubChannel + } + if parentTC, ok := parent.(tracedChannel); ok { + parentTC.getChannelTrace().append(&TraceEvent{ + Desc: desc.Parent.Desc, + Severity: desc.Parent.Severity, + Timestamp: time.Now(), + RefID: id, + RefName: childTC.getRefName(), + RefType: chanType, + }) + childTC.incrTraceRefCount() + } + } + c.mu.Unlock() +} + type int64Slice []int64 func (s int64Slice) Len() int { return len(s) } @@ -361,29 +480,32 @@ func copyMap(m map[int64]string) map[int64]string { return n } -func min(a, b int) int { +func min(a, b int64) int64 { if a < b { return a } return b } -func (c *channelMap) GetTopChannels(id int64) ([]*ChannelMetric, bool) { +func (c *channelMap) GetTopChannels(id int64, maxResults int64) ([]*ChannelMetric, bool) { + if maxResults <= 0 { + maxResults = EntryPerPage + } c.mu.RLock() - l := len(c.topLevelChannels) + l := int64(len(c.topLevelChannels)) ids := make([]int64, 0, l) - cns := make([]*channel, 0, min(l, EntryPerPage)) + cns := make([]*channel, 0, min(l, maxResults)) for k := range c.topLevelChannels { ids = append(ids, k) } sort.Sort(int64Slice(ids)) idx := sort.Search(len(ids), func(i int) bool { return ids[i] >= id }) - count := 0 + count := int64(0) var end bool var t []*ChannelMetric for i, v := range ids[idx:] { - if count == EntryPerPage { + if count == maxResults { break } if cn, ok := c.channels[v]; ok { @@ -408,25 +530,29 @@ func (c *channelMap) GetTopChannels(id int64) ([]*ChannelMetric, bool) { t[i].ChannelData = cn.c.ChannelzMetric() t[i].ID = cn.id t[i].RefName = cn.refName + t[i].Trace = cn.trace.dumpData() } return t, end } -func (c *channelMap) GetServers(id int64) ([]*ServerMetric, bool) { +func (c *channelMap) GetServers(id, maxResults int64) ([]*ServerMetric, bool) { + if maxResults <= 0 { + maxResults = EntryPerPage + } c.mu.RLock() - l := len(c.servers) + l := int64(len(c.servers)) ids := make([]int64, 0, l) - ss := make([]*server, 0, min(l, EntryPerPage)) + ss := make([]*server, 0, min(l, maxResults)) for k := range c.servers { ids = append(ids, k) } sort.Sort(int64Slice(ids)) idx := sort.Search(len(ids), func(i int) bool { return ids[i] >= id }) - count := 0 + count := int64(0) var end bool var s []*ServerMetric for i, v := range ids[idx:] { - if count == EntryPerPage { + if count == maxResults { break } if svr, ok := c.servers[v]; ok { @@ -454,7 +580,10 @@ func (c *channelMap) GetServers(id int64) ([]*ServerMetric, bool) { return s, end } -func (c *channelMap) GetServerSockets(id int64, startID int64) ([]*SocketMetric, bool) { +func (c *channelMap) GetServerSockets(id int64, startID int64, maxResults int64) ([]*SocketMetric, bool) { + if maxResults <= 0 { + maxResults = EntryPerPage + } var svr *server var ok bool c.mu.RLock() @@ -464,18 +593,18 @@ func (c *channelMap) GetServerSockets(id int64, startID int64) ([]*SocketMetric, return nil, true } svrskts := svr.sockets - l := len(svrskts) + l := int64(len(svrskts)) ids := make([]int64, 0, l) - sks := make([]*normalSocket, 0, min(l, EntryPerPage)) + sks := make([]*normalSocket, 0, min(l, maxResults)) for k := range svrskts { ids = append(ids, k) } - sort.Sort((int64Slice(ids))) - idx := sort.Search(len(ids), func(i int) bool { return ids[i] >= id }) - count := 0 + sort.Sort(int64Slice(ids)) + idx := sort.Search(len(ids), func(i int) bool { return ids[i] >= startID }) + count := int64(0) var end bool for i, v := range ids[idx:] { - if count == EntryPerPage { + if count == maxResults { break } if ns, ok := c.normalSockets[v]; ok { @@ -514,10 +643,14 @@ func (c *channelMap) GetChannel(id int64) *ChannelMetric { } cm.NestedChans = copyMap(cn.nestedChans) cm.SubChans = copyMap(cn.subChans) + // cn.c can be set to &dummyChannel{} when deleteSelfFromMap is called. Save a copy of cn.c when + // holding the lock to prevent potential data race. + chanCopy := cn.c c.mu.RUnlock() - cm.ChannelData = cn.c.ChannelzMetric() + cm.ChannelData = chanCopy.ChannelzMetric() cm.ID = cn.id cm.RefName = cn.refName + cm.Trace = cn.trace.dumpData() return cm } @@ -532,10 +665,14 @@ func (c *channelMap) GetSubChannel(id int64) *SubChannelMetric { return nil } cm.Sockets = copyMap(sc.sockets) + // sc.c can be set to &dummyChannel{} when deleteSelfFromMap is called. Save a copy of sc.c when + // holding the lock to prevent potential data race. + chanCopy := sc.c c.mu.RUnlock() - cm.ChannelData = sc.c.ChannelzMetric() + cm.ChannelData = chanCopy.ChannelzMetric() cm.ID = sc.id cm.RefName = sc.refName + cm.Trace = sc.trace.dumpData() return cm } @@ -560,6 +697,23 @@ func (c *channelMap) GetSocket(id int64) *SocketMetric { return nil } +func (c *channelMap) GetServer(id int64) *ServerMetric { + sm := &ServerMetric{} + var svr *server + var ok bool + c.mu.RLock() + if svr, ok = c.servers[id]; !ok { + c.mu.RUnlock() + return nil + } + sm.ListenSockets = copyMap(svr.listenSockets) + c.mu.RUnlock() + sm.ID = svr.id + sm.RefName = svr.refName + sm.ServerData = svr.s.ChannelzMetric() + return sm +} + type idGenerator struct { id int64 } diff --git a/vendor/google.golang.org/grpc/internal/channelz/types.go b/vendor/google.golang.org/grpc/internal/channelz/types.go index 6fd6bb388af..17c2274cb3d 100644 --- a/vendor/google.golang.org/grpc/internal/channelz/types.go +++ b/vendor/google.golang.org/grpc/internal/channelz/types.go @@ -20,6 +20,8 @@ package channelz import ( "net" + "sync" + "sync/atomic" "time" "google.golang.org/grpc/connectivity" @@ -40,6 +42,8 @@ type entry interface { // deleteSelfIfReady check whether triggerDelete() has been called before, and whether child // list is now empty. If both conditions are met, then delete self from database. deleteSelfIfReady() + // getParentID returns parent ID of the entry. 0 value parent ID means no parent. + getParentID() int64 } // dummyEntry is a fake entry to handle entry not found case. @@ -73,6 +77,10 @@ func (*dummyEntry) deleteSelfIfReady() { // code should not reach here. deleteSelfIfReady is always called on an existing entry. } +func (*dummyEntry) getParentID() int64 { + return 0 +} + // ChannelMetric defines the info channelz provides for a specific Channel, which // includes ChannelInternalMetric and channelz-specific data, such as channelz id, // child list, etc. @@ -95,6 +103,8 @@ type ChannelMetric struct { // Note current grpc implementation doesn't allow channel having sockets directly, // therefore, this is field is unused. Sockets map[int64]string + // Trace contains the most recent traced events. + Trace *ChannelTrace } // SubChannelMetric defines the info channelz provides for a specific SubChannel, @@ -121,6 +131,8 @@ type SubChannelMetric struct { // Sockets tracks the socket type children of this subchannel in the format of a map // from socket channelz id to corresponding reference string. Sockets map[int64]string + // Trace contains the most recent traced events. + Trace *ChannelTrace } // ChannelInternalMetric defines the struct that the implementor of Channel interface @@ -138,7 +150,35 @@ type ChannelInternalMetric struct { CallsFailed int64 // The last time a call was started on the channel. LastCallStartedTimestamp time.Time - //TODO: trace +} + +// ChannelTrace stores traced events on a channel/subchannel and related info. +type ChannelTrace struct { + // EventNum is the number of events that ever got traced (i.e. including those that have been deleted) + EventNum int64 + // CreationTime is the creation time of the trace. + CreationTime time.Time + // Events stores the most recent trace events (up to $maxTraceEntry, newer event will overwrite the + // oldest one) + Events []*TraceEvent +} + +// TraceEvent represent a single trace event +type TraceEvent struct { + // Desc is a simple description of the trace event. + Desc string + // Severity states the severity of this trace event. + Severity Severity + // Timestamp is the event time. + Timestamp time.Time + // RefID is the id of the entity that gets referenced in the event. RefID is 0 if no other entity is + // involved in this event. + // e.g. SubChannel (id: 4[]) Created. --> RefID = 4, RefName = "" (inside []) + RefID int64 + // RefName is the reference name for the entity that gets referenced in the event. + RefName string + // RefType indicates the referenced entity type, i.e Channel or SubChannel. + RefType RefChannelType } // Channel is the interface that should be satisfied in order to be tracked by @@ -147,6 +187,12 @@ type Channel interface { ChannelzMetric() *ChannelInternalMetric } +type dummyChannel struct{} + +func (d *dummyChannel) ChannelzMetric() *ChannelInternalMetric { + return &ChannelInternalMetric{} +} + type channel struct { refName string c Channel @@ -156,6 +202,10 @@ type channel struct { id int64 pid int64 cm *channelMap + trace *channelTrace + // traceRefCount is the number of trace events that reference this channel. + // Non-zero traceRefCount means the trace of this channel cannot be deleted. + traceRefCount int32 } func (c *channel) addChild(id int64, e entry) { @@ -180,25 +230,96 @@ func (c *channel) triggerDelete() { c.deleteSelfIfReady() } -func (c *channel) deleteSelfIfReady() { +func (c *channel) getParentID() int64 { + return c.pid +} + +// deleteSelfFromTree tries to delete the channel from the channelz entry relation tree, which means +// deleting the channel reference from its parent's child list. +// +// In order for a channel to be deleted from the tree, it must meet the criteria that, removal of the +// corresponding grpc object has been invoked, and the channel does not have any children left. +// +// The returned boolean value indicates whether the channel has been successfully deleted from tree. +func (c *channel) deleteSelfFromTree() (deleted bool) { if !c.closeCalled || len(c.subChans)+len(c.nestedChans) != 0 { - return + return false } - c.cm.deleteEntry(c.id) // not top channel if c.pid != 0 { c.cm.findEntry(c.pid).deleteChild(c.id) } + return true +} + +// deleteSelfFromMap checks whether it is valid to delete the channel from the map, which means +// deleting the channel from channelz's tracking entirely. Users can no longer use id to query the +// channel, and its memory will be garbage collected. +// +// The trace reference count of the channel must be 0 in order to be deleted from the map. This is +// specified in the channel tracing gRFC that as long as some other trace has reference to an entity, +// the trace of the referenced entity must not be deleted. In order to release the resource allocated +// by grpc, the reference to the grpc object is reset to a dummy object. +// +// deleteSelfFromMap must be called after deleteSelfFromTree returns true. +// +// It returns a bool to indicate whether the channel can be safely deleted from map. +func (c *channel) deleteSelfFromMap() (delete bool) { + if c.getTraceRefCount() != 0 { + c.c = &dummyChannel{} + return false + } + return true +} + +// deleteSelfIfReady tries to delete the channel itself from the channelz database. +// The delete process includes two steps: +// 1. delete the channel from the entry relation tree, i.e. delete the channel reference from its +// parent's child list. +// 2. delete the channel from the map, i.e. delete the channel entirely from channelz. Lookup by id +// will return entry not found error. +func (c *channel) deleteSelfIfReady() { + if !c.deleteSelfFromTree() { + return + } + if !c.deleteSelfFromMap() { + return + } + c.cm.deleteEntry(c.id) + c.trace.clear() +} + +func (c *channel) getChannelTrace() *channelTrace { + return c.trace +} + +func (c *channel) incrTraceRefCount() { + atomic.AddInt32(&c.traceRefCount, 1) +} + +func (c *channel) decrTraceRefCount() { + atomic.AddInt32(&c.traceRefCount, -1) +} + +func (c *channel) getTraceRefCount() int { + i := atomic.LoadInt32(&c.traceRefCount) + return int(i) +} + +func (c *channel) getRefName() string { + return c.refName } type subChannel struct { - refName string - c Channel - closeCalled bool - sockets map[int64]string - id int64 - pid int64 - cm *channelMap + refName string + c Channel + closeCalled bool + sockets map[int64]string + id int64 + pid int64 + cm *channelMap + trace *channelTrace + traceRefCount int32 } func (sc *subChannel) addChild(id int64, e entry) { @@ -219,12 +340,82 @@ func (sc *subChannel) triggerDelete() { sc.deleteSelfIfReady() } -func (sc *subChannel) deleteSelfIfReady() { +func (sc *subChannel) getParentID() int64 { + return sc.pid +} + +// deleteSelfFromTree tries to delete the subchannel from the channelz entry relation tree, which +// means deleting the subchannel reference from its parent's child list. +// +// In order for a subchannel to be deleted from the tree, it must meet the criteria that, removal of +// the corresponding grpc object has been invoked, and the subchannel does not have any children left. +// +// The returned boolean value indicates whether the channel has been successfully deleted from tree. +func (sc *subChannel) deleteSelfFromTree() (deleted bool) { if !sc.closeCalled || len(sc.sockets) != 0 { + return false + } + sc.cm.findEntry(sc.pid).deleteChild(sc.id) + return true +} + +// deleteSelfFromMap checks whether it is valid to delete the subchannel from the map, which means +// deleting the subchannel from channelz's tracking entirely. Users can no longer use id to query +// the subchannel, and its memory will be garbage collected. +// +// The trace reference count of the subchannel must be 0 in order to be deleted from the map. This is +// specified in the channel tracing gRFC that as long as some other trace has reference to an entity, +// the trace of the referenced entity must not be deleted. In order to release the resource allocated +// by grpc, the reference to the grpc object is reset to a dummy object. +// +// deleteSelfFromMap must be called after deleteSelfFromTree returns true. +// +// It returns a bool to indicate whether the channel can be safely deleted from map. +func (sc *subChannel) deleteSelfFromMap() (delete bool) { + if sc.getTraceRefCount() != 0 { + // free the grpc struct (i.e. addrConn) + sc.c = &dummyChannel{} + return false + } + return true +} + +// deleteSelfIfReady tries to delete the subchannel itself from the channelz database. +// The delete process includes two steps: +// 1. delete the subchannel from the entry relation tree, i.e. delete the subchannel reference from +// its parent's child list. +// 2. delete the subchannel from the map, i.e. delete the subchannel entirely from channelz. Lookup +// by id will return entry not found error. +func (sc *subChannel) deleteSelfIfReady() { + if !sc.deleteSelfFromTree() { + return + } + if !sc.deleteSelfFromMap() { return } sc.cm.deleteEntry(sc.id) - sc.cm.findEntry(sc.pid).deleteChild(sc.id) + sc.trace.clear() +} + +func (sc *subChannel) getChannelTrace() *channelTrace { + return sc.trace +} + +func (sc *subChannel) incrTraceRefCount() { + atomic.AddInt32(&sc.traceRefCount, 1) +} + +func (sc *subChannel) decrTraceRefCount() { + atomic.AddInt32(&sc.traceRefCount, -1) +} + +func (sc *subChannel) getTraceRefCount() int { + i := atomic.LoadInt32(&sc.traceRefCount) + return int(i) +} + +func (sc *subChannel) getRefName() string { + return sc.refName } // SocketMetric defines the info channelz provides for a specific Socket, which @@ -318,6 +509,10 @@ func (ls *listenSocket) deleteSelfIfReady() { grpclog.Errorf("cannot call deleteSelfIfReady on a listen socket") } +func (ls *listenSocket) getParentID() int64 { + return ls.pid +} + type normalSocket struct { refName string s Socket @@ -343,6 +538,10 @@ func (ns *normalSocket) deleteSelfIfReady() { grpclog.Errorf("cannot call deleteSelfIfReady on a normal socket") } +func (ns *normalSocket) getParentID() int64 { + return ns.pid +} + // ServerMetric defines the info channelz provides for a specific Server, which // includes ServerInternalMetric and channelz-specific data, such as channelz id, // child list, etc. @@ -370,7 +569,6 @@ type ServerInternalMetric struct { CallsFailed int64 // The last time a call was started on the server. LastCallStartedTimestamp time.Time - //TODO: trace } // Server is the interface to be satisfied in order to be tracked by channelz as @@ -417,3 +615,88 @@ func (s *server) deleteSelfIfReady() { } s.cm.deleteEntry(s.id) } + +func (s *server) getParentID() int64 { + return 0 +} + +type tracedChannel interface { + getChannelTrace() *channelTrace + incrTraceRefCount() + decrTraceRefCount() + getRefName() string +} + +type channelTrace struct { + cm *channelMap + createdTime time.Time + eventCount int64 + mu sync.Mutex + events []*TraceEvent +} + +func (c *channelTrace) append(e *TraceEvent) { + c.mu.Lock() + if len(c.events) == getMaxTraceEntry() { + del := c.events[0] + c.events = c.events[1:] + if del.RefID != 0 { + // start recursive cleanup in a goroutine to not block the call originated from grpc. + go func() { + // need to acquire c.cm.mu lock to call the unlocked attemptCleanup func. + c.cm.mu.Lock() + c.cm.decrTraceRefCount(del.RefID) + c.cm.mu.Unlock() + }() + } + } + e.Timestamp = time.Now() + c.events = append(c.events, e) + c.eventCount++ + c.mu.Unlock() +} + +func (c *channelTrace) clear() { + c.mu.Lock() + for _, e := range c.events { + if e.RefID != 0 { + // caller should have already held the c.cm.mu lock. + c.cm.decrTraceRefCount(e.RefID) + } + } + c.mu.Unlock() +} + +// Severity is the severity level of a trace event. +// The canonical enumeration of all valid values is here: +// https://github.com/grpc/grpc-proto/blob/9b13d199cc0d4703c7ea26c9c330ba695866eb23/grpc/channelz/v1/channelz.proto#L126. +type Severity int + +const ( + // CtUNKNOWN indicates unknown severity of a trace event. + CtUNKNOWN Severity = iota + // CtINFO indicates info level severity of a trace event. + CtINFO + // CtWarning indicates warning level severity of a trace event. + CtWarning + // CtError indicates error level severity of a trace event. + CtError +) + +// RefChannelType is the type of the entity being referenced in a trace event. +type RefChannelType int + +const ( + // RefChannel indicates the referenced entity is a Channel. + RefChannel RefChannelType = iota + // RefSubChannel indicates the referenced entity is a SubChannel. + RefSubChannel +) + +func (c *channelTrace) dumpData() *ChannelTrace { + c.mu.Lock() + ct := &ChannelTrace{EventNum: c.eventCount, CreationTime: c.createdTime} + ct.Events = c.events[:len(c.events)] + c.mu.Unlock() + return ct +} diff --git a/vendor/google.golang.org/grpc/internal/channelz/types_linux.go b/vendor/google.golang.org/grpc/internal/channelz/types_linux.go index 9801c3e2ca0..692dd618177 100644 --- a/vendor/google.golang.org/grpc/internal/channelz/types_linux.go +++ b/vendor/google.golang.org/grpc/internal/channelz/types_linux.go @@ -1,4 +1,4 @@ -// +build !appengine,go1.7 +// +build !appengine /* * @@ -50,5 +50,4 @@ func (s *SocketOptionData) Getsockopt(fd uintptr) { if v, err := unix.GetsockoptTCPInfo(int(fd), syscall.SOL_TCP, syscall.TCP_INFO); err == nil { s.TCPInfo = v } - return } diff --git a/vendor/google.golang.org/grpc/internal/channelz/types_nonlinux.go b/vendor/google.golang.org/grpc/internal/channelz/types_nonlinux.go index 884910c4e02..79edbefc433 100644 --- a/vendor/google.golang.org/grpc/internal/channelz/types_nonlinux.go +++ b/vendor/google.golang.org/grpc/internal/channelz/types_nonlinux.go @@ -1,4 +1,4 @@ -// +build !linux appengine !go1.7 +// +build !linux appengine /* * @@ -20,11 +20,13 @@ package channelz -import "google.golang.org/grpc/grpclog" +import ( + "sync" -func init() { - grpclog.Infof("Channelz: socket options are not supported on non-linux os and appengine.") -} + "google.golang.org/grpc/grpclog" +) + +var once sync.Once // SocketOptionData defines the struct to hold socket option data, and related // getter function to obtain info from fd. @@ -35,4 +37,8 @@ type SocketOptionData struct { // Getsockopt defines the function to get socket options requested by channelz. // It is to be passed to syscall.RawConn.Control(). // Windows OS doesn't support Socket Option -func (s *SocketOptionData) Getsockopt(fd uintptr) {} +func (s *SocketOptionData) Getsockopt(fd uintptr) { + once.Do(func() { + grpclog.Warningln("Channelz: socket options are not supported on non-linux os and appengine.") + }) +} diff --git a/vendor/google.golang.org/grpc/internal/channelz/util_linux_go19.go b/vendor/google.golang.org/grpc/internal/channelz/util_linux.go similarity index 96% rename from vendor/google.golang.org/grpc/internal/channelz/util_linux_go19.go rename to vendor/google.golang.org/grpc/internal/channelz/util_linux.go index e1e9e32d739..fdf409d55de 100644 --- a/vendor/google.golang.org/grpc/internal/channelz/util_linux_go19.go +++ b/vendor/google.golang.org/grpc/internal/channelz/util_linux.go @@ -1,4 +1,4 @@ -// +build linux,go1.9,!appengine +// +build linux,!appengine /* * diff --git a/vendor/google.golang.org/grpc/internal/channelz/util_nonlinux_pre_go19.go b/vendor/google.golang.org/grpc/internal/channelz/util_nonlinux.go similarity index 95% rename from vendor/google.golang.org/grpc/internal/channelz/util_nonlinux_pre_go19.go rename to vendor/google.golang.org/grpc/internal/channelz/util_nonlinux.go index 1d4da952d85..8864a081116 100644 --- a/vendor/google.golang.org/grpc/internal/channelz/util_nonlinux_pre_go19.go +++ b/vendor/google.golang.org/grpc/internal/channelz/util_nonlinux.go @@ -1,4 +1,4 @@ -// +build !linux !go1.9 appengine +// +build !linux appengine /* * diff --git a/vendor/google.golang.org/grpc/internal/grpcsync/event.go b/vendor/google.golang.org/grpc/internal/grpcsync/event.go new file mode 100644 index 00000000000..fbe697c3768 --- /dev/null +++ b/vendor/google.golang.org/grpc/internal/grpcsync/event.go @@ -0,0 +1,61 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package grpcsync implements additional synchronization primitives built upon +// the sync package. +package grpcsync + +import ( + "sync" + "sync/atomic" +) + +// Event represents a one-time event that may occur in the future. +type Event struct { + fired int32 + c chan struct{} + o sync.Once +} + +// Fire causes e to complete. It is safe to call multiple times, and +// concurrently. It returns true iff this call to Fire caused the signaling +// channel returned by Done to close. +func (e *Event) Fire() bool { + ret := false + e.o.Do(func() { + atomic.StoreInt32(&e.fired, 1) + close(e.c) + ret = true + }) + return ret +} + +// Done returns a channel that will be closed when Fire is called. +func (e *Event) Done() <-chan struct{} { + return e.c +} + +// HasFired returns true if Fire has been called. +func (e *Event) HasFired() bool { + return atomic.LoadInt32(&e.fired) == 1 +} + +// NewEvent returns a new, ready-to-use Event. +func NewEvent() *Event { + return &Event{c: make(chan struct{})} +} diff --git a/vendor/google.golang.org/grpc/internal/internal.go b/vendor/google.golang.org/grpc/internal/internal.go index cd34267f7f3..bc1f99ac803 100644 --- a/vendor/google.golang.org/grpc/internal/internal.go +++ b/vendor/google.golang.org/grpc/internal/internal.go @@ -20,17 +20,52 @@ // symbols to avoid circular dependencies. package internal -var ( +import ( + "context" + "time" - // TestingUseHandlerImpl enables the http.Handler-based server implementation. - // It must be called before Serve and requires TLS credentials. - // - // The provided grpcServer must be of type *grpc.Server. It is untyped - // for circular dependency reasons. - TestingUseHandlerImpl func(grpcServer interface{}) - - // WithContextDialer is exported by clientconn.go - WithContextDialer interface{} // func(context.Context, string) (net.Conn, error) grpc.DialOption - // WithResolverBuilder is exported by clientconn.go - WithResolverBuilder interface{} // func (resolver.Builder) grpc.DialOption + "google.golang.org/grpc/connectivity" +) + +var ( + // WithResolverBuilder is exported by dialoptions.go + WithResolverBuilder interface{} // func (resolver.Builder) grpc.DialOption + // WithHealthCheckFunc is not exported by dialoptions.go + WithHealthCheckFunc interface{} // func (HealthChecker) DialOption + // HealthCheckFunc is used to provide client-side LB channel health checking + HealthCheckFunc HealthChecker + // BalancerUnregister is exported by package balancer to unregister a balancer. + BalancerUnregister func(name string) + // KeepaliveMinPingTime is the minimum ping interval. This must be 10s by + // default, but tests may wish to set it lower for convenience. + KeepaliveMinPingTime = 10 * time.Second + // ParseServiceConfig is a function to parse JSON service configs into + // opaque data structures. + ParseServiceConfig func(sc string) (interface{}, error) + // StatusRawProto is exported by status/status.go. This func returns a + // pointer to the wrapped Status proto for a given status.Status without a + // call to proto.Clone(). The returned Status proto should not be mutated by + // the caller. + StatusRawProto interface{} // func (*status.Status) *spb.Status +) + +// HealthChecker defines the signature of the client-side LB channel health checking function. +// +// The implementation is expected to create a health checking RPC stream by +// calling newStream(), watch for the health status of serviceName, and report +// it's health back by calling setConnectivityState(). +// +// The health checking protocol is defined at: +// https://github.com/grpc/grpc/blob/master/doc/health-checking.md +type HealthChecker func(ctx context.Context, newStream func(string) (interface{}, error), setConnectivityState func(connectivity.State), serviceName string) error + +const ( + // CredsBundleModeFallback switches GoogleDefaultCreds to fallback mode. + CredsBundleModeFallback = "fallback" + // CredsBundleModeBalancer switches GoogleDefaultCreds to grpclb balancer + // mode. + CredsBundleModeBalancer = "balancer" + // CredsBundleModeBackendFromBalancer switches GoogleDefaultCreds to mode + // that supports backend returned by grpclb balancer. + CredsBundleModeBackendFromBalancer = "backend-from-balancer" ) diff --git a/vendor/google.golang.org/grpc/internal/syscall/syscall_linux.go b/vendor/google.golang.org/grpc/internal/syscall/syscall_linux.go new file mode 100644 index 00000000000..43281a3e078 --- /dev/null +++ b/vendor/google.golang.org/grpc/internal/syscall/syscall_linux.go @@ -0,0 +1,114 @@ +// +build !appengine + +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package syscall provides functionalities that grpc uses to get low-level operating system +// stats/info. +package syscall + +import ( + "fmt" + "net" + "syscall" + "time" + + "golang.org/x/sys/unix" + "google.golang.org/grpc/grpclog" +) + +// GetCPUTime returns the how much CPU time has passed since the start of this process. +func GetCPUTime() int64 { + var ts unix.Timespec + if err := unix.ClockGettime(unix.CLOCK_PROCESS_CPUTIME_ID, &ts); err != nil { + grpclog.Fatal(err) + } + return ts.Nano() +} + +// Rusage is an alias for syscall.Rusage under linux non-appengine environment. +type Rusage syscall.Rusage + +// GetRusage returns the resource usage of current process. +func GetRusage() (rusage *Rusage) { + rusage = new(Rusage) + syscall.Getrusage(syscall.RUSAGE_SELF, (*syscall.Rusage)(rusage)) + return +} + +// CPUTimeDiff returns the differences of user CPU time and system CPU time used +// between two Rusage structs. +func CPUTimeDiff(first *Rusage, latest *Rusage) (float64, float64) { + f := (*syscall.Rusage)(first) + l := (*syscall.Rusage)(latest) + var ( + utimeDiffs = l.Utime.Sec - f.Utime.Sec + utimeDiffus = l.Utime.Usec - f.Utime.Usec + stimeDiffs = l.Stime.Sec - f.Stime.Sec + stimeDiffus = l.Stime.Usec - f.Stime.Usec + ) + + uTimeElapsed := float64(utimeDiffs) + float64(utimeDiffus)*1.0e-6 + sTimeElapsed := float64(stimeDiffs) + float64(stimeDiffus)*1.0e-6 + + return uTimeElapsed, sTimeElapsed +} + +// SetTCPUserTimeout sets the TCP user timeout on a connection's socket +func SetTCPUserTimeout(conn net.Conn, timeout time.Duration) error { + tcpconn, ok := conn.(*net.TCPConn) + if !ok { + // not a TCP connection. exit early + return nil + } + rawConn, err := tcpconn.SyscallConn() + if err != nil { + return fmt.Errorf("error getting raw connection: %v", err) + } + err = rawConn.Control(func(fd uintptr) { + err = syscall.SetsockoptInt(int(fd), syscall.IPPROTO_TCP, unix.TCP_USER_TIMEOUT, int(timeout/time.Millisecond)) + }) + if err != nil { + return fmt.Errorf("error setting option on socket: %v", err) + } + + return nil +} + +// GetTCPUserTimeout gets the TCP user timeout on a connection's socket +func GetTCPUserTimeout(conn net.Conn) (opt int, err error) { + tcpconn, ok := conn.(*net.TCPConn) + if !ok { + err = fmt.Errorf("conn is not *net.TCPConn. got %T", conn) + return + } + rawConn, err := tcpconn.SyscallConn() + if err != nil { + err = fmt.Errorf("error getting raw connection: %v", err) + return + } + err = rawConn.Control(func(fd uintptr) { + opt, err = syscall.GetsockoptInt(int(fd), syscall.IPPROTO_TCP, unix.TCP_USER_TIMEOUT) + }) + if err != nil { + err = fmt.Errorf("error getting option on socket: %v", err) + return + } + + return +} diff --git a/vendor/google.golang.org/grpc/internal/syscall/syscall_nonlinux.go b/vendor/google.golang.org/grpc/internal/syscall/syscall_nonlinux.go new file mode 100644 index 00000000000..d3fd9dab333 --- /dev/null +++ b/vendor/google.golang.org/grpc/internal/syscall/syscall_nonlinux.go @@ -0,0 +1,73 @@ +// +build !linux appengine + +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package syscall + +import ( + "net" + "sync" + "time" + + "google.golang.org/grpc/grpclog" +) + +var once sync.Once + +func log() { + once.Do(func() { + grpclog.Info("CPU time info is unavailable on non-linux or appengine environment.") + }) +} + +// GetCPUTime returns the how much CPU time has passed since the start of this process. +// It always returns 0 under non-linux or appengine environment. +func GetCPUTime() int64 { + log() + return 0 +} + +// Rusage is an empty struct under non-linux or appengine environment. +type Rusage struct{} + +// GetRusage is a no-op function under non-linux or appengine environment. +func GetRusage() (rusage *Rusage) { + log() + return nil +} + +// CPUTimeDiff returns the differences of user CPU time and system CPU time used +// between two Rusage structs. It a no-op function for non-linux or appengine environment. +func CPUTimeDiff(first *Rusage, latest *Rusage) (float64, float64) { + log() + return 0, 0 +} + +// SetTCPUserTimeout is a no-op function under non-linux or appengine environments +func SetTCPUserTimeout(conn net.Conn, timeout time.Duration) error { + log() + return nil +} + +// GetTCPUserTimeout is a no-op function under non-linux or appengine environments +// a negative return value indicates the operation is not supported +func GetTCPUserTimeout(conn net.Conn) (int, error) { + log() + return -1, nil +} diff --git a/vendor/google.golang.org/grpc/internal/transport/bdp_estimator.go b/vendor/google.golang.org/grpc/internal/transport/bdp_estimator.go index 63cd2627c87..070680edbac 100644 --- a/vendor/google.golang.org/grpc/internal/transport/bdp_estimator.go +++ b/vendor/google.golang.org/grpc/internal/transport/bdp_estimator.go @@ -24,9 +24,10 @@ import ( ) const ( - // bdpLimit is the maximum value the flow control windows - // will be increased to. - bdpLimit = (1 << 20) * 4 + // bdpLimit is the maximum value the flow control windows will be increased + // to. TCP typically limits this to 4MB, but some systems go up to 16MB. + // Since this is only a limit, it is safe to make it optimistic. + bdpLimit = (1 << 20) * 16 // alpha is a constant factor used to keep a moving average // of RTTs. alpha = 0.9 diff --git a/vendor/google.golang.org/grpc/internal/transport/controlbuf.go b/vendor/google.golang.org/grpc/internal/transport/controlbuf.go index ce135c4d162..b8e0aa4db27 100644 --- a/vendor/google.golang.org/grpc/internal/transport/controlbuf.go +++ b/vendor/google.golang.org/grpc/internal/transport/controlbuf.go @@ -23,6 +23,7 @@ import ( "fmt" "runtime" "sync" + "sync/atomic" "golang.org/x/net/http2" "golang.org/x/net/http2/hpack" @@ -84,12 +85,24 @@ func (il *itemList) isEmpty() bool { // the control buffer of transport. They represent different aspects of // control tasks, e.g., flow control, settings, streaming resetting, etc. +// maxQueuedTransportResponseFrames is the most queued "transport response" +// frames we will buffer before preventing new reads from occurring on the +// transport. These are control frames sent in response to client requests, +// such as RST_STREAM due to bad headers or settings acks. +const maxQueuedTransportResponseFrames = 50 + +type cbItem interface { + isTransportResponseFrame() bool +} + // registerStream is used to register an incoming stream with loopy writer. type registerStream struct { streamID uint32 wq *writeQuota } +func (*registerStream) isTransportResponseFrame() bool { return false } + // headerFrame is also used to register stream on the client-side. type headerFrame struct { streamID uint32 @@ -102,14 +115,19 @@ type headerFrame struct { onOrphaned func(error) // Valid on client-side } +func (h *headerFrame) isTransportResponseFrame() bool { + return h.cleanup != nil && h.cleanup.rst // Results in a RST_STREAM +} + type cleanupStream struct { streamID uint32 - idPtr *uint32 rst bool rstCode http2.ErrCode onWrite func() } +func (c *cleanupStream) isTransportResponseFrame() bool { return c.rst } // Results in a RST_STREAM + type dataFrame struct { streamID uint32 endStream bool @@ -120,30 +138,41 @@ type dataFrame struct { onEachWrite func() } +func (*dataFrame) isTransportResponseFrame() bool { return false } + type incomingWindowUpdate struct { streamID uint32 increment uint32 } +func (*incomingWindowUpdate) isTransportResponseFrame() bool { return false } + type outgoingWindowUpdate struct { streamID uint32 increment uint32 } +func (*outgoingWindowUpdate) isTransportResponseFrame() bool { + return false // window updates are throttled by thresholds +} + type incomingSettings struct { ss []http2.Setting } +func (*incomingSettings) isTransportResponseFrame() bool { return true } // Results in a settings ACK + type outgoingSettings struct { ss []http2.Setting } -type settingsAck struct { -} +func (*outgoingSettings) isTransportResponseFrame() bool { return false } type incomingGoAway struct { } +func (*incomingGoAway) isTransportResponseFrame() bool { return false } + type goAway struct { code http2.ErrCode debugData []byte @@ -151,15 +180,21 @@ type goAway struct { closeConn bool } +func (*goAway) isTransportResponseFrame() bool { return false } + type ping struct { ack bool data [8]byte } +func (*ping) isTransportResponseFrame() bool { return true } + type outFlowControlSizeRequest struct { resp chan uint32 } +func (*outFlowControlSizeRequest) isTransportResponseFrame() bool { return false } + type outStreamState int const ( @@ -242,6 +277,14 @@ type controlBuffer struct { consumerWaiting bool list *itemList err error + + // transportResponseFrames counts the number of queued items that represent + // the response of an action initiated by the peer. trfChan is created + // when transportResponseFrames >= maxQueuedTransportResponseFrames and is + // closed and nilled when transportResponseFrames drops below the + // threshold. Both fields are protected by mu. + transportResponseFrames int + trfChan atomic.Value // *chan struct{} } func newControlBuffer(done <-chan struct{}) *controlBuffer { @@ -252,12 +295,24 @@ func newControlBuffer(done <-chan struct{}) *controlBuffer { } } -func (c *controlBuffer) put(it interface{}) error { +// throttle blocks if there are too many incomingSettings/cleanupStreams in the +// controlbuf. +func (c *controlBuffer) throttle() { + ch, _ := c.trfChan.Load().(*chan struct{}) + if ch != nil { + select { + case <-*ch: + case <-c.done: + } + } +} + +func (c *controlBuffer) put(it cbItem) error { _, err := c.executeAndPut(nil, it) return err } -func (c *controlBuffer) executeAndPut(f func(it interface{}) bool, it interface{}) (bool, error) { +func (c *controlBuffer) executeAndPut(f func(it interface{}) bool, it cbItem) (bool, error) { var wakeUp bool c.mu.Lock() if c.err != nil { @@ -275,6 +330,15 @@ func (c *controlBuffer) executeAndPut(f func(it interface{}) bool, it interface{ c.consumerWaiting = false } c.list.enqueue(it) + if it.isTransportResponseFrame() { + c.transportResponseFrames++ + if c.transportResponseFrames == maxQueuedTransportResponseFrames { + // We are adding the frame that puts us over the threshold; create + // a throttling channel. + ch := make(chan struct{}) + c.trfChan.Store(&ch) + } + } c.mu.Unlock() if wakeUp { select { @@ -308,7 +372,17 @@ func (c *controlBuffer) get(block bool) (interface{}, error) { return nil, c.err } if !c.list.isEmpty() { - h := c.list.dequeue() + h := c.list.dequeue().(cbItem) + if h.isTransportResponseFrame() { + if c.transportResponseFrames == maxQueuedTransportResponseFrames { + // We are removing the frame that put us over the + // threshold; close and clear the throttling channel. + ch := c.trfChan.Load().(*chan struct{}) + close(*ch) + c.trfChan.Store((*chan struct{})(nil)) + } + c.transportResponseFrames-- + } c.mu.Unlock() return h, nil } diff --git a/vendor/google.golang.org/grpc/internal/transport/flowcontrol.go b/vendor/google.golang.org/grpc/internal/transport/flowcontrol.go index 5ea997a7e45..f262edd8ecd 100644 --- a/vendor/google.golang.org/grpc/internal/transport/flowcontrol.go +++ b/vendor/google.golang.org/grpc/internal/transport/flowcontrol.go @@ -149,6 +149,7 @@ func (f *inFlow) maybeAdjust(n uint32) uint32 { n = uint32(math.MaxInt32) } f.mu.Lock() + defer f.mu.Unlock() // estSenderQuota is the receiver's view of the maximum number of bytes the sender // can send without a window update. estSenderQuota := int32(f.limit - (f.pendingData + f.pendingUpdate)) @@ -169,10 +170,8 @@ func (f *inFlow) maybeAdjust(n uint32) uint32 { // is padded; We will fallback on the current available window(at least a 1/4th of the limit). f.delta = n } - f.mu.Unlock() return f.delta } - f.mu.Unlock() return 0 } diff --git a/vendor/google.golang.org/grpc/internal/transport/go16.go b/vendor/google.golang.org/grpc/internal/transport/go16.go deleted file mode 100644 index e0d00115d8e..00000000000 --- a/vendor/google.golang.org/grpc/internal/transport/go16.go +++ /dev/null @@ -1,52 +0,0 @@ -// +build go1.6,!go1.7 - -/* - * - * Copyright 2016 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package transport - -import ( - "net" - "net/http" - - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" - - "golang.org/x/net/context" -) - -// dialContext connects to the address on the named network. -func dialContext(ctx context.Context, network, address string) (net.Conn, error) { - return (&net.Dialer{Cancel: ctx.Done()}).Dial(network, address) -} - -// ContextErr converts the error from context package into a status error. -func ContextErr(err error) error { - switch err { - case context.DeadlineExceeded: - return status.Error(codes.DeadlineExceeded, err.Error()) - case context.Canceled: - return status.Error(codes.Canceled, err.Error()) - } - return status.Errorf(codes.Internal, "Unexpected error from context packet: %v", err) -} - -// contextFromRequest returns a background context. -func contextFromRequest(r *http.Request) context.Context { - return context.Background() -} diff --git a/vendor/google.golang.org/grpc/internal/transport/go17.go b/vendor/google.golang.org/grpc/internal/transport/go17.go deleted file mode 100644 index 4d515b00dcf..00000000000 --- a/vendor/google.golang.org/grpc/internal/transport/go17.go +++ /dev/null @@ -1,53 +0,0 @@ -// +build go1.7 - -/* - * - * Copyright 2016 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package transport - -import ( - "context" - "net" - "net/http" - - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" - - netctx "golang.org/x/net/context" -) - -// dialContext connects to the address on the named network. -func dialContext(ctx context.Context, network, address string) (net.Conn, error) { - return (&net.Dialer{}).DialContext(ctx, network, address) -} - -// ContextErr converts the error from context package into a status error. -func ContextErr(err error) error { - switch err { - case context.DeadlineExceeded, netctx.DeadlineExceeded: - return status.Error(codes.DeadlineExceeded, err.Error()) - case context.Canceled, netctx.Canceled: - return status.Error(codes.Canceled, err.Error()) - } - return status.Errorf(codes.Internal, "Unexpected error from context packet: %v", err) -} - -// contextFromRequest returns a context from the HTTP Request. -func contextFromRequest(r *http.Request) context.Context { - return r.Context() -} diff --git a/vendor/google.golang.org/grpc/internal/transport/handler_server.go b/vendor/google.golang.org/grpc/internal/transport/handler_server.go index bc8564345f5..78f9ddc3d3a 100644 --- a/vendor/google.golang.org/grpc/internal/transport/handler_server.go +++ b/vendor/google.golang.org/grpc/internal/transport/handler_server.go @@ -24,6 +24,8 @@ package transport import ( + "bytes" + "context" "errors" "fmt" "io" @@ -34,7 +36,6 @@ import ( "time" "github.com/golang/protobuf/proto" - "golang.org/x/net/context" "golang.org/x/net/http2" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials" @@ -63,9 +64,6 @@ func NewServerHandlerTransport(w http.ResponseWriter, r *http.Request, stats sta if _, ok := w.(http.Flusher); !ok { return nil, errors.New("gRPC requires a ResponseWriter supporting http.Flusher") } - if _, ok := w.(http.CloseNotifier); !ok { - return nil, errors.New("gRPC requires a ResponseWriter supporting http.CloseNotifier") - } st := &serverHandlerTransport{ rw: w, @@ -176,17 +174,11 @@ func (a strAddr) String() string { return string(a) } // do runs fn in the ServeHTTP goroutine. func (ht *serverHandlerTransport) do(fn func()) error { - // Avoid a panic writing to closed channel. Imperfect but maybe good enough. select { case <-ht.closedCh: return ErrConnClosing - default: - select { - case ht.writes <- fn: - return nil - case <-ht.closedCh: - return ErrConnClosing - } + case ht.writes <- fn: + return nil } } @@ -237,9 +229,8 @@ func (ht *serverHandlerTransport) WriteStatus(s *Stream, st *status.Status) erro if ht.stats != nil { ht.stats.HandleRPC(s.Context(), &stats.OutTrailer{}) } - ht.Close() - close(ht.writes) } + ht.Close() return err } @@ -307,7 +298,7 @@ func (ht *serverHandlerTransport) WriteHeader(s *Stream, md metadata.MD) error { func (ht *serverHandlerTransport) HandleStreams(startStream func(*Stream), traceCtx func(context.Context, string) context.Context) { // With this transport type there will be exactly 1 stream: this HTTP request. - ctx := contextFromRequest(ht.req) + ctx := ht.req.Context() var cancel context.CancelFunc if ht.timeoutSet { ctx, cancel = context.WithTimeout(ctx, ht.timeout) @@ -315,22 +306,16 @@ func (ht *serverHandlerTransport) HandleStreams(startStream func(*Stream), trace ctx, cancel = context.WithCancel(ctx) } - // requestOver is closed when either the request's context is done - // or the status has been written via WriteStatus. + // requestOver is closed when the status has been written via WriteStatus. requestOver := make(chan struct{}) - - // clientGone receives a single value if peer is gone, either - // because the underlying connection is dead or because the - // peer sends an http2 RST_STREAM. - clientGone := ht.rw.(http.CloseNotifier).CloseNotify() go func() { select { case <-requestOver: - return case <-ht.closedCh: - case <-clientGone: + case <-ht.req.Context().Done(): } cancel() + ht.Close() }() req := ht.req @@ -363,7 +348,7 @@ func (ht *serverHandlerTransport) HandleStreams(startStream func(*Stream), trace ht.stats.HandleRPC(s.ctx, inHeader) } s.trReader = &transportReader{ - reader: &recvBufferReader{ctx: s.ctx, ctxDone: s.ctx.Done(), recv: s.buf}, + reader: &recvBufferReader{ctx: s.ctx, ctxDone: s.ctx.Done(), recv: s.buf, freeBuffer: func(*bytes.Buffer) {}}, windowHandler: func(int) {}, } @@ -377,7 +362,7 @@ func (ht *serverHandlerTransport) HandleStreams(startStream func(*Stream), trace for buf := make([]byte, readSize); ; { n, err := req.Body.Read(buf) if n > 0 { - s.buf.put(recvMsg{data: buf[:n:n]}) + s.buf.put(recvMsg{buffer: bytes.NewBuffer(buf[:n:n])}) buf = buf[n:] } if err != nil { @@ -407,10 +392,7 @@ func (ht *serverHandlerTransport) HandleStreams(startStream func(*Stream), trace func (ht *serverHandlerTransport) runStream() { for { select { - case fn, ok := <-ht.writes: - if !ok { - return - } + case fn := <-ht.writes: fn() case <-ht.closedCh: return @@ -442,5 +424,8 @@ func mapRecvMsgError(err error) error { return status.Error(code, se.Error()) } } + if strings.Contains(err.Error(), "body closed by handler") { + return status.Error(codes.Canceled, err.Error()) + } return connectionErrorf(true, err, err.Error()) } diff --git a/vendor/google.golang.org/grpc/internal/transport/http2_client.go b/vendor/google.golang.org/grpc/internal/transport/http2_client.go index 88d1c1612bf..41a79c56702 100644 --- a/vendor/google.golang.org/grpc/internal/transport/http2_client.go +++ b/vendor/google.golang.org/grpc/internal/transport/http2_client.go @@ -19,6 +19,8 @@ package transport import ( + "context" + "fmt" "io" "math" "net" @@ -28,13 +30,13 @@ import ( "sync/atomic" "time" - "golang.org/x/net/context" "golang.org/x/net/http2" "golang.org/x/net/http2/hpack" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials" "google.golang.org/grpc/internal/channelz" + "google.golang.org/grpc/internal/syscall" "google.golang.org/grpc/keepalive" "google.golang.org/grpc/metadata" "google.golang.org/grpc/peer" @@ -73,7 +75,7 @@ type http2Client struct { isSecure bool - creds []credentials.PerRPCCredentials + perRPCCreds []credentials.PerRPCCredentials // Boolean to keep track of reading activity on transport. // 1 is true and 0 is false. @@ -89,10 +91,10 @@ type http2Client struct { maxSendHeaderListSize *uint32 bdpEst *bdpEstimator - // onSuccess is a callback that client transport calls upon + // onPrefaceReceipt is a callback that client transport calls upon // receiving server preface to signal that a succefull HTTP2 // connection was established. - onSuccess func() + onPrefaceReceipt func() maxConcurrentStreams uint32 streamQuota int64 @@ -111,26 +113,19 @@ type http2Client struct { // Fields below are for channelz metric collection. channelzID int64 // channelz unique identification number - czmu sync.RWMutex - kpCount int64 - // The number of streams that have started, including already finished ones. - streamsStarted int64 - // The number of streams that have ended successfully by receiving EoS bit set - // frame from server. - streamsSucceeded int64 - streamsFailed int64 - lastStreamCreated time.Time - msgSent int64 - msgRecv int64 - lastMsgSent time.Time - lastMsgRecv time.Time + czData *channelzData + + onGoAway func(GoAwayReason) + onClose func() + + bufferPool *bufferPool } func dial(ctx context.Context, fn func(context.Context, string) (net.Conn, error), addr string) (net.Conn, error) { if fn != nil { return fn(ctx, addr) } - return dialContext(ctx, "tcp", addr) + return (&net.Dialer{}).DialContext(ctx, "tcp", addr) } func isTemporary(err error) bool { @@ -152,7 +147,7 @@ func isTemporary(err error) bool { // newHTTP2Client constructs a connected ClientTransport to addr based on HTTP2 // and starts to receive messages on it. Non-nil error returns if construction // fails. -func newHTTP2Client(connectCtx, ctx context.Context, addr TargetInfo, opts ConnectOptions, onSuccess func()) (_ *http2Client, err error) { +func newHTTP2Client(connectCtx, ctx context.Context, addr TargetInfo, opts ConnectOptions, onPrefaceReceipt func(), onGoAway func(GoAwayReason), onClose func()) (_ *http2Client, err error) { scheme := "http" ctx, cancel := context.WithCancel(ctx) defer func() { @@ -174,18 +169,6 @@ func newHTTP2Client(connectCtx, ctx context.Context, addr TargetInfo, opts Conne conn.Close() } }(conn) - var ( - isSecure bool - authInfo credentials.AuthInfo - ) - if creds := opts.TransportCredentials; creds != nil { - scheme = "https" - conn, authInfo, err = creds.ClientHandshake(connectCtx, addr.Authority, conn) - if err != nil { - return nil, connectionErrorf(isTemporary(err), err, "transport: authentication handshake failed: %v", err) - } - isSecure = true - } kp := opts.KeepaliveParams // Validate keepalive parameters. if kp.Time == 0 { @@ -194,6 +177,36 @@ func newHTTP2Client(connectCtx, ctx context.Context, addr TargetInfo, opts Conne if kp.Timeout == 0 { kp.Timeout = defaultClientKeepaliveTimeout } + keepaliveEnabled := false + if kp.Time != infinity { + if err = syscall.SetTCPUserTimeout(conn, kp.Timeout); err != nil { + return nil, connectionErrorf(false, err, "transport: failed to set TCP_USER_TIMEOUT: %v", err) + } + keepaliveEnabled = true + } + var ( + isSecure bool + authInfo credentials.AuthInfo + ) + transportCreds := opts.TransportCredentials + perRPCCreds := opts.PerRPCCredentials + + if b := opts.CredsBundle; b != nil { + if t := b.TransportCredentials(); t != nil { + transportCreds = t + } + if t := b.PerRPCCredentials(); t != nil { + perRPCCreds = append(perRPCCreds, t) + } + } + if transportCreds != nil { + scheme = "https" + conn, authInfo, err = transportCreds.ClientHandshake(connectCtx, addr.Authority, conn) + if err != nil { + return nil, connectionErrorf(isTemporary(err), err, "transport: authentication handshake failed: %v", err) + } + isSecure = true + } dynamicWindow := true icwz := int32(initialWindowSize) if opts.InitialConnWindowSize >= defaultWindowSize { @@ -225,15 +238,20 @@ func newHTTP2Client(connectCtx, ctx context.Context, addr TargetInfo, opts Conne scheme: scheme, activeStreams: make(map[uint32]*Stream), isSecure: isSecure, - creds: opts.PerRPCCredentials, + perRPCCreds: perRPCCreds, kp: kp, statsHandler: opts.StatsHandler, initialWindowSize: initialWindowSize, - onSuccess: onSuccess, + onPrefaceReceipt: onPrefaceReceipt, nextID: 1, maxConcurrentStreams: defaultMaxStreamsClient, streamQuota: defaultMaxStreamsClient, streamsQuotaAvailable: make(chan struct{}, 1), + czData: new(channelzData), + onGoAway: onGoAway, + onClose: onClose, + keepaliveEnabled: keepaliveEnabled, + bufferPool: newBufferPool(), } t.controlBuf = newControlBuffer(t.ctxDone) if opts.InitialWindowSize >= defaultWindowSize { @@ -260,16 +278,16 @@ func newHTTP2Client(connectCtx, ctx context.Context, addr TargetInfo, opts Conne t.statsHandler.HandleConn(t.ctx, connBegin) } if channelz.IsOn() { - t.channelzID = channelz.RegisterNormalSocket(t, opts.ChannelzParentID, "") + t.channelzID = channelz.RegisterNormalSocket(t, opts.ChannelzParentID, fmt.Sprintf("%s -> %s", t.localAddr, t.remoteAddr)) } - if t.kp.Time != infinity { - t.keepaliveEnabled = true + if t.keepaliveEnabled { go t.keepalive() } // Start the reader goroutine for incoming message. Each transport has // a dedicated goroutine which reads HTTP2 frame from network. Then it // dispatches the frame to the corresponding stream entity. go t.reader() + // Send connection preface to server. n, err := t.conn.Write(clientPreface) if err != nil { @@ -306,7 +324,10 @@ func newHTTP2Client(connectCtx, ctx context.Context, addr TargetInfo, opts Conne return nil, connectionErrorf(true, err, "transport: failed to write window update: %v", err) } } - t.framer.writer.Flush() + + if err := t.framer.writer.Flush(); err != nil { + return nil, err + } go func() { t.loopy = newLoopyWriter(clientSide, t.framer, t.controlBuf, t.bdpEst) err := t.loopy.run() @@ -346,6 +367,10 @@ func (t *http2Client) newStream(ctx context.Context, callHdr *CallHdr) *Stream { ctx: s.ctx, ctxDone: s.ctx.Done(), recv: s.buf, + closeStream: func(err error) { + t.CloseStream(s, err) + }, + freeBuffer: t.bufferPool.put, }, windowHandler: func(n int) { t.updateWindow(s, uint32(n)) @@ -398,7 +423,7 @@ func (t *http2Client) createHeaderFields(ctx context.Context, callHdr *CallHdr) if dl, ok := ctx.Deadline(); ok { // Send out timeout regardless its value. The server can detect timeout context by itself. // TODO(mmukhi): Perhaps this field should be updated when actually writing out to the wire. - timeout := dl.Sub(time.Now()) + timeout := time.Until(dl) headerFields = append(headerFields, hpack.HeaderField{Name: "grpc-timeout", Value: encodeTimeout(timeout)}) } for k, v := range authData { @@ -416,6 +441,15 @@ func (t *http2Client) createHeaderFields(ctx context.Context, callHdr *CallHdr) if md, added, ok := metadata.FromOutgoingContextRaw(ctx); ok { var k string + for k, vv := range md { + // HTTP doesn't allow you to set pseudoheaders after non pseudoheaders were set. + if isReservedHeader(k) { + continue + } + for _, v := range vv { + headerFields = append(headerFields, hpack.HeaderField{Name: k, Value: encodeMetadataHeader(k, v)}) + } + } for _, vv := range added { for i, v := range vv { if i%2 == 0 { @@ -429,15 +463,6 @@ func (t *http2Client) createHeaderFields(ctx context.Context, callHdr *CallHdr) headerFields = append(headerFields, hpack.HeaderField{Name: strings.ToLower(k), Value: encodeMetadataHeader(k, v)}) } } - for k, vv := range md { - // HTTP doesn't allow you to set pseudoheaders after non pseudoheaders were set. - if isReservedHeader(k) { - continue - } - for _, v := range vv { - headerFields = append(headerFields, hpack.HeaderField{Name: k, Value: encodeMetadataHeader(k, v)}) - } - } } if md, ok := t.md.(*metadata.MD); ok { for k, vv := range *md { @@ -454,7 +479,7 @@ func (t *http2Client) createHeaderFields(ctx context.Context, callHdr *CallHdr) func (t *http2Client) createAudience(callHdr *CallHdr) string { // Create an audience string only if needed. - if len(t.creds) == 0 && callHdr.Creds == nil { + if len(t.perRPCCreds) == 0 && callHdr.Creds == nil { return "" } // Construct URI required to get auth request metadata. @@ -468,8 +493,11 @@ func (t *http2Client) createAudience(callHdr *CallHdr) string { } func (t *http2Client) getTrAuthData(ctx context.Context, audience string) (map[string]string, error) { + if len(t.perRPCCreds) == 0 { + return nil, nil + } authData := map[string]string{} - for _, c := range t.creds { + for _, c := range t.perRPCCreds { data, err := c.GetRequestMetadata(ctx, audience) if err != nil { if _, ok := status.FromError(err); ok { @@ -488,7 +516,7 @@ func (t *http2Client) getTrAuthData(ctx context.Context, audience string) (map[s } func (t *http2Client) getCallAuthData(ctx context.Context, audience string, callHdr *CallHdr) (map[string]string, error) { - callAuthData := map[string]string{} + var callAuthData map[string]string // Check if credentials.PerRPCCredentials were provided via call options. // Note: if these credentials are provided both via dial options and call // options, then both sets of credentials will be applied. @@ -500,6 +528,7 @@ func (t *http2Client) getCallAuthData(ctx context.Context, audience string, call if err != nil { return nil, status.Errorf(codes.Internal, "transport: %v", err) } + callAuthData = make(map[string]string, len(data)) for k, v := range data { // Capital header names are illegal in HTTP/2 k = strings.ToLower(k) @@ -528,10 +557,9 @@ func (t *http2Client) NewStream(ctx context.Context, callHdr *CallHdr) (_ *Strea s.write(recvMsg{err: err}) close(s.done) // If headerChan isn't closed, then close it. - if atomic.SwapUint32(&s.headerDone, 1) == 0 { + if atomic.CompareAndSwapUint32(&s.headerChanClosed, 0, 1) { close(s.headerChan) } - } hdr := &headerFrame{ hf: headerFields, @@ -550,10 +578,8 @@ func (t *http2Client) NewStream(ctx context.Context, callHdr *CallHdr) (_ *Strea } t.activeStreams[id] = s if channelz.IsOn() { - t.czmu.Lock() - t.streamsStarted++ - t.lastStreamCreated = time.Now() - t.czmu.Unlock() + atomic.AddInt64(&t.czData.streamsStarted, 1) + atomic.StoreInt64(&t.czData.lastStreamCreatedTime, time.Now().UnixNano()) } var sendPing bool // If the number of active streams change from 0 to 1, then check if keepalive @@ -677,7 +703,9 @@ func (t *http2Client) CloseStream(s *Stream, err error) { func (t *http2Client) closeStream(s *Stream, err error, rst bool, rstCode http2.ErrCode, st *status.Status, mdata map[string][]string, eosReceived bool) { // Set stream status to done. if s.swapState(streamDone) == streamDone { - // If it was already done, return. + // If it was already done, return. If multiple closeStream calls + // happen simultaneously, wait for the first to finish. + <-s.done return } // status and trailers can be updated here without any synchronization because the stream goroutine will @@ -691,10 +719,8 @@ func (t *http2Client) closeStream(s *Stream, err error, rst bool, rstCode http2. // This will unblock reads eventually. s.write(recvMsg{err: err}) } - // This will unblock write. - close(s.done) // If headerChan isn't closed, then close it. - if atomic.SwapUint32(&s.headerDone, 1) == 0 { + if atomic.CompareAndSwapUint32(&s.headerChanClosed, 0, 1) { s.noHeaders = true close(s.headerChan) } @@ -707,13 +733,11 @@ func (t *http2Client) closeStream(s *Stream, err error, rst bool, rstCode http2. } t.mu.Unlock() if channelz.IsOn() { - t.czmu.Lock() if eosReceived { - t.streamsSucceeded++ + atomic.AddInt64(&t.czData.streamsSucceeded, 1) } else { - t.streamsFailed++ + atomic.AddInt64(&t.czData.streamsFailed, 1) } - t.czmu.Unlock() } }, rst: rst, @@ -730,11 +754,17 @@ func (t *http2Client) closeStream(s *Stream, err error, rst bool, rstCode http2. return true } t.controlBuf.executeAndPut(addBackStreamQuota, cleanup) + // This will unblock write. + close(s.done) } // Close kicks off the shutdown process of the transport. This should be called // only once on a transport. Once it is called, the transport should not be // accessed any more. +// +// This method blocks until the addrConn that initiated this transport is +// re-connected. This happens because t.onClose() begins reconnect logic at the +// addrConn level and blocks until the addrConn is successfully connected. func (t *http2Client) Close() error { t.mu.Lock() // Make sure we only Close once. @@ -742,6 +772,9 @@ func (t *http2Client) Close() error { t.mu.Unlock() return nil } + // Call t.onClose before setting the state to closing to prevent the client + // from attempting to create new streams ASAP. + t.onClose() t.state = closing streams := t.activeStreams t.activeStreams = nil @@ -770,21 +803,21 @@ func (t *http2Client) Close() error { // stream is closed. If there are no active streams, the transport is closed // immediately. This does nothing if the transport is already draining or // closing. -func (t *http2Client) GracefulClose() error { +func (t *http2Client) GracefulClose() { t.mu.Lock() // Make sure we move to draining only from active. if t.state == draining || t.state == closing { t.mu.Unlock() - return nil + return } t.state = draining active := len(t.activeStreams) t.mu.Unlock() if active == 0 { - return t.Close() + t.Close() + return } t.controlBuf.put(&incomingGoAway{}) - return nil } // Write formats the data into HTTP2 data frame(s) and sends it out. The caller @@ -922,9 +955,10 @@ func (t *http2Client) handleData(f *http2.DataFrame) { // guarantee f.Data() is consumed before the arrival of next frame. // Can this copy be eliminated? if len(f.Data()) > 0 { - data := make([]byte, len(f.Data())) - copy(data, f.Data()) - s.write(recvMsg{data: data}) + buffer := t.bufferPool.get() + buffer.Reset() + buffer.Write(f.Data()) + s.write(recvMsg{buffer: buffer}) } } // The server has closed the stream without sending trailers. Record that @@ -949,9 +983,9 @@ func (t *http2Client) handleRSTStream(f *http2.RSTStreamFrame) { statusCode = codes.Unknown } if statusCode == codes.Canceled { - // Our deadline was already exceeded, and that was likely the cause of - // this cancelation. Alter the status code accordingly. - if d, ok := s.ctx.Deadline(); ok && d.After(time.Now()) { + if d, ok := s.ctx.Deadline(); ok && !d.After(time.Now()) { + // Our deadline was already exceeded, and that was likely the cause + // of this cancelation. Alter the status code accordingly. statusCode = codes.DeadlineExceeded } } @@ -1056,8 +1090,12 @@ func (t *http2Client) handleGoAway(f *http2.GoAwayFrame) { default: t.setGoAwayReason(f) close(t.goAway) - t.state = draining t.controlBuf.put(&incomingGoAway{}) + // Notify the clientconn about the GOAWAY before we set the state to + // draining, to allow the client to stop attempting to create streams + // before disallowing new streams on this connection. + t.onGoAway(t.goAwayReason) + t.state = draining } // All streams with IDs greater than the GoAwayId // and smaller than the previous GoAway ID should be killed. @@ -1113,16 +1151,26 @@ func (t *http2Client) operateHeaders(frame *http2.MetaHeadersFrame) { if !ok { return } + endStream := frame.StreamEnded() atomic.StoreUint32(&s.bytesReceived, 1) - var state decodeState - if err := state.decodeHeader(frame); err != nil { - t.closeStream(s, err, true, http2.ErrCodeProtocol, status.New(codes.Internal, err.Error()), nil, false) - // Something wrong. Stops reading even when there is remaining. + initialHeader := atomic.LoadUint32(&s.headerChanClosed) == 0 + + if !initialHeader && !endStream { + // As specified by gRPC over HTTP2, a HEADERS frame (and associated CONTINUATION frames) can only appear at the start or end of a stream. Therefore, second HEADERS frame must have EOS bit set. + st := status.New(codes.Internal, "a HEADERS frame cannot appear in the middle of a stream") + t.closeStream(s, st.Err(), true, http2.ErrCodeProtocol, st, nil, false) return } - endStream := frame.StreamEnded() - var isHeader bool + state := &decodeState{} + // Initialize isGRPC value to be !initialHeader, since if a gRPC Response-Headers has already been received, then it means that the peer is speaking gRPC and we are in gRPC mode. + state.data.isGRPC = !initialHeader + if err := state.decodeHeader(frame); err != nil { + t.closeStream(s, err, true, http2.ErrCodeProtocol, status.Convert(err), nil, endStream) + return + } + + isHeader := false defer func() { if t.statsHandler != nil { if isHeader { @@ -1140,27 +1188,33 @@ func (t *http2Client) operateHeaders(frame *http2.MetaHeadersFrame) { } } }() - // If headers haven't been received yet. - if atomic.SwapUint32(&s.headerDone, 1) == 0 { + + // If headerChan hasn't been closed yet + if atomic.CompareAndSwapUint32(&s.headerChanClosed, 0, 1) { if !endStream { - // Headers frame is not actually a trailers-only frame. + // HEADERS frame block carries a Response-Headers. isHeader = true // These values can be set without any synchronization because // stream goroutine will read it only after seeing a closed // headerChan which we'll close after setting this. - s.recvCompress = state.encoding - if len(state.mdata) > 0 { - s.header = state.mdata + s.recvCompress = state.data.encoding + if len(state.data.mdata) > 0 { + s.header = state.data.mdata } } else { + // HEADERS frame block carries a Trailers-Only. s.noHeaders = true } close(s.headerChan) } + if !endStream { return } - t.closeStream(s, io.EOF, false, http2.ErrCodeNo, state.status(), state.mdata, true) + + // if client received END_STREAM from server while stream was still active, send RST_STREAM + rst := s.getState() == streamActive + t.closeStream(s, io.EOF, rst, http2.ErrCodeNo, state.status(), state.data.mdata, true) } // reader runs as a separate goroutine in charge of reading data from network @@ -1174,22 +1228,24 @@ func (t *http2Client) reader() { // Check the validity of server preface. frame, err := t.framer.fr.ReadFrame() if err != nil { - t.Close() + t.Close() // this kicks off resetTransport, so must be last before return return } + t.conn.SetReadDeadline(time.Time{}) // reset deadline once we get the settings frame (we didn't time out, yay!) if t.keepaliveEnabled { atomic.CompareAndSwapUint32(&t.activity, 0, 1) } sf, ok := frame.(*http2.SettingsFrame) if !ok { - t.Close() + t.Close() // this kicks off resetTransport, so must be last before return return } - t.onSuccess() + t.onPrefaceReceipt() t.handleSettings(sf, true) // loop to keep reading incoming messages on this transport. for { + t.controlBuf.throttle() frame, err := t.framer.fr.ReadFrame() if t.keepaliveEnabled { atomic.CompareAndSwapUint32(&t.activity, 0, 1) @@ -1263,9 +1319,7 @@ func (t *http2Client) keepalive() { } else { t.mu.Unlock() if channelz.IsOn() { - t.czmu.Lock() - t.kpCount++ - t.czmu.Unlock() + atomic.AddInt64(&t.czData.kpCount, 1) } // Send ping. t.controlBuf.put(p) @@ -1279,6 +1333,7 @@ func (t *http2Client) keepalive() { timer.Reset(t.kp.Time) continue } + infof("transport: closing client transport due to idleness.") t.Close() return case <-t.ctx.Done(): @@ -1305,17 +1360,16 @@ func (t *http2Client) GoAway() <-chan struct{} { } func (t *http2Client) ChannelzMetric() *channelz.SocketInternalMetric { - t.czmu.RLock() s := channelz.SocketInternalMetric{ - StreamsStarted: t.streamsStarted, - StreamsSucceeded: t.streamsSucceeded, - StreamsFailed: t.streamsFailed, - MessagesSent: t.msgSent, - MessagesReceived: t.msgRecv, - KeepAlivesSent: t.kpCount, - LastLocalStreamCreatedTimestamp: t.lastStreamCreated, - LastMessageSentTimestamp: t.lastMsgSent, - LastMessageReceivedTimestamp: t.lastMsgRecv, + StreamsStarted: atomic.LoadInt64(&t.czData.streamsStarted), + StreamsSucceeded: atomic.LoadInt64(&t.czData.streamsSucceeded), + StreamsFailed: atomic.LoadInt64(&t.czData.streamsFailed), + MessagesSent: atomic.LoadInt64(&t.czData.msgSent), + MessagesReceived: atomic.LoadInt64(&t.czData.msgRecv), + KeepAlivesSent: atomic.LoadInt64(&t.czData.kpCount), + LastLocalStreamCreatedTimestamp: time.Unix(0, atomic.LoadInt64(&t.czData.lastStreamCreatedTime)), + LastMessageSentTimestamp: time.Unix(0, atomic.LoadInt64(&t.czData.lastMsgSentTime)), + LastMessageReceivedTimestamp: time.Unix(0, atomic.LoadInt64(&t.czData.lastMsgRecvTime)), LocalFlowControlWindow: int64(t.fc.getSize()), SocketOptions: channelz.GetSocketOption(t.conn), LocalAddr: t.localAddr, @@ -1325,23 +1379,20 @@ func (t *http2Client) ChannelzMetric() *channelz.SocketInternalMetric { if au, ok := t.authInfo.(credentials.ChannelzSecurityInfo); ok { s.Security = au.GetSecurityValue() } - t.czmu.RUnlock() s.RemoteFlowControlWindow = t.getOutFlowWindow() return &s } +func (t *http2Client) RemoteAddr() net.Addr { return t.remoteAddr } + func (t *http2Client) IncrMsgSent() { - t.czmu.Lock() - t.msgSent++ - t.lastMsgSent = time.Now() - t.czmu.Unlock() + atomic.AddInt64(&t.czData.msgSent, 1) + atomic.StoreInt64(&t.czData.lastMsgSentTime, time.Now().UnixNano()) } func (t *http2Client) IncrMsgRecv() { - t.czmu.Lock() - t.msgRecv++ - t.lastMsgRecv = time.Now() - t.czmu.Unlock() + atomic.AddInt64(&t.czData.msgRecv, 1) + atomic.StoreInt64(&t.czData.lastMsgRecvTime, time.Now().UnixNano()) } func (t *http2Client) getOutFlowWindow() int64 { diff --git a/vendor/google.golang.org/grpc/internal/transport/http2_server.go b/vendor/google.golang.org/grpc/internal/transport/http2_server.go index a8a09270b84..4e26f6a1d6b 100644 --- a/vendor/google.golang.org/grpc/internal/transport/http2_server.go +++ b/vendor/google.golang.org/grpc/internal/transport/http2_server.go @@ -20,6 +20,7 @@ package transport import ( "bytes" + "context" "errors" "fmt" "io" @@ -31,13 +32,14 @@ import ( "time" "github.com/golang/protobuf/proto" - "golang.org/x/net/context" "golang.org/x/net/http2" "golang.org/x/net/http2/hpack" + spb "google.golang.org/genproto/googleapis/rpc/status" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials" "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/internal" "google.golang.org/grpc/internal/channelz" "google.golang.org/grpc/internal/grpcrand" "google.golang.org/grpc/keepalive" @@ -55,6 +57,9 @@ var ( // ErrHeaderListSizeLimitViolation indicates that the header list size is larger // than the limit set by peer. ErrHeaderListSizeLimitViolation = errors.New("transport: trying to send header list size larger than the limit set by peer") + // statusRawProto is a function to get to the raw status proto wrapped in a + // status.Status without a proto.Clone(). + statusRawProto = internal.StatusRawProto.(func(*status.Status) *spb.Status) ) // http2Server implements the ServerTransport interface with HTTP2. @@ -118,19 +123,8 @@ type http2Server struct { // Fields below are for channelz metric collection. channelzID int64 // channelz unique identification number - czmu sync.RWMutex - kpCount int64 - // The number of streams that have started, including already finished ones. - streamsStarted int64 - // The number of streams that have ended successfully by sending frame with - // EoS bit set. - streamsSucceeded int64 - streamsFailed int64 - lastStreamCreated time.Time - msgSent int64 - msgRecv int64 - lastMsgSent time.Time - lastMsgRecv time.Time + czData *channelzData + bufferPool *bufferPool } // newHTTP2Server constructs a ServerTransport based on HTTP2. ConnectionError is @@ -144,7 +138,10 @@ func newHTTP2Server(conn net.Conn, config *ServerConfig) (_ ServerTransport, err } framer := newFramer(conn, writeBufSize, readBufSize, maxHeaderListSize) // Send initial settings as connection preface to client. - var isettings []http2.Setting + isettings := []http2.Setting{{ + ID: http2.SettingMaxFrameSize, + Val: http2MaxFrameLen, + }} // TODO(zhaoq): Have a better way to signal "no limit" because 0 is // permitted in the HTTP2 spec. maxStreams := config.MaxStreams @@ -231,6 +228,8 @@ func newHTTP2Server(conn net.Conn, config *ServerConfig) (_ ServerTransport, err idle: time.Now(), kep: kep, initialWindowSize: iwz, + czData: new(channelzData), + bufferPool: newBufferPool(), } t.controlBuf = newControlBuffer(t.ctxDone) if dynamicWindow { @@ -248,7 +247,7 @@ func newHTTP2Server(conn net.Conn, config *ServerConfig) (_ ServerTransport, err t.stats.HandleConn(t.ctx, connBegin) } if channelz.IsOn() { - t.channelzID = channelz.RegisterNormalSocket(t, config.ChannelzParentID, "") + t.channelzID = channelz.RegisterNormalSocket(t, config.ChannelzParentID, fmt.Sprintf("%s -> %s", t.remoteAddr, t.localAddr)) } t.framer.writer.Flush() @@ -295,9 +294,11 @@ func newHTTP2Server(conn net.Conn, config *ServerConfig) (_ ServerTransport, err } // operateHeader takes action on the decoded headers. -func (t *http2Server) operateHeaders(frame *http2.MetaHeadersFrame, handle func(*Stream), traceCtx func(context.Context, string) context.Context) (close bool) { +func (t *http2Server) operateHeaders(frame *http2.MetaHeadersFrame, handle func(*Stream), traceCtx func(context.Context, string) context.Context) (fatal bool) { streamID := frame.Header().StreamID - state := decodeState{serverSide: true} + state := &decodeState{ + serverSide: true, + } if err := state.decodeHeader(frame); err != nil { if se, ok := status.FromError(err); ok { t.controlBuf.put(&cleanupStream{ @@ -307,7 +308,7 @@ func (t *http2Server) operateHeaders(frame *http2.MetaHeadersFrame, handle func( onWrite: func() {}, }) } - return + return false } buf := newRecvBuffer() @@ -316,16 +317,16 @@ func (t *http2Server) operateHeaders(frame *http2.MetaHeadersFrame, handle func( st: t, buf: buf, fc: &inFlow{limit: uint32(t.initialWindowSize)}, - recvCompress: state.encoding, - method: state.method, - contentSubtype: state.contentSubtype, + recvCompress: state.data.encoding, + method: state.data.method, + contentSubtype: state.data.contentSubtype, } if frame.StreamEnded() { // s is just created by the caller. No lock needed. s.state = streamReadDone } - if state.timeoutSet { - s.ctx, s.cancel = context.WithTimeout(t.ctx, state.timeout) + if state.data.timeoutSet { + s.ctx, s.cancel = context.WithTimeout(t.ctx, state.data.timeout) } else { s.ctx, s.cancel = context.WithCancel(t.ctx) } @@ -338,19 +339,19 @@ func (t *http2Server) operateHeaders(frame *http2.MetaHeadersFrame, handle func( } s.ctx = peer.NewContext(s.ctx, pr) // Attach the received metadata to the context. - if len(state.mdata) > 0 { - s.ctx = metadata.NewIncomingContext(s.ctx, state.mdata) + if len(state.data.mdata) > 0 { + s.ctx = metadata.NewIncomingContext(s.ctx, state.data.mdata) } - if state.statsTags != nil { - s.ctx = stats.SetIncomingTags(s.ctx, state.statsTags) + if state.data.statsTags != nil { + s.ctx = stats.SetIncomingTags(s.ctx, state.data.statsTags) } - if state.statsTrace != nil { - s.ctx = stats.SetIncomingTrace(s.ctx, state.statsTrace) + if state.data.statsTrace != nil { + s.ctx = stats.SetIncomingTrace(s.ctx, state.data.statsTrace) } if t.inTapHandle != nil { var err error info := &tap.Info{ - FullMethodName: state.method, + FullMethodName: state.data.method, } s.ctx, err = t.inTapHandle(s.ctx, info) if err != nil { @@ -361,13 +362,13 @@ func (t *http2Server) operateHeaders(frame *http2.MetaHeadersFrame, handle func( rstCode: http2.ErrCodeRefusedStream, onWrite: func() {}, }) - return + return false } } t.mu.Lock() if t.state != reachable { t.mu.Unlock() - return + return false } if uint32(len(t.activeStreams)) >= t.maxStreams { t.mu.Unlock() @@ -377,7 +378,7 @@ func (t *http2Server) operateHeaders(frame *http2.MetaHeadersFrame, handle func( rstCode: http2.ErrCodeRefusedStream, onWrite: func() {}, }) - return + return false } if streamID%2 != 1 || streamID <= t.maxStreamID { t.mu.Unlock() @@ -392,10 +393,8 @@ func (t *http2Server) operateHeaders(frame *http2.MetaHeadersFrame, handle func( } t.mu.Unlock() if channelz.IsOn() { - t.czmu.Lock() - t.streamsStarted++ - t.lastStreamCreated = time.Now() - t.czmu.Unlock() + atomic.AddInt64(&t.czData.streamsStarted, 1) + atomic.StoreInt64(&t.czData.lastStreamCreatedTime, time.Now().UnixNano()) } s.requestRead = func(n int) { t.adjustWindow(s, uint32(n)) @@ -416,9 +415,10 @@ func (t *http2Server) operateHeaders(frame *http2.MetaHeadersFrame, handle func( s.wq = newWriteQuota(defaultWriteQuota, s.ctxDone) s.trReader = &transportReader{ reader: &recvBufferReader{ - ctx: s.ctx, - ctxDone: s.ctxDone, - recv: s.buf, + ctx: s.ctx, + ctxDone: s.ctxDone, + recv: s.buf, + freeBuffer: t.bufferPool.put, }, windowHandler: func(n int) { t.updateWindow(s, uint32(n)) @@ -430,7 +430,7 @@ func (t *http2Server) operateHeaders(frame *http2.MetaHeadersFrame, handle func( wq: s.wq, }) handle(s) - return + return false } // HandleStreams receives incoming streams using the given handler. This is @@ -439,6 +439,7 @@ func (t *http2Server) operateHeaders(frame *http2.MetaHeadersFrame, handle func( func (t *http2Server) HandleStreams(handle func(*Stream), traceCtx func(context.Context, string) context.Context) { defer close(t.readerDone) for { + t.controlBuf.throttle() frame, err := t.framer.fr.ReadFrame() atomic.StoreUint32(&t.activity, 1) if err != nil { @@ -448,7 +449,7 @@ func (t *http2Server) HandleStreams(handle func(*Stream), traceCtx func(context. s := t.activeStreams[se.StreamID] t.mu.Unlock() if s != nil { - t.closeStream(s, true, se.Code, nil, false) + t.closeStream(s, true, se.Code, false) } else { t.controlBuf.put(&cleanupStream{ streamID: se.StreamID, @@ -590,7 +591,7 @@ func (t *http2Server) handleData(f *http2.DataFrame) { } if size > 0 { if err := s.fc.onData(size); err != nil { - t.closeStream(s, true, http2.ErrCodeFlowControl, nil, false) + t.closeStream(s, true, http2.ErrCodeFlowControl, false) return } if f.Header().Flags.Has(http2.FlagDataPadded) { @@ -602,9 +603,10 @@ func (t *http2Server) handleData(f *http2.DataFrame) { // guarantee f.Data() is consumed before the arrival of next frame. // Can this copy be eliminated? if len(f.Data()) > 0 { - data := make([]byte, len(f.Data())) - copy(data, f.Data()) - s.write(recvMsg{data: data}) + buffer := t.bufferPool.get() + buffer.Reset() + buffer.Write(f.Data()) + s.write(recvMsg{buffer: buffer}) } } if f.Header().Flags.Has(http2.FlagDataEndStream) { @@ -615,11 +617,18 @@ func (t *http2Server) handleData(f *http2.DataFrame) { } func (t *http2Server) handleRSTStream(f *http2.RSTStreamFrame) { - s, ok := t.getStream(f) - if !ok { + // If the stream is not deleted from the transport's active streams map, then do a regular close stream. + if s, ok := t.getStream(f); ok { + t.closeStream(s, false, 0, false) return } - t.closeStream(s, false, 0, nil, false) + // If the stream is already deleted from the active streams map, then put a cleanupStream item into controlbuf to delete the stream from loopy writer's established streams map. + t.controlBuf.put(&cleanupStream{ + streamID: f.Header().StreamID, + rst: false, + rstCode: 0, + onWrite: func() {}, + }) } func (t *http2Server) handleSettings(f *http2.SettingsFrame) { @@ -761,6 +770,10 @@ func (t *http2Server) WriteHeader(s *Stream, md metadata.MD) error { return nil } +func (t *http2Server) setResetPingStrikes() { + atomic.StoreUint32(&t.resetPingStrikes, 1) +} + func (t *http2Server) writeHeaderLocked(s *Stream) error { // TODO(mmukhi): Benchmark if the performance gets better if count the metadata and other header fields // first and create a slice of that exact size. @@ -775,15 +788,13 @@ func (t *http2Server) writeHeaderLocked(s *Stream) error { streamID: s.id, hf: headerFields, endStream: false, - onWrite: func() { - atomic.StoreUint32(&t.resetPingStrikes, 1) - }, + onWrite: t.setResetPingStrikes, }) if !success { if err != nil { return err } - t.closeStream(s, true, http2.ErrCodeInternal, nil, false) + t.closeStream(s, true, http2.ErrCodeInternal, false) return ErrHeaderListSizeLimitViolation } if t.stats != nil { @@ -821,7 +832,7 @@ func (t *http2Server) WriteStatus(s *Stream, st *status.Status) error { headerFields = append(headerFields, hpack.HeaderField{Name: "grpc-status", Value: strconv.Itoa(int(st.Code()))}) headerFields = append(headerFields, hpack.HeaderField{Name: "grpc-message", Value: encodeGrpcMessage(st.Message())}) - if p := st.Proto(); p != nil && len(p.Details) > 0 { + if p := statusRawProto(st); p != nil && len(p.Details) > 0 { stBytes, err := proto.Marshal(p) if err != nil { // TODO: return error instead, when callers are able to handle it. @@ -837,9 +848,7 @@ func (t *http2Server) WriteStatus(s *Stream, st *status.Status) error { streamID: s.id, hf: headerFields, endStream: true, - onWrite: func() { - atomic.StoreUint32(&t.resetPingStrikes, 1) - }, + onWrite: t.setResetPingStrikes, } s.hdrMu.Unlock() success, err := t.controlBuf.execute(t.checkForHeaderListSize, trailingHeader) @@ -847,10 +856,12 @@ func (t *http2Server) WriteStatus(s *Stream, st *status.Status) error { if err != nil { return err } - t.closeStream(s, true, http2.ErrCodeInternal, nil, false) + t.closeStream(s, true, http2.ErrCodeInternal, false) return ErrHeaderListSizeLimitViolation } - t.closeStream(s, false, 0, trailingHeader, true) + // Send a RST_STREAM after the trailers if the client has not already half-closed. + rst := s.getState() == streamActive + t.finishStream(s, rst, http2.ErrCodeNo, trailingHeader, true) if t.stats != nil { t.stats.HandleRPC(s.Context(), &stats.OutTrailer{}) } @@ -862,6 +873,9 @@ func (t *http2Server) WriteStatus(s *Stream, st *status.Status) error { func (t *http2Server) Write(s *Stream, hdr []byte, data []byte, opts *Options) error { if !s.isHeaderSent() { // Headers haven't been written yet. if err := t.WriteHeader(s, nil); err != nil { + if _, ok := err.(ConnectionError); ok { + return err + } // TODO(mmukhi, dfawley): Make sure this is the right code to return. return status.Errorf(codes.Internal, "transport: %v", err) } @@ -886,12 +900,10 @@ func (t *http2Server) Write(s *Stream, hdr []byte, data []byte, opts *Options) e hdr = append(hdr, data[:emptyLen]...) data = data[emptyLen:] df := &dataFrame{ - streamID: s.id, - h: hdr, - d: data, - onEachWrite: func() { - atomic.StoreUint32(&t.resetPingStrikes, 1) - }, + streamID: s.id, + h: hdr, + d: data, + onEachWrite: t.setResetPingStrikes, } if err := s.wq.get(int32(len(hdr) + len(data))); err != nil { select { @@ -957,6 +969,7 @@ func (t *http2Server) keepalive() { select { case <-maxAge.C: // Close the connection after grace period. + infof("transport: closing server transport due to maximum connection age.") t.Close() // Resetting the timer so that the clean-up doesn't deadlock. maxAge.Reset(infinity) @@ -970,6 +983,7 @@ func (t *http2Server) keepalive() { continue } if pingSent { + infof("transport: closing server transport due to idleness.") t.Close() // Resetting the timer so that the clean-up doesn't deadlock. keepalive.Reset(infinity) @@ -977,9 +991,7 @@ func (t *http2Server) keepalive() { } pingSent = true if channelz.IsOn() { - t.czmu.Lock() - t.kpCount++ - t.czmu.Unlock() + atomic.AddInt64(&t.czData.kpCount, 1) } t.controlBuf.put(p) keepalive.Reset(t.kp.Timeout) @@ -1019,47 +1031,61 @@ func (t *http2Server) Close() error { return err } -// closeStream clears the footprint of a stream when the stream is not needed -// any more. -func (t *http2Server) closeStream(s *Stream, rst bool, rstCode http2.ErrCode, hdr *headerFrame, eosReceived bool) { - if s.swapState(streamDone) == streamDone { - // If the stream was already done, return. - return - } +// deleteStream deletes the stream s from transport's active streams. +func (t *http2Server) deleteStream(s *Stream, eosReceived bool) { // In case stream sending and receiving are invoked in separate // goroutines (e.g., bi-directional streaming), cancel needs to be // called to interrupt the potential blocking on other goroutines. s.cancel() - cleanup := &cleanupStream{ + + t.mu.Lock() + if _, ok := t.activeStreams[s.id]; ok { + delete(t.activeStreams, s.id) + if len(t.activeStreams) == 0 { + t.idle = time.Now() + } + } + t.mu.Unlock() + + if channelz.IsOn() { + if eosReceived { + atomic.AddInt64(&t.czData.streamsSucceeded, 1) + } else { + atomic.AddInt64(&t.czData.streamsFailed, 1) + } + } +} + +// finishStream closes the stream and puts the trailing headerFrame into controlbuf. +func (t *http2Server) finishStream(s *Stream, rst bool, rstCode http2.ErrCode, hdr *headerFrame, eosReceived bool) { + oldState := s.swapState(streamDone) + if oldState == streamDone { + // If the stream was already done, return. + return + } + + hdr.cleanup = &cleanupStream{ streamID: s.id, rst: rst, rstCode: rstCode, onWrite: func() { - t.mu.Lock() - if t.activeStreams != nil { - delete(t.activeStreams, s.id) - if len(t.activeStreams) == 0 { - t.idle = time.Now() - } - } - t.mu.Unlock() - if channelz.IsOn() { - t.czmu.Lock() - if eosReceived { - t.streamsSucceeded++ - } else { - t.streamsFailed++ - } - t.czmu.Unlock() - } + t.deleteStream(s, eosReceived) }, } - if hdr != nil { - hdr.cleanup = cleanup - t.controlBuf.put(hdr) - } else { - t.controlBuf.put(cleanup) - } + t.controlBuf.put(hdr) +} + +// closeStream clears the footprint of a stream when the stream is not needed any more. +func (t *http2Server) closeStream(s *Stream, rst bool, rstCode http2.ErrCode, eosReceived bool) { + s.swapState(streamDone) + t.deleteStream(s, eosReceived) + + t.controlBuf.put(&cleanupStream{ + streamID: s.id, + rst: rst, + rstCode: rstCode, + onWrite: func() {}, + }) } func (t *http2Server) RemoteAddr() net.Addr { @@ -1138,17 +1164,16 @@ func (t *http2Server) outgoingGoAwayHandler(g *goAway) (bool, error) { } func (t *http2Server) ChannelzMetric() *channelz.SocketInternalMetric { - t.czmu.RLock() s := channelz.SocketInternalMetric{ - StreamsStarted: t.streamsStarted, - StreamsSucceeded: t.streamsSucceeded, - StreamsFailed: t.streamsFailed, - MessagesSent: t.msgSent, - MessagesReceived: t.msgRecv, - KeepAlivesSent: t.kpCount, - LastRemoteStreamCreatedTimestamp: t.lastStreamCreated, - LastMessageSentTimestamp: t.lastMsgSent, - LastMessageReceivedTimestamp: t.lastMsgRecv, + StreamsStarted: atomic.LoadInt64(&t.czData.streamsStarted), + StreamsSucceeded: atomic.LoadInt64(&t.czData.streamsSucceeded), + StreamsFailed: atomic.LoadInt64(&t.czData.streamsFailed), + MessagesSent: atomic.LoadInt64(&t.czData.msgSent), + MessagesReceived: atomic.LoadInt64(&t.czData.msgRecv), + KeepAlivesSent: atomic.LoadInt64(&t.czData.kpCount), + LastRemoteStreamCreatedTimestamp: time.Unix(0, atomic.LoadInt64(&t.czData.lastStreamCreatedTime)), + LastMessageSentTimestamp: time.Unix(0, atomic.LoadInt64(&t.czData.lastMsgSentTime)), + LastMessageReceivedTimestamp: time.Unix(0, atomic.LoadInt64(&t.czData.lastMsgRecvTime)), LocalFlowControlWindow: int64(t.fc.getSize()), SocketOptions: channelz.GetSocketOption(t.conn), LocalAddr: t.localAddr, @@ -1158,27 +1183,22 @@ func (t *http2Server) ChannelzMetric() *channelz.SocketInternalMetric { if au, ok := t.authInfo.(credentials.ChannelzSecurityInfo); ok { s.Security = au.GetSecurityValue() } - t.czmu.RUnlock() s.RemoteFlowControlWindow = t.getOutFlowWindow() return &s } func (t *http2Server) IncrMsgSent() { - t.czmu.Lock() - t.msgSent++ - t.lastMsgSent = time.Now() - t.czmu.Unlock() + atomic.AddInt64(&t.czData.msgSent, 1) + atomic.StoreInt64(&t.czData.lastMsgSentTime, time.Now().UnixNano()) } func (t *http2Server) IncrMsgRecv() { - t.czmu.Lock() - t.msgRecv++ - t.lastMsgRecv = time.Now() - t.czmu.Unlock() + atomic.AddInt64(&t.czData.msgRecv, 1) + atomic.StoreInt64(&t.czData.lastMsgRecvTime, time.Now().UnixNano()) } func (t *http2Server) getOutFlowWindow() int64 { - resp := make(chan uint32) + resp := make(chan uint32, 1) timer := time.NewTimer(time.Second) defer timer.Stop() t.controlBuf.put(&outFlowControlSizeRequest{resp}) diff --git a/vendor/google.golang.org/grpc/internal/transport/http_util.go b/vendor/google.golang.org/grpc/internal/transport/http_util.go index 21da6e80b3b..8f5f3349d90 100644 --- a/vendor/google.golang.org/grpc/internal/transport/http_util.go +++ b/vendor/google.golang.org/grpc/internal/transport/http_util.go @@ -24,6 +24,7 @@ import ( "encoding/base64" "fmt" "io" + "math" "net" "net/http" "strconv" @@ -77,7 +78,8 @@ var ( codes.ResourceExhausted: http2.ErrCodeEnhanceYourCalm, codes.PermissionDenied: http2.ErrCodeInadequateSecurity, } - httpStatusConvTab = map[int]codes.Code{ + // HTTPStatusConvTab is the HTTP status code to gRPC error code conversion table. + HTTPStatusConvTab = map[int]codes.Code{ // 400 Bad Request - INTERNAL. http.StatusBadRequest: codes.Internal, // 401 Unauthorized - UNAUTHENTICATED. @@ -97,9 +99,7 @@ var ( } ) -// Records the states during HPACK decoding. Must be reset once the -// decoding of the entire headers are finished. -type decodeState struct { +type parsedHeaderData struct { encoding string // statusGen caches the stream status received from the trailer the server // sent. Client side only. Do not access directly. After all trailers are @@ -119,8 +119,30 @@ type decodeState struct { statsTags []byte statsTrace []byte contentSubtype string + + // isGRPC field indicates whether the peer is speaking gRPC (otherwise HTTP). + // + // We are in gRPC mode (peer speaking gRPC) if: + // * We are client side and have already received a HEADER frame that indicates gRPC peer. + // * The header contains valid a content-type, i.e. a string starts with "application/grpc" + // And we should handle error specific to gRPC. + // + // Otherwise (i.e. a content-type string starts without "application/grpc", or does not exist), we + // are in HTTP fallback mode, and should handle error specific to HTTP. + isGRPC bool + grpcErr error + httpErr error + contentTypeErr string +} + +// decodeState configures decoding criteria and records the decoded data. +type decodeState struct { // whether decoding on server side or not serverSide bool + + // Records the states during HPACK decoding. It will be filled with info parsed from HTTP HEADERS + // frame once decodeHeader function has been invoked and returned. + data parsedHeaderData } // isReservedHeader checks whether hdr belongs to HTTP2 headers @@ -201,11 +223,11 @@ func contentType(contentSubtype string) string { } func (d *decodeState) status() *status.Status { - if d.statusGen == nil { + if d.data.statusGen == nil { // No status-details were provided; generate status using code/msg. - d.statusGen = status.New(codes.Code(int32(*(d.rawStatusCode))), d.rawStatusMsg) + d.data.statusGen = status.New(codes.Code(int32(*(d.data.rawStatusCode))), d.data.rawStatusMsg) } - return d.statusGen + return d.data.statusGen } const binHdrSuffix = "-bin" @@ -243,113 +265,146 @@ func (d *decodeState) decodeHeader(frame *http2.MetaHeadersFrame) error { if frame.Truncated { return status.Error(codes.Internal, "peer header list size exceeded limit") } + for _, hf := range frame.Fields { - if err := d.processHeaderField(hf); err != nil { - return err + d.processHeaderField(hf) + } + + if d.data.isGRPC { + if d.data.grpcErr != nil { + return d.data.grpcErr + } + if d.serverSide { + return nil + } + if d.data.rawStatusCode == nil && d.data.statusGen == nil { + // gRPC status doesn't exist. + // Set rawStatusCode to be unknown and return nil error. + // So that, if the stream has ended this Unknown status + // will be propagated to the user. + // Otherwise, it will be ignored. In which case, status from + // a later trailer, that has StreamEnded flag set, is propagated. + code := int(codes.Unknown) + d.data.rawStatusCode = &code } - } - - if d.serverSide { return nil } - // If grpc status exists, no need to check further. - if d.rawStatusCode != nil || d.statusGen != nil { - return nil + // HTTP fallback mode + if d.data.httpErr != nil { + return d.data.httpErr } - // If grpc status doesn't exist and http status doesn't exist, - // then it's a malformed header. - if d.httpStatus == nil { - return status.Error(codes.Internal, "malformed header: doesn't contain status(gRPC or HTTP)") - } + var ( + code = codes.Internal // when header does not include HTTP status, return INTERNAL + ok bool + ) - if *(d.httpStatus) != http.StatusOK { - code, ok := httpStatusConvTab[*(d.httpStatus)] + if d.data.httpStatus != nil { + code, ok = HTTPStatusConvTab[*(d.data.httpStatus)] if !ok { code = codes.Unknown } - return status.Error(code, http.StatusText(*(d.httpStatus))) } - // gRPC status doesn't exist and http status is OK. - // Set rawStatusCode to be unknown and return nil error. - // So that, if the stream has ended this Unknown status - // will be propagated to the user. - // Otherwise, it will be ignored. In which case, status from - // a later trailer, that has StreamEnded flag set, is propagated. - code := int(codes.Unknown) - d.rawStatusCode = &code - return nil + return status.Error(code, d.constructHTTPErrMsg()) +} + +// constructErrMsg constructs error message to be returned in HTTP fallback mode. +// Format: HTTP status code and its corresponding message + content-type error message. +func (d *decodeState) constructHTTPErrMsg() string { + var errMsgs []string + + if d.data.httpStatus == nil { + errMsgs = append(errMsgs, "malformed header: missing HTTP status") + } else { + errMsgs = append(errMsgs, fmt.Sprintf("%s: HTTP status code %d", http.StatusText(*(d.data.httpStatus)), *d.data.httpStatus)) + } + + if d.data.contentTypeErr == "" { + errMsgs = append(errMsgs, "transport: missing content-type field") + } else { + errMsgs = append(errMsgs, d.data.contentTypeErr) + } + + return strings.Join(errMsgs, "; ") } func (d *decodeState) addMetadata(k, v string) { - if d.mdata == nil { - d.mdata = make(map[string][]string) + if d.data.mdata == nil { + d.data.mdata = make(map[string][]string) } - d.mdata[k] = append(d.mdata[k], v) + d.data.mdata[k] = append(d.data.mdata[k], v) } -func (d *decodeState) processHeaderField(f hpack.HeaderField) error { +func (d *decodeState) processHeaderField(f hpack.HeaderField) { switch f.Name { case "content-type": contentSubtype, validContentType := contentSubtype(f.Value) if !validContentType { - return status.Errorf(codes.Internal, "transport: received the unexpected content-type %q", f.Value) + d.data.contentTypeErr = fmt.Sprintf("transport: received the unexpected content-type %q", f.Value) + return } - d.contentSubtype = contentSubtype + d.data.contentSubtype = contentSubtype // TODO: do we want to propagate the whole content-type in the metadata, // or come up with a way to just propagate the content-subtype if it was set? // ie {"content-type": "application/grpc+proto"} or {"content-subtype": "proto"} // in the metadata? d.addMetadata(f.Name, f.Value) + d.data.isGRPC = true case "grpc-encoding": - d.encoding = f.Value + d.data.encoding = f.Value case "grpc-status": code, err := strconv.Atoi(f.Value) if err != nil { - return status.Errorf(codes.Internal, "transport: malformed grpc-status: %v", err) + d.data.grpcErr = status.Errorf(codes.Internal, "transport: malformed grpc-status: %v", err) + return } - d.rawStatusCode = &code + d.data.rawStatusCode = &code case "grpc-message": - d.rawStatusMsg = decodeGrpcMessage(f.Value) + d.data.rawStatusMsg = decodeGrpcMessage(f.Value) case "grpc-status-details-bin": v, err := decodeBinHeader(f.Value) if err != nil { - return status.Errorf(codes.Internal, "transport: malformed grpc-status-details-bin: %v", err) + d.data.grpcErr = status.Errorf(codes.Internal, "transport: malformed grpc-status-details-bin: %v", err) + return } s := &spb.Status{} if err := proto.Unmarshal(v, s); err != nil { - return status.Errorf(codes.Internal, "transport: malformed grpc-status-details-bin: %v", err) + d.data.grpcErr = status.Errorf(codes.Internal, "transport: malformed grpc-status-details-bin: %v", err) + return } - d.statusGen = status.FromProto(s) + d.data.statusGen = status.FromProto(s) case "grpc-timeout": - d.timeoutSet = true + d.data.timeoutSet = true var err error - if d.timeout, err = decodeTimeout(f.Value); err != nil { - return status.Errorf(codes.Internal, "transport: malformed time-out: %v", err) + if d.data.timeout, err = decodeTimeout(f.Value); err != nil { + d.data.grpcErr = status.Errorf(codes.Internal, "transport: malformed time-out: %v", err) } case ":path": - d.method = f.Value + d.data.method = f.Value case ":status": code, err := strconv.Atoi(f.Value) if err != nil { - return status.Errorf(codes.Internal, "transport: malformed http-status: %v", err) + d.data.httpErr = status.Errorf(codes.Internal, "transport: malformed http-status: %v", err) + return } - d.httpStatus = &code + d.data.httpStatus = &code case "grpc-tags-bin": v, err := decodeBinHeader(f.Value) if err != nil { - return status.Errorf(codes.Internal, "transport: malformed grpc-tags-bin: %v", err) + d.data.grpcErr = status.Errorf(codes.Internal, "transport: malformed grpc-tags-bin: %v", err) + return } - d.statsTags = v + d.data.statsTags = v d.addMetadata(f.Name, string(v)) case "grpc-trace-bin": v, err := decodeBinHeader(f.Value) if err != nil { - return status.Errorf(codes.Internal, "transport: malformed grpc-trace-bin: %v", err) + d.data.grpcErr = status.Errorf(codes.Internal, "transport: malformed grpc-trace-bin: %v", err) + return } - d.statsTrace = v + d.data.statsTrace = v d.addMetadata(f.Name, string(v)) default: if isReservedHeader(f.Name) && !isWhitelistedHeader(f.Name) { @@ -358,11 +413,10 @@ func (d *decodeState) processHeaderField(f hpack.HeaderField) error { v, err := decodeMetadataHeader(f.Name, f.Value) if err != nil { errorf("Failed to decode metadata header (%q, %q): %v", f.Name, f.Value, err) - return nil + return } d.addMetadata(f.Name, v) } - return nil } type timeoutUnit uint8 @@ -435,6 +489,10 @@ func decodeTimeout(s string) (time.Duration, error) { if size < 2 { return 0, fmt.Errorf("transport: timeout string is too short: %q", s) } + if size > 9 { + // Spec allows for 8 digits plus the unit. + return 0, fmt.Errorf("transport: timeout string is too long: %q", s) + } unit := timeoutUnit(s[size-1]) d, ok := timeoutUnitToDuration(unit) if !ok { @@ -444,6 +502,11 @@ func decodeTimeout(s string) (time.Duration, error) { if err != nil { return 0, err } + const maxHours = math.MaxInt64 / int64(time.Hour) + if d == time.Hour && t > maxHours { + // This timeout would overflow math.MaxInt64; clamp it. + return time.Duration(math.MaxInt64), nil + } return d * time.Duration(t), nil } @@ -604,6 +667,7 @@ func newFramer(conn net.Conn, writeBufferSize, readBufferSize int, maxHeaderList writer: w, fr: http2.NewFramer(w, r), } + f.fr.SetMaxReadFrameSize(http2MaxFrameLen) // Opt-in to Frame reuse API on framer to reduce garbage. // Frames aren't safe to read from after a subsequent call to ReadFrame. f.fr.SetReuseFrames() diff --git a/vendor/google.golang.org/grpc/internal/transport/log.go b/vendor/google.golang.org/grpc/internal/transport/log.go index ac8e358c5c8..879df80c4de 100644 --- a/vendor/google.golang.org/grpc/internal/transport/log.go +++ b/vendor/google.golang.org/grpc/internal/transport/log.go @@ -42,9 +42,3 @@ func errorf(format string, args ...interface{}) { grpclog.Errorf(format, args...) } } - -func fatalf(format string, args ...interface{}) { - if grpclog.V(logLevel) { - grpclog.Fatalf(format, args...) - } -} diff --git a/vendor/google.golang.org/grpc/internal/transport/transport.go b/vendor/google.golang.org/grpc/internal/transport/transport.go index 9775eeb813e..1c1d106709a 100644 --- a/vendor/google.golang.org/grpc/internal/transport/transport.go +++ b/vendor/google.golang.org/grpc/internal/transport/transport.go @@ -22,6 +22,8 @@ package transport import ( + "bytes" + "context" "errors" "fmt" "io" @@ -29,7 +31,6 @@ import ( "sync" "sync/atomic" - "golang.org/x/net/context" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials" "google.golang.org/grpc/keepalive" @@ -39,10 +40,32 @@ import ( "google.golang.org/grpc/tap" ) +type bufferPool struct { + pool sync.Pool +} + +func newBufferPool() *bufferPool { + return &bufferPool{ + pool: sync.Pool{ + New: func() interface{} { + return new(bytes.Buffer) + }, + }, + } +} + +func (p *bufferPool) get() *bytes.Buffer { + return p.pool.Get().(*bytes.Buffer) +} + +func (p *bufferPool) put(b *bytes.Buffer) { + p.pool.Put(b) +} + // recvMsg represents the received msg from the transport. All transport // protocol specific info has been removed. type recvMsg struct { - data []byte + buffer *bytes.Buffer // nil: received some data // io.EOF: stream is completed. data is nil. // other non-nil error: transport failure. data is nil. @@ -110,15 +133,16 @@ func (b *recvBuffer) get() <-chan recvMsg { return b.c } -// // recvBufferReader implements io.Reader interface to read the data from // recvBuffer. type recvBufferReader struct { - ctx context.Context - ctxDone <-chan struct{} // cache of ctx.Done() (for performance). - recv *recvBuffer - last []byte // Stores the remaining data in the previous calls. - err error + closeStream func(error) // Closes the client transport stream with the given error and nil trailer metadata. + ctx context.Context + ctxDone <-chan struct{} // cache of ctx.Done() (for performance). + recv *recvBuffer + last *bytes.Buffer // Stores the remaining data in the previous calls. + err error + freeBuffer func(*bytes.Buffer) } // Read reads the next len(p) bytes from last. If last is drained, it tries to @@ -128,31 +152,74 @@ func (r *recvBufferReader) Read(p []byte) (n int, err error) { if r.err != nil { return 0, r.err } - n, r.err = r.read(p) + if r.last != nil { + // Read remaining data left in last call. + copied, _ := r.last.Read(p) + if r.last.Len() == 0 { + r.freeBuffer(r.last) + r.last = nil + } + return copied, nil + } + if r.closeStream != nil { + n, r.err = r.readClient(p) + } else { + n, r.err = r.read(p) + } return n, r.err } func (r *recvBufferReader) read(p []byte) (n int, err error) { - if r.last != nil && len(r.last) > 0 { - // Read remaining data left in last call. - copied := copy(p, r.last) - r.last = r.last[copied:] - return copied, nil - } select { case <-r.ctxDone: return 0, ContextErr(r.ctx.Err()) case m := <-r.recv.get(): - r.recv.load() - if m.err != nil { - return 0, m.err - } - copied := copy(p, m.data) - r.last = m.data[copied:] - return copied, nil + return r.readAdditional(m, p) } } +func (r *recvBufferReader) readClient(p []byte) (n int, err error) { + // If the context is canceled, then closes the stream with nil metadata. + // closeStream writes its error parameter to r.recv as a recvMsg. + // r.readAdditional acts on that message and returns the necessary error. + select { + case <-r.ctxDone: + // Note that this adds the ctx error to the end of recv buffer, and + // reads from the head. This will delay the error until recv buffer is + // empty, thus will delay ctx cancellation in Recv(). + // + // It's done this way to fix a race between ctx cancel and trailer. The + // race was, stream.Recv() may return ctx error if ctxDone wins the + // race, but stream.Trailer() may return a non-nil md because the stream + // was not marked as done when trailer is received. This closeStream + // call will mark stream as done, thus fix the race. + // + // TODO: delaying ctx error seems like a unnecessary side effect. What + // we really want is to mark the stream as done, and return ctx error + // faster. + r.closeStream(ContextErr(r.ctx.Err())) + m := <-r.recv.get() + return r.readAdditional(m, p) + case m := <-r.recv.get(): + return r.readAdditional(m, p) + } +} + +func (r *recvBufferReader) readAdditional(m recvMsg, p []byte) (n int, err error) { + r.recv.load() + if m.err != nil { + return 0, m.err + } + copied, _ := m.buffer.Read(p) + if m.buffer.Len() == 0 { + r.freeBuffer(m.buffer) + r.last = nil + } else { + r.last = m.buffer + } + return copied, nil +} + type streamState uint32 const ( @@ -176,19 +243,22 @@ type Stream struct { buf *recvBuffer trReader io.Reader fc *inFlow - recvQuota uint32 wq *writeQuota // Callback to state application's intentions to read data. This // is used to adjust flow control, if needed. requestRead func(int) - headerChan chan struct{} // closed to indicate the end of header metadata. - headerDone uint32 // set when headerChan is closed. Used to avoid closing headerChan multiple times. + headerChan chan struct{} // closed to indicate the end of header metadata. + headerChanClosed uint32 // set when headerChan is closed. Used to avoid closing headerChan multiple times. // hdrMu protects header and trailer metadata on the server-side. - hdrMu sync.Mutex - header metadata.MD // the received header metadata. + hdrMu sync.Mutex + // On client side, header keeps the received header metadata. + // + // On server side, header keeps the header set by SetHeader(). The complete + // header will merged into this after t.WriteHeader() is called. + header metadata.MD trailer metadata.MD // the key-value map of trailer metadata. noHeaders bool // set if the client never received headers (set only after the stream is done). @@ -241,6 +311,14 @@ func (s *Stream) waitOnHeader() error { } select { case <-s.ctx.Done(): + // We prefer success over failure when reading messages because we delay + // context error in stream.Read(). To keep behavior consistent, we also + // prefer success here. + select { + case <-s.headerChan: + return nil + default: + } return ContextErr(s.ctx.Err()) case <-s.headerChan: return nil @@ -267,10 +345,19 @@ func (s *Stream) Done() <-chan struct{} { return s.done } -// Header acquires the key-value pairs of header metadata once it -// is available. It blocks until i) the metadata is ready or ii) there is no -// header metadata or iii) the stream is canceled/expired. +// Header returns the header metadata of the stream. +// +// On client side, it acquires the key-value pairs of header metadata once it is +// available. It blocks until i) the metadata is ready or ii) there is no header +// metadata or iii) the stream is canceled/expired. +// +// On server side, it returns the out header after t.WriteHeader is called. func (s *Stream) Header() (metadata.MD, error) { + if s.headerChan == nil && s.header != nil { + // On server side, return the header in stream. It will be the out + // header after t.WriteHeader is called. + return s.header.Copy(), nil + } err := s.waitOnHeader() // Even if the stream is closed, header is returned if available. select { @@ -293,8 +380,7 @@ func (s *Stream) TrailersOnly() (bool, error) { if err != nil { return false, err } - // if !headerDone, some other connection error occurred. - return s.noHeaders && atomic.LoadUint32(&s.headerDone) == 1, nil + return s.noHeaders, nil } // Trailer returns the cached trailer metedata. Note that if it is not called @@ -466,8 +552,12 @@ type ConnectOptions struct { FailOnNonTempDialError bool // PerRPCCredentials stores the PerRPCCredentials required to issue RPCs. PerRPCCredentials []credentials.PerRPCCredentials - // TransportCredentials stores the Authenticator required to setup a client connection. + // TransportCredentials stores the Authenticator required to setup a client + // connection. Only one of TransportCredentials and CredsBundle is non-nil. TransportCredentials credentials.TransportCredentials + // CredsBundle is the credentials bundle to be used. Only one of + // TransportCredentials and CredsBundle is non-nil. + CredsBundle credentials.Bundle // KeepaliveParams stores the keepalive parameters. KeepaliveParams keepalive.ClientParameters // StatsHandler stores the handler for stats. @@ -495,8 +585,8 @@ type TargetInfo struct { // NewClientTransport establishes the transport with the required ConnectOptions // and returns it to the caller. -func NewClientTransport(connectCtx, ctx context.Context, target TargetInfo, opts ConnectOptions, onSuccess func()) (ClientTransport, error) { - return newHTTP2Client(connectCtx, ctx, target, opts, onSuccess) +func NewClientTransport(connectCtx, ctx context.Context, target TargetInfo, opts ConnectOptions, onPrefaceReceipt func(), onGoAway func(GoAwayReason), onClose func()) (ClientTransport, error) { + return newHTTP2Client(connectCtx, ctx, target, opts, onPrefaceReceipt, onGoAway, onClose) } // Options provides additional hints and information for message @@ -541,9 +631,12 @@ type ClientTransport interface { // is called only once. Close() error - // GracefulClose starts to tear down the transport. It stops accepting - // new RPCs and wait the completion of the pending RPCs. - GracefulClose() error + // GracefulClose starts to tear down the transport: the transport will stop + // accepting new RPCs and NewStream will return error. Once all streams are + // finished, the transport will close. + // + // It does not block. + GracefulClose() // Write sends the data for the given stream. A nil stream indicates // the write is to be performed on the transport as a whole. @@ -573,6 +666,9 @@ type ClientTransport interface { // GetGoAwayReason returns the reason why GoAway frame was received. GetGoAwayReason() GoAwayReason + // RemoteAddr returns the remote network address. + RemoteAddr() net.Addr + // IncrMsgSent increments the number of message sent through this transport. IncrMsgSent() @@ -683,3 +779,38 @@ const ( // "too_many_pings". GoAwayTooManyPings GoAwayReason = 2 ) + +// channelzData is used to store channelz related data for http2Client and http2Server. +// These fields cannot be embedded in the original structs (e.g. http2Client), since to do atomic +// operation on int64 variable on 32-bit machine, user is responsible to enforce memory alignment. +// Here, by grouping those int64 fields inside a struct, we are enforcing the alignment. +type channelzData struct { + kpCount int64 + // The number of streams that have started, including already finished ones. + streamsStarted int64 + // Client side: The number of streams that have ended successfully by receiving + // EoS bit set frame from server. + // Server side: The number of streams that have ended successfully by sending + // frame with EoS bit set. + streamsSucceeded int64 + streamsFailed int64 + // lastStreamCreatedTime stores the timestamp that the last stream gets created. It is of int64 type + // instead of time.Time since it's more costly to atomically update time.Time variable than int64 + // variable. The same goes for lastMsgSentTime and lastMsgRecvTime. + lastStreamCreatedTime int64 + msgSent int64 + msgRecv int64 + lastMsgSentTime int64 + lastMsgRecvTime int64 +} + +// ContextErr converts the error from context package into a status error. +func ContextErr(err error) error { + switch err { + case context.DeadlineExceeded: + return status.Error(codes.DeadlineExceeded, err.Error()) + case context.Canceled: + return status.Error(codes.Canceled, err.Error()) + } + return status.Errorf(codes.Internal, "Unexpected error from context packet: %v", err) +} diff --git a/vendor/google.golang.org/grpc/keepalive/keepalive.go b/vendor/google.golang.org/grpc/keepalive/keepalive.go index f8adc7e6d4f..34d31b5e7d3 100644 --- a/vendor/google.golang.org/grpc/keepalive/keepalive.go +++ b/vendor/google.golang.org/grpc/keepalive/keepalive.go @@ -16,7 +16,8 @@ * */ -// Package keepalive defines configurable parameters for point-to-point healthcheck. +// Package keepalive defines configurable parameters for point-to-point +// healthcheck. package keepalive import ( @@ -24,42 +25,61 @@ import ( ) // ClientParameters is used to set keepalive parameters on the client-side. -// These configure how the client will actively probe to notice when a connection is broken -// and send pings so intermediaries will be aware of the liveness of the connection. -// Make sure these parameters are set in coordination with the keepalive policy on the server, -// as incompatible settings can result in closing of connection. +// These configure how the client will actively probe to notice when a +// connection is broken and send pings so intermediaries will be aware of the +// liveness of the connection. Make sure these parameters are set in +// coordination with the keepalive policy on the server, as incompatible +// settings can result in closing of connection. type ClientParameters struct { - // After a duration of this time if the client doesn't see any activity it pings the server to see if the transport is still alive. + // After a duration of this time if the client doesn't see any activity it + // pings the server to see if the transport is still alive. + // If set below 10s, a minimum value of 10s will be used instead. Time time.Duration // The current default value is infinity. - // After having pinged for keepalive check, the client waits for a duration of Timeout and if no activity is seen even after that - // the connection is closed. + // After having pinged for keepalive check, the client waits for a duration + // of Timeout and if no activity is seen even after that the connection is + // closed. Timeout time.Duration // The current default value is 20 seconds. - // If true, client runs keepalive checks even with no active RPCs. + // If true, client sends keepalive pings even with no active RPCs. If false, + // when there are no active RPCs, Time and Timeout will be ignored and no + // keepalive pings will be sent. PermitWithoutStream bool // false by default. } -// ServerParameters is used to set keepalive and max-age parameters on the server-side. +// ServerParameters is used to set keepalive and max-age parameters on the +// server-side. type ServerParameters struct { - // MaxConnectionIdle is a duration for the amount of time after which an idle connection would be closed by sending a GoAway. - // Idleness duration is defined since the most recent time the number of outstanding RPCs became zero or the connection establishment. + // MaxConnectionIdle is a duration for the amount of time after which an + // idle connection would be closed by sending a GoAway. Idleness duration is + // defined since the most recent time the number of outstanding RPCs became + // zero or the connection establishment. MaxConnectionIdle time.Duration // The current default value is infinity. - // MaxConnectionAge is a duration for the maximum amount of time a connection may exist before it will be closed by sending a GoAway. - // A random jitter of +/-10% will be added to MaxConnectionAge to spread out connection storms. + // MaxConnectionAge is a duration for the maximum amount of time a + // connection may exist before it will be closed by sending a GoAway. A + // random jitter of +/-10% will be added to MaxConnectionAge to spread out + // connection storms. MaxConnectionAge time.Duration // The current default value is infinity. - // MaxConnectinoAgeGrace is an additive period after MaxConnectionAge after which the connection will be forcibly closed. + // MaxConnectionAgeGrace is an additive period after MaxConnectionAge after + // which the connection will be forcibly closed. MaxConnectionAgeGrace time.Duration // The current default value is infinity. - // After a duration of this time if the server doesn't see any activity it pings the client to see if the transport is still alive. + // After a duration of this time if the server doesn't see any activity it + // pings the client to see if the transport is still alive. + // If set below 1s, a minimum value of 1s will be used instead. Time time.Duration // The current default value is 2 hours. - // After having pinged for keepalive check, the server waits for a duration of Timeout and if no activity is seen even after that - // the connection is closed. + // After having pinged for keepalive check, the server waits for a duration + // of Timeout and if no activity is seen even after that the connection is + // closed. Timeout time.Duration // The current default value is 20 seconds. } -// EnforcementPolicy is used to set keepalive enforcement policy on the server-side. -// Server will close connection with a client that violates this policy. +// EnforcementPolicy is used to set keepalive enforcement policy on the +// server-side. Server will close connection with a client that violates this +// policy. type EnforcementPolicy struct { - // MinTime is the minimum amount of time a client should wait before sending a keepalive ping. + // MinTime is the minimum amount of time a client should wait before sending + // a keepalive ping. MinTime time.Duration // The current default value is 5 minutes. - // If true, server expects keepalive pings even when there are no active streams(RPCs). + // If true, server allows keepalive pings even when there are no active + // streams(RPCs). If false, and client sends ping when there are no active + // streams, server will send GOAWAY and close the connection. PermitWithoutStream bool // false by default. } diff --git a/vendor/google.golang.org/grpc/metadata/metadata.go b/vendor/google.golang.org/grpc/metadata/metadata.go index bd2eaf40837..cf6d1b94781 100644 --- a/vendor/google.golang.org/grpc/metadata/metadata.go +++ b/vendor/google.golang.org/grpc/metadata/metadata.go @@ -22,10 +22,9 @@ package metadata // import "google.golang.org/grpc/metadata" import ( + "context" "fmt" "strings" - - "golang.org/x/net/context" ) // DecodeKeyValue returns k, v, nil. diff --git a/vendor/google.golang.org/grpc/naming/dns_resolver.go b/vendor/google.golang.org/grpc/naming/dns_resolver.go index 0f8a908ea9c..c9f79dc5336 100644 --- a/vendor/google.golang.org/grpc/naming/dns_resolver.go +++ b/vendor/google.golang.org/grpc/naming/dns_resolver.go @@ -19,13 +19,13 @@ package naming import ( + "context" "errors" "fmt" "net" "strconv" "time" - "golang.org/x/net/context" "google.golang.org/grpc/grpclog" ) @@ -37,6 +37,9 @@ const ( var ( errMissingAddr = errors.New("missing address") errWatcherClose = errors.New("watcher has been closed") + + lookupHost = net.DefaultResolver.LookupHost + lookupSRV = net.DefaultResolver.LookupSRV ) // NewDNSResolverWithFreq creates a DNS Resolver that can resolve DNS names, and @@ -73,8 +76,8 @@ func formatIP(addr string) (addrIP string, ok bool) { // parseTarget takes the user input target string, returns formatted host and port info. // If target doesn't specify a port, set the port to be the defaultPort. -// If target is in IPv6 format and host-name is enclosed in sqarue brackets, brackets -// are strippd when setting the host. +// If target is in IPv6 format and host-name is enclosed in square brackets, brackets +// are stripped when setting the host. // examples: // target: "www.google.com" returns host: "www.google.com", port: "443" // target: "ipv4-host:80" returns host: "ipv4-host", port: "80" @@ -218,7 +221,7 @@ func (w *dnsWatcher) lookupSRV() map[string]*Update { for _, s := range srvs { lbAddrs, err := lookupHost(w.ctx, s.Target) if err != nil { - grpclog.Warningf("grpc: failed load banlacer address dns lookup due to %v.\n", err) + grpclog.Warningf("grpc: failed load balancer address dns lookup due to %v.\n", err) continue } for _, a := range lbAddrs { diff --git a/vendor/google.golang.org/grpc/naming/go17.go b/vendor/google.golang.org/grpc/naming/go17.go deleted file mode 100644 index 57b65d7b889..00000000000 --- a/vendor/google.golang.org/grpc/naming/go17.go +++ /dev/null @@ -1,34 +0,0 @@ -// +build go1.6,!go1.8 - -/* - * - * Copyright 2017 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package naming - -import ( - "net" - - "golang.org/x/net/context" -) - -var ( - lookupHost = func(ctx context.Context, host string) ([]string, error) { return net.LookupHost(host) } - lookupSRV = func(ctx context.Context, service, proto, name string) (string, []*net.SRV, error) { - return net.LookupSRV(service, proto, name) - } -) diff --git a/vendor/google.golang.org/grpc/naming/naming.go b/vendor/google.golang.org/grpc/naming/naming.go index 8cc39e93758..f4c1c8b6894 100644 --- a/vendor/google.golang.org/grpc/naming/naming.go +++ b/vendor/google.golang.org/grpc/naming/naming.go @@ -17,9 +17,8 @@ */ // Package naming defines the naming API and related data structures for gRPC. -// The interface is EXPERIMENTAL and may be suject to change. // -// Deprecated: please use package resolver. +// This package is deprecated: please use package resolver instead. package naming // Operation defines the corresponding operations for a name resolution change. diff --git a/vendor/google.golang.org/grpc/peer/peer.go b/vendor/google.golang.org/grpc/peer/peer.go index 317b8b9d09a..e01d219ffbc 100644 --- a/vendor/google.golang.org/grpc/peer/peer.go +++ b/vendor/google.golang.org/grpc/peer/peer.go @@ -21,9 +21,9 @@ package peer import ( + "context" "net" - "golang.org/x/net/context" "google.golang.org/grpc/credentials" ) diff --git a/vendor/google.golang.org/grpc/picker_wrapper.go b/vendor/google.golang.org/grpc/picker_wrapper.go index 76cc456aa7e..45baa2ae13d 100644 --- a/vendor/google.golang.org/grpc/picker_wrapper.go +++ b/vendor/google.golang.org/grpc/picker_wrapper.go @@ -19,10 +19,10 @@ package grpc import ( + "context" "io" "sync" - "golang.org/x/net/context" "google.golang.org/grpc/balancer" "google.golang.org/grpc/codes" "google.golang.org/grpc/grpclog" @@ -101,10 +101,7 @@ func doneChannelzWrapper(acw *acBalancerWrapper, done func(balancer.DoneInfo)) f // - the subConn returned by the current picker is not READY // When one of these situations happens, pick blocks until the picker gets updated. func (bp *pickerWrapper) pick(ctx context.Context, failfast bool, opts balancer.PickOptions) (transport.ClientTransport, func(balancer.DoneInfo), error) { - var ( - p balancer.Picker - ch chan struct{} - ) + var ch chan struct{} for { bp.mu.Lock() @@ -123,6 +120,14 @@ func (bp *pickerWrapper) pick(ctx context.Context, failfast bool, opts balancer. bp.mu.Unlock() select { case <-ctx.Done(): + if connectionErr := bp.connectionError(); connectionErr != nil { + switch ctx.Err() { + case context.DeadlineExceeded: + return nil, nil, status.Errorf(codes.DeadlineExceeded, "latest connection error: %v", connectionErr) + case context.Canceled: + return nil, nil, status.Errorf(codes.Canceled, "latest connection error: %v", connectionErr) + } + } return nil, nil, ctx.Err() case <-ch: } @@ -130,7 +135,7 @@ func (bp *pickerWrapper) pick(ctx context.Context, failfast bool, opts balancer. } ch = bp.blockingCh - p = bp.picker + p := bp.picker bp.mu.Unlock() subConn, done, err := p.Pick(ctx, opts) @@ -144,15 +149,22 @@ func (bp *pickerWrapper) pick(ctx context.Context, failfast bool, opts balancer. continue } return nil, nil, status.Errorf(codes.Unavailable, "%v, latest connection error: %v", err, bp.connectionError()) + case context.DeadlineExceeded: + return nil, nil, status.Error(codes.DeadlineExceeded, err.Error()) + case context.Canceled: + return nil, nil, status.Error(codes.Canceled, err.Error()) default: + if _, ok := status.FromError(err); ok { + return nil, nil, err + } // err is some other error. - return nil, nil, toRPCErr(err) + return nil, nil, status.Error(codes.Unknown, err.Error()) } } acw, ok := subConn.(*acBalancerWrapper) if !ok { - grpclog.Infof("subconn returned from pick is not *acBalancerWrapper") + grpclog.Error("subconn returned from pick is not *acBalancerWrapper") continue } if t, ok := acw.getAddrConn().getReadyTransport(); ok { @@ -161,6 +173,11 @@ func (bp *pickerWrapper) pick(ctx context.Context, failfast bool, opts balancer. } return t, done, nil } + if done != nil { + // Calling done with nil error, no bytes sent and no bytes received. + // DoneInfo with default value works. + done(balancer.DoneInfo{}) + } grpclog.Infof("blockingPicker: the picked transport is not ready, loop back to repick") // If ok == false, ac.state is not READY. // A valid picker always returns READY subConn. This means the state of ac diff --git a/vendor/google.golang.org/grpc/pickfirst.go b/vendor/google.golang.org/grpc/pickfirst.go index bf659d49d2f..ed05b02ed96 100644 --- a/vendor/google.golang.org/grpc/pickfirst.go +++ b/vendor/google.golang.org/grpc/pickfirst.go @@ -19,7 +19,8 @@ package grpc import ( - "golang.org/x/net/context" + "context" + "google.golang.org/grpc/balancer" "google.golang.org/grpc/connectivity" "google.golang.org/grpc/grpclog" @@ -50,13 +51,18 @@ type pickfirstBalancer struct { func (b *pickfirstBalancer) HandleResolvedAddrs(addrs []resolver.Address, err error) { if err != nil { - grpclog.Infof("pickfirstBalancer: HandleResolvedAddrs called with error %v", err) + if grpclog.V(2) { + grpclog.Infof("pickfirstBalancer: HandleResolvedAddrs called with error %v", err) + } return } if b.sc == nil { b.sc, err = b.cc.NewSubConn(addrs, balancer.NewSubConnOptions{}) if err != nil { - grpclog.Errorf("pickfirstBalancer: failed to NewSubConn: %v", err) + //TODO(yuxuanli): why not change the cc state to Idle? + if grpclog.V(2) { + grpclog.Errorf("pickfirstBalancer: failed to NewSubConn: %v", err) + } return } b.cc.UpdateBalancerState(connectivity.Idle, &picker{sc: b.sc}) @@ -68,9 +74,13 @@ func (b *pickfirstBalancer) HandleResolvedAddrs(addrs []resolver.Address, err er } func (b *pickfirstBalancer) HandleSubConnStateChange(sc balancer.SubConn, s connectivity.State) { - grpclog.Infof("pickfirstBalancer: HandleSubConnStateChange: %p, %v", sc, s) + if grpclog.V(2) { + grpclog.Infof("pickfirstBalancer: HandleSubConnStateChange: %p, %v", sc, s) + } if b.sc != sc { - grpclog.Infof("pickfirstBalancer: ignored state change because sc is not recognized") + if grpclog.V(2) { + grpclog.Infof("pickfirstBalancer: ignored state change because sc is not recognized") + } return } if s == connectivity.Shutdown { diff --git a/vendor/google.golang.org/grpc/preloader.go b/vendor/google.golang.org/grpc/preloader.go new file mode 100644 index 00000000000..76acbbcc93b --- /dev/null +++ b/vendor/google.golang.org/grpc/preloader.go @@ -0,0 +1,64 @@ +/* + * + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpc + +import ( + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// PreparedMsg is responsible for creating a Marshalled and Compressed object. +// +// This API is EXPERIMENTAL. +type PreparedMsg struct { + // Struct for preparing msg before sending them + encodedData []byte + hdr []byte + payload []byte +} + +// Encode marshalls and compresses the message using the codec and compressor for the stream. +func (p *PreparedMsg) Encode(s Stream, msg interface{}) error { + ctx := s.Context() + rpcInfo, ok := rpcInfoFromContext(ctx) + if !ok { + return status.Errorf(codes.Internal, "grpc: unable to get rpcInfo") + } + + // check if the context has the relevant information to prepareMsg + if rpcInfo.preloaderInfo == nil { + return status.Errorf(codes.Internal, "grpc: rpcInfo.preloaderInfo is nil") + } + if rpcInfo.preloaderInfo.codec == nil { + return status.Errorf(codes.Internal, "grpc: rpcInfo.preloaderInfo.codec is nil") + } + + // prepare the msg + data, err := encode(rpcInfo.preloaderInfo.codec, msg) + if err != nil { + return err + } + p.encodedData = data + compData, err := compress(data, rpcInfo.preloaderInfo.cp, rpcInfo.preloaderInfo.comp) + if err != nil { + return err + } + p.hdr, p.payload = msgHeader(data, compData) + return nil +} diff --git a/vendor/google.golang.org/grpc/proxy.go b/vendor/google.golang.org/grpc/proxy.go index 2d40236e218..f8f69bfb70f 100644 --- a/vendor/google.golang.org/grpc/proxy.go +++ b/vendor/google.golang.org/grpc/proxy.go @@ -20,6 +20,8 @@ package grpc import ( "bufio" + "context" + "encoding/base64" "errors" "fmt" "io" @@ -27,10 +29,10 @@ import ( "net/http" "net/http/httputil" "net/url" - - "golang.org/x/net/context" ) +const proxyAuthHeaderKey = "Proxy-Authorization" + var ( // errDisabled indicates that proxy is disabled for the address. errDisabled = errors.New("proxy is disabled for the address") @@ -38,7 +40,7 @@ var ( httpProxyFromEnvironment = http.ProxyFromEnvironment ) -func mapAddress(ctx context.Context, address string) (string, error) { +func mapAddress(ctx context.Context, address string) (*url.URL, error) { req := &http.Request{ URL: &url.URL{ Scheme: "https", @@ -47,12 +49,12 @@ func mapAddress(ctx context.Context, address string) (string, error) { } url, err := httpProxyFromEnvironment(req) if err != nil { - return "", err + return nil, err } if url == nil { - return "", errDisabled + return nil, errDisabled } - return url.Host, nil + return url, nil } // To read a response from a net.Conn, http.ReadResponse() takes a bufio.Reader. @@ -69,18 +71,28 @@ func (c *bufConn) Read(b []byte) (int, error) { return c.r.Read(b) } -func doHTTPConnectHandshake(ctx context.Context, conn net.Conn, addr string) (_ net.Conn, err error) { +func basicAuth(username, password string) string { + auth := username + ":" + password + return base64.StdEncoding.EncodeToString([]byte(auth)) +} + +func doHTTPConnectHandshake(ctx context.Context, conn net.Conn, backendAddr string, proxyURL *url.URL) (_ net.Conn, err error) { defer func() { if err != nil { conn.Close() } }() - req := (&http.Request{ + req := &http.Request{ Method: http.MethodConnect, - URL: &url.URL{Host: addr}, + URL: &url.URL{Host: backendAddr}, Header: map[string][]string{"User-Agent": {grpcUA}}, - }) + } + if t := proxyURL.User; t != nil { + u := t.Username() + p, _ := t.Password() + req.Header.Add(proxyAuthHeaderKey, "Basic "+basicAuth(u, p)) + } if err := sendHTTPRequest(ctx, req, conn); err != nil { return nil, fmt.Errorf("failed to write the HTTP request: %v", err) @@ -108,23 +120,33 @@ func doHTTPConnectHandshake(ctx context.Context, conn net.Conn, addr string) (_ // provided dialer, does HTTP CONNECT handshake and returns the connection. func newProxyDialer(dialer func(context.Context, string) (net.Conn, error)) func(context.Context, string) (net.Conn, error) { return func(ctx context.Context, addr string) (conn net.Conn, err error) { - var skipHandshake bool - newAddr, err := mapAddress(ctx, addr) + var newAddr string + proxyURL, err := mapAddress(ctx, addr) if err != nil { if err != errDisabled { return nil, err } - skipHandshake = true newAddr = addr + } else { + newAddr = proxyURL.Host } conn, err = dialer(ctx, newAddr) if err != nil { return } - if !skipHandshake { - conn, err = doHTTPConnectHandshake(ctx, conn, addr) + if proxyURL != nil { + // proxy is disabled if proxyURL is nil. + conn, err = doHTTPConnectHandshake(ctx, conn, addr, proxyURL) } return } } + +func sendHTTPRequest(ctx context.Context, req *http.Request, conn net.Conn) error { + req = req.WithContext(ctx) + if err := req.Write(conn); err != nil { + return fmt.Errorf("failed to write the HTTP request: %v", err) + } + return nil +} diff --git a/vendor/google.golang.org/grpc/resolver/dns/dns_resolver.go b/vendor/google.golang.org/grpc/resolver/dns/dns_resolver.go index 084bdbfe605..297492e87af 100644 --- a/vendor/google.golang.org/grpc/resolver/dns/dns_resolver.go +++ b/vendor/google.golang.org/grpc/resolver/dns/dns_resolver.go @@ -1,6 +1,6 @@ /* * - * Copyright 2017 gRPC authors. + * Copyright 2018 gRPC authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,6 +21,7 @@ package dns import ( + "context" "encoding/json" "errors" "fmt" @@ -31,7 +32,6 @@ import ( "sync" "time" - "golang.org/x/net/context" "google.golang.org/grpc/grpclog" "google.golang.org/grpc/internal/backoff" "google.golang.org/grpc/internal/grpcrand" @@ -43,9 +43,12 @@ func init() { } const ( - defaultPort = "443" - defaultFreq = time.Minute * 30 - golang = "GO" + defaultPort = "443" + defaultFreq = time.Minute * 30 + defaultDNSSvrPort = "53" + golang = "GO" + // txtPrefix is the prefix string to be prepended to the host name for txt record lookup. + txtPrefix = "_grpc_config." // In DNS, service config is encoded in a TXT record via the mechanism // described in RFC-1464 using the attribute name grpc_config. txtAttribute = "grpc_config=" @@ -61,6 +64,34 @@ var ( errEndsWithColon = errors.New("dns resolver: missing port after port-separator colon") ) +var ( + defaultResolver netResolver = net.DefaultResolver + // To prevent excessive re-resolution, we enforce a rate limit on DNS + // resolution requests. + minDNSResRate = 30 * time.Second +) + +var customAuthorityDialler = func(authority string) func(ctx context.Context, network, address string) (net.Conn, error) { + return func(ctx context.Context, network, address string) (net.Conn, error) { + var dialer net.Dialer + return dialer.DialContext(ctx, network, authority) + } +} + +var customAuthorityResolver = func(authority string) (netResolver, error) { + host, port, err := parseTarget(authority, defaultDNSSvrPort) + if err != nil { + return nil, err + } + + authorityWithPort := net.JoinHostPort(host, port) + + return &net.Resolver{ + PreferGo: true, + Dial: customAuthorityDialler(authorityWithPort), + }, nil +} + // NewBuilder creates a dnsBuilder which is used to factory DNS resolvers. func NewBuilder() resolver.Builder { return &dnsBuilder{minFreq: defaultFreq} @@ -73,10 +104,7 @@ type dnsBuilder struct { // Build creates and starts a DNS resolver that watches the name resolution of the target. func (b *dnsBuilder) Build(target resolver.Target, cc resolver.ClientConn, opts resolver.BuildOption) (resolver.Resolver, error) { - if target.Authority != "" { - return nil, fmt.Errorf("Default DNS resolver does not support custom DNS server") - } - host, port, err := parseTarget(target.Endpoint) + host, port, err := parseTarget(target.Endpoint, defaultPort) if err != nil { return nil, err } @@ -111,6 +139,15 @@ func (b *dnsBuilder) Build(target resolver.Target, cc resolver.ClientConn, opts disableServiceConfig: opts.DisableServiceConfig, } + if target.Authority == "" { + d.resolver = defaultResolver + } else { + d.resolver, err = customAuthorityResolver(target.Authority) + if err != nil { + return nil, err + } + } + d.wg.Add(1) go d.watcher() return d, nil @@ -121,6 +158,12 @@ func (b *dnsBuilder) Scheme() string { return "dns" } +type netResolver interface { + LookupHost(ctx context.Context, host string) (addrs []string, err error) + LookupSRV(ctx context.Context, service, proto, name string) (cname string, addrs []*net.SRV, err error) + LookupTXT(ctx context.Context, name string) (txts []string, err error) +} + // ipResolver watches for the name resolution update for an IP address. type ipResolver struct { cc resolver.ClientConn @@ -161,6 +204,7 @@ type dnsResolver struct { retryCount int host string port string + resolver netResolver ctx context.Context cancel context.CancelFunc cc resolver.ClientConn @@ -200,7 +244,13 @@ func (d *dnsResolver) watcher() { return case <-d.t.C: case <-d.rn: + if !d.t.Stop() { + // Before resetting a timer, it should be stopped to prevent racing with + // reads on it's channel. + <-d.t.C + } } + result, sc := d.lookup() // Next lookup should happen within an interval defined by d.freq. It may be // more often due to exponential retry on empty address list. @@ -213,18 +263,28 @@ func (d *dnsResolver) watcher() { } d.cc.NewServiceConfig(sc) d.cc.NewAddress(result) + + // Sleep to prevent excessive re-resolutions. Incoming resolution requests + // will be queued in d.rn. + t := time.NewTimer(minDNSResRate) + select { + case <-t.C: + case <-d.ctx.Done(): + t.Stop() + return + } } } func (d *dnsResolver) lookupSRV() []resolver.Address { var newAddrs []resolver.Address - _, srvs, err := lookupSRV(d.ctx, "grpclb", "tcp", d.host) + _, srvs, err := d.resolver.LookupSRV(d.ctx, "grpclb", "tcp", d.host) if err != nil { grpclog.Infof("grpc: failed dns SRV record lookup due to %v.\n", err) return nil } for _, s := range srvs { - lbAddrs, err := lookupHost(d.ctx, s.Target) + lbAddrs, err := d.resolver.LookupHost(d.ctx, s.Target) if err != nil { grpclog.Infof("grpc: failed load balancer address dns lookup due to %v.\n", err) continue @@ -243,7 +303,7 @@ func (d *dnsResolver) lookupSRV() []resolver.Address { } func (d *dnsResolver) lookupTXT() string { - ss, err := lookupTXT(d.ctx, d.host) + ss, err := d.resolver.LookupTXT(d.ctx, txtPrefix+d.host) if err != nil { grpclog.Infof("grpc: failed dns TXT record lookup due to %v.\n", err) return "" @@ -263,7 +323,7 @@ func (d *dnsResolver) lookupTXT() string { func (d *dnsResolver) lookupHost() []resolver.Address { var newAddrs []resolver.Address - addrs, err := lookupHost(d.ctx, d.host) + addrs, err := d.resolver.LookupHost(d.ctx, d.host) if err != nil { grpclog.Warningf("grpc: failed dns A record lookup due to %v.\n", err) return nil @@ -305,16 +365,16 @@ func formatIP(addr string) (addrIP string, ok bool) { return "[" + addr + "]", true } -// parseTarget takes the user input target string, returns formatted host and port info. +// parseTarget takes the user input target string and default port, returns formatted host and port info. // If target doesn't specify a port, set the port to be the defaultPort. -// If target is in IPv6 format and host-name is enclosed in sqarue brackets, brackets -// are strippd when setting the host. +// If target is in IPv6 format and host-name is enclosed in square brackets, brackets +// are stripped when setting the host. // examples: -// target: "www.google.com" returns host: "www.google.com", port: "443" -// target: "ipv4-host:80" returns host: "ipv4-host", port: "80" -// target: "[ipv6-host]" returns host: "ipv6-host", port: "443" -// target: ":80" returns host: "localhost", port: "80" -func parseTarget(target string) (host, port string, err error) { +// target: "www.google.com" defaultPort: "443" returns host: "www.google.com", port: "443" +// target: "ipv4-host:80" defaultPort: "443" returns host: "ipv4-host", port: "80" +// target: "[ipv6-host]" defaultPort: "443" returns host: "ipv6-host", port: "443" +// target: ":80" defaultPort: "443" returns host: "localhost", port: "80" +func parseTarget(target, defaultPort string) (host, port string, err error) { if target == "" { return "", "", errMissingAddr } diff --git a/vendor/google.golang.org/grpc/resolver/dns/go17.go b/vendor/google.golang.org/grpc/resolver/dns/go17.go deleted file mode 100644 index b466bc8f6d4..00000000000 --- a/vendor/google.golang.org/grpc/resolver/dns/go17.go +++ /dev/null @@ -1,35 +0,0 @@ -// +build go1.6, !go1.8 - -/* - * - * Copyright 2017 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package dns - -import ( - "net" - - "golang.org/x/net/context" -) - -var ( - lookupHost = func(ctx context.Context, host string) ([]string, error) { return net.LookupHost(host) } - lookupSRV = func(ctx context.Context, service, proto, name string) (string, []*net.SRV, error) { - return net.LookupSRV(service, proto, name) - } - lookupTXT = func(ctx context.Context, name string) ([]string, error) { return net.LookupTXT(name) } -) diff --git a/vendor/google.golang.org/grpc/resolver/dns/go18.go b/vendor/google.golang.org/grpc/resolver/dns/go18.go deleted file mode 100644 index fa34f14cad4..00000000000 --- a/vendor/google.golang.org/grpc/resolver/dns/go18.go +++ /dev/null @@ -1,29 +0,0 @@ -// +build go1.8 - -/* - * - * Copyright 2017 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package dns - -import "net" - -var ( - lookupHost = net.DefaultResolver.LookupHost - lookupSRV = net.DefaultResolver.LookupSRV - lookupTXT = net.DefaultResolver.LookupTXT -) diff --git a/vendor/google.golang.org/grpc/resolver/passthrough/passthrough.go b/vendor/google.golang.org/grpc/resolver/passthrough/passthrough.go index b76010d74d1..893d5d12cb0 100644 --- a/vendor/google.golang.org/grpc/resolver/passthrough/passthrough.go +++ b/vendor/google.golang.org/grpc/resolver/passthrough/passthrough.go @@ -45,7 +45,7 @@ type passthroughResolver struct { } func (r *passthroughResolver) start() { - r.cc.NewAddress([]resolver.Address{{Addr: r.target.Endpoint}}) + r.cc.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: r.target.Endpoint}}}) } func (*passthroughResolver) ResolveNow(o resolver.ResolveNowOption) {} diff --git a/vendor/google.golang.org/grpc/resolver/resolver.go b/vendor/google.golang.org/grpc/resolver/resolver.go index 145cf477edb..e83da346a5c 100644 --- a/vendor/google.golang.org/grpc/resolver/resolver.go +++ b/vendor/google.golang.org/grpc/resolver/resolver.go @@ -20,6 +20,10 @@ // All APIs in this package are experimental. package resolver +import ( + "google.golang.org/grpc/serviceconfig" +) + var ( // m is a map from scheme to resolver builder. m = make(map[string]Builder) @@ -98,6 +102,16 @@ type BuildOption struct { DisableServiceConfig bool } +// State contains the current Resolver state relevant to the ClientConn. +type State struct { + Addresses []Address // Resolved addresses for the target + // ServiceConfig is the parsed service config; obtained from + // serviceconfig.Parse. + ServiceConfig serviceconfig.Config + + // TODO: add Err error +} + // ClientConn contains the callbacks for resolver to notify any updates // to the gRPC ClientConn. // @@ -106,17 +120,38 @@ type BuildOption struct { // testing, the new implementation should embed this interface. This allows // gRPC to add new methods to this interface. type ClientConn interface { + // UpdateState updates the state of the ClientConn appropriately. + UpdateState(State) // NewAddress is called by resolver to notify ClientConn a new list // of resolved addresses. // The address list should be the complete list of resolved addresses. + // + // Deprecated: Use UpdateState instead. NewAddress(addresses []Address) // NewServiceConfig is called by resolver to notify ClientConn a new // service config. The service config should be provided as a json string. + // + // Deprecated: Use UpdateState instead. NewServiceConfig(serviceConfig string) } // Target represents a target for gRPC, as specified in: // https://github.com/grpc/grpc/blob/master/doc/naming.md. +// It is parsed from the target string that gets passed into Dial or DialContext by the user. And +// grpc passes it to the resolver and the balancer. +// +// If the target follows the naming spec, and the parsed scheme is registered with grpc, we will +// parse the target string according to the spec. e.g. "dns://some_authority/foo.bar" will be parsed +// into &Target{Scheme: "dns", Authority: "some_authority", Endpoint: "foo.bar"} +// +// If the target does not contain a scheme, we will apply the default scheme, and set the Target to +// be the full target string. e.g. "foo.bar" will be parsed into +// &Target{Scheme: resolver.GetDefaultScheme(), Endpoint: "foo.bar"}. +// +// If the parsed scheme is not registered (i.e. no corresponding resolver available to resolve the +// endpoint), we set the Scheme to be the default scheme, and set the Endpoint to be the full target +// string. e.g. target string "unknown_scheme://authority/endpoint" will be parsed into +// &Target{Scheme: resolver.GetDefaultScheme(), Endpoint: "unknown_scheme://authority/endpoint"}. type Target struct { Scheme string Authority string diff --git a/vendor/google.golang.org/grpc/resolver_conn_wrapper.go b/vendor/google.golang.org/grpc/resolver_conn_wrapper.go index 494d6931ebb..6934905b0f6 100644 --- a/vendor/google.golang.org/grpc/resolver_conn_wrapper.go +++ b/vendor/google.golang.org/grpc/resolver_conn_wrapper.go @@ -21,8 +21,10 @@ package grpc import ( "fmt" "strings" + "sync/atomic" "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/internal/channelz" "google.golang.org/grpc/resolver" ) @@ -33,11 +35,12 @@ type ccResolverWrapper struct { resolver resolver.Resolver addrCh chan []resolver.Address scCh chan string - done chan struct{} + done uint32 // accessed atomically; set to 1 when closed. + curState resolver.State } // split2 returns the values from strings.SplitN(s, sep, 2). -// If sep is not found, it returns ("", s, false) instead. +// If sep is not found, it returns ("", "", false) instead. func split2(s, sep string) (string, string, bool) { spl := strings.SplitN(s, sep, 2) if len(spl) < 2 { @@ -80,7 +83,6 @@ func newCCResolverWrapper(cc *ClientConn) (*ccResolverWrapper, error) { cc: cc, addrCh: make(chan []resolver.Address, 1), scCh: make(chan string, 1), - done: make(chan struct{}), } var err error @@ -91,68 +93,76 @@ func newCCResolverWrapper(cc *ClientConn) (*ccResolverWrapper, error) { return ccr, nil } -func (ccr *ccResolverWrapper) start() { - go ccr.watcher() -} - -// watcher processes address updates and service config updates sequentially. -// Otherwise, we need to resolve possible races between address and service -// config (e.g. they specify different balancer types). -func (ccr *ccResolverWrapper) watcher() { - for { - select { - case <-ccr.done: - return - default: - } - - select { - case addrs := <-ccr.addrCh: - select { - case <-ccr.done: - return - default: - } - grpclog.Infof("ccResolverWrapper: sending new addresses to cc: %v", addrs) - ccr.cc.handleResolvedAddrs(addrs, nil) - case sc := <-ccr.scCh: - select { - case <-ccr.done: - return - default: - } - grpclog.Infof("ccResolverWrapper: got new service config: %v", sc) - ccr.cc.handleServiceConfig(sc) - case <-ccr.done: - return - } - } -} - func (ccr *ccResolverWrapper) resolveNow(o resolver.ResolveNowOption) { ccr.resolver.ResolveNow(o) } func (ccr *ccResolverWrapper) close() { ccr.resolver.Close() - close(ccr.done) + atomic.StoreUint32(&ccr.done, 1) } -// NewAddress is called by the resolver implemenetion to send addresses to gRPC. -func (ccr *ccResolverWrapper) NewAddress(addrs []resolver.Address) { - select { - case <-ccr.addrCh: - default: +func (ccr *ccResolverWrapper) isDone() bool { + return atomic.LoadUint32(&ccr.done) == 1 +} + +func (ccr *ccResolverWrapper) UpdateState(s resolver.State) { + if ccr.isDone() { + return } - ccr.addrCh <- addrs + grpclog.Infof("ccResolverWrapper: sending update to cc: %v", s) + if channelz.IsOn() { + ccr.addChannelzTraceEvent(s) + } + ccr.cc.updateResolverState(s) + ccr.curState = s } -// NewServiceConfig is called by the resolver implemenetion to send service +// NewAddress is called by the resolver implementation to send addresses to gRPC. +func (ccr *ccResolverWrapper) NewAddress(addrs []resolver.Address) { + if ccr.isDone() { + return + } + grpclog.Infof("ccResolverWrapper: sending new addresses to cc: %v", addrs) + if channelz.IsOn() { + ccr.addChannelzTraceEvent(resolver.State{Addresses: addrs, ServiceConfig: ccr.curState.ServiceConfig}) + } + ccr.curState.Addresses = addrs + ccr.cc.updateResolverState(ccr.curState) +} + +// NewServiceConfig is called by the resolver implementation to send service // configs to gRPC. func (ccr *ccResolverWrapper) NewServiceConfig(sc string) { - select { - case <-ccr.scCh: - default: + if ccr.isDone() { + return } - ccr.scCh <- sc + grpclog.Infof("ccResolverWrapper: got new service config: %v", sc) + c, err := parseServiceConfig(sc) + if err != nil { + return + } + if channelz.IsOn() { + ccr.addChannelzTraceEvent(resolver.State{Addresses: ccr.curState.Addresses, ServiceConfig: c}) + } + ccr.curState.ServiceConfig = c + ccr.cc.updateResolverState(ccr.curState) +} + +func (ccr *ccResolverWrapper) addChannelzTraceEvent(s resolver.State) { + var updates []string + oldSC, oldOK := ccr.curState.ServiceConfig.(*ServiceConfig) + newSC, newOK := s.ServiceConfig.(*ServiceConfig) + if oldOK != newOK || (oldOK && newOK && oldSC.rawJSONString != newSC.rawJSONString) { + updates = append(updates, "service config updated") + } + if len(ccr.curState.Addresses) > 0 && len(s.Addresses) == 0 { + updates = append(updates, "resolver returned an empty address list") + } else if len(ccr.curState.Addresses) == 0 && len(s.Addresses) > 0 { + updates = append(updates, "resolver returned new addresses") + } + channelz.AddTraceEvent(ccr.cc.channelzID, &channelz.TraceEventDesc{ + Desc: fmt.Sprintf("Resolver state updated: %+v (%v)", s, strings.Join(updates, "; ")), + Severity: channelz.CtINFO, + }) } diff --git a/vendor/google.golang.org/grpc/rpc_util.go b/vendor/google.golang.org/grpc/rpc_util.go index 61342c9cd88..088c3f1b252 100644 --- a/vendor/google.golang.org/grpc/rpc_util.go +++ b/vendor/google.golang.org/grpc/rpc_util.go @@ -21,6 +21,7 @@ package grpc import ( "bytes" "compress/gzip" + "context" "encoding/binary" "fmt" "io" @@ -31,7 +32,6 @@ import ( "sync" "time" - "golang.org/x/net/context" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials" "google.golang.org/grpc/encoding" @@ -155,14 +155,12 @@ func (d *gzipDecompressor) Type() string { type callInfo struct { compressorType string failFast bool - stream *clientStream - traceInfo traceInfo // in trace.go + stream ClientStream maxReceiveMessageSize *int maxSendMessageSize *int creds credentials.PerRPCCredentials contentSubtype string codec baseCodec - disableRetry bool maxRetryRPCBufferSize int } @@ -255,8 +253,8 @@ func (o PeerCallOption) after(c *callInfo) { } } -// FailFast configures the action to take when an RPC is attempted on broken -// connections or unreachable servers. If failFast is true, the RPC will fail +// WaitForReady configures the action to take when an RPC is attempted on broken +// connections or unreachable servers. If waitForReady is false, the RPC will fail // immediately. Otherwise, the RPC client will block the call until a // connection is available (or the call is canceled or times out) and will // retry the call if it fails due to a transient error. gRPC will not retry if @@ -264,7 +262,14 @@ func (o PeerCallOption) after(c *callInfo) { // the data. Please refer to // https://github.com/grpc/grpc/blob/master/doc/wait-for-ready.md. // -// By default, RPCs are "Fail Fast". +// By default, RPCs don't "wait for ready". +func WaitForReady(waitForReady bool) CallOption { + return FailFastCallOption{FailFast: !waitForReady} +} + +// FailFast is the opposite of WaitForReady. +// +// Deprecated: use WaitForReady. func FailFast(failFast bool) CallOption { return FailFastCallOption{FailFast: failFast} } @@ -365,13 +370,13 @@ func (o CompressorCallOption) after(c *callInfo) {} // https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md#requests for // more details. // -// If CallCustomCodec is not also used, the content-subtype will be used to -// look up the Codec to use in the registry controlled by RegisterCodec. See -// the documentation on RegisterCodec for details on registration. The lookup -// of content-subtype is case-insensitive. If no such Codec is found, the call +// If ForceCodec is not also used, the content-subtype will be used to look up +// the Codec to use in the registry controlled by RegisterCodec. See the +// documentation on RegisterCodec for details on registration. The lookup of +// content-subtype is case-insensitive. If no such Codec is found, the call // will result in an error with code codes.Internal. // -// If CallCustomCodec is also used, that Codec will be used for all request and +// If ForceCodec is also used, that Codec will be used for all request and // response messages, with the content-subtype set to the given contentSubtype // here for requests. func CallContentSubtype(contentSubtype string) CallOption { @@ -391,7 +396,7 @@ func (o ContentSubtypeCallOption) before(c *callInfo) error { } func (o ContentSubtypeCallOption) after(c *callInfo) {} -// CallCustomCodec returns a CallOption that will set the given Codec to be +// ForceCodec returns a CallOption that will set the given Codec to be // used for all request and response messages for a call. The result of calling // String() will be used as the content-subtype in a case-insensitive manner. // @@ -403,12 +408,37 @@ func (o ContentSubtypeCallOption) after(c *callInfo) {} // // This function is provided for advanced users; prefer to use only // CallContentSubtype to select a registered codec instead. +// +// This is an EXPERIMENTAL API. +func ForceCodec(codec encoding.Codec) CallOption { + return ForceCodecCallOption{Codec: codec} +} + +// ForceCodecCallOption is a CallOption that indicates the codec used for +// marshaling messages. +// +// This is an EXPERIMENTAL API. +type ForceCodecCallOption struct { + Codec encoding.Codec +} + +func (o ForceCodecCallOption) before(c *callInfo) error { + c.codec = o.Codec + return nil +} +func (o ForceCodecCallOption) after(c *callInfo) {} + +// CallCustomCodec behaves like ForceCodec, but accepts a grpc.Codec instead of +// an encoding.Codec. +// +// Deprecated: use ForceCodec instead. func CallCustomCodec(codec Codec) CallOption { return CustomCodecCallOption{Codec: codec} } // CustomCodecCallOption is a CallOption that indicates the codec used for // marshaling messages. +// // This is an EXPERIMENTAL API. type CustomCodecCallOption struct { Codec Codec @@ -533,7 +563,10 @@ func compress(in []byte, cp Compressor, compressor encoding.Compressor) ([]byte, } cbuf := &bytes.Buffer{} if compressor != nil { - z, _ := compressor.Compress(cbuf) + z, err := compressor.Compress(cbuf) + if err != nil { + return nil, wrapErr(err) + } if _, err := z.Write(in); err != nil { return nil, wrapErr(err) } @@ -597,20 +630,22 @@ func checkRecvPayload(pf payloadFormat, recvCompress string, haveCompressor bool return nil } -// For the two compressor parameters, both should not be set, but if they are, -// dc takes precedence over compressor. -// TODO(dfawley): wrap the old compressor/decompressor using the new API? -func recv(p *parser, c baseCodec, s *transport.Stream, dc Decompressor, m interface{}, maxReceiveMessageSize int, inPayload *stats.InPayload, compressor encoding.Compressor) error { +type payloadInfo struct { + wireLength int // The compressed length got from wire. + uncompressedBytes []byte +} + +func recvAndDecompress(p *parser, s *transport.Stream, dc Decompressor, maxReceiveMessageSize int, payInfo *payloadInfo, compressor encoding.Compressor) ([]byte, error) { pf, d, err := p.recvMsg(maxReceiveMessageSize) if err != nil { - return err + return nil, err } - if inPayload != nil { - inPayload.WireLength = len(d) + if payInfo != nil { + payInfo.wireLength = len(d) } if st := checkRecvPayload(pf, s.RecvCompress(), compressor != nil || dc != nil); st != nil { - return st.Err() + return nil, st.Err() } if pf == compressionMade { @@ -619,45 +654,74 @@ func recv(p *parser, c baseCodec, s *transport.Stream, dc Decompressor, m interf if dc != nil { d, err = dc.Do(bytes.NewReader(d)) if err != nil { - return status.Errorf(codes.Internal, "grpc: failed to decompress the received message %v", err) + return nil, status.Errorf(codes.Internal, "grpc: failed to decompress the received message %v", err) } } else { dcReader, err := compressor.Decompress(bytes.NewReader(d)) if err != nil { - return status.Errorf(codes.Internal, "grpc: failed to decompress the received message %v", err) + return nil, status.Errorf(codes.Internal, "grpc: failed to decompress the received message %v", err) } - d, err = ioutil.ReadAll(dcReader) + // Read from LimitReader with limit max+1. So if the underlying + // reader is over limit, the result will be bigger than max. + d, err = ioutil.ReadAll(io.LimitReader(dcReader, int64(maxReceiveMessageSize)+1)) if err != nil { - return status.Errorf(codes.Internal, "grpc: failed to decompress the received message %v", err) + return nil, status.Errorf(codes.Internal, "grpc: failed to decompress the received message %v", err) } } } if len(d) > maxReceiveMessageSize { // TODO: Revisit the error code. Currently keep it consistent with java // implementation. - return status.Errorf(codes.ResourceExhausted, "grpc: received message larger than max (%d vs. %d)", len(d), maxReceiveMessageSize) + return nil, status.Errorf(codes.ResourceExhausted, "grpc: received message larger than max (%d vs. %d)", len(d), maxReceiveMessageSize) + } + return d, nil +} + +// For the two compressor parameters, both should not be set, but if they are, +// dc takes precedence over compressor. +// TODO(dfawley): wrap the old compressor/decompressor using the new API? +func recv(p *parser, c baseCodec, s *transport.Stream, dc Decompressor, m interface{}, maxReceiveMessageSize int, payInfo *payloadInfo, compressor encoding.Compressor) error { + d, err := recvAndDecompress(p, s, dc, maxReceiveMessageSize, payInfo, compressor) + if err != nil { + return err } if err := c.Unmarshal(d, m); err != nil { return status.Errorf(codes.Internal, "grpc: failed to unmarshal the received message %v", err) } - if inPayload != nil { - inPayload.RecvTime = time.Now() - inPayload.Payload = m - // TODO truncate large payload. - inPayload.Data = d - inPayload.Length = len(d) + if payInfo != nil { + payInfo.uncompressedBytes = d } return nil } +// Information about RPC type rpcInfo struct { - failfast bool + failfast bool + preloaderInfo *compressorInfo +} + +// Information about Preloader +// Responsible for storing codec, and compressors +// If stream (s) has context s.Context which stores rpcInfo that has non nil +// pointers to codec, and compressors, then we can use preparedMsg for Async message prep +// and reuse marshalled bytes +type compressorInfo struct { + codec baseCodec + cp Compressor + comp encoding.Compressor } type rpcInfoContextKey struct{} -func newContextWithRPCInfo(ctx context.Context, failfast bool) context.Context { - return context.WithValue(ctx, rpcInfoContextKey{}, &rpcInfo{failfast: failfast}) +func newContextWithRPCInfo(ctx context.Context, failfast bool, codec baseCodec, cp Compressor, comp encoding.Compressor) context.Context { + return context.WithValue(ctx, rpcInfoContextKey{}, &rpcInfo{ + failfast: failfast, + preloaderInfo: &compressorInfo{ + codec: codec, + cp: cp, + comp: comp, + }, + }) } func rpcInfoFromContext(ctx context.Context) (s *rpcInfo, ok bool) { @@ -668,23 +732,17 @@ func rpcInfoFromContext(ctx context.Context) (s *rpcInfo, ok bool) { // Code returns the error code for err if it was produced by the rpc system. // Otherwise, it returns codes.Unknown. // -// Deprecated: use status.FromError and Code method instead. +// Deprecated: use status.Code instead. func Code(err error) codes.Code { - if s, ok := status.FromError(err); ok { - return s.Code() - } - return codes.Unknown + return status.Code(err) } // ErrorDesc returns the error description of err if it was produced by the rpc system. // Otherwise, it returns err.Error() or empty string when err is nil. // -// Deprecated: use status.FromError and Message method instead. +// Deprecated: use status.Convert and Message method instead. func ErrorDesc(err error) string { - if s, ok := status.FromError(err); ok { - return s.Message() - } - return err.Error() + return status.Convert(err).Message() } // Errorf returns an error containing an error code and a description; @@ -695,6 +753,31 @@ func Errorf(c codes.Code, format string, a ...interface{}) error { return status.Errorf(c, format, a...) } +// toRPCErr converts an error into an error from the status package. +func toRPCErr(err error) error { + if err == nil || err == io.EOF { + return err + } + if err == io.ErrUnexpectedEOF { + return status.Error(codes.Internal, err.Error()) + } + if _, ok := status.FromError(err); ok { + return err + } + switch e := err.(type) { + case transport.ConnectionError: + return status.Error(codes.Unavailable, e.Desc) + default: + switch err { + case context.DeadlineExceeded: + return status.Error(codes.DeadlineExceeded, err.Error()) + case context.Canceled: + return status.Error(codes.Canceled, err.Error()) + } + } + return status.Error(codes.Unknown, err.Error()) +} + // setCallInfoCodec should only be called after CallOptions have been applied. func setCallInfoCodec(c *callInfo) error { if c.codec != nil { @@ -750,6 +833,19 @@ func parseDialTarget(target string) (net string, addr string) { return net, target } +// channelzData is used to store channelz related data for ClientConn, addrConn and Server. +// These fields cannot be embedded in the original structs (e.g. ClientConn), since to do atomic +// operation on int64 variable on 32-bit machine, user is responsible to enforce memory alignment. +// Here, by grouping those int64 fields inside a struct, we are enforcing the alignment. +type channelzData struct { + callsStarted int64 + callsFailed int64 + callsSucceeded int64 + // lastCallStartedTime stores the timestamp that last call starts. It is of int64 type instead of + // time.Time since it's more costly to atomically update time.Time variable than int64 variable. + lastCallStartedTime int64 +} + // The SupportPackageIsVersion variables are referenced from generated protocol // buffer files to ensure compatibility with the gRPC version used. The latest // support package version is 5. diff --git a/vendor/google.golang.org/grpc/server.go b/vendor/google.golang.org/grpc/server.go index f5bea72383b..f064b73e555 100644 --- a/vendor/google.golang.org/grpc/server.go +++ b/vendor/google.golang.org/grpc/server.go @@ -19,7 +19,7 @@ package grpc import ( - "bytes" + "context" "errors" "fmt" "io" @@ -30,12 +30,9 @@ import ( "runtime" "strings" "sync" + "sync/atomic" "time" - "io/ioutil" - - "golang.org/x/net/context" - "golang.org/x/net/http2" "golang.org/x/net/trace" "google.golang.org/grpc/codes" @@ -43,11 +40,13 @@ import ( "google.golang.org/grpc/encoding" "google.golang.org/grpc/encoding/proto" "google.golang.org/grpc/grpclog" - "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/binarylog" "google.golang.org/grpc/internal/channelz" + "google.golang.org/grpc/internal/grpcsync" "google.golang.org/grpc/internal/transport" "google.golang.org/grpc/keepalive" "google.golang.org/grpc/metadata" + "google.golang.org/grpc/peer" "google.golang.org/grpc/stats" "google.golang.org/grpc/status" "google.golang.org/grpc/tap" @@ -58,6 +57,8 @@ const ( defaultServerMaxSendMessageSize = math.MaxInt32 ) +var statusOK = status.New(codes.OK, "") + type methodHandler func(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor UnaryServerInterceptor) (interface{}, error) // MethodDesc represents an RPC service's method specification. @@ -88,33 +89,27 @@ type service struct { // Server is a gRPC server to serve RPC requests. type Server struct { - opts options + opts serverOptions mu sync.Mutex // guards following lis map[net.Listener]bool - conns map[io.Closer]bool + conns map[transport.ServerTransport]bool serve bool drain bool cv *sync.Cond // signaled when connections close for GracefulStop m map[string]*service // service name -> service info events trace.EventLog - quit chan struct{} - done chan struct{} - quitOnce sync.Once - doneOnce sync.Once + quit *grpcsync.Event + done *grpcsync.Event channelzRemoveOnce sync.Once serveWG sync.WaitGroup // counts active Serve goroutines for GracefulStop - channelzID int64 // channelz unique identification number - czmu sync.RWMutex - callsStarted int64 - callsFailed int64 - callsSucceeded int64 - lastCallStartedTime time.Time + channelzID int64 // channelz unique identification number + czData *channelzData } -type options struct { +type serverOptions struct { creds credentials.TransportCredentials codec baseCodec cp Compressor @@ -126,7 +121,6 @@ type options struct { maxConcurrentStreams uint32 maxReceiveMessageSize int maxSendMessageSize int - useHandlerImpl bool // use http.Handler-based server unknownStreamDesc *StreamDesc keepaliveParams keepalive.ServerParameters keepalivePolicy keepalive.EnforcementPolicy @@ -138,7 +132,7 @@ type options struct { maxHeaderListSize *uint32 } -var defaultServerOptions = options{ +var defaultServerOptions = serverOptions{ maxReceiveMessageSize: defaultServerMaxReceiveMessageSize, maxSendMessageSize: defaultServerMaxSendMessageSize, connectionTimeout: 120 * time.Second, @@ -147,7 +141,33 @@ var defaultServerOptions = options{ } // A ServerOption sets options such as credentials, codec and keepalive parameters, etc. -type ServerOption func(*options) +type ServerOption interface { + apply(*serverOptions) +} + +// EmptyServerOption does not alter the server configuration. It can be embedded +// in another structure to build custom server options. +// +// This API is EXPERIMENTAL. +type EmptyServerOption struct{} + +func (EmptyServerOption) apply(*serverOptions) {} + +// funcServerOption wraps a function that modifies serverOptions into an +// implementation of the ServerOption interface. +type funcServerOption struct { + f func(*serverOptions) +} + +func (fdo *funcServerOption) apply(do *serverOptions) { + fdo.f(do) +} + +func newFuncServerOption(f func(*serverOptions)) *funcServerOption { + return &funcServerOption{ + f: f, + } +} // WriteBufferSize determines how much data can be batched before doing a write on the wire. // The corresponding memory allocation for this buffer will be twice the size to keep syscalls low. @@ -155,9 +175,9 @@ type ServerOption func(*options) // Zero will disable the write buffer such that each write will be on underlying connection. // Note: A Send call may not directly translate to a write. func WriteBufferSize(s int) ServerOption { - return func(o *options) { + return newFuncServerOption(func(o *serverOptions) { o.writeBufferSize = s - } + }) } // ReadBufferSize lets you set the size of read buffer, this determines how much data can be read at most @@ -166,48 +186,53 @@ func WriteBufferSize(s int) ServerOption { // Zero will disable read buffer for a connection so data framer can access the underlying // conn directly. func ReadBufferSize(s int) ServerOption { - return func(o *options) { + return newFuncServerOption(func(o *serverOptions) { o.readBufferSize = s - } + }) } // InitialWindowSize returns a ServerOption that sets window size for stream. // The lower bound for window size is 64K and any value smaller than that will be ignored. func InitialWindowSize(s int32) ServerOption { - return func(o *options) { + return newFuncServerOption(func(o *serverOptions) { o.initialWindowSize = s - } + }) } // InitialConnWindowSize returns a ServerOption that sets window size for a connection. // The lower bound for window size is 64K and any value smaller than that will be ignored. func InitialConnWindowSize(s int32) ServerOption { - return func(o *options) { + return newFuncServerOption(func(o *serverOptions) { o.initialConnWindowSize = s - } + }) } // KeepaliveParams returns a ServerOption that sets keepalive and max-age parameters for the server. func KeepaliveParams(kp keepalive.ServerParameters) ServerOption { - return func(o *options) { - o.keepaliveParams = kp + if kp.Time > 0 && kp.Time < time.Second { + grpclog.Warning("Adjusting keepalive ping interval to minimum period of 1s") + kp.Time = time.Second } + + return newFuncServerOption(func(o *serverOptions) { + o.keepaliveParams = kp + }) } // KeepaliveEnforcementPolicy returns a ServerOption that sets keepalive enforcement policy for the server. func KeepaliveEnforcementPolicy(kep keepalive.EnforcementPolicy) ServerOption { - return func(o *options) { + return newFuncServerOption(func(o *serverOptions) { o.keepalivePolicy = kep - } + }) } // CustomCodec returns a ServerOption that sets a codec for message marshaling and unmarshaling. // // This will override any lookups by content-subtype for Codecs registered with RegisterCodec. func CustomCodec(codec Codec) ServerOption { - return func(o *options) { + return newFuncServerOption(func(o *serverOptions) { o.codec = codec - } + }) } // RPCCompressor returns a ServerOption that sets a compressor for outbound @@ -218,9 +243,9 @@ func CustomCodec(codec Codec) ServerOption { // // Deprecated: use encoding.RegisterCompressor instead. func RPCCompressor(cp Compressor) ServerOption { - return func(o *options) { + return newFuncServerOption(func(o *serverOptions) { o.cp = cp - } + }) } // RPCDecompressor returns a ServerOption that sets a decompressor for inbound @@ -229,9 +254,9 @@ func RPCCompressor(cp Compressor) ServerOption { // // Deprecated: use encoding.RegisterCompressor instead. func RPCDecompressor(dc Decompressor) ServerOption { - return func(o *options) { + return newFuncServerOption(func(o *serverOptions) { o.dc = dc - } + }) } // MaxMsgSize returns a ServerOption to set the max message size in bytes the server can receive. @@ -245,73 +270,73 @@ func MaxMsgSize(m int) ServerOption { // MaxRecvMsgSize returns a ServerOption to set the max message size in bytes the server can receive. // If this is not set, gRPC uses the default 4MB. func MaxRecvMsgSize(m int) ServerOption { - return func(o *options) { + return newFuncServerOption(func(o *serverOptions) { o.maxReceiveMessageSize = m - } + }) } // MaxSendMsgSize returns a ServerOption to set the max message size in bytes the server can send. -// If this is not set, gRPC uses the default 4MB. +// If this is not set, gRPC uses the default `math.MaxInt32`. func MaxSendMsgSize(m int) ServerOption { - return func(o *options) { + return newFuncServerOption(func(o *serverOptions) { o.maxSendMessageSize = m - } + }) } // MaxConcurrentStreams returns a ServerOption that will apply a limit on the number // of concurrent streams to each ServerTransport. func MaxConcurrentStreams(n uint32) ServerOption { - return func(o *options) { + return newFuncServerOption(func(o *serverOptions) { o.maxConcurrentStreams = n - } + }) } // Creds returns a ServerOption that sets credentials for server connections. func Creds(c credentials.TransportCredentials) ServerOption { - return func(o *options) { + return newFuncServerOption(func(o *serverOptions) { o.creds = c - } + }) } // UnaryInterceptor returns a ServerOption that sets the UnaryServerInterceptor for the // server. Only one unary interceptor can be installed. The construction of multiple // interceptors (e.g., chaining) can be implemented at the caller. func UnaryInterceptor(i UnaryServerInterceptor) ServerOption { - return func(o *options) { + return newFuncServerOption(func(o *serverOptions) { if o.unaryInt != nil { panic("The unary server interceptor was already set and may not be reset.") } o.unaryInt = i - } + }) } // StreamInterceptor returns a ServerOption that sets the StreamServerInterceptor for the // server. Only one stream interceptor can be installed. func StreamInterceptor(i StreamServerInterceptor) ServerOption { - return func(o *options) { + return newFuncServerOption(func(o *serverOptions) { if o.streamInt != nil { panic("The stream server interceptor was already set and may not be reset.") } o.streamInt = i - } + }) } // InTapHandle returns a ServerOption that sets the tap handle for all the server // transport to be created. Only one can be installed. func InTapHandle(h tap.ServerInHandle) ServerOption { - return func(o *options) { + return newFuncServerOption(func(o *serverOptions) { if o.inTapHandle != nil { panic("The tap handle was already set and may not be reset.") } o.inTapHandle = h - } + }) } // StatsHandler returns a ServerOption that sets the stats handler for the server. func StatsHandler(h stats.Handler) ServerOption { - return func(o *options) { + return newFuncServerOption(func(o *serverOptions) { o.statsHandler = h - } + }) } // UnknownServiceHandler returns a ServerOption that allows for adding a custom @@ -321,7 +346,7 @@ func StatsHandler(h stats.Handler) ServerOption { // The handling function has full access to the Context of the request and the // stream, and the invocation bypasses interceptors. func UnknownServiceHandler(streamHandler StreamHandler) ServerOption { - return func(o *options) { + return newFuncServerOption(func(o *serverOptions) { o.unknownStreamDesc = &StreamDesc{ StreamName: "unknown_service_handler", Handler: streamHandler, @@ -329,7 +354,7 @@ func UnknownServiceHandler(streamHandler StreamHandler) ServerOption { ClientStreams: true, ServerStreams: true, } - } + }) } // ConnectionTimeout returns a ServerOption that sets the timeout for @@ -339,17 +364,17 @@ func UnknownServiceHandler(streamHandler StreamHandler) ServerOption { // // This API is EXPERIMENTAL. func ConnectionTimeout(d time.Duration) ServerOption { - return func(o *options) { + return newFuncServerOption(func(o *serverOptions) { o.connectionTimeout = d - } + }) } // MaxHeaderListSize returns a ServerOption that sets the max (uncompressed) size // of header list that the server is prepared to accept. func MaxHeaderListSize(s uint32) ServerOption { - return func(o *options) { + return newFuncServerOption(func(o *serverOptions) { o.maxHeaderListSize = &s - } + }) } // NewServer creates a gRPC server which has no service registered and has not @@ -357,15 +382,16 @@ func MaxHeaderListSize(s uint32) ServerOption { func NewServer(opt ...ServerOption) *Server { opts := defaultServerOptions for _, o := range opt { - o(&opts) + o.apply(&opts) } s := &Server{ - lis: make(map[net.Listener]bool), - opts: opts, - conns: make(map[io.Closer]bool), - m: make(map[string]*service), - quit: make(chan struct{}), - done: make(chan struct{}), + lis: make(map[net.Listener]bool), + opts: opts, + conns: make(map[transport.ServerTransport]bool), + m: make(map[string]*service), + quit: grpcsync.NewEvent(), + done: grpcsync.NewEvent(), + czData: new(channelzData), } s.cv = sync.NewCond(&s.mu) if EnableTracing { @@ -374,7 +400,7 @@ func NewServer(opt ...ServerOption) *Server { } if channelz.IsOn() { - s.channelzID = channelz.RegisterServer(s, "") + s.channelzID = channelz.RegisterServer(&channelzServer{s}, "") } return s } @@ -531,11 +557,9 @@ func (s *Server) Serve(lis net.Listener) error { s.serveWG.Add(1) defer func() { s.serveWG.Done() - select { - // Stop or GracefulStop called; block until done and return nil. - case <-s.quit: - <-s.done - default: + if s.quit.HasFired() { + // Stop or GracefulStop called; block until done and return nil. + <-s.done.Done() } }() @@ -543,7 +567,7 @@ func (s *Server) Serve(lis net.Listener) error { s.lis[ls] = true if channelz.IsOn() { - ls.channelzID = channelz.RegisterListenSocket(ls, s.channelzID, "") + ls.channelzID = channelz.RegisterListenSocket(ls, s.channelzID, lis.Addr().String()) } s.mu.Unlock() @@ -578,7 +602,7 @@ func (s *Server) Serve(lis net.Listener) error { timer := time.NewTimer(tempDelay) select { case <-timer.C: - case <-s.quit: + case <-s.quit.Done(): timer.Stop() return nil } @@ -588,10 +612,8 @@ func (s *Server) Serve(lis net.Listener) error { s.printf("done serving; Accept = %v", err) s.mu.Unlock() - select { - case <-s.quit: + if s.quit.HasFired() { return nil - default: } return err } @@ -612,50 +634,39 @@ func (s *Server) Serve(lis net.Listener) error { // handleRawConn forks a goroutine to handle a just-accepted connection that // has not had any I/O performed on it yet. func (s *Server) handleRawConn(rawConn net.Conn) { + if s.quit.HasFired() { + rawConn.Close() + return + } rawConn.SetDeadline(time.Now().Add(s.opts.connectionTimeout)) conn, authInfo, err := s.useTransportAuthenticator(rawConn) if err != nil { - s.mu.Lock() - s.errorf("ServerHandshake(%q) failed: %v", rawConn.RemoteAddr(), err) - s.mu.Unlock() - grpclog.Warningf("grpc: Server.Serve failed to complete security handshake from %q: %v", rawConn.RemoteAddr(), err) - // If serverHandshake returns ErrConnDispatched, keep rawConn open. + // ErrConnDispatched means that the connection was dispatched away from + // gRPC; those connections should be left open. if err != credentials.ErrConnDispatched { + s.mu.Lock() + s.errorf("ServerHandshake(%q) failed: %v", rawConn.RemoteAddr(), err) + s.mu.Unlock() + grpclog.Warningf("grpc: Server.Serve failed to complete security handshake from %q: %v", rawConn.RemoteAddr(), err) rawConn.Close() } rawConn.SetDeadline(time.Time{}) return } - s.mu.Lock() - if s.conns == nil { - s.mu.Unlock() - conn.Close() + // Finish handshaking (HTTP2) + st := s.newHTTP2Transport(conn, authInfo) + if st == nil { return } - s.mu.Unlock() - - var serve func() - c := conn.(io.Closer) - if s.opts.useHandlerImpl { - serve = func() { s.serveUsingHandler(conn) } - } else { - // Finish handshaking (HTTP2) - st := s.newHTTP2Transport(conn, authInfo) - if st == nil { - return - } - c = st - serve = func() { s.serveStreams(st) } - } rawConn.SetDeadline(time.Time{}) - if !s.addConn(c) { + if !s.addConn(st) { return } go func() { - serve() - s.removeConn(c) + s.serveStreams(st) + s.removeConn(st) }() } @@ -710,27 +721,6 @@ func (s *Server) serveStreams(st transport.ServerTransport) { var _ http.Handler = (*Server)(nil) -// serveUsingHandler is called from handleRawConn when s is configured -// to handle requests via the http.Handler interface. It sets up a -// net/http.Server to handle the just-accepted conn. The http.Server -// is configured to route all incoming requests (all HTTP/2 streams) -// to ServeHTTP, which creates a new ServerTransport for each stream. -// serveUsingHandler blocks until conn closes. -// -// This codepath is only used when Server.TestingUseHandlerImpl has -// been configured. This lets the end2end tests exercise the ServeHTTP -// method as one of the environment types. -// -// conn is the *tls.Conn that's already been authenticated. -func (s *Server) serveUsingHandler(conn net.Conn) { - h2s := &http2.Server{ - MaxConcurrentStreams: s.opts.maxConcurrentStreams, - } - h2s.ServeConn(conn, &http2.ServeConnOpts{ - Handler: s, - }) -} - // ServeHTTP implements the Go standard library's http.Handler // interface by responding to the gRPC request r, by looking up // the requested gRPC method in the gRPC server s. @@ -771,6 +761,9 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { // traceInfo returns a traceInfo and associates it with stream, if tracing is enabled. // If tracing is not enabled, it returns nil. func (s *Server) traceInfo(st transport.ServerTransport, stream *transport.Stream) (trInfo *traceInfo) { + if !EnableTracing { + return nil + } tr, ok := trace.FromContext(stream.Context()) if !ok { return nil @@ -778,71 +771,62 @@ func (s *Server) traceInfo(st transport.ServerTransport, stream *transport.Strea trInfo = &traceInfo{ tr: tr, + firstLine: firstLine{ + client: false, + remoteAddr: st.RemoteAddr(), + }, } - trInfo.firstLine.client = false - trInfo.firstLine.remoteAddr = st.RemoteAddr() - if dl, ok := stream.Context().Deadline(); ok { - trInfo.firstLine.deadline = dl.Sub(time.Now()) + trInfo.firstLine.deadline = time.Until(dl) } return trInfo } -func (s *Server) addConn(c io.Closer) bool { +func (s *Server) addConn(st transport.ServerTransport) bool { s.mu.Lock() defer s.mu.Unlock() if s.conns == nil { - c.Close() + st.Close() return false } if s.drain { // Transport added after we drained our existing conns: drain it // immediately. - c.(transport.ServerTransport).Drain() + st.Drain() } - s.conns[c] = true + s.conns[st] = true return true } -func (s *Server) removeConn(c io.Closer) { +func (s *Server) removeConn(st transport.ServerTransport) { s.mu.Lock() defer s.mu.Unlock() if s.conns != nil { - delete(s.conns, c) + delete(s.conns, st) s.cv.Broadcast() } } -// ChannelzMetric returns ServerInternalMetric of current server. -// This is an EXPERIMENTAL API. -func (s *Server) ChannelzMetric() *channelz.ServerInternalMetric { - s.czmu.RLock() - defer s.czmu.RUnlock() +func (s *Server) channelzMetric() *channelz.ServerInternalMetric { return &channelz.ServerInternalMetric{ - CallsStarted: s.callsStarted, - CallsSucceeded: s.callsSucceeded, - CallsFailed: s.callsFailed, - LastCallStartedTimestamp: s.lastCallStartedTime, + CallsStarted: atomic.LoadInt64(&s.czData.callsStarted), + CallsSucceeded: atomic.LoadInt64(&s.czData.callsSucceeded), + CallsFailed: atomic.LoadInt64(&s.czData.callsFailed), + LastCallStartedTimestamp: time.Unix(0, atomic.LoadInt64(&s.czData.lastCallStartedTime)), } } func (s *Server) incrCallsStarted() { - s.czmu.Lock() - s.callsStarted++ - s.lastCallStartedTime = time.Now() - s.czmu.Unlock() + atomic.AddInt64(&s.czData.callsStarted, 1) + atomic.StoreInt64(&s.czData.lastCallStartedTime, time.Now().UnixNano()) } func (s *Server) incrCallsSucceeded() { - s.czmu.Lock() - s.callsSucceeded++ - s.czmu.Unlock() + atomic.AddInt64(&s.czData.callsSucceeded, 1) } func (s *Server) incrCallsFailed() { - s.czmu.Lock() - s.callsFailed++ - s.czmu.Unlock() + atomic.AddInt64(&s.czData.callsFailed, 1) } func (s *Server) sendResponse(t transport.ServerTransport, stream *transport.Stream, msg interface{}, cp Compressor, opts *transport.Options, comp encoding.Compressor) error { @@ -899,7 +883,6 @@ func (s *Server) processUnaryRPC(t transport.ServerTransport, stream *transport. } if trInfo != nil { defer trInfo.tr.Finish() - trInfo.firstLine.client = false trInfo.tr.LazyLog(&trInfo.firstLine, false) defer func() { if err != nil && err != io.EOF { @@ -909,6 +892,30 @@ func (s *Server) processUnaryRPC(t transport.ServerTransport, stream *transport. }() } + binlog := binarylog.GetMethodLogger(stream.Method()) + if binlog != nil { + ctx := stream.Context() + md, _ := metadata.FromIncomingContext(ctx) + logEntry := &binarylog.ClientHeader{ + Header: md, + MethodName: stream.Method(), + PeerAddr: nil, + } + if deadline, ok := ctx.Deadline(); ok { + logEntry.Timeout = time.Until(deadline) + if logEntry.Timeout < 0 { + logEntry.Timeout = 0 + } + } + if a := md[":authority"]; len(a) > 0 { + logEntry.Authority = a[0] + } + if peer, ok := peer.FromContext(ctx); ok { + logEntry.PeerAddr = peer.Addr + } + binlog.Log(logEntry) + } + // comp and cp are used for compression. decomp and dc are used for // decompression. If comp and decomp are both set, they are the same; // however they are kept separate to ensure that at most one of the @@ -945,77 +952,39 @@ func (s *Server) processUnaryRPC(t transport.ServerTransport, stream *transport. } } - p := &parser{r: stream} - pf, req, err := p.recvMsg(s.opts.maxReceiveMessageSize) - if err == io.EOF { - // The entire stream is done (for unary RPC only). - return err - } - if err == io.ErrUnexpectedEOF { - err = status.Errorf(codes.Internal, io.ErrUnexpectedEOF.Error()) + var payInfo *payloadInfo + if sh != nil || binlog != nil { + payInfo = &payloadInfo{} } + d, err := recvAndDecompress(&parser{r: stream}, stream, dc, s.opts.maxReceiveMessageSize, payInfo, decomp) if err != nil { if st, ok := status.FromError(err); ok { if e := t.WriteStatus(stream, st); e != nil { grpclog.Warningf("grpc: Server.processUnaryRPC failed to write status %v", e) } - } else { - switch st := err.(type) { - case transport.ConnectionError: - // Nothing to do here. - default: - panic(fmt.Sprintf("grpc: Unexpected error (%T) from recvMsg: %v", st, st)) - } } return err } if channelz.IsOn() { t.IncrMsgRecv() } - if st := checkRecvPayload(pf, stream.RecvCompress(), dc != nil || decomp != nil); st != nil { - if e := t.WriteStatus(stream, st); e != nil { - grpclog.Warningf("grpc: Server.processUnaryRPC failed to write status %v", e) - } - return st.Err() - } - var inPayload *stats.InPayload - if sh != nil { - inPayload = &stats.InPayload{ - RecvTime: time.Now(), - } - } df := func(v interface{}) error { - if inPayload != nil { - inPayload.WireLength = len(req) - } - if pf == compressionMade { - var err error - if dc != nil { - req, err = dc.Do(bytes.NewReader(req)) - if err != nil { - return status.Errorf(codes.Internal, err.Error()) - } - } else { - tmp, _ := decomp.Decompress(bytes.NewReader(req)) - req, err = ioutil.ReadAll(tmp) - if err != nil { - return status.Errorf(codes.Internal, "grpc: failed to decompress the received message %v", err) - } - } - } - if len(req) > s.opts.maxReceiveMessageSize { - // TODO: Revisit the error code. Currently keep it consistent with - // java implementation. - return status.Errorf(codes.ResourceExhausted, "grpc: received message larger than max (%d vs. %d)", len(req), s.opts.maxReceiveMessageSize) - } - if err := s.getCodec(stream.ContentSubtype()).Unmarshal(req, v); err != nil { + if err := s.getCodec(stream.ContentSubtype()).Unmarshal(d, v); err != nil { return status.Errorf(codes.Internal, "grpc: error unmarshalling request: %v", err) } - if inPayload != nil { - inPayload.Payload = v - inPayload.Data = req - inPayload.Length = len(req) - sh.HandleRPC(stream.Context(), inPayload) + if sh != nil { + sh.HandleRPC(stream.Context(), &stats.InPayload{ + RecvTime: time.Now(), + Payload: v, + WireLength: payInfo.wireLength, + Data: d, + Length: len(d), + }) + } + if binlog != nil { + binlog.Log(&binarylog.ClientMessage{ + Message: d, + }) } if trInfo != nil { trInfo.tr.LazyLog(&payload{sent: false, msg: v}, true) @@ -1038,6 +1007,19 @@ func (s *Server) processUnaryRPC(t transport.ServerTransport, stream *transport. if e := t.WriteStatus(stream, appStatus); e != nil { grpclog.Warningf("grpc: Server.processUnaryRPC failed to write status: %v", e) } + if binlog != nil { + if h, _ := stream.Header(); h.Len() > 0 { + // Only log serverHeader if there was header. Otherwise it can + // be trailer only. + binlog.Log(&binarylog.ServerHeader{ + Header: h, + }) + } + binlog.Log(&binarylog.ServerTrailer{ + Trailer: stream.Trailer(), + Err: appErr, + }) + } return appErr } if trInfo != nil { @@ -1062,8 +1044,27 @@ func (s *Server) processUnaryRPC(t transport.ServerTransport, stream *transport. panic(fmt.Sprintf("grpc: Unexpected error (%T) from sendResponse: %v", st, st)) } } + if binlog != nil { + h, _ := stream.Header() + binlog.Log(&binarylog.ServerHeader{ + Header: h, + }) + binlog.Log(&binarylog.ServerTrailer{ + Trailer: stream.Trailer(), + Err: appErr, + }) + } return err } + if binlog != nil { + h, _ := stream.Header() + binlog.Log(&binarylog.ServerHeader{ + Header: h, + }) + binlog.Log(&binarylog.ServerMessage{ + Message: reply, + }) + } if channelz.IsOn() { t.IncrMsgSent() } @@ -1073,7 +1074,14 @@ func (s *Server) processUnaryRPC(t transport.ServerTransport, stream *transport. // TODO: Should we be logging if writing status failed here, like above? // Should the logging be in WriteStatus? Should we ignore the WriteStatus // error or allow the stats handler to see it? - return t.WriteStatus(stream, status.New(codes.OK, "")) + err = t.WriteStatus(stream, statusOK) + if binlog != nil { + binlog.Log(&binarylog.ServerTrailer{ + Trailer: stream.Trailer(), + Err: appErr, + }) + } + return err } func (s *Server) processStreamingRPC(t transport.ServerTransport, stream *transport.Stream, srv *service, sd *StreamDesc, trInfo *traceInfo) (err error) { @@ -1107,17 +1115,40 @@ func (s *Server) processStreamingRPC(t transport.ServerTransport, stream *transp } ctx := NewContextWithServerTransportStream(stream.Context(), stream) ss := &serverStream{ - ctx: ctx, - t: t, - s: stream, - p: &parser{r: stream}, - codec: s.getCodec(stream.ContentSubtype()), + ctx: ctx, + t: t, + s: stream, + p: &parser{r: stream}, + codec: s.getCodec(stream.ContentSubtype()), maxReceiveMessageSize: s.opts.maxReceiveMessageSize, maxSendMessageSize: s.opts.maxSendMessageSize, trInfo: trInfo, statsHandler: sh, } + ss.binlog = binarylog.GetMethodLogger(stream.Method()) + if ss.binlog != nil { + md, _ := metadata.FromIncomingContext(ctx) + logEntry := &binarylog.ClientHeader{ + Header: md, + MethodName: stream.Method(), + PeerAddr: nil, + } + if deadline, ok := ctx.Deadline(); ok { + logEntry.Timeout = time.Until(deadline) + if logEntry.Timeout < 0 { + logEntry.Timeout = 0 + } + } + if a := md[":authority"]; len(a) > 0 { + logEntry.Authority = a[0] + } + if peer, ok := peer.FromContext(ss.Context()); ok { + logEntry.PeerAddr = peer.Addr + } + ss.binlog.Log(logEntry) + } + // If dc is set and matches the stream's compression, use it. Otherwise, try // to find a matching registered compressor for decomp. if rc := stream.RecvCompress(); s.opts.dc != nil && s.opts.dc.Type() == rc { @@ -1187,6 +1218,12 @@ func (s *Server) processStreamingRPC(t transport.ServerTransport, stream *transp ss.mu.Unlock() } t.WriteStatus(ss.s, appStatus) + if ss.binlog != nil { + ss.binlog.Log(&binarylog.ServerTrailer{ + Trailer: ss.s.Trailer(), + Err: appErr, + }) + } // TODO: Should we log an error from WriteStatus here and below? return appErr } @@ -1195,7 +1232,14 @@ func (s *Server) processStreamingRPC(t transport.ServerTransport, stream *transp ss.trInfo.tr.LazyLog(stringer("OK"), false) ss.mu.Unlock() } - return t.WriteStatus(ss.s, status.New(codes.OK, "")) + err = t.WriteStatus(ss.s, statusOK) + if ss.binlog != nil { + ss.binlog.Log(&binarylog.ServerTrailer{ + Trailer: ss.s.Trailer(), + Err: appErr, + }) + } + return err } func (s *Server) handleStream(t transport.ServerTransport, stream *transport.Stream, trInfo *traceInfo) { @@ -1224,47 +1268,33 @@ func (s *Server) handleStream(t transport.ServerTransport, stream *transport.Str } service := sm[:pos] method := sm[pos+1:] - srv, ok := s.m[service] - if !ok { - if unknownDesc := s.opts.unknownStreamDesc; unknownDesc != nil { - s.processStreamingRPC(t, stream, nil, unknownDesc, trInfo) + + srv, knownService := s.m[service] + if knownService { + if md, ok := srv.md[method]; ok { + s.processUnaryRPC(t, stream, srv, md, trInfo) return } - if trInfo != nil { - trInfo.tr.LazyLog(&fmtStringer{"Unknown service %v", []interface{}{service}}, true) - trInfo.tr.SetError() + if sd, ok := srv.sd[method]; ok { + s.processStreamingRPC(t, stream, srv, sd, trInfo) + return } - errDesc := fmt.Sprintf("unknown service %v", service) - if err := t.WriteStatus(stream, status.New(codes.Unimplemented, errDesc)); err != nil { - if trInfo != nil { - trInfo.tr.LazyLog(&fmtStringer{"%v", []interface{}{err}}, true) - trInfo.tr.SetError() - } - grpclog.Warningf("grpc: Server.handleStream failed to write status: %v", err) - } - if trInfo != nil { - trInfo.tr.Finish() - } - return - } - // Unary RPC or Streaming RPC? - if md, ok := srv.md[method]; ok { - s.processUnaryRPC(t, stream, srv, md, trInfo) - return - } - if sd, ok := srv.sd[method]; ok { - s.processStreamingRPC(t, stream, srv, sd, trInfo) - return - } - if trInfo != nil { - trInfo.tr.LazyLog(&fmtStringer{"Unknown method %v", []interface{}{method}}, true) - trInfo.tr.SetError() } + // Unknown service, or known server unknown method. if unknownDesc := s.opts.unknownStreamDesc; unknownDesc != nil { s.processStreamingRPC(t, stream, nil, unknownDesc, trInfo) return } - errDesc := fmt.Sprintf("unknown method %v", method) + var errDesc string + if !knownService { + errDesc = fmt.Sprintf("unknown service %v", service) + } else { + errDesc = fmt.Sprintf("unknown method %v for service %v", method, service) + } + if trInfo != nil { + trInfo.tr.LazyPrintf("%s", errDesc) + trInfo.tr.SetError() + } if err := t.WriteStatus(stream, status.New(codes.Unimplemented, errDesc)); err != nil { if trInfo != nil { trInfo.tr.LazyLog(&fmtStringer{"%v", []interface{}{err}}, true) @@ -1319,15 +1349,11 @@ func ServerTransportStreamFromContext(ctx context.Context) ServerTransportStream // pending RPCs on the client side will get notified by connection // errors. func (s *Server) Stop() { - s.quitOnce.Do(func() { - close(s.quit) - }) + s.quit.Fire() defer func() { s.serveWG.Wait() - s.doneOnce.Do(func() { - close(s.done) - }) + s.done.Fire() }() s.channelzRemoveOnce.Do(func() { @@ -1364,15 +1390,8 @@ func (s *Server) Stop() { // accepting new connections and RPCs and blocks until all the pending RPCs are // finished. func (s *Server) GracefulStop() { - s.quitOnce.Do(func() { - close(s.quit) - }) - - defer func() { - s.doneOnce.Do(func() { - close(s.done) - }) - }() + s.quit.Fire() + defer s.done.Fire() s.channelzRemoveOnce.Do(func() { if channelz.IsOn() { @@ -1390,8 +1409,8 @@ func (s *Server) GracefulStop() { } s.lis = nil if !s.drain { - for c := range s.conns { - c.(transport.ServerTransport).Drain() + for st := range s.conns { + st.Drain() } s.drain = true } @@ -1413,12 +1432,6 @@ func (s *Server) GracefulStop() { s.mu.Unlock() } -func init() { - internal.TestingUseHandlerImpl = func(arg interface{}) { - arg.(*Server).opts.useHandlerImpl = true - } -} - // contentSubtype must be lowercase // cannot return nil func (s *Server) getCodec(contentSubtype string) baseCodec { @@ -1487,3 +1500,11 @@ func Method(ctx context.Context) (string, bool) { } return s.Method(), true } + +type channelzServer struct { + s *Server +} + +func (c *channelzServer) ChannelzMetric() *channelz.ServerInternalMetric { + return c.s.channelzMetric() +} diff --git a/vendor/google.golang.org/grpc/service_config.go b/vendor/google.golang.org/grpc/service_config.go index e0d735265f4..d0787f1e2a1 100644 --- a/vendor/google.golang.org/grpc/service_config.go +++ b/vendor/google.golang.org/grpc/service_config.go @@ -25,8 +25,11 @@ import ( "strings" "time" + "google.golang.org/grpc/balancer" "google.golang.org/grpc/codes" "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/serviceconfig" ) const maxInt = int(^uint(0) >> 1) @@ -61,6 +64,11 @@ type MethodConfig struct { retryPolicy *retryPolicy } +type lbConfig struct { + name string + cfg serviceconfig.LoadBalancingConfig +} + // ServiceConfig is provided by the service provider and contains parameters for how // clients that connect to the service should behave. // @@ -68,10 +76,18 @@ type MethodConfig struct { // through name resolver, as specified here // https://github.com/grpc/grpc/blob/master/doc/service_config.md type ServiceConfig struct { - // LB is the load balancer the service providers recommends. The balancer specified - // via grpc.WithBalancer will override this. + serviceconfig.Config + + // LB is the load balancer the service providers recommends. The balancer + // specified via grpc.WithBalancer will override this. This is deprecated; + // lbConfigs is preferred. If lbConfig and LB are both present, lbConfig + // will be used. LB *string + // lbConfig is the service config's load balancing configuration. If + // lbConfig and LB are both present, lbConfig will be used. + lbConfig *lbConfig + // Methods contains a map for the methods in this service. If there is an // exact match for a method (i.e. /service/method) in the map, use the // corresponding MethodConfig. If there's no exact match, look for the @@ -96,6 +112,18 @@ type ServiceConfig struct { // If token_count is less than or equal to maxTokens / 2, then RPCs will not // be retried and hedged RPCs will not be sent. retryThrottling *retryThrottlingPolicy + // healthCheckConfig must be set as one of the requirement to enable LB channel + // health check. + healthCheckConfig *healthCheckConfig + // rawJSONString stores service config json string that get parsed into + // this service config struct. + rawJSONString string +} + +// healthCheckConfig defines the go-native version of the LB channel health check config. +type healthCheckConfig struct { + // serviceName is the service name to use in the health-checking request. + ServiceName string } // retryPolicy defines the go-native version of the retry policy defined by the @@ -221,29 +249,72 @@ type jsonMC struct { RetryPolicy *jsonRetryPolicy } +type loadBalancingConfig map[string]json.RawMessage + // TODO(lyuxuan): delete this struct after cleaning up old service config implementation. type jsonSC struct { LoadBalancingPolicy *string + LoadBalancingConfig *[]loadBalancingConfig MethodConfig *[]jsonMC RetryThrottling *retryThrottlingPolicy + HealthCheckConfig *healthCheckConfig } -func parseServiceConfig(js string) (ServiceConfig, error) { +func init() { + internal.ParseServiceConfig = func(sc string) (interface{}, error) { + return parseServiceConfig(sc) + } +} + +func parseServiceConfig(js string) (*ServiceConfig, error) { + if len(js) == 0 { + return nil, fmt.Errorf("no JSON service config provided") + } var rsc jsonSC err := json.Unmarshal([]byte(js), &rsc) if err != nil { grpclog.Warningf("grpc: parseServiceConfig error unmarshaling %s due to %v", js, err) - return ServiceConfig{}, err + return nil, err } sc := ServiceConfig{ - LB: rsc.LoadBalancingPolicy, - Methods: make(map[string]MethodConfig), - retryThrottling: rsc.RetryThrottling, + LB: rsc.LoadBalancingPolicy, + Methods: make(map[string]MethodConfig), + retryThrottling: rsc.RetryThrottling, + healthCheckConfig: rsc.HealthCheckConfig, + rawJSONString: js, } - if rsc.MethodConfig == nil { - return sc, nil + if rsc.LoadBalancingConfig != nil { + for i, lbcfg := range *rsc.LoadBalancingConfig { + if len(lbcfg) != 1 { + err := fmt.Errorf("invalid loadBalancingConfig: entry %v does not contain exactly 1 policy/config pair: %q", i, lbcfg) + grpclog.Warningf(err.Error()) + return nil, err + } + var name string + var jsonCfg json.RawMessage + for name, jsonCfg = range lbcfg { + } + builder := balancer.Get(name) + if builder == nil { + continue + } + sc.lbConfig = &lbConfig{name: name} + if parser, ok := builder.(balancer.ConfigParser); ok { + var err error + sc.lbConfig.cfg, err = parser.ParseConfig(jsonCfg) + if err != nil { + return nil, fmt.Errorf("error parsing loadBalancingConfig for policy %q: %v", name, err) + } + } else if string(jsonCfg) != "{}" { + grpclog.Warningf("non-empty balancer configuration %q, but balancer does not implement ParseConfig", string(jsonCfg)) + } + break + } } + if rsc.MethodConfig == nil { + return &sc, nil + } for _, m := range *rsc.MethodConfig { if m.Name == nil { continue @@ -251,7 +322,7 @@ func parseServiceConfig(js string) (ServiceConfig, error) { d, err := parseDuration(m.Timeout) if err != nil { grpclog.Warningf("grpc: parseServiceConfig error unmarshaling %s due to %v", js, err) - return ServiceConfig{}, err + return nil, err } mc := MethodConfig{ @@ -260,7 +331,7 @@ func parseServiceConfig(js string) (ServiceConfig, error) { } if mc.retryPolicy, err = convertRetryPolicy(m.RetryPolicy); err != nil { grpclog.Warningf("grpc: parseServiceConfig error unmarshaling %s due to %v", js, err) - return ServiceConfig{}, err + return nil, err } if m.MaxRequestMessageBytes != nil { if *m.MaxRequestMessageBytes > int64(maxInt) { @@ -284,14 +355,14 @@ func parseServiceConfig(js string) (ServiceConfig, error) { } if sc.retryThrottling != nil { - if sc.retryThrottling.MaxTokens <= 0 || - sc.retryThrottling.MaxTokens >= 1000 || - sc.retryThrottling.TokenRatio <= 0 { - // Illegal throttling config; disable throttling. - sc.retryThrottling = nil + if mt := sc.retryThrottling.MaxTokens; mt <= 0 || mt > 1000 { + return nil, fmt.Errorf("invalid retry throttling config: maxTokens (%v) out of range (0, 1000]", mt) + } + if tr := sc.retryThrottling.TokenRatio; tr <= 0 { + return nil, fmt.Errorf("invalid retry throttling config: tokenRatio (%v) may not be negative", tr) } } - return sc, nil + return &sc, nil } func convertRetryPolicy(jrp *jsonRetryPolicy) (p *retryPolicy, err error) { diff --git a/vendor/google.golang.org/grpc/serviceconfig/serviceconfig.go b/vendor/google.golang.org/grpc/serviceconfig/serviceconfig.go new file mode 100644 index 00000000000..53b27875a1a --- /dev/null +++ b/vendor/google.golang.org/grpc/serviceconfig/serviceconfig.go @@ -0,0 +1,48 @@ +/* + * + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package serviceconfig defines types and methods for operating on gRPC +// service configs. +// +// This package is EXPERIMENTAL. +package serviceconfig + +import ( + "google.golang.org/grpc/internal" +) + +// Config represents an opaque data structure holding a service config. +type Config interface { + isConfig() +} + +// LoadBalancingConfig represents an opaque data structure holding a load +// balancer config. +type LoadBalancingConfig interface { + isLoadBalancingConfig() +} + +// Parse parses the JSON service config provided into an internal form or +// returns an error if the config is invalid. +func Parse(ServiceConfigJSON string) (Config, error) { + c, err := internal.ParseServiceConfig(ServiceConfigJSON) + if err != nil { + return nil, err + } + return c.(Config), err +} diff --git a/vendor/google.golang.org/grpc/stats/handlers.go b/vendor/google.golang.org/grpc/stats/handlers.go index 05b384c6931..dc03731e45e 100644 --- a/vendor/google.golang.org/grpc/stats/handlers.go +++ b/vendor/google.golang.org/grpc/stats/handlers.go @@ -19,9 +19,8 @@ package stats import ( + "context" "net" - - "golang.org/x/net/context" ) // ConnTagInfo defines the relevant information needed by connection context tagger. diff --git a/vendor/google.golang.org/grpc/stats/stats.go b/vendor/google.golang.org/grpc/stats/stats.go index 3f13190a0ac..f3f593c8443 100644 --- a/vendor/google.golang.org/grpc/stats/stats.go +++ b/vendor/google.golang.org/grpc/stats/stats.go @@ -24,10 +24,11 @@ package stats // import "google.golang.org/grpc/stats" import ( + "context" "net" "time" - "golang.org/x/net/context" + "google.golang.org/grpc/metadata" ) // RPCStats contains stats information about RPCs. @@ -173,6 +174,9 @@ type End struct { BeginTime time.Time // EndTime is the time when the RPC ends. EndTime time.Time + // Trailer contains the trailer metadata received from the server. This + // field is only valid if this End is from the client side. + Trailer metadata.MD // Error is the error the RPC ended with. It is an error generated from // status.Status and can be converted back to status.Status using // status.FromError if non-nil. diff --git a/vendor/google.golang.org/grpc/status/go17.go b/vendor/google.golang.org/grpc/status/go17.go deleted file mode 100644 index 090215149cf..00000000000 --- a/vendor/google.golang.org/grpc/status/go17.go +++ /dev/null @@ -1,44 +0,0 @@ -// +build go1.7 - -/* - * - * Copyright 2018 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package status - -import ( - "context" - - netctx "golang.org/x/net/context" - "google.golang.org/grpc/codes" -) - -// FromContextError converts a context error into a Status. It returns a -// Status with codes.OK if err is nil, or a Status with codes.Unknown if err is -// non-nil and not a context error. -func FromContextError(err error) *Status { - switch err { - case nil: - return New(codes.OK, "") - case context.DeadlineExceeded, netctx.DeadlineExceeded: - return New(codes.DeadlineExceeded, err.Error()) - case context.Canceled, netctx.Canceled: - return New(codes.Canceled, err.Error()) - default: - return New(codes.Unknown, err.Error()) - } -} diff --git a/vendor/google.golang.org/grpc/status/status.go b/vendor/google.golang.org/grpc/status/status.go index 9c61b094508..a1348e9b16b 100644 --- a/vendor/google.golang.org/grpc/status/status.go +++ b/vendor/google.golang.org/grpc/status/status.go @@ -28,6 +28,7 @@ package status import ( + "context" "errors" "fmt" @@ -35,8 +36,15 @@ import ( "github.com/golang/protobuf/ptypes" spb "google.golang.org/genproto/googleapis/rpc/status" "google.golang.org/grpc/codes" + "google.golang.org/grpc/internal" ) +func init() { + internal.StatusRawProto = statusRawProto +} + +func statusRawProto(s *Status) *spb.Status { return s.s } + // statusError is an alias of a status proto. It implements error and Status, // and a nil statusError should never be returned by this package. type statusError spb.Status @@ -50,6 +58,17 @@ func (se *statusError) GRPCStatus() *Status { return &Status{s: (*spb.Status)(se)} } +// Is implements future error.Is functionality. +// A statusError is equivalent if the code and message are identical. +func (se *statusError) Is(target error) bool { + tse, ok := target.(*statusError) + if !ok { + return false + } + + return proto.Equal((*spb.Status)(se), (*spb.Status)(tse)) +} + // Status represents an RPC status code, message, and details. It is immutable // and should be created with New, Newf, or FromProto. type Status struct { @@ -124,9 +143,11 @@ func FromProto(s *spb.Status) *Status { // Status is returned with codes.Unknown and the original error message. func FromError(err error) (s *Status, ok bool) { if err == nil { - return &Status{s: &spb.Status{Code: int32(codes.OK)}}, true + return nil, true } - if se, ok := err.(interface{ GRPCStatus() *Status }); ok { + if se, ok := err.(interface { + GRPCStatus() *Status + }); ok { return se.GRPCStatus(), true } return New(codes.Unknown, err.Error()), false @@ -182,8 +203,26 @@ func Code(err error) codes.Code { if err == nil { return codes.OK } - if se, ok := err.(interface{ GRPCStatus() *Status }); ok { + if se, ok := err.(interface { + GRPCStatus() *Status + }); ok { return se.GRPCStatus().Code() } return codes.Unknown } + +// FromContextError converts a context error into a Status. It returns a +// Status with codes.OK if err is nil, or a Status with codes.Unknown if err is +// non-nil and not a context error. +func FromContextError(err error) *Status { + switch err { + case nil: + return nil + case context.DeadlineExceeded: + return New(codes.DeadlineExceeded, err.Error()) + case context.Canceled: + return New(codes.Canceled, err.Error()) + default: + return New(codes.Unknown, err.Error()) + } +} diff --git a/vendor/google.golang.org/grpc/stream.go b/vendor/google.golang.org/grpc/stream.go index 65d45a1d95d..134a624a15d 100644 --- a/vendor/google.golang.org/grpc/stream.go +++ b/vendor/google.golang.org/grpc/stream.go @@ -19,6 +19,7 @@ package grpc import ( + "context" "errors" "io" "math" @@ -26,16 +27,18 @@ import ( "sync" "time" - "golang.org/x/net/context" "golang.org/x/net/trace" "google.golang.org/grpc/balancer" "google.golang.org/grpc/codes" "google.golang.org/grpc/encoding" "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/internal/balancerload" + "google.golang.org/grpc/internal/binarylog" "google.golang.org/grpc/internal/channelz" "google.golang.org/grpc/internal/grpcrand" "google.golang.org/grpc/internal/transport" "google.golang.org/grpc/metadata" + "google.golang.org/grpc/peer" "google.golang.org/grpc/stats" "google.golang.org/grpc/status" ) @@ -82,7 +85,8 @@ type ClientStream interface { // stream.Recv has returned a non-nil error (including io.EOF). Trailer() metadata.MD // CloseSend closes the send direction of the stream. It closes the stream - // when non-nil error is met. + // when non-nil error is met. It is also not safe to call CloseSend + // concurrently with SendMsg. CloseSend() error // Context returns the context for this stream. // @@ -105,7 +109,8 @@ type ClientStream interface { // // It is safe to have a goroutine calling SendMsg and another goroutine // calling RecvMsg on the same stream at the same time, but it is not safe - // to call SendMsg on the same stream in different goroutines. + // to call SendMsg on the same stream in different goroutines. It is also + // not safe to call CloseSend concurrently with SendMsg. SendMsg(m interface{}) error // RecvMsg blocks until it receives a message into m or the stream is // done. It returns io.EOF when the stream completes successfully. On @@ -160,6 +165,11 @@ func newClientStream(ctx context.Context, desc *StreamDesc, cc *ClientConn, meth }() } c := defaultCallInfo() + // Provide an opportunity for the first RPC to see the first service config + // provided by the resolver. + if err := cc.waitForResolvedAddrs(ctx); err != nil { + return nil, err + } mc := cc.GetMethodConfig(method) if mc.WaitForReady != nil { c.failFast = !*mc.WaitForReady @@ -220,17 +230,21 @@ func newClientStream(ctx context.Context, desc *StreamDesc, cc *ClientConn, meth if c.creds != nil { callHdr.Creds = c.creds } - var trInfo traceInfo + var trInfo *traceInfo if EnableTracing { - trInfo.tr = trace.New("grpc.Sent."+methodFamily(method), method) - trInfo.firstLine.client = true + trInfo = &traceInfo{ + tr: trace.New("grpc.Sent."+methodFamily(method), method), + firstLine: firstLine{ + client: true, + }, + } if deadline, ok := ctx.Deadline(); ok { - trInfo.firstLine.deadline = deadline.Sub(time.Now()) + trInfo.firstLine.deadline = time.Until(deadline) } trInfo.tr.LazyLog(&trInfo.firstLine, false) ctx = trace.NewContext(ctx, trInfo.tr) } - ctx = newContextWithRPCInfo(ctx, c.failFast) + ctx = newContextWithRPCInfo(ctx, c.failFast, c.codec, cp, comp) sh := cc.dopts.copts.StatsHandler var beginTime time.Time if sh != nil { @@ -262,6 +276,7 @@ func newClientStream(ctx context.Context, desc *StreamDesc, cc *ClientConn, meth if !cc.dopts.disableRetry { cs.retryThrottler = cc.retryThrottler.Load().(*retryThrottler) } + cs.binlog = binarylog.GetMethodLogger(method) cs.callInfo.stream = cs // Only this initial attempt has stats/tracing. @@ -277,6 +292,23 @@ func newClientStream(ctx context.Context, desc *StreamDesc, cc *ClientConn, meth return nil, err } + if cs.binlog != nil { + md, _ := metadata.FromOutgoingContext(ctx) + logEntry := &binarylog.ClientHeader{ + OnClientSide: true, + Header: md, + MethodName: method, + Authority: cs.cc.authority, + } + if deadline, ok := ctx.Deadline(); ok { + logEntry.Timeout = time.Until(deadline) + if logEntry.Timeout < 0 { + logEntry.Timeout = 0 + } + } + cs.binlog.Log(logEntry) + } + if desc != unaryStreamDesc { // Listen on cc and stream contexts to cleanup when the user closes the // ClientConn or cancels the stream context. In all other cases, an error @@ -295,13 +327,23 @@ func newClientStream(ctx context.Context, desc *StreamDesc, cc *ClientConn, meth return cs, nil } -func (cs *clientStream) newAttemptLocked(sh stats.Handler, trInfo traceInfo) error { - cs.attempt = &csAttempt{ +// newAttemptLocked creates a new attempt with a transport. +// If it succeeds, then it replaces clientStream's attempt with this new attempt. +func (cs *clientStream) newAttemptLocked(sh stats.Handler, trInfo *traceInfo) (retErr error) { + newAttempt := &csAttempt{ cs: cs, dc: cs.cc.dopts.dc, statsHandler: sh, trInfo: trInfo, } + defer func() { + if retErr != nil { + // This attempt is not set in the clientStream, so it's finish won't + // be called. Call it here for stats and trace in case they are not + // nil. + newAttempt.finish(retErr) + } + }() if err := cs.ctx.Err(); err != nil { return toRPCErr(err) @@ -310,8 +352,12 @@ func (cs *clientStream) newAttemptLocked(sh stats.Handler, trInfo traceInfo) err if err != nil { return err } - cs.attempt.t = t - cs.attempt.done = done + if trInfo != nil { + trInfo.firstLine.SetRemoteAddr(t.RemoteAddr()) + } + newAttempt.t = t + newAttempt.done = done + cs.attempt = newAttempt return nil } @@ -350,12 +396,28 @@ type clientStream struct { retryThrottler *retryThrottler // The throttler active when the RPC began. + binlog *binarylog.MethodLogger // Binary logger, can be nil. + // serverHeaderBinlogged is a boolean for whether server header has been + // logged. Server header will be logged when the first time one of those + // happens: stream.Header(), stream.Recv(). + // + // It's only read and used by Recv() and Header(), so it doesn't need to be + // synchronized. + serverHeaderBinlogged bool + mu sync.Mutex - firstAttempt bool // if true, transparent retry is valid - numRetries int // exclusive of transparent retry attempt(s) - numRetriesSincePushback int // retries since pushback; to reset backoff - finished bool // TODO: replace with atomic cmpxchg or sync.Once? - attempt *csAttempt // the active client stream attempt + firstAttempt bool // if true, transparent retry is valid + numRetries int // exclusive of transparent retry attempt(s) + numRetriesSincePushback int // retries since pushback; to reset backoff + finished bool // TODO: replace with atomic cmpxchg or sync.Once? + // attempt is the active client stream attempt. + // The only place where it is written is the newAttemptLocked method and this method never writes nil. + // So, attempt can be nil only inside newClientStream function when clientStream is first created. + // One of the first things done after clientStream's creation, is to call newAttemptLocked which either + // assigns a non nil value to the attempt or returns an error. If an error is returned from newAttemptLocked, + // then newClientStream calls finish on the clientStream and returns. So, finish method is the only + // place where we need to check if the attempt is nil. + attempt *csAttempt // TODO(hedging): hedging will have multiple attempts simultaneously. committed bool // active attempt committed for retry? buffer []func(a *csAttempt) error // operations to replay on retry @@ -377,9 +439,10 @@ type csAttempt struct { decompSet bool mu sync.Mutex // guards trInfo.tr + // trInfo may be nil (if EnableTracing is false). // trInfo.tr is set when created (if EnableTracing is true), // and cleared when the finish method is called. - trInfo traceInfo + trInfo *traceInfo statsHandler stats.Handler } @@ -412,8 +475,8 @@ func (cs *clientStream) shouldRetry(err error) error { if cs.attempt.s != nil { <-cs.attempt.s.Done() } - if cs.firstAttempt && !cs.callInfo.failFast && (cs.attempt.s == nil || cs.attempt.s.Unprocessed()) { - // First attempt, wait-for-ready, stream unprocessed: transparently retry. + if cs.firstAttempt && (cs.attempt.s == nil || cs.attempt.s.Unprocessed()) { + // First attempt, stream unprocessed: transparently retry. cs.firstAttempt = false return nil } @@ -425,10 +488,7 @@ func (cs *clientStream) shouldRetry(err error) error { pushback := 0 hasPushback := false if cs.attempt.s != nil { - if to, toErr := cs.attempt.s.TrailersOnly(); toErr != nil { - // Context error; stop now. - return toErr - } else if !to { + if to, toErr := cs.attempt.s.TrailersOnly(); toErr != nil || !to { return err } @@ -506,7 +566,7 @@ func (cs *clientStream) retryLocked(lastErr error) error { cs.commitAttemptLocked() return err } - if err := cs.newAttemptLocked(nil, traceInfo{}); err != nil { + if err := cs.newAttemptLocked(nil, nil); err != nil { return err } if lastErr = cs.replayBufferLocked(); lastErr == nil { @@ -561,6 +621,20 @@ func (cs *clientStream) Header() (metadata.MD, error) { }, cs.commitAttemptLocked) if err != nil { cs.finish(err) + return nil, err + } + if cs.binlog != nil && !cs.serverHeaderBinlogged { + // Only log if binary log is on and header has not been logged. + logEntry := &binarylog.ServerHeader{ + OnClientSide: true, + Header: m, + PeerAddr: nil, + } + if peer, ok := peer.FromContext(cs.Context()); ok { + logEntry.PeerAddr = peer.Addr + } + cs.binlog.Log(logEntry) + cs.serverHeaderBinlogged = true } return m, err } @@ -620,19 +694,18 @@ func (cs *clientStream) SendMsg(m interface{}) (err error) { if !cs.desc.ClientStreams { cs.sentLast = true } - data, err := encode(cs.codec, m) + + // load hdr, payload, data + hdr, payload, data, err := prepareMsg(m, cs.codec, cs.cp, cs.comp) if err != nil { return err } - compData, err := compress(data, cs.cp, cs.comp) - if err != nil { - return err - } - hdr, payload := msgHeader(data, compData) + // TODO(dfawley): should we be checking len(data) instead? if len(payload) > *cs.callInfo.maxSendMessageSize { return status.Errorf(codes.ResourceExhausted, "trying to send message larger than max (%d vs. %d)", len(payload), *cs.callInfo.maxSendMessageSize) } + msgBytes := data // Store the pointer before setting to nil. For binary logging. op := func(a *csAttempt) error { err := a.sendMsg(m, hdr, payload, data) // nil out the message and uncomp when replaying; they are only needed for @@ -640,16 +713,53 @@ func (cs *clientStream) SendMsg(m interface{}) (err error) { m, data = nil, nil return err } - return cs.withRetry(op, func() { cs.bufferForRetryLocked(len(hdr)+len(payload), op) }) + err = cs.withRetry(op, func() { cs.bufferForRetryLocked(len(hdr)+len(payload), op) }) + if cs.binlog != nil && err == nil { + cs.binlog.Log(&binarylog.ClientMessage{ + OnClientSide: true, + Message: msgBytes, + }) + } + return } func (cs *clientStream) RecvMsg(m interface{}) error { + if cs.binlog != nil && !cs.serverHeaderBinlogged { + // Call Header() to binary log header if it's not already logged. + cs.Header() + } + var recvInfo *payloadInfo + if cs.binlog != nil { + recvInfo = &payloadInfo{} + } err := cs.withRetry(func(a *csAttempt) error { - return a.recvMsg(m) + return a.recvMsg(m, recvInfo) }, cs.commitAttemptLocked) + if cs.binlog != nil && err == nil { + cs.binlog.Log(&binarylog.ServerMessage{ + OnClientSide: true, + Message: recvInfo.uncompressedBytes, + }) + } if err != nil || !cs.desc.ServerStreams { // err != nil or non-server-streaming indicates end of stream. cs.finish(err) + + if cs.binlog != nil { + // finish will not log Trailer. Log Trailer here. + logEntry := &binarylog.ServerTrailer{ + OnClientSide: true, + Trailer: cs.Trailer(), + Err: err, + } + if logEntry.Err == io.EOF { + logEntry.Err = nil + } + if peer, ok := peer.FromContext(cs.Context()); ok { + logEntry.PeerAddr = peer.Addr + } + cs.binlog.Log(logEntry) + } } return err } @@ -660,8 +770,20 @@ func (cs *clientStream) CloseSend() error { return nil } cs.sentLast = true - op := func(a *csAttempt) error { return a.t.Write(a.s, nil, nil, &transport.Options{Last: true}) } + op := func(a *csAttempt) error { + a.t.Write(a.s, nil, nil, &transport.Options{Last: true}) + // Always return nil; io.EOF is the only error that might make sense + // instead, but there is no need to signal the client to call RecvMsg + // as the only use left for the stream after CloseSend is to call + // RecvMsg. This also matches historical behavior. + return nil + } cs.withRetry(op, func() { cs.bufferForRetryLocked(0, op) }) + if cs.binlog != nil { + cs.binlog.Log(&binarylog.ClientHalfClose{ + OnClientSide: true, + }) + } // We never returned an error here for reasons. return nil } @@ -679,6 +801,16 @@ func (cs *clientStream) finish(err error) { cs.finished = true cs.commitAttemptLocked() cs.mu.Unlock() + // For binary logging. only log cancel in finish (could be caused by RPC ctx + // canceled or ClientConn closed). Trailer will be logged in RecvMsg. + // + // Only one of cancel or trailer needs to be logged. In the cases where + // users don't call RecvMsg, users must have already canceled the RPC. + if cs.binlog != nil && status.Code(err) == codes.Canceled { + cs.binlog.Log(&binarylog.Cancel{ + OnClientSide: true, + }) + } if err == nil { cs.retryThrottler.successfulRPC() } @@ -691,11 +823,11 @@ func (cs *clientStream) finish(err error) { } if cs.attempt != nil { cs.attempt.finish(err) - } - // after functions all rely upon having a stream. - if cs.attempt.s != nil { - for _, o := range cs.opts { - o.after(cs.callInfo) + // after functions all rely upon having a stream. + if cs.attempt.s != nil { + for _, o := range cs.opts { + o.after(cs.callInfo) + } } } cs.cancel() @@ -703,7 +835,7 @@ func (cs *clientStream) finish(err error) { func (a *csAttempt) sendMsg(m interface{}, hdr, payld, data []byte) error { cs := a.cs - if EnableTracing { + if a.trInfo != nil { a.mu.Lock() if a.trInfo.tr != nil { a.trInfo.tr.LazyLog(&payload{sent: true, msg: m}, true) @@ -728,14 +860,12 @@ func (a *csAttempt) sendMsg(m interface{}, hdr, payld, data []byte) error { return nil } -func (a *csAttempt) recvMsg(m interface{}) (err error) { +func (a *csAttempt) recvMsg(m interface{}, payInfo *payloadInfo) (err error) { cs := a.cs - var inPayload *stats.InPayload - if a.statsHandler != nil { - inPayload = &stats.InPayload{ - Client: true, - } + if a.statsHandler != nil && payInfo == nil { + payInfo = &payloadInfo{} } + if !a.decompSet { // Block until we receive headers containing received message encoding. if ct := a.s.RecvCompress(); ct != "" && ct != encoding.Identity { @@ -752,7 +882,7 @@ func (a *csAttempt) recvMsg(m interface{}) (err error) { // Only initialize this state once per stream. a.decompSet = true } - err = recv(a.p, cs.codec, a.s, a.dc, m, *cs.callInfo.maxReceiveMessageSize, inPayload, a.decomp) + err = recv(a.p, cs.codec, a.s, a.dc, m, *cs.callInfo.maxReceiveMessageSize, payInfo, a.decomp) if err != nil { if err == io.EOF { if statusErr := a.s.Status().Err(); statusErr != nil { @@ -762,15 +892,23 @@ func (a *csAttempt) recvMsg(m interface{}) (err error) { } return toRPCErr(err) } - if EnableTracing { + if a.trInfo != nil { a.mu.Lock() if a.trInfo.tr != nil { a.trInfo.tr.LazyLog(&payload{sent: false, msg: m}, true) } a.mu.Unlock() } - if inPayload != nil { - a.statsHandler.HandleRPC(cs.ctx, inPayload) + if a.statsHandler != nil { + a.statsHandler.HandleRPC(cs.ctx, &stats.InPayload{ + Client: true, + RecvTime: time.Now(), + Payload: m, + // TODO truncate large payload. + Data: payInfo.uncompressedBytes, + WireLength: payInfo.wireLength, + Length: len(payInfo.uncompressedBytes), + }) } if channelz.IsOn() { a.t.IncrMsgRecv() @@ -779,7 +917,6 @@ func (a *csAttempt) recvMsg(m interface{}) (err error) { // Subsequent messages should be received by subsequent RecvMsg calls. return nil } - // Special handling for non-server-stream rpcs. // This recv expects EOF or errors, so we don't collect inPayload. err = recv(a.p, cs.codec, a.s, a.dc, m, *cs.callInfo.maxReceiveMessageSize, nil, a.decomp) @@ -803,8 +940,10 @@ func (a *csAttempt) finish(err error) { // Ending a stream with EOF indicates a success. err = nil } + var tr metadata.MD if a.s != nil { a.t.CloseStream(a.s, err) + tr = a.s.Trailer() } if a.done != nil { @@ -814,8 +953,10 @@ func (a *csAttempt) finish(err error) { } a.done(balancer.DoneInfo{ Err: err, + Trailer: tr, BytesSent: a.s != nil, BytesReceived: br, + ServerLoad: balancerload.Parse(tr), }) } if a.statsHandler != nil { @@ -823,11 +964,12 @@ func (a *csAttempt) finish(err error) { Client: true, BeginTime: a.cs.beginTime, EndTime: time.Now(), + Trailer: tr, Error: err, } a.statsHandler.HandleRPC(a.cs.ctx, end) } - if a.trInfo.tr != nil { + if a.trInfo != nil && a.trInfo.tr != nil { if err == nil { a.trInfo.tr.LazyPrintf("RPC: [OK]") } else { @@ -840,6 +982,295 @@ func (a *csAttempt) finish(err error) { a.mu.Unlock() } +// newClientStream creates a ClientStream with the specified transport, on the +// given addrConn. +// +// It's expected that the given transport is either the same one in addrConn, or +// is already closed. To avoid race, transport is specified separately, instead +// of using ac.transpot. +// +// Main difference between this and ClientConn.NewStream: +// - no retry +// - no service config (or wait for service config) +// - no tracing or stats +func newNonRetryClientStream(ctx context.Context, desc *StreamDesc, method string, t transport.ClientTransport, ac *addrConn, opts ...CallOption) (_ ClientStream, err error) { + if t == nil { + // TODO: return RPC error here? + return nil, errors.New("transport provided is nil") + } + // defaultCallInfo contains unnecessary info(i.e. failfast, maxRetryRPCBufferSize), so we just initialize an empty struct. + c := &callInfo{} + + // Possible context leak: + // The cancel function for the child context we create will only be called + // when RecvMsg returns a non-nil error, if the ClientConn is closed, or if + // an error is generated by SendMsg. + // https://github.com/grpc/grpc-go/issues/1818. + ctx, cancel := context.WithCancel(ctx) + defer func() { + if err != nil { + cancel() + } + }() + + for _, o := range opts { + if err := o.before(c); err != nil { + return nil, toRPCErr(err) + } + } + c.maxReceiveMessageSize = getMaxSize(nil, c.maxReceiveMessageSize, defaultClientMaxReceiveMessageSize) + c.maxSendMessageSize = getMaxSize(nil, c.maxSendMessageSize, defaultServerMaxSendMessageSize) + if err := setCallInfoCodec(c); err != nil { + return nil, err + } + + callHdr := &transport.CallHdr{ + Host: ac.cc.authority, + Method: method, + ContentSubtype: c.contentSubtype, + } + + // Set our outgoing compression according to the UseCompressor CallOption, if + // set. In that case, also find the compressor from the encoding package. + // Otherwise, use the compressor configured by the WithCompressor DialOption, + // if set. + var cp Compressor + var comp encoding.Compressor + if ct := c.compressorType; ct != "" { + callHdr.SendCompress = ct + if ct != encoding.Identity { + comp = encoding.GetCompressor(ct) + if comp == nil { + return nil, status.Errorf(codes.Internal, "grpc: Compressor is not installed for requested grpc-encoding %q", ct) + } + } + } else if ac.cc.dopts.cp != nil { + callHdr.SendCompress = ac.cc.dopts.cp.Type() + cp = ac.cc.dopts.cp + } + if c.creds != nil { + callHdr.Creds = c.creds + } + + // Use a special addrConnStream to avoid retry. + as := &addrConnStream{ + callHdr: callHdr, + ac: ac, + ctx: ctx, + cancel: cancel, + opts: opts, + callInfo: c, + desc: desc, + codec: c.codec, + cp: cp, + comp: comp, + t: t, + } + + as.callInfo.stream = as + s, err := as.t.NewStream(as.ctx, as.callHdr) + if err != nil { + err = toRPCErr(err) + return nil, err + } + as.s = s + as.p = &parser{r: s} + ac.incrCallsStarted() + if desc != unaryStreamDesc { + // Listen on cc and stream contexts to cleanup when the user closes the + // ClientConn or cancels the stream context. In all other cases, an error + // should already be injected into the recv buffer by the transport, which + // the client will eventually receive, and then we will cancel the stream's + // context in clientStream.finish. + go func() { + select { + case <-ac.ctx.Done(): + as.finish(status.Error(codes.Canceled, "grpc: the SubConn is closing")) + case <-ctx.Done(): + as.finish(toRPCErr(ctx.Err())) + } + }() + } + return as, nil +} + +type addrConnStream struct { + s *transport.Stream + ac *addrConn + callHdr *transport.CallHdr + cancel context.CancelFunc + opts []CallOption + callInfo *callInfo + t transport.ClientTransport + ctx context.Context + sentLast bool + desc *StreamDesc + codec baseCodec + cp Compressor + comp encoding.Compressor + decompSet bool + dc Decompressor + decomp encoding.Compressor + p *parser + mu sync.Mutex + finished bool +} + +func (as *addrConnStream) Header() (metadata.MD, error) { + m, err := as.s.Header() + if err != nil { + as.finish(toRPCErr(err)) + } + return m, err +} + +func (as *addrConnStream) Trailer() metadata.MD { + return as.s.Trailer() +} + +func (as *addrConnStream) CloseSend() error { + if as.sentLast { + // TODO: return an error and finish the stream instead, due to API misuse? + return nil + } + as.sentLast = true + + as.t.Write(as.s, nil, nil, &transport.Options{Last: true}) + // Always return nil; io.EOF is the only error that might make sense + // instead, but there is no need to signal the client to call RecvMsg + // as the only use left for the stream after CloseSend is to call + // RecvMsg. This also matches historical behavior. + return nil +} + +func (as *addrConnStream) Context() context.Context { + return as.s.Context() +} + +func (as *addrConnStream) SendMsg(m interface{}) (err error) { + defer func() { + if err != nil && err != io.EOF { + // Call finish on the client stream for errors generated by this SendMsg + // call, as these indicate problems created by this client. (Transport + // errors are converted to an io.EOF error in csAttempt.sendMsg; the real + // error will be returned from RecvMsg eventually in that case, or be + // retried.) + as.finish(err) + } + }() + if as.sentLast { + return status.Errorf(codes.Internal, "SendMsg called after CloseSend") + } + if !as.desc.ClientStreams { + as.sentLast = true + } + + // load hdr, payload, data + hdr, payld, _, err := prepareMsg(m, as.codec, as.cp, as.comp) + if err != nil { + return err + } + + // TODO(dfawley): should we be checking len(data) instead? + if len(payld) > *as.callInfo.maxSendMessageSize { + return status.Errorf(codes.ResourceExhausted, "trying to send message larger than max (%d vs. %d)", len(payld), *as.callInfo.maxSendMessageSize) + } + + if err := as.t.Write(as.s, hdr, payld, &transport.Options{Last: !as.desc.ClientStreams}); err != nil { + if !as.desc.ClientStreams { + // For non-client-streaming RPCs, we return nil instead of EOF on error + // because the generated code requires it. finish is not called; RecvMsg() + // will call it with the stream's status independently. + return nil + } + return io.EOF + } + + if channelz.IsOn() { + as.t.IncrMsgSent() + } + return nil +} + +func (as *addrConnStream) RecvMsg(m interface{}) (err error) { + defer func() { + if err != nil || !as.desc.ServerStreams { + // err != nil or non-server-streaming indicates end of stream. + as.finish(err) + } + }() + + if !as.decompSet { + // Block until we receive headers containing received message encoding. + if ct := as.s.RecvCompress(); ct != "" && ct != encoding.Identity { + if as.dc == nil || as.dc.Type() != ct { + // No configured decompressor, or it does not match the incoming + // message encoding; attempt to find a registered compressor that does. + as.dc = nil + as.decomp = encoding.GetCompressor(ct) + } + } else { + // No compression is used; disable our decompressor. + as.dc = nil + } + // Only initialize this state once per stream. + as.decompSet = true + } + err = recv(as.p, as.codec, as.s, as.dc, m, *as.callInfo.maxReceiveMessageSize, nil, as.decomp) + if err != nil { + if err == io.EOF { + if statusErr := as.s.Status().Err(); statusErr != nil { + return statusErr + } + return io.EOF // indicates successful end of stream. + } + return toRPCErr(err) + } + + if channelz.IsOn() { + as.t.IncrMsgRecv() + } + if as.desc.ServerStreams { + // Subsequent messages should be received by subsequent RecvMsg calls. + return nil + } + + // Special handling for non-server-stream rpcs. + // This recv expects EOF or errors, so we don't collect inPayload. + err = recv(as.p, as.codec, as.s, as.dc, m, *as.callInfo.maxReceiveMessageSize, nil, as.decomp) + if err == nil { + return toRPCErr(errors.New("grpc: client streaming protocol violation: get , want ")) + } + if err == io.EOF { + return as.s.Status().Err() // non-server streaming Recv returns nil on success + } + return toRPCErr(err) +} + +func (as *addrConnStream) finish(err error) { + as.mu.Lock() + if as.finished { + as.mu.Unlock() + return + } + as.finished = true + if err == io.EOF { + // Ending a stream with EOF indicates a success. + err = nil + } + if as.s != nil { + as.t.CloseStream(as.s, err) + } + + if err != nil { + as.ac.incrCallsFailed() + } else { + as.ac.incrCallsSucceeded() + } + as.cancel() + as.mu.Unlock() +} + // ServerStream defines the server-side behavior of a streaming RPC. // // All errors returned from ServerStream methods are compatible with the @@ -906,6 +1337,15 @@ type serverStream struct { statsHandler stats.Handler + binlog *binarylog.MethodLogger + // serverHeaderBinlogged indicates whether server header has been logged. It + // will happen when one of the following two happens: stream.SendHeader(), + // stream.Send(). + // + // It's only checked in send and sendHeader, doesn't need to be + // synchronized. + serverHeaderBinlogged bool + mu sync.Mutex // protects trInfo.tr after the service handler runs. } @@ -921,7 +1361,15 @@ func (ss *serverStream) SetHeader(md metadata.MD) error { } func (ss *serverStream) SendHeader(md metadata.MD) error { - return ss.t.WriteHeader(ss.s, md) + err := ss.t.WriteHeader(ss.s, md) + if ss.binlog != nil && !ss.serverHeaderBinlogged { + h, _ := ss.s.Header() + ss.binlog.Log(&binarylog.ServerHeader{ + Header: h, + }) + ss.serverHeaderBinlogged = true + } + return err } func (ss *serverStream) SetTrailer(md metadata.MD) { @@ -948,20 +1396,24 @@ func (ss *serverStream) SendMsg(m interface{}) (err error) { if err != nil && err != io.EOF { st, _ := status.FromError(toRPCErr(err)) ss.t.WriteStatus(ss.s, st) + // Non-user specified status was sent out. This should be an error + // case (as a server side Cancel maybe). + // + // This is not handled specifically now. User will return a final + // status from the service handler, we will log that error instead. + // This behavior is similar to an interceptor. } if channelz.IsOn() && err == nil { ss.t.IncrMsgSent() } }() - data, err := encode(ss.codec, m) + + // load hdr, payload, data + hdr, payload, data, err := prepareMsg(m, ss.codec, ss.cp, ss.comp) if err != nil { return err } - compData, err := compress(data, ss.cp, ss.comp) - if err != nil { - return err - } - hdr, payload := msgHeader(data, compData) + // TODO(dfawley): should we be checking len(data) instead? if len(payload) > ss.maxSendMessageSize { return status.Errorf(codes.ResourceExhausted, "trying to send message larger than max (%d vs. %d)", len(payload), ss.maxSendMessageSize) @@ -969,6 +1421,18 @@ func (ss *serverStream) SendMsg(m interface{}) (err error) { if err := ss.t.Write(ss.s, hdr, payload, &transport.Options{Last: false}); err != nil { return toRPCErr(err) } + if ss.binlog != nil { + if !ss.serverHeaderBinlogged { + h, _ := ss.s.Header() + ss.binlog.Log(&binarylog.ServerHeader{ + Header: h, + }) + ss.serverHeaderBinlogged = true + } + ss.binlog.Log(&binarylog.ServerMessage{ + Message: data, + }) + } if ss.statsHandler != nil { ss.statsHandler.HandleRPC(ss.s.Context(), outPayload(false, m, data, payload, time.Now())) } @@ -992,17 +1456,26 @@ func (ss *serverStream) RecvMsg(m interface{}) (err error) { if err != nil && err != io.EOF { st, _ := status.FromError(toRPCErr(err)) ss.t.WriteStatus(ss.s, st) + // Non-user specified status was sent out. This should be an error + // case (as a server side Cancel maybe). + // + // This is not handled specifically now. User will return a final + // status from the service handler, we will log that error instead. + // This behavior is similar to an interceptor. } if channelz.IsOn() && err == nil { ss.t.IncrMsgRecv() } }() - var inPayload *stats.InPayload - if ss.statsHandler != nil { - inPayload = &stats.InPayload{} + var payInfo *payloadInfo + if ss.statsHandler != nil || ss.binlog != nil { + payInfo = &payloadInfo{} } - if err := recv(ss.p, ss.codec, ss.s, ss.dc, m, ss.maxReceiveMessageSize, inPayload, ss.decomp); err != nil { + if err := recv(ss.p, ss.codec, ss.s, ss.dc, m, ss.maxReceiveMessageSize, payInfo, ss.decomp); err != nil { if err == io.EOF { + if ss.binlog != nil { + ss.binlog.Log(&binarylog.ClientHalfClose{}) + } return err } if err == io.ErrUnexpectedEOF { @@ -1010,8 +1483,20 @@ func (ss *serverStream) RecvMsg(m interface{}) (err error) { } return toRPCErr(err) } - if inPayload != nil { - ss.statsHandler.HandleRPC(ss.s.Context(), inPayload) + if ss.statsHandler != nil { + ss.statsHandler.HandleRPC(ss.s.Context(), &stats.InPayload{ + RecvTime: time.Now(), + Payload: m, + // TODO truncate large payload. + Data: payInfo.uncompressedBytes, + WireLength: payInfo.wireLength, + Length: len(payInfo.uncompressedBytes), + }) + } + if ss.binlog != nil { + ss.binlog.Log(&binarylog.ClientMessage{ + Message: payInfo.uncompressedBytes, + }) } return nil } @@ -1021,3 +1506,24 @@ func (ss *serverStream) RecvMsg(m interface{}) (err error) { func MethodFromServerStream(stream ServerStream) (string, bool) { return Method(stream.Context()) } + +// prepareMsg returns the hdr, payload and data +// using the compressors passed or using the +// passed preparedmsg +func prepareMsg(m interface{}, codec baseCodec, cp Compressor, comp encoding.Compressor) (hdr, payload, data []byte, err error) { + if preparedMsg, ok := m.(*PreparedMsg); ok { + return preparedMsg.hdr, preparedMsg.payload, preparedMsg.encodedData, nil + } + // The input interface is not a prepared msg. + // Marshal and Compress the data at this point + data, err = encode(codec, m) + if err != nil { + return nil, nil, nil, err + } + compData, err := compress(data, cp, comp) + if err != nil { + return nil, nil, nil, err + } + hdr, payload = msgHeader(data, compData) + return hdr, payload, data, nil +} diff --git a/vendor/google.golang.org/grpc/tap/tap.go b/vendor/google.golang.org/grpc/tap/tap.go index 22b8fb50dea..584360f681b 100644 --- a/vendor/google.golang.org/grpc/tap/tap.go +++ b/vendor/google.golang.org/grpc/tap/tap.go @@ -21,7 +21,7 @@ package tap import ( - "golang.org/x/net/context" + "context" ) // Info defines the relevant information needed by the handles. diff --git a/vendor/google.golang.org/grpc/trace.go b/vendor/google.golang.org/grpc/trace.go index c1c96dedcb7..0a57b999481 100644 --- a/vendor/google.golang.org/grpc/trace.go +++ b/vendor/google.golang.org/grpc/trace.go @@ -24,6 +24,7 @@ import ( "io" "net" "strings" + "sync" "time" "golang.org/x/net/trace" @@ -53,13 +54,25 @@ type traceInfo struct { } // firstLine is the first line of an RPC trace. +// It may be mutated after construction; remoteAddr specifically may change +// during client-side use. type firstLine struct { + mu sync.Mutex client bool // whether this is a client (outgoing) RPC remoteAddr net.Addr deadline time.Duration // may be zero } +func (f *firstLine) SetRemoteAddr(addr net.Addr) { + f.mu.Lock() + f.remoteAddr = addr + f.mu.Unlock() +} + func (f *firstLine) String() string { + f.mu.Lock() + defer f.mu.Unlock() + var line bytes.Buffer io.WriteString(&line, "RPC: ") if f.client { diff --git a/vendor/google.golang.org/grpc/version.go b/vendor/google.golang.org/grpc/version.go index d668a424dd1..58885056385 100644 --- a/vendor/google.golang.org/grpc/version.go +++ b/vendor/google.golang.org/grpc/version.go @@ -19,4 +19,4 @@ package grpc // Version is the current grpc version. -const Version = "1.14.0" +const Version = "1.23.1" diff --git a/vendor/google.golang.org/grpc/vet.sh b/vendor/google.golang.org/grpc/vet.sh index 44a310e55b4..661e1e1de9b 100644 --- a/vendor/google.golang.org/grpc/vet.sh +++ b/vendor/google.golang.org/grpc/vet.sh @@ -13,19 +13,43 @@ die() { exit 1 } -PATH="$GOPATH/bin:$GOROOT/bin:$PATH" +fail_on_output() { + tee /dev/stderr | (! read) +} -if [ "$1" = "-install" ]; then - go get -d \ - google.golang.org/grpc/... - go get -u \ - github.com/golang/lint/golint \ - golang.org/x/tools/cmd/goimports \ - honnef.co/go/tools/cmd/staticcheck \ - github.com/client9/misspell/cmd/misspell \ - github.com/golang/protobuf/protoc-gen-go - if [[ -z "$VET_SKIP_PROTO" ]]; then - if [[ "$TRAVIS" = "true" ]]; then +# Check to make sure it's safe to modify the user's git repo. +git status --porcelain | fail_on_output + +# Undo any edits made by this script. +cleanup() { + git reset --hard HEAD +} +trap cleanup EXIT + +PATH="${GOPATH}/bin:${GOROOT}/bin:${PATH}" + +if [[ "$1" = "-install" ]]; then + # Check for module support + if go help mod >& /dev/null; then + go install \ + golang.org/x/lint/golint \ + golang.org/x/tools/cmd/goimports \ + honnef.co/go/tools/cmd/staticcheck \ + github.com/client9/misspell/cmd/misspell \ + github.com/golang/protobuf/protoc-gen-go + else + # Ye olde `go get` incantation. + # Note: this gets the latest version of all tools (vs. the pinned versions + # with Go modules). + go get -u \ + golang.org/x/lint/golint \ + golang.org/x/tools/cmd/goimports \ + honnef.co/go/tools/cmd/staticcheck \ + github.com/client9/misspell/cmd/misspell \ + github.com/golang/protobuf/protoc-gen-go + fi + if [[ -z "${VET_SKIP_PROTO}" ]]; then + if [[ "${TRAVIS}" = "true" ]]; then PROTOBUF_VERSION=3.3.0 PROTOC_FILENAME=protoc-${PROTOBUF_VERSION}-linux-x86_64.zip pushd /home/travis @@ -42,48 +66,69 @@ elif [[ "$#" -ne 0 ]]; then die "Unknown argument(s): $*" fi -# TODO: Remove this check and the mangling below once "context" is imported -# directly. -if git status --porcelain | read; then - die "Uncommitted or untracked files found; commit changes first" -fi +# - Ensure all source files contain a copyright message. +git ls-files "*.go" | xargs grep -L "\(Copyright [0-9]\{4,\} gRPC authors\)\|DO NOT EDIT" 2>&1 | fail_on_output -git ls-files "*.go" | xargs grep -L "\(Copyright [0-9]\{4,\} gRPC authors\)\|DO NOT EDIT" 2>&1 | tee /dev/stderr | (! read) -git ls-files "*.go" | xargs grep -l '"unsafe"' 2>&1 | (! grep -v '_test.go') | tee /dev/stderr | (! read) -git ls-files "*.go" | xargs grep -l '"math/rand"' 2>&1 | (! grep -v '^examples\|^stress\|grpcrand') | tee /dev/stderr | (! read) -gofmt -s -d -l . 2>&1 | tee /dev/stderr | (! read) -goimports -l . 2>&1 | tee /dev/stderr | (! read) -golint ./... 2>&1 | (grep -vE "(_mock|\.pb)\.go:" || true) | tee /dev/stderr | (! read) +# - Make sure all tests in grpc and grpc/test use leakcheck via Teardown. +(! grep 'func Test[^(]' *_test.go) +(! grep 'func Test[^(]' test/*.go) -# Undo any edits made by this script. -cleanup() { - git reset --hard HEAD -} -trap cleanup EXIT +# - Do not import math/rand for real library code. Use internal/grpcrand for +# thread safety. +git ls-files "*.go" | xargs grep -l '"math/rand"' 2>&1 | (! grep -v '^examples\|^stress\|grpcrand\|wrr_test') -# Rewrite golang.org/x/net/context -> context imports (see grpc/grpc-go#1484). -# TODO: Remove this mangling once "context" is imported directly (grpc/grpc-go#711). -git ls-files "*.go" | xargs sed -i 's:"golang.org/x/net/context":"context":' -set +o pipefail -# TODO: Stop filtering pb.go files once golang/protobuf#214 is fixed. -go tool vet -all . 2>&1 | grep -vE '(clientconn|transport\/transport_test).go:.*cancel (function|var)' | grep -vF '.pb.go:' | tee /dev/stderr | (! read) -set -o pipefail -git reset --hard HEAD +# - Ensure all ptypes proto packages are renamed when importing. +git ls-files "*.go" | (! xargs grep "\(import \|^\s*\)\"github.com/golang/protobuf/ptypes/") -if [[ -z "$VET_SKIP_PROTO" ]]; then - PATH="/home/travis/bin:$PATH" make proto && \ - git status --porcelain 2>&1 | (! read) || \ +# - Check imports that are illegal in appengine (until Go 1.11). +# TODO: Remove when we drop Go 1.10 support +go list -f {{.Dir}} ./... | xargs go run test/go_vet/vet.go + +# - gofmt, goimports, golint (with exceptions for generated code), go vet. +gofmt -s -d -l . 2>&1 | fail_on_output +goimports -l . 2>&1 | (! grep -vE "(_mock|\.pb)\.go:") | fail_on_output +golint ./... 2>&1 | (! grep -vE "(_mock|\.pb)\.go:") +go vet -all . + +# - Check that generated proto files are up to date. +if [[ -z "${VET_SKIP_PROTO}" ]]; then + PATH="/home/travis/bin:${PATH}" make proto && \ + git status --porcelain 2>&1 | fail_on_output || \ (git status; git --no-pager diff; exit 1) fi -# TODO(menghanl): fix errors in transport_test. -staticcheck -ignore ' -google.golang.org/grpc/internal/transport/transport_test.go:SA2002 -google.golang.org/grpc/benchmark/benchmain/main.go:SA1019 -google.golang.org/grpc/stats/stats_test.go:SA1019 -google.golang.org/grpc/test/end2end_test.go:SA1019 -google.golang.org/grpc/balancer_test.go:SA1019 +# - Check that our module is tidy. +if go help mod >& /dev/null; then + go mod tidy && \ + git status --porcelain 2>&1 | fail_on_output || \ + (git status; git --no-pager diff; exit 1) +fi + +# - Collection of static analysis checks +# TODO(dfawley): don't use deprecated functions in examples. +staticcheck -go 1.9 -checks 'inherit,-ST1015' -ignore ' google.golang.org/grpc/balancer.go:SA1019 +google.golang.org/grpc/balancer/grpclb/grpclb_remote_balancer.go:SA1019 +google.golang.org/grpc/balancer/roundrobin/roundrobin_test.go:SA1019 +google.golang.org/grpc/xds/internal/balancer/edsbalancer/balancergroup.go:SA1019 +google.golang.org/grpc/xds/internal/balancer/xds.go:SA1019 +google.golang.org/grpc/xds/internal/balancer/xds_client.go:SA1019 +google.golang.org/grpc/balancer_conn_wrappers.go:SA1019 +google.golang.org/grpc/balancer_test.go:SA1019 +google.golang.org/grpc/benchmark/benchmain/main.go:SA1019 +google.golang.org/grpc/benchmark/worker/benchmark_client.go:SA1019 +google.golang.org/grpc/clientconn.go:S1024 +google.golang.org/grpc/clientconn_state_transition_test.go:SA1019 google.golang.org/grpc/clientconn_test.go:SA1019 +google.golang.org/grpc/examples/features/debugging/client/main.go:SA1019 +google.golang.org/grpc/examples/features/load_balancing/client/main.go:SA1019 +google.golang.org/grpc/internal/transport/handler_server.go:SA1019 +google.golang.org/grpc/internal/transport/handler_server_test.go:SA1019 +google.golang.org/grpc/resolver/dns/dns_resolver.go:SA1019 +google.golang.org/grpc/stats/stats_test.go:SA1019 +google.golang.org/grpc/test/balancer_test.go:SA1019 +google.golang.org/grpc/test/channelz_test.go:SA1019 +google.golang.org/grpc/test/end2end_test.go:SA1019 +google.golang.org/grpc/test/healthcheck_test.go:SA1019 ' ./... misspell -error . diff --git a/vendor/modules.txt b/vendor/modules.txt index a6d3f5294ef..346e94a0169 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -5,6 +5,18 @@ cloud.google.com/go/compute/metadata github.com/BurntSushi/toml # github.com/VividCortex/mysqlerr v0.0.0-20170204212430-6c6b55f8796f github.com/VividCortex/mysqlerr +# github.com/apache/arrow/go/arrow v0.0.0-20190716210558-5f564424c71c +github.com/apache/arrow/go/arrow +github.com/apache/arrow/go/arrow/array +github.com/apache/arrow/go/arrow/arrio +github.com/apache/arrow/go/arrow/decimal128 +github.com/apache/arrow/go/arrow/float16 +github.com/apache/arrow/go/arrow/internal/bitutil +github.com/apache/arrow/go/arrow/internal/cpu +github.com/apache/arrow/go/arrow/internal/debug +github.com/apache/arrow/go/arrow/internal/flatbuf +github.com/apache/arrow/go/arrow/ipc +github.com/apache/arrow/go/arrow/memory # github.com/aws/aws-sdk-go v1.25.6 github.com/aws/aws-sdk-go/aws github.com/aws/aws-sdk-go/aws/awserr @@ -108,6 +120,8 @@ github.com/golang/protobuf/ptypes github.com/golang/protobuf/ptypes/any github.com/golang/protobuf/ptypes/duration github.com/golang/protobuf/ptypes/timestamp +# github.com/google/flatbuffers v1.11.0 +github.com/google/flatbuffers/go # github.com/gopherjs/gopherjs v0.0.0-20190430165422-3e4dfb77656c github.com/gopherjs/gopherjs/js # github.com/gorilla/websocket v1.4.0 @@ -117,6 +131,10 @@ github.com/gosimple/slug # github.com/grafana/grafana-plugin-model v0.0.0-20190930120109-1fc953a61fb4 github.com/grafana/grafana-plugin-model/go/datasource github.com/grafana/grafana-plugin-model/go/renderer +# github.com/grafana/grafana-plugin-sdk-go v0.0.0-20191024130641-6756418f682c +github.com/grafana/grafana-plugin-sdk-go +github.com/grafana/grafana-plugin-sdk-go/dataframe +github.com/grafana/grafana-plugin-sdk-go/genproto/datasource # github.com/hashicorp/go-hclog v0.8.0 github.com/hashicorp/go-hclog # github.com/hashicorp/go-plugin v1.0.1 @@ -145,6 +163,8 @@ github.com/klauspost/cpuid github.com/lib/pq github.com/lib/pq/oid github.com/lib/pq/scram +# github.com/mattetti/filebuffer v1.0.0 +github.com/mattetti/filebuffer # github.com/mattn/go-colorable v0.1.1 github.com/mattn/go-colorable # github.com/mattn/go-isatty v0.0.7 @@ -287,14 +307,16 @@ google.golang.org/appengine/internal/urlfetch google.golang.org/appengine/urlfetch # google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8 google.golang.org/genproto/googleapis/rpc/status -# google.golang.org/grpc v1.14.0 +# google.golang.org/grpc v1.23.1 google.golang.org/grpc google.golang.org/grpc/balancer google.golang.org/grpc/balancer/base google.golang.org/grpc/balancer/roundrobin +google.golang.org/grpc/binarylog/grpc_binarylog_v1 google.golang.org/grpc/codes google.golang.org/grpc/connectivity google.golang.org/grpc/credentials +google.golang.org/grpc/credentials/internal google.golang.org/grpc/encoding google.golang.org/grpc/encoding/proto google.golang.org/grpc/grpclog @@ -302,9 +324,13 @@ google.golang.org/grpc/health google.golang.org/grpc/health/grpc_health_v1 google.golang.org/grpc/internal google.golang.org/grpc/internal/backoff +google.golang.org/grpc/internal/balancerload +google.golang.org/grpc/internal/binarylog google.golang.org/grpc/internal/channelz google.golang.org/grpc/internal/envconfig google.golang.org/grpc/internal/grpcrand +google.golang.org/grpc/internal/grpcsync +google.golang.org/grpc/internal/syscall google.golang.org/grpc/internal/transport google.golang.org/grpc/keepalive google.golang.org/grpc/metadata @@ -313,6 +339,7 @@ google.golang.org/grpc/peer google.golang.org/grpc/resolver google.golang.org/grpc/resolver/dns google.golang.org/grpc/resolver/passthrough +google.golang.org/grpc/serviceconfig google.golang.org/grpc/stats google.golang.org/grpc/status google.golang.org/grpc/tap