Merge pull request #1137 from BruceWangNo1/s3-select

followed @chrislusf and @iliul 's suggestions
This commit is contained in:
Chris Lu 2019-11-25 19:37:22 -08:00 committed by GitHub
commit 8d068dc5d6
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 626 additions and 20 deletions

View file

@ -1,8 +1,6 @@
sudo: false
language: go
go:
- 1.11.x
- 1.12.x
- 1.13.x
before_install:

22
go.mod
View file

@ -10,10 +10,9 @@ require (
contrib.go.opencensus.io/resource v0.1.2 // indirect
github.com/Azure/azure-amqp-common-go v1.1.4 // indirect
github.com/Azure/azure-pipeline-go v0.2.2 // indirect
github.com/Azure/azure-sdk-for-go v33.0.0+incompatible // indirect
github.com/Azure/azure-storage-blob-go v0.8.0
github.com/Azure/go-autorest v13.0.0+incompatible // indirect
github.com/Azure/go-autorest/tracing v0.5.0 // indirect
github.com/Azure/go-autorest/autorest v0.9.2 // indirect
github.com/DataDog/zstd v1.4.1 // indirect
github.com/GoogleCloudPlatform/cloudsql-proxy v0.0.0-20190828224159-d93c53a4824c // indirect
github.com/Shopify/sarama v1.23.1
@ -22,12 +21,10 @@ require (
github.com/aws/aws-sdk-go v1.23.13
github.com/chrislusf/raft v0.0.0-20190225081310-10d6e2182d92
github.com/coreos/etcd v3.3.15+incompatible // indirect
github.com/coreos/go-semver v0.3.0 // indirect
github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f // indirect
github.com/dgrijalva/jwt-go v3.2.0+incompatible
github.com/disintegration/imaging v1.6.1
github.com/dustin/go-humanize v1.0.0
github.com/eapache/go-resiliency v1.2.0 // indirect
github.com/gabriel-vasile/mimetype v0.3.17
github.com/go-kit/kit v0.9.0 // indirect
github.com/go-redis/redis v6.15.2+incompatible
@ -39,28 +36,23 @@ require (
github.com/google/pprof v0.0.0-20190723021845-34ac40c74b70 // indirect
github.com/google/uuid v1.1.1
github.com/gorilla/mux v1.7.3
github.com/gorilla/websocket v1.4.1 // indirect
github.com/grpc-ecosystem/grpc-gateway v1.11.0 // indirect
github.com/jacobsa/daemonize v0.0.0-20160101105449-e460293e890f
github.com/jcmturner/gofork v1.0.0 // indirect
github.com/juju/errors v0.0.0-20190930114154-d42613fe1ab9 // indirect
github.com/karlseguin/ccache v2.0.3+incompatible
github.com/karlseguin/expect v1.0.1 // indirect
github.com/klauspost/cpuid v1.2.1 // indirect
github.com/klauspost/crc32 v1.2.0
github.com/klauspost/reedsolomon v1.9.2
github.com/klauspost/reedsolomon v1.9.3
github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
github.com/kr/pty v1.1.8 // indirect
github.com/kurin/blazer v0.5.3
github.com/kurin/blazer v0.5.4-0.20190613185654-cf2f27cc0be3
github.com/lib/pq v1.2.0
github.com/magiconair/properties v1.8.1 // indirect
github.com/mattn/go-ieproxy v0.0.0-20190805055040-f9202b1cfdeb // indirect
github.com/mattn/go-isatty v0.0.9 // indirect
github.com/mattn/go-runewidth v0.0.4 // indirect
github.com/minio/minio v0.0.0-20191122214605-0a56e33ce1b5
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect
github.com/nats-io/gnatsd v1.4.1 // indirect
github.com/nats-io/go-nats v1.7.2 // indirect
github.com/nats-io/nats-server/v2 v2.0.4 // indirect
github.com/onsi/ginkgo v1.10.1 // indirect
github.com/onsi/gomega v1.7.0 // indirect
github.com/opentracing/opentracing-go v1.1.0 // indirect
@ -81,7 +73,6 @@ require (
github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd
github.com/satori/go.uuid v1.2.0
github.com/seaweedfs/fuse v0.0.0-20190510212405-310228904eff
github.com/sirupsen/logrus v1.4.2 // indirect
github.com/spaolacci/murmur3 v1.1.0 // indirect
github.com/spf13/afero v1.2.2 // indirect
github.com/spf13/jwalterweatherman v1.1.0 // indirect
@ -107,13 +98,12 @@ require (
golang.org/x/exp v0.0.0-20190829153037-c13cbed26979 // indirect
golang.org/x/image v0.0.0-20190829233526-b3c06291d021 // indirect
golang.org/x/mobile v0.0.0-20190830201351-c6da95954960 // indirect
golang.org/x/net v0.0.0-20190909003024-a7b16738d86b
golang.org/x/sys v0.0.0-20190910064555-bbd175535a8b
golang.org/x/net v0.0.0-20190923162816-aa69164e4478
golang.org/x/sys v0.0.0-20190922100055-0a153f010e69
golang.org/x/tools v0.0.0-20190911022129-16c5e0f7d110
google.golang.org/api v0.9.0
google.golang.org/appengine v1.6.2 // indirect
google.golang.org/grpc v1.23.0
gopkg.in/jcmturner/goidentity.v3 v3.0.0 // indirect
gopkg.in/jcmturner/gokrb5.v7 v7.3.0 // indirect
gopkg.in/karlseguin/expect.v1 v1.0.1 // indirect
honnef.co/go/tools v0.0.1-2019.2.2 // indirect

548
go.sum

File diff suppressed because it is too large Load diff

View file

@ -4,6 +4,8 @@ import (
"crypto/md5"
"encoding/json"
"fmt"
"github.com/minio/minio/cmd"
"github.com/minio/minio/pkg/s3select"
"io"
"io/ioutil"
"net/http"
@ -12,6 +14,7 @@ import (
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/server"
"github.com/gorilla/mux"
xhttp "github.com/minio/minio/cmd/http"
)
var (
@ -115,10 +118,15 @@ func (s3a *S3ApiServer) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *h
}
func (s3a *S3ApiServer) proxyToFiler(w http.ResponseWriter, r *http.Request, destUrl string, responseFn func(proxyResonse *http.Response, w http.ResponseWriter)) {
var method string
method = r.Method
glog.V(2).Infof("s3 proxying %s to %s", r.Method, destUrl)
if r.URL.RawQuery == "select&select-type=2" {
method = "GET"
}
glog.V(2).Infof("s3 proxying %s to %s", method, destUrl)
proxyReq, err := http.NewRequest(r.Method, destUrl, r.Body)
proxyReq, err := http.NewRequest(method, destUrl, r.Body)
if err != nil {
glog.Errorf("NewRequest %s: %v", destUrl, err)
@ -224,3 +232,63 @@ func getObject(vars map[string]string) string {
}
return object
}
func (s3a *S3ApiServer) SelectObjectContent(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
bucket := vars["bucket"]
object := getObject(vars)
s3Select, err := s3select.NewS3Select(r.Body)
if err != nil {
if serr, ok := err.(s3select.SelectError); ok {
encodedErrorResponse := encodeResponse(cmd.APIErrorResponse{
Code: serr.ErrorCode(),
Message: serr.ErrorMessage(),
BucketName: bucket,
Key: object,
Resource: r.URL.Path,
RequestID: w.Header().Get(xhttp.AmzRequestID),
HostID: "",
})
writeResponse(w, serr.HTTPStatusCode(), encodedErrorResponse, "application/xml")
} else {
writeErrorResponse(w, ErrInternalError, r.URL)
}
return
}
if strings.HasSuffix(r.URL.Path, "/") {
writeErrorResponse(w, ErrNotImplemented, r.URL)
return
}
destUrl := fmt.Sprintf("http://%s%s/%s%s",
s3a.option.Filer, s3a.option.BucketsPath, bucket, object)
passThroughResponseSelectObjectContent := func(proxyResonse *http.Response,
w http.ResponseWriter) {
getObject := func(offset, length int64) (io.ReadCloser, error) {
return proxyResonse.Body, nil
}
if err = s3Select.Open(getObject); err != nil {
if serr, ok := err.(s3select.SelectError); ok {
encodedErrorResponse := encodeResponse(cmd.APIErrorResponse{
Code: serr.ErrorCode(),
Message: serr.ErrorMessage(),
BucketName: bucket,
Key: object,
Resource: r.URL.Path,
RequestID: w.Header().Get(xhttp.AmzRequestID),
HostID: "",
})
writeResponse(w, serr.HTTPStatusCode(), encodedErrorResponse, mimeXML)
} else {
writeResponse(w, http.StatusInternalServerError, encodeResponse("not s3select.SelectError"), mimeXML)
}
return
}
s3Select.Evaluate(w)
s3Select.Close()
}
s3a.proxyToFiler(w, r, destUrl, passThroughResponseSelectObjectContent)
}

View file

@ -79,6 +79,8 @@ func (s3a *S3ApiServer) registerRouter(router *mux.Router) {
bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(s3a.GetObjectHandler)
// ListObjectsV1 (Legacy)
bucket.Methods("GET").HandlerFunc(s3a.ListObjectsV1Handler)
// SelectObjectContent
bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(s3a.SelectObjectContent).Queries("select", "", "select-type", "2")
// DeleteMultipleObjects
bucket.Methods("POST").HandlerFunc(s3a.DeleteMultipleObjectsHandler).Queries("delete", "")