diff --git a/Makefile b/Makefile index 52667ea..b7c6da9 100644 --- a/Makefile +++ b/Makefile @@ -38,8 +38,10 @@ bench: ## Run benchmarks go test -run '^$$' -bench . -benchmem ./... .PHONY: fuzz -fuzz: ## Fuzz the filter parser for a minute +fuzz: ## Fuzz the filter parser and the bucket servers for a minute each go test -run '^$$' -fuzz '^FuzzParseFilter$$' -fuzztime 60s ./gcp/compute/ + go test -run '^$$' -fuzz '^FuzzServeHTTP$$' -fuzztime 60s ./aws/s3/ + go test -run '^$$' -fuzz '^FuzzServeHTTP$$' -fuzztime 60s ./gcp/storage/ .PHONY: tidy tidy: ## Tidy go.mod files diff --git a/README.md b/README.md index e5d38c7..b746baf 100644 --- a/README.md +++ b/README.md @@ -22,8 +22,10 @@ failure replays from its seed. | --- | --- | | [`fault`](https://pkg.go.dev/github.com/parallelworks/squall/fault) | Decides which calls fail and how: rejected, applied with the response lost, applied with an error, an operation that ends in an error, or slow. Scripted rules and a seeded random rate. | | [`aws/ec2`](https://pkg.go.dev/github.com/parallelworks/squall/aws/ec2) | Amazon EC2: instances (run with client tokens, describe with filters and pages, start, stop, terminate, change type), images (describe, register from snapshots, deregister), instance types, EBS volumes (create, describe, attach, detach, delete), EBS snapshots (create, describe with filters and pages, delete), placement groups, tags, Elastic IP addresses (allocate, describe, release, associate with an interface or instance, disassociate) and network interfaces (create with client tokens, describe, delete). | +| [`aws/s3`](https://pkg.go.dev/github.com/parallelworks/squall/aws/s3) | Amazon S3: general purpose buckets (create, head, list with pages and filters, delete), their tags, versioning, policy and CORS rules, and objects (put, delete one or many, list versions and delete markers). | | [`azure/arm`](https://pkg.go.dev/github.com/parallelworks/squall/azure/arm) | Azure Resource Manager: resource groups (get, create or update, list with tag filters, delete with everything in them that nothing outside uses), virtual machines (create or update, get, list, instance view, deallocate, start, delete), Uniform scale sets (create or update, update, get, list, delete, delete instances) and their VMs, network security groups, network interfaces and public IP addresses, the IP configurations of scale set VMs, private DNS zones, virtual network links and A record sets, and long-running operations to poll. | | [`gcp/compute`](https://pkg.go.dev/github.com/parallelworks/squall/gcp/compute) | Compute Engine: instances (bulkInsert, insert, get, list, aggregatedList, delete, stop, start, setLabels, setMachineType, addAccessConfig, attachDisk, detachDisk), disks (get, list, aggregatedList, insert, delete), snapshots (insert, get, list, delete), zones, regional addresses (get, list, insert, delete, setLabels), resource policies, instance templates, routes, and zone, region and global operations (get, wait). | +| [`gcp/storage`](https://pkg.go.dev/github.com/parallelworks/squall/gcp/storage) | Cloud Storage: buckets (insert, get, list, patch labels, versioning and CORS, delete) and objects (multipart and media uploads, list with versions, delete by generation). | Coverage grows with use: each stand-in implements the methods someone needed, answers anything else with 404, and documents how faithful it is. Contributions @@ -160,6 +162,51 @@ For Amazon EC2: first instance, for `RunInstances`). A rejected call has no effect, so a rejected describe does not move an instance along. +For Amazon S3: + +- Requests are path-style, which the SDK uses for an IP endpoint such as + httptest's. Signatures are not checked; the region comes from the SigV4 + credential scope. A call on a bucket in another region fails with 301 + `PermanentRedirect` and `x-amz-bucket-region`, as S3 answers it. +- There is one account. `PutForeignBucket` holds a name for another one, so + creating it fails with `BucketAlreadyExists` and calls on it with + `AccessDenied` (403 for `HeadBucket`). Creating an owned bucket again + succeeds in us-east-1 and fails with `BucketAlreadyOwnedByYou` elsewhere. +- `ListBuckets` with any parameter pages (10,000 buckets by default) and + reports each bucket's region, as S3 does; with none it lists every bucket + without regions. `ListObjectVersions` orders by key and then newest first, and its markers + are positions, so the next page still follows one whose versions were just + deleted. `DeleteBucket` fails with `BucketNotEmpty` while any version or + delete marker remains. +- Objects keep their size and ETag, not their data. Checksums are not + verified, and multipart uploads, copies, object tags and lifecycle are not + supported. Policies and CORS rules are recorded, not enforced. S3 documents + in-progress multipart uploads as blocking `DeleteBucket` only for directory + buckets. +- A fault rule's `Resource` is the bucket name, or `bucket/key` for an object + call. A lost `HeadBucket` breaks off inside its status line, since a sent + header would read as an answer. + +For Cloud Storage: + +- Point the client at `srv.URL + "/storage/v1/"`, or set + `STORAGE_EMULATOR_HOST`. There is one caller, which reaches every project; + `PutForeignBucket` holds a name it cannot reach, so inserting it fails with + 409 `conflict` and calls on it with 403 `forbidden`. A repeated insert of + the caller's own bucket fails with 409, as the client's retry of a create + that took effect does. +- Bucket names and labels follow the API's rules; locations are upper-cased + but not checked. Fields the server does not model, such as lifecycle rules + or retention, are rejected with 400. A patch with a field set to null + clears it, and a label set to null is deleted. +- Objects keep their size, MD5 and CRC32C, which an upload's checksums must + match. With versioning on, overwrites and deletes keep the old generation as + noncurrent, and a bucket with any generation left cannot be deleted. Page + tokens are positions, so a listing survives deletes between pages. + Resumable uploads, reads, IAM policies and HMAC keys are not supported. +- A fault rule's `Resource` is the path below `/storage/v1/`, such as + `b/my-bucket/o/my-object`, and for an insert the bucket or object it makes. + For Azure Resource Manager: - Long-running operations finish within the call that starts them, so the diff --git a/aws/s3/bench_test.go b/aws/s3/bench_test.go new file mode 100644 index 0000000..9279ec2 --- /dev/null +++ b/aws/s3/bench_test.go @@ -0,0 +1,37 @@ +package s3_test + +import ( + "net/http" + "net/http/httptest" + "strconv" + "strings" + "testing" + + "github.com/parallelworks/squall/aws/s3" +) + +func BenchmarkPutListDeleteVersions(b *testing.B) { + s := s3.New() + if _, err := s.PutBucket(s3.Bucket{Name: "bench", Region: "us-east-1", Versioning: "Enabled"}); err != nil { + b.Fatal(err) + } + for i := range 1000 { + if _, err := s.PutObject("bench", "key-"+strconv.Itoa(i), []byte("data")); err != nil { + b.Fatal(err) + } + } + do := func(method, target, body string) { + req := httptest.NewRequestWithContext(b.Context(), method, target, strings.NewReader(body)) + req.Header.Set("Authorization", authorization) + rec := httptest.NewRecorder() + s.ServeHTTP(rec, req) + if rec.Code >= http.StatusBadRequest { + b.Fatalf("%s %s = %d %s", method, target, rec.Code, rec.Body) + } + } + for b.Loop() { + do(http.MethodPut, "/bench/new", "data") + do(http.MethodGet, "/bench?versions&max-keys=100&prefix=key-1", "") + do(http.MethodPost, "/bench?delete", "new") + } +} diff --git a/aws/s3/buckets.go b/aws/s3/buckets.go new file mode 100644 index 0000000..7f95f8f --- /dev/null +++ b/aws/s3/buckets.go @@ -0,0 +1,523 @@ +package s3 + +import ( + "encoding/base64" + "encoding/json" + "encoding/xml" + "errors" + "fmt" + "maps" + "net/http" + "net/netip" + "slices" + "strconv" + "strings" + "time" + "unicode" + "unicode/utf8" +) + +const ( + versioningEnabled = "Enabled" + versioningSuspended = "Suspended" + + maxTags = 50 +) + +// Bucket is a general purpose S3 bucket. +type Bucket struct { + Name string + // Region is where the bucket is. + Region string + CreationDate time.Time + // Tags is nil while the bucket has no tag set. + Tags map[string]string + // Versioning is "" for a bucket never versioned, "Enabled" or "Suspended". + Versioning string + // Policy is the bucket policy JSON as sent, empty for none. + Policy string + // CORS is the CORSConfiguration XML as sent, empty for none. + CORS string +} + +type bucket struct { + Bucket + // objects holds each key's versions, newest first. + objects map[string][]*version + // nullSeqs holds each key's last "null" version's sequence number, kept after a delete so a marker naming it keeps its place. + nullSeqs map[string]uint64 +} + +func (b *bucket) clone() Bucket { + c := b.Bucket + c.Tags = maps.Clone(b.Tags) + return c +} + +// lookup returns the bucket c names as the account sees it. The caller holds s.mu. +func (s *Server) lookup(c *call) (*bucket, *apiError) { + if b, ok := s.buckets[c.bucket]; ok { + if b.Region != c.region { + return nil, redirect(c.bucket, b.Region) + } + return b, nil + } + if region, ok := s.foreign[c.bucket]; ok { + if region != c.region { + return nil, redirect(c.bucket, region) + } + denied := accessDenied(c.bucket) + if c.r.Method == http.MethodHead { + denied.Region = region + } + return nil, denied + } + return nil, noSuchBucket(c.bucket) +} + +// validBucketName reports whether name follows the general purpose bucket naming rules. +func validBucketName(name string) bool { + if len(name) < 3 || len(name) > 63 || strings.Contains(name, "..") { + return false + } + for _, c := range []byte(name) { + if (c < 'a' || c > 'z') && (c < '0' || c > '9') && c != '.' && c != '-' { + return false + } + } + first, last := name[0], name[len(name)-1] + if first == '.' || first == '-' || last == '.' || last == '-' { + return false + } + if _, err := netip.ParseAddr(name); err == nil { + return false + } + for _, prefix := range []string{"xn--", "sthree-", "amzn-s3-demo-"} { + if strings.HasPrefix(name, prefix) { + return false + } + } + for _, suffix := range []string{"-s3alias", "--ol-s3", ".mrap", "--x-s3", "--table-s3"} { + if strings.HasSuffix(name, suffix) { + return false + } + } + return true +} + +func invalidBucketName(name string) *apiError { + return &apiError{Status: http.StatusBadRequest, Code: "InvalidBucketName", Message: "The specified bucket is not valid.", Bucket: name} +} + +type createBucketConfiguration struct { + LocationConstraint string `xml:"LocationConstraint"` + Location *struct{} `xml:"Location"` + Bucket *struct{} `xml:"Bucket"` + Tags *struct{} `xml:"Tags"` +} + +func (s *Server) createBucket(c *call) (*reply, *apiError) { + if !validBucketName(c.bucket) { + return nil, invalidBucketName(c.bucket) + } + if acl := c.r.Header.Get("x-amz-acl"); acl != "" && acl != "private" { + return nil, &apiError{ + Status: http.StatusBadRequest, Code: "InvalidBucketAclWithObjectOwnership", + Message: "Bucket cannot have ACLs set with ObjectOwnership's BucketOwnerEnforced setting", + } + } + if o := c.r.Header.Get("x-amz-object-ownership"); o != "" && o != "BucketOwnerEnforced" { + return nil, invalidArgument("squall: object ownership %s is not supported", o) + } + if strings.EqualFold(c.r.Header.Get("x-amz-bucket-object-lock-enabled"), "true") { + return nil, invalidArgument("squall: object lock is not supported") + } + var cfg createBucketConfiguration + if len(c.body) > 0 { + if err := decodeXML(c.body, &cfg); err != nil { + return nil, err + } + if cfg.Location != nil || cfg.Bucket != nil || cfg.Tags != nil { + return nil, invalidArgument("squall: directory buckets and tags at creation are not supported") + } + } + s.mu.Lock() + defer s.mu.Unlock() + if _, ok := s.foreign[c.bucket]; ok { + return nil, &apiError{ + Status: http.StatusConflict, Code: "BucketAlreadyExists", + Message: "The requested bucket name is not available. The bucket namespace is shared by all users of the system. Please select a different name and try again.", + Bucket: c.bucket, + } + } + if b, ok := s.buckets[c.bucket]; ok { + if b.Region == "us-east-1" && c.region == "us-east-1" { + return &reply{header: map[string]string{"Location": "/" + c.bucket}}, nil + } + return nil, &apiError{ + Status: http.StatusConflict, Code: "BucketAlreadyOwnedByYou", + Message: "Your previous request to create the named bucket succeeded and you already own it.", Bucket: c.bucket, + } + } + switch constraint := cfg.LocationConstraint; { + case constraint == "us-east-1": + return nil, &apiError{Status: http.StatusBadRequest, Code: "InvalidLocationConstraint", Message: "The specified location-constraint is not valid"} + case constraint == "" && c.region != "us-east-1": + return nil, &apiError{ + Status: http.StatusBadRequest, Code: "IllegalLocationConstraintException", + Message: "The unspecified location constraint is incompatible for the region specific endpoint this request was sent to.", + } + case constraint != "" && constraint != c.region: + return nil, &apiError{ + Status: http.StatusBadRequest, Code: "IllegalLocationConstraintException", + Message: fmt.Sprintf("The %s location constraint is incompatible for the region specific endpoint this request was sent to.", constraint), + } + } + s.buckets[c.bucket] = &bucket{Name: c.bucket, Region: c.region, CreationDate: s.now().UTC(), objects: map[string][]*version{}} + location := "/" + c.bucket + if c.region != "us-east-1" { + location = "http://" + c.bucket + ".s3.amazonaws.com/" + } + return &reply{header: map[string]string{"Location": location}}, nil +} + +func (s *Server) headBucket(c *call) (*reply, *apiError) { + s.mu.Lock() + defer s.mu.Unlock() + b, err := s.lookup(c) + if err != nil { + return nil, err + } + return &reply{header: map[string]string{"x-amz-bucket-region": b.Region, "x-amz-access-point-alias": "false"}}, nil +} + +func (s *Server) deleteBucket(c *call) (*reply, *apiError) { + s.mu.Lock() + defer s.mu.Unlock() + b, err := s.lookup(c) + if err != nil { + return nil, err + } + if len(b.objects) > 0 { + message := "The bucket you tried to delete is not empty" + if b.Versioning != "" { + message += ". You must delete all versions in the bucket." + } + return nil, &apiError{Status: http.StatusConflict, Code: "BucketNotEmpty", Message: message, Bucket: c.bucket} + } + delete(s.buckets, c.bucket) + return &reply{status: http.StatusNoContent}, nil +} + +type ownerXML struct { + ID string `xml:"ID"` +} + +type listBucketsXML struct { + XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListAllMyBucketsResult"` + Owner ownerXML `xml:"Owner"` + Buckets []bucketXML `xml:"Buckets>Bucket"` + ContinuationToken string `xml:"ContinuationToken,omitempty"` + Prefix string `xml:"Prefix,omitempty"` +} + +type bucketXML struct { + Name string `xml:"Name"` + CreationDate string `xml:"CreationDate"` + BucketRegion string `xml:"BucketRegion,omitempty"` +} + +// listBucketsParams are the ListBuckets parameters that make it page and report regions. +var listBucketsParams = []string{"max-buckets", "continuation-token", "prefix", "bucket-region"} + +func (s *Server) listBuckets(c *call) (*reply, *apiError) { + paged := slices.ContainsFunc(listBucketsParams, c.query.Has) + limit := 10_000 + if c.query.Has("max-buckets") { + n, err := strconv.Atoi(c.query.Get("max-buckets")) + if err != nil || n < 1 || n > 10_000 { + return nil, invalidArgument("Argument max-buckets must be an integer between 1 and 10000") + } + limit = n + } + after := "" + if token := c.query.Get("continuation-token"); token != "" { + b, err := base64.RawURLEncoding.DecodeString(token) + if err != nil || len(b) == 0 { + return nil, invalidArgument("The continuation token provided is incorrect") + } + after = string(b) + } + prefix, region := c.query.Get("prefix"), c.query.Get("bucket-region") + if region != "" && region != c.region { + return nil, invalidArgument("squall: bucket-region %s differs from the request's region %s, which S3 does not support", region, c.region) + } + s.mu.Lock() + defer s.mu.Unlock() + out := listBucketsXML{Owner: ownerXML{ID: ownerID}, Prefix: prefix} + for _, name := range slices.Sorted(maps.Keys(s.buckets)) { + b := s.buckets[name] + if name <= after || !strings.HasPrefix(name, prefix) || region != "" && b.Region != region { + continue + } + if paged && len(out.Buckets) == limit { + out.ContinuationToken = base64.RawURLEncoding.EncodeToString([]byte(out.Buckets[limit-1].Name)) + break + } + entry := bucketXML{Name: name, CreationDate: b.CreationDate.Format(timeFormat)} + if paged { + entry.BucketRegion = b.Region + } + out.Buckets = append(out.Buckets, entry) + } + return &reply{body: out}, nil +} + +type taggingXML struct { + XMLName xml.Name `xml:"Tagging"` + TagSet []tagXML `xml:"TagSet>Tag"` +} + +type tagXML struct { + Key string `xml:"Key"` + Value string `xml:"Value"` +} + +func (s *Server) getBucketTagging(c *call) (*reply, *apiError) { + s.mu.Lock() + defer s.mu.Unlock() + b, err := s.lookup(c) + if err != nil { + return nil, err + } + if b.Tags == nil { + return nil, &apiError{Status: http.StatusNotFound, Code: "NoSuchTagSet", Message: "The TagSet does not exist", Bucket: c.bucket} + } + out := taggingXML{XMLName: xml.Name{Space: namespace, Local: "Tagging"}} + for _, k := range slices.Sorted(maps.Keys(b.Tags)) { + out.TagSet = append(out.TagSet, tagXML{k, b.Tags[k]}) + } + return &reply{body: out}, nil +} + +func validTagText(s string) bool { + for _, r := range s { + if !unicode.In(r, unicode.L, unicode.Z, unicode.N) && !strings.ContainsRune("+-=._:/@", r) { + return false + } + } + return true +} + +func parseTags(body []byte) (map[string]string, *apiError) { + var in taggingXML + if err := decodeXML(body, &in); err != nil { + return nil, err + } + if len(in.TagSet) > maxTags { + return nil, invalidTag(fmt.Sprintf("Bucket tag count cannot be greater than %d", maxTags)) + } + tags := map[string]string{} + for _, t := range in.TagSet { + switch { + case t.Key == "" || utf8.RuneCountInString(t.Key) > 128 || !validTagText(t.Key): + return nil, invalidTag("The TagKey you have provided is invalid") + case utf8.RuneCountInString(t.Value) > 256 || !validTagText(t.Value): + return nil, invalidTag("The TagValue you have provided is invalid") + case strings.HasPrefix(strings.ToLower(t.Key), "aws:"): + return nil, invalidTag("System tags cannot be added/updated by requester") + } + if _, dup := tags[t.Key]; dup { + return nil, invalidTag("Cannot provide multiple Tags with the same key") + } + tags[t.Key] = t.Value + } + if len(tags) == 0 { + return nil, nil + } + return tags, nil +} + +func (s *Server) putBucketTagging(c *call) (*reply, *apiError) { + tags, err := parseTags(c.body) + if err != nil { + return nil, err + } + s.mu.Lock() + defer s.mu.Unlock() + b, err := s.lookup(c) + if err != nil { + return nil, err + } + b.Tags = tags + return &reply{status: http.StatusNoContent}, nil +} + +func (s *Server) deleteBucketTagging(c *call) (*reply, *apiError) { + s.mu.Lock() + defer s.mu.Unlock() + b, err := s.lookup(c) + if err != nil { + return nil, err + } + b.Tags = nil + return &reply{status: http.StatusNoContent}, nil +} + +type versioningXML struct { + XMLName xml.Name `xml:"VersioningConfiguration"` + Status string `xml:"Status,omitempty"` + MfaDelete string `xml:"MfaDelete,omitempty"` +} + +func (s *Server) getBucketVersioning(c *call) (*reply, *apiError) { + s.mu.Lock() + defer s.mu.Unlock() + b, err := s.lookup(c) + if err != nil { + return nil, err + } + return &reply{body: versioningXML{XMLName: xml.Name{Space: namespace, Local: "VersioningConfiguration"}, Status: b.Versioning}}, nil +} + +func (s *Server) putBucketVersioning(c *call) (*reply, *apiError) { + var in versioningXML + if err := decodeXML(c.body, &in); err != nil { + return nil, err + } + if in.MfaDelete != "" { + return nil, invalidArgument("squall: MFA delete is not supported") + } + if in.Status != versioningEnabled && in.Status != versioningSuspended { + return nil, malformedXML() + } + s.mu.Lock() + defer s.mu.Unlock() + b, err := s.lookup(c) + if err != nil { + return nil, err + } + b.Versioning = in.Status + return &reply{}, nil +} + +func (s *Server) putBucketPolicy(c *call) (*reply, *apiError) { + var policy map[string]any + if len(c.body) == 0 || c.body[0] != '{' || json.Unmarshal(c.body, &policy) != nil { + return nil, &apiError{Status: http.StatusBadRequest, Code: "MalformedPolicy", Message: "Policies must be valid JSON and the first byte must be '{'"} + } + s.mu.Lock() + defer s.mu.Unlock() + b, err := s.lookup(c) + if err != nil { + return nil, err + } + b.Policy = string(c.body) + return &reply{status: http.StatusNoContent}, nil +} + +type corsXML struct { + Rules []struct { + AllowedMethods []string `xml:"AllowedMethod"` + AllowedOrigins []string `xml:"AllowedOrigin"` + } `xml:"CORSRule"` +} + +func (s *Server) putBucketCors(c *call) (*reply, *apiError) { + var in corsXML + if err := decodeXML(c.body, &in); err != nil { + return nil, err + } + if len(in.Rules) == 0 || len(in.Rules) > 100 { + return nil, malformedXML() + } + for _, rule := range in.Rules { + if len(rule.AllowedMethods) == 0 || len(rule.AllowedOrigins) == 0 { + return nil, malformedXML() + } + for _, m := range rule.AllowedMethods { + if !slices.Contains([]string{"GET", "PUT", "HEAD", "POST", "DELETE"}, m) { + return nil, &apiError{ + Status: http.StatusBadRequest, Code: "InvalidRequest", + Message: "Found unsupported HTTP method in CORS config. Unsupported method is " + m, + } + } + } + } + s.mu.Lock() + defer s.mu.Unlock() + b, err := s.lookup(c) + if err != nil { + return nil, err + } + b.CORS = string(c.body) + return &reply{}, nil +} + +// Buckets returns every bucket the account owns, ordered by name. +func (s *Server) Buckets() []Bucket { + s.mu.Lock() + defer s.mu.Unlock() + out := make([]Bucket, 0, len(s.buckets)) + for _, name := range slices.Sorted(maps.Keys(s.buckets)) { + out = append(out, s.buckets[name].clone()) + } + return out +} + +// LookupBucket returns the named bucket, if the account owns it. +func (s *Server) LookupBucket(name string) (Bucket, bool) { + s.mu.Lock() + defer s.mu.Unlock() + b, ok := s.buckets[name] + if !ok { + return Bucket{}, false + } + return b.clone(), true +} + +// PutBucket stores b, which needs a valid name and a region, keeping the objects of a bucket it replaces. +func (s *Server) PutBucket(b Bucket) (Bucket, error) { + if !validBucketName(b.Name) { + return Bucket{}, invalidBucketName(b.Name) + } + if b.Region == "" { + return Bucket{}, errors.New("squall: a bucket needs a region") + } + if b.Versioning != "" && b.Versioning != versioningEnabled && b.Versioning != versioningSuspended { + return Bucket{}, fmt.Errorf("squall: versioning %q is not one of Enabled and Suspended", b.Versioning) + } + s.mu.Lock() + defer s.mu.Unlock() + if _, ok := s.foreign[b.Name]; ok { + return Bucket{}, fmt.Errorf("squall: bucket %s belongs to another account", b.Name) + } + if b.CreationDate.IsZero() { + b.CreationDate = s.now() + } + b.CreationDate = b.CreationDate.UTC() + b.Tags = maps.Clone(b.Tags) + stored := &bucket{Bucket: b, objects: map[string][]*version{}} + if old, ok := s.buckets[b.Name]; ok { + stored.objects, stored.nullSeqs = old.objects, old.nullSeqs + } + s.buckets[b.Name] = stored + return stored.clone(), nil +} + +// PutForeignBucket holds name in region for another account, so creating it conflicts and calls on it are denied. +func (s *Server) PutForeignBucket(name, region string) error { + if !validBucketName(name) { + return invalidBucketName(name) + } + if region == "" { + return errors.New("squall: a bucket needs a region") + } + s.mu.Lock() + defer s.mu.Unlock() + if _, ok := s.buckets[name]; ok { + return fmt.Errorf("squall: the account owns bucket %s", name) + } + s.foreign[name] = region + return nil +} diff --git a/aws/s3/errors.go b/aws/s3/errors.go new file mode 100644 index 0000000..7c4dc3f --- /dev/null +++ b/aws/s3/errors.go @@ -0,0 +1,96 @@ +package s3 + +import ( + "cmp" + "encoding/xml" + "fmt" + "net/http" + + "github.com/parallelworks/squall/fault" +) + +type apiError struct { + Status int + Code string + Message string + // Bucket names the bucket the error is about, when there is one. + Bucket string + // Region is the region the bucket is in, sent in x-amz-bucket-region for a redirect or a denied HeadBucket. + Region string +} + +func (e *apiError) Error() string { return fmt.Sprintf("%d %s: %s", e.Status, e.Code, e.Message) } + +func invalidArgument(format string, args ...any) *apiError { + return &apiError{Status: http.StatusBadRequest, Code: "InvalidArgument", Message: fmt.Sprintf(format, args...)} +} + +func malformedXML() *apiError { + return &apiError{ + Status: http.StatusBadRequest, Code: "MalformedXML", + Message: "The XML you provided was not well-formed or did not validate against our published schema", + } +} + +func noSuchBucket(name string) *apiError { + return &apiError{Status: http.StatusNotFound, Code: "NoSuchBucket", Message: "The specified bucket does not exist", Bucket: name} +} + +func accessDenied(name string) *apiError { + return &apiError{Status: http.StatusForbidden, Code: "AccessDenied", Message: "Access Denied", Bucket: name} +} + +func invalidTag(message string) *apiError { + return &apiError{Status: http.StatusBadRequest, Code: "InvalidTag", Message: message} +} + +func redirect(name, region string) *apiError { + return &apiError{ + Status: http.StatusMovedPermanently, Code: "PermanentRedirect", + Message: "The bucket you are attempting to access must be addressed using the specified endpoint. Please send all future requests to this endpoint.", + Bucket: name, Region: region, + } +} + +// faultError maps the fault package's generic reasons to S3 codes; any other reason is the code itself. +func faultError(f fault.Fault) *apiError { + status := cmp.Or(f.Status, http.StatusServiceUnavailable) + code := f.Reason + switch code { + case "", "backendError": + code = "InternalError" + if status == http.StatusServiceUnavailable { + code = "ServiceUnavailable" + } + case "rateLimitExceeded": + code, status = "SlowDown", http.StatusServiceUnavailable + } + return &apiError{Status: status, Code: code, Message: fmt.Sprintf("squall: injected fault (%s)", f)} +} + +type errorXML struct { + XMLName xml.Name `xml:"Error"` + Code string `xml:"Code"` + Message string `xml:"Message"` + BucketName string `xml:"BucketName,omitempty"` + Bucket string `xml:"Bucket,omitempty"` + Endpoint string `xml:"Endpoint,omitempty"` + RequestID string `xml:"RequestId"` + HostID string `xml:"HostId"` +} + +// writeError answers with e, with no body for a HEAD, as S3 answers one. +func writeError(w http.ResponseWriter, r *http.Request, requestID string, e *apiError) { + if e.Region != "" { + w.Header().Set("x-amz-bucket-region", e.Region) + } + if r.Method == http.MethodHead { + w.WriteHeader(e.Status) + return + } + body := errorXML{Code: e.Code, Message: e.Message, BucketName: e.Bucket, RequestID: requestID, HostID: "squall/" + requestID} + if e.Status == http.StatusMovedPermanently { + body.BucketName, body.Bucket, body.Endpoint = "", e.Bucket, "s3."+e.Region+".amazonaws.com" + } + writeXML(w, e.Status, body) +} diff --git a/aws/s3/example_test.go b/aws/s3/example_test.go new file mode 100644 index 0000000..d0748c3 --- /dev/null +++ b/aws/s3/example_test.go @@ -0,0 +1,38 @@ +package s3_test + +import ( + "fmt" + "net/http" + "net/http/httptest" + + "github.com/parallelworks/squall/aws/s3" + "github.com/parallelworks/squall/fault" +) + +// The first create takes effect but its response is lost, so the caller sees +// an error even though the bucket now exists. Code under test must find and +// claim it rather than take it for someone else's. Real tests point the +// official SDK at srv.URL with its base endpoint setting. +func Example() { + faults := fault.New().Add(fault.Rule{Method: "s3.CreateBucket", Nth: 1, Fault: fault.LoseResponse()}) + s := s3.New(s3.WithFaults(faults)) + srv := httptest.NewServer(s) + defer srv.Close() + + req, _ := http.NewRequest(http.MethodPut, srv.URL+"/my-bucket", http.NoBody) + req.Header.Set("Authorization", "AWS4-HMAC-SHA256 Credential=AKID/20260101/us-east-1/s3/aws4_request, SignedHeaders=host, Signature=0") + resp, err := http.DefaultClient.Do(req) + if err == nil { + _, err = resp.Body.Read(make([]byte, 1<<10)) + resp.Body.Close() + } + fmt.Println("create failed:", err != nil) + + b, _ := s.LookupBucket("my-bucket") + fmt.Println("bucket exists:", b.Name, b.Region) + fmt.Println(faults.Injected()[0]) + // Output: + // create failed: true + // bucket exists: my-bucket us-east-1 + // #1 s3.CreateBucket my-bucket: lose response +} diff --git a/aws/s3/http_test.go b/aws/s3/http_test.go new file mode 100644 index 0000000..12088a5 --- /dev/null +++ b/aws/s3/http_test.go @@ -0,0 +1,224 @@ +package s3_test + +import ( + "bytes" + "encoding/xml" + "errors" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/parallelworks/squall/aws/s3" +) + +const authorization = "AWS4-HMAC-SHA256 Credential=AKID/20260101/us-east-1/s3/aws4_request, SignedHeaders=host, Signature=0" + +// do sends one request straight to s and returns the status and body. +func do(t *testing.T, s *s3.Server, method, target, auth, body string) (int, http.Header, string) { + t.Helper() + req := httptest.NewRequestWithContext(t.Context(), method, target, strings.NewReader(body)) + if auth != "" { + req.Header.Set("Authorization", auth) + } + rec := httptest.NewRecorder() + s.ServeHTTP(rec, req) + return rec.Code, rec.Header(), rec.Body.String() +} + +func TestResponsesUseTheS3Shape(t *testing.T) { + s := s3.New() + if code, h, body := do(t, s, http.MethodPut, "/shape", authorization, ""); code != http.StatusOK || h.Get("Location") != "/shape" || + h.Get("x-amz-request-id") == "" || body != "" { + t.Fatalf("create = %d %v %q", code, h, body) + } + code, _, body := do(t, s, http.MethodGet, "/", authorization, "") + for _, want := range []string{ + ``, + "", "shape", + } { + if code != http.StatusOK || !strings.Contains(body, want) { + t.Fatalf("list = %d %s, want %s", code, body, want) + } + } + code, _, body = do(t, s, http.MethodGet, "/missing?tagging", authorization, "") + if code != http.StatusNotFound || !strings.Contains(body, + "NoSuchBucketThe specified bucket does not existmissing") { + t.Fatalf("missing = %d %s", code, body) + } + code, h, body := do(t, s, http.MethodHead, "/missing", authorization, "") + if code != http.StatusNotFound || body != "" || h.Get("Content-Type") != "" { + t.Fatalf("head = %d %v %q", code, h, body) + } +} + +func TestRequestsTheServerCannotRead(t *testing.T) { + s := s3.New() + for _, tc := range []struct { + name, method, target, auth, body string + status int + code string + }{ + {"no credentials", http.MethodGet, "/", "", "", http.StatusForbidden, "AccessDenied"}, + {"a malformed credential scope", http.MethodGet, "/", "AWS4-HMAC-SHA256 Credential=AKID/us-east-1", "", http.StatusBadRequest, "AuthorizationHeaderMalformed"}, + {"another signing algorithm", http.MethodGet, "/", "AWS AKID:sig", "", http.StatusBadRequest, "InvalidArgument"}, + {"an unimplemented operation", http.MethodGet, "/b?lifecycle", authorization, "", http.StatusNotFound, "NotImplemented"}, + {"an unknown parameter", http.MethodGet, "/?bogus=1", authorization, "", http.StatusBadRequest, "InvalidArgument"}, + {"malformed XML", http.MethodPut, "/b?tagging", authorization, "", http.StatusBadRequest, "MalformedXML"}, + {"an empty delete", http.MethodPost, "/b?delete", authorization, "", http.StatusBadRequest, "MalformedXML"}, + {"a delete of an empty key", http.MethodPost, "/b?delete", authorization, "", http.StatusBadRequest, "MalformedXML"}, + {"a bad versioning status", http.MethodPut, "/b?versioning", authorization, "On", http.StatusBadRequest, "MalformedXML"}, + {"a bad continuation token", http.MethodGet, "/?continuation-token=%25", authorization, "", http.StatusBadRequest, "InvalidArgument"}, + {"a malformed chunked body", http.MethodPut, "/b?tagging", authorization, "", http.StatusBadRequest, "IncompleteBody"}, + } { + t.Run(tc.name, func(t *testing.T) { + req := httptest.NewRequestWithContext(t.Context(), tc.method, tc.target, strings.NewReader(tc.body)) + if tc.auth != "" { + req.Header.Set("Authorization", tc.auth) + } + if tc.code == "IncompleteBody" { + req.Header.Set("Content-Encoding", "aws-chunked") + req.Body = http.NoBody + req.Header.Set("X-Amz-Content-Sha256", "STREAMING-UNSIGNED-PAYLOAD-TRAILER") + } + rec := httptest.NewRecorder() + s.ServeHTTP(rec, req) + if rec.Code != tc.status || !strings.Contains(rec.Body.String(), ""+tc.code+"") { + t.Fatalf("= %d %s, want %d %s", rec.Code, rec.Body, tc.status, tc.code) + } + }) + } +} + +func TestChunkedBodies(t *testing.T) { + s := s3.New() + if _, err := s.PutBucket(s3.Bucket{Name: "chunks", Region: "us-east-1"}); err != nil { + t.Fatal(err) + } + body := "5;chunk-signature=0\r\nhello\r\n6\r\n world\r\n0\r\nx-amz-checksum-crc32:AAAAAA==\r\n\r\n" + req := httptest.NewRequestWithContext(t.Context(), http.MethodPut, "/chunks/greeting?x-id=PutObject", strings.NewReader(body)) + req.Header.Set("Authorization", authorization) + req.Header.Set("Content-Encoding", "aws-chunked") + rec := httptest.NewRecorder() + s.ServeHTTP(rec, req) + if rec.Code != http.StatusOK || rec.Header().Get("ETag") != `"5eb63bbbe01eeed093cb22bb8f5acdc3"` { + t.Fatalf("put = %d %v %s", rec.Code, rec.Header(), rec.Body) + } +} + +func TestPresignedCredentialsNameTheRegion(t *testing.T) { + s := s3.New() + target := "/presigned?X-Amz-Credential=AKID%2F20260101%2Fap-south-1%2Fs3%2Faws4_request&X-Amz-Signature=0" + code, _, body := do(t, s, http.MethodPut, target, "", "ap-south-1") + if code != http.StatusOK { + t.Fatalf("a presigned create = %d %s", code, body) + } + if b, ok := s.LookupBucket("presigned"); !ok || b.Region != "ap-south-1" { + t.Fatalf("bucket = %+v, %v", b, ok) + } +} + +func TestHostileBodies(t *testing.T) { + s := s3.New() + if _, err := s.PutBucket(s3.Bucket{Name: "hostile", Region: "us-east-1"}); err != nil { + t.Fatal(err) + } + for _, tc := range []struct { + name, target, body string + chunked bool + code string + }{ + {"a chunk size near the integer limit", "/hostile/k", "7fffffffffffffff\r\nab", true, "IncompleteBody"}, + {"a chunk size with a 0x prefix", "/hostile/k", "0x5\r\nhello\r\n0\r\n\r\n", true, "IncompleteBody"}, + {"a chunk longer than the body", "/hostile/k", "10\r\nhello\r\n0\r\n\r\n", true, "IncompleteBody"}, + {"a tag set over 1 MiB", "/hostile?tagging", strings.Repeat(" ", 2<<20), false, "MaxMessageLengthExceeded"}, + {"an object over 16 MiB", "/hostile/k", strings.Repeat("x", 17<<20), false, "EntityTooLarge"}, + {"a key over 1024 bytes", "/hostile/" + strings.Repeat("k", 1025), "x", false, "KeyTooLongError"}, + } { + t.Run(tc.name, func(t *testing.T) { + req := httptest.NewRequestWithContext(t.Context(), http.MethodPut, tc.target, strings.NewReader(tc.body)) + req.Header.Set("Authorization", authorization) + if tc.chunked { + req.Header.Set("Content-Encoding", "aws-chunked") + } + rec := httptest.NewRecorder() + s.ServeHTTP(rec, req) + if rec.Code != http.StatusBadRequest || !strings.Contains(rec.Body.String(), ""+tc.code+"") { + t.Fatalf("= %d %s, want 400 %s", rec.Code, rec.Body, tc.code) + } + }) + } + if _, err := s.PutBucket(s3.Bucket{Name: "hostile", Region: "us-east-1", Versioning: "Enabled"}); err != nil { + t.Fatal(err) + } + long := strings.Repeat("k", 1025) + code, _, body := do(t, s, http.MethodPost, "/hostile?delete", authorization, ""+long+"") + if code != http.StatusOK || !strings.Contains(body, "KeyTooLongError") || strings.Contains(body, "") { + t.Fatalf("a delete of a long key = %d %s", code, body) + } + for _, tag := range []string{"a\tb", "a\nb"} { + code, _, body := do(t, s, http.MethodPut, "/hostile?tagging", authorization, ""+tag+"v") + if code != http.StatusBadRequest || !strings.Contains(body, "InvalidTag") { + t.Fatalf("a tag key %q = %d %s", tag, code, body) + } + } + if got := s.ObjectVersions("hostile"); len(got) != 0 { + t.Fatalf("versions = %+v", got) + } +} + +func FuzzServeHTTP(f *testing.F) { + for _, seed := range []struct{ method, target, header, body string }{ + {http.MethodPut, "/fuzz/k", "Content-Encoding: aws-chunked", "5;chunk-signature=0\r\nhello\r\n0\r\n\r\n"}, + {http.MethodGet, "/fuzz?versions&key-marker=k&version-id-marker=null&max-keys=1", "", ""}, + {http.MethodPost, "/fuzz?delete", "", "knull"}, + {http.MethodPut, "/fuzz?tagging", "X-Amz-Content-Sha256: STREAMING-UNSIGNED-PAYLOAD-TRAILER", "0\r\n\r\n"}, + {http.MethodGet, "/?max-buckets=1&continuation-token=YQ&bucket-region=us-east-1", "", ""}, + {http.MethodPut, "/other", "", "eu-west-1"}, + } { + f.Add(seed.method, seed.target, seed.header, []byte(seed.body)) + } + f.Fuzz(func(t *testing.T, method, target, header string, body []byte) { + req, err := http.NewRequestWithContext(t.Context(), method, "http://s3.test"+target, bytes.NewReader(body)) + if err != nil { + return + } + req.Header.Set("Authorization", authorization) + if name, value, ok := strings.Cut(header, ":"); ok && validHeaderName(name) { + req.Header.Set(name, strings.TrimSpace(value)) + } + s := s3.New() + if _, err := s.PutBucket(s3.Bucket{Name: "fuzz", Region: "us-east-1", Versioning: "Enabled"}); err != nil { + t.Fatal(err) + } + if _, err := s.PutObject("fuzz", "k", []byte("v")); err != nil { + t.Fatal(err) + } + rec := httptest.NewRecorder() + s.ServeHTTP(rec, req) + if rec.Code < 200 || rec.Code > 599 { + t.Fatalf("%s %s = %d", method, target, rec.Code) + } + if rec.Body.Len() > 0 && !wellFormed(rec.Body.Bytes()) { + t.Fatalf("%s %s = %d %q, not XML", method, target, rec.Code, rec.Body) + } + }) +} + +func validHeaderName(name string) bool { + return name != "" && strings.Trim(name, "-0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz") == "" +} + +func wellFormed(b []byte) bool { + dec := xml.NewDecoder(bytes.NewReader(b)) + for { + _, err := dec.Token() + if errors.Is(err, io.EOF) { + return true + } + if err != nil { + return false + } + } +} diff --git a/aws/s3/objects.go b/aws/s3/objects.go new file mode 100644 index 0000000..dd111a1 --- /dev/null +++ b/aws/s3/objects.go @@ -0,0 +1,382 @@ +package s3 + +import ( + "cmp" + "crypto/md5" //nolint:gosec // S3 ETags are MD5 digests; nothing here relies on them for security + "encoding/hex" + "encoding/xml" + "errors" + "fmt" + "maps" + "net/http" + "slices" + "strconv" + "strings" + "time" +) + +const ( + nullVersion = "null" + maxKeys = 1000 + maxKeyBytes = 1024 +) + +func keyTooLong() *apiError { + return &apiError{Status: http.StatusBadRequest, Code: "KeyTooLongError", Message: "Your key is too long"} +} + +// ObjectVersion is one version of an object, or a delete marker. +type ObjectVersion struct { + Key string + // VersionID is "null" for a version written while versioning was off or suspended. + VersionID string + IsLatest bool + DeleteMarker bool + Size int64 + // ETag is the quoted hex MD5 of the object's data; empty for a delete marker. + ETag string + LastModified time.Time +} + +type version struct { + // seq orders a key's versions, and is what a version ID other than "null" encodes. + seq uint64 + id string + deleteMarker bool + size int64 + etag string + modified time.Time +} + +func (v *version) export(key string, latest bool) ObjectVersion { + return ObjectVersion{Key: key, VersionID: v.id, IsLatest: latest, DeleteMarker: v.deleteMarker, Size: v.size, ETag: v.etag, LastModified: v.modified} +} + +// newVersion returns a version written now, "null" unless versioning is on. The caller holds s.mu. +func (s *Server) newVersion(b *bucket) *version { + s.nextVersion++ + v := &version{seq: s.nextVersion, id: nullVersion, modified: s.now().UTC()} + if b.Versioning == versioningEnabled { + v.id = fmt.Sprintf("%032x", v.seq) + } + return v +} + +// validVersionID reports whether the server could have made id, and the sequence number it encodes. +func validVersionID(id string) (uint64, bool) { + if id == nullVersion { + return 0, true + } + if len(id) != 32 { + return 0, false + } + n, err := strconv.ParseUint(id, 16, 64) + return n, err == nil +} + +func invalidVersion() *apiError { return invalidArgument("Invalid version id specified") } + +// write adds v as key's newest version, replacing its "null" version unless v has an ID. The caller holds s.mu. +func (b *bucket) write(key string, v *version) { + versions := b.objects[key] + if v.id == nullVersion { + versions = slices.DeleteFunc(versions, func(old *version) bool { return old.id == nullVersion }) + if b.nullSeqs == nil { + b.nullSeqs = map[string]uint64{} + } + b.nullSeqs[key] = v.seq + } + b.objects[key] = append([]*version{v}, versions...) +} + +// forget removes every version of key. The caller holds s.mu. +func (b *bucket) forget(key string) { + delete(b.objects, key) + delete(b.nullSeqs, key) +} + +// deleted is the result of deleting one object or version. +type deleted struct { + Key string `xml:"Key"` + VersionID string `xml:"VersionId,omitempty"` + DeleteMarker bool `xml:"DeleteMarker,omitempty"` + DeleteMarkerVersionID string `xml:"DeleteMarkerVersionId,omitempty"` +} + +// remove deletes version id of key, or with no id deletes key as the bucket's versioning says. The caller holds s.mu. +func (s *Server) remove(b *bucket, key, id string) deleted { + if id != "" { + out := deleted{Key: key, VersionID: id} + versions := b.objects[key] + i := slices.IndexFunc(versions, func(v *version) bool { return v.id == id }) + if i < 0 { + return out + } + if versions[i].deleteMarker { + out.DeleteMarker, out.DeleteMarkerVersionID = true, id + } + versions = slices.Delete(versions, i, i+1) + if len(versions) == 0 { + b.forget(key) + } else { + b.objects[key] = versions + } + return out + } + if b.Versioning == "" { + b.forget(key) + return deleted{Key: key} + } + marker := s.newVersion(b) + marker.deleteMarker = true + b.write(key, marker) + return deleted{Key: key, DeleteMarker: true, DeleteMarkerVersionID: marker.id} +} + +func (s *Server) putObject(c *call) (*reply, *apiError) { + if len(c.key) > maxKeyBytes { + return nil, keyTooLong() + } + if c.r.Header.Get("x-amz-copy-source") != "" { + return nil, &apiError{Status: http.StatusNotFound, Code: "NotImplemented", Message: "squall: CopyObject is not implemented"} + } + if acl := c.r.Header.Get("x-amz-acl"); acl != "" && acl != "private" && acl != "bucket-owner-full-control" { + return nil, &apiError{Status: http.StatusBadRequest, Code: "AccessControlListNotSupported", Message: "The bucket does not allow ACLs"} + } + for h := range c.r.Header { + if strings.HasPrefix(strings.ToLower(h), "x-amz-object-lock-") || strings.EqualFold(h, "x-amz-tagging") { + return nil, invalidArgument("squall: the %s header is not supported", h) + } + } + s.mu.Lock() + defer s.mu.Unlock() + b, err := s.lookup(c) + if err != nil { + return nil, err + } + v := s.storeObject(b, c.key, c.body) + header := map[string]string{"ETag": v.etag} + if b.Versioning != "" { + header["x-amz-version-id"] = v.id + } + return &reply{header: header}, nil +} + +// storeObject writes data as key's newest version. The caller holds s.mu. +func (s *Server) storeObject(b *bucket, key string, data []byte) *version { + sum := md5.Sum(data) //nolint:gosec // an ETag, not a security check + v := s.newVersion(b) + v.size, v.etag = int64(len(data)), `"`+hex.EncodeToString(sum[:])+`"` + b.write(key, v) + return v +} + +func (s *Server) deleteObject(c *call) (*reply, *apiError) { + id := c.query.Get("versionId") + if c.query.Has("versionId") { + if _, ok := validVersionID(id); !ok { + return nil, invalidVersion() + } + } + s.mu.Lock() + defer s.mu.Unlock() + b, err := s.lookup(c) + if err != nil { + return nil, err + } + d := s.remove(b, c.key, id) + header := map[string]string{} + if d.DeleteMarker { + header["x-amz-delete-marker"] = "true" + } + if v := cmp.Or(d.DeleteMarkerVersionID, d.VersionID); v != "" { + header["x-amz-version-id"] = v + } + return &reply{status: http.StatusNoContent, header: header}, nil +} + +type deleteXML struct { + Quiet bool `xml:"Quiet"` + Objects []objectIdentifierXML `xml:"Object"` +} + +type objectIdentifierXML struct { + Key string `xml:"Key"` + VersionID string `xml:"VersionId"` +} + +type deleteResultXML struct { + XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ DeleteResult"` + Deleted []deleted `xml:"Deleted"` + Errors []deleteErrorXML `xml:"Error"` +} + +type deleteErrorXML struct { + Key string `xml:"Key"` + VersionID string `xml:"VersionId,omitempty"` + Code string `xml:"Code"` + Message string `xml:"Message"` +} + +func (s *Server) deleteObjects(c *call) (*reply, *apiError) { + var in deleteXML + if err := decodeXML(c.body, &in); err != nil { + return nil, err + } + emptyKey := slices.ContainsFunc(in.Objects, func(o objectIdentifierXML) bool { return o.Key == "" }) + if len(in.Objects) == 0 || len(in.Objects) > maxKeys || emptyKey { + return nil, malformedXML() + } + s.mu.Lock() + defer s.mu.Unlock() + b, err := s.lookup(c) + if err != nil { + return nil, err + } + out := deleteResultXML{} + for _, o := range in.Objects { + if len(o.Key) > maxKeyBytes { + out.Errors = append(out.Errors, deleteErrorXML{o.Key, o.VersionID, "KeyTooLongError", "Your key is too long"}) + continue + } + if o.VersionID != "" { + if _, ok := validVersionID(o.VersionID); !ok { + out.Errors = append(out.Errors, deleteErrorXML{o.Key, o.VersionID, "NoSuchVersion", "The specified version does not exist."}) + continue + } + } + d := s.remove(b, o.Key, o.VersionID) + if !in.Quiet { + out.Deleted = append(out.Deleted, d) + } + } + return &reply{body: out}, nil +} + +type listVersionsXML struct { + XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListVersionsResult"` + Name string `xml:"Name"` + Prefix string `xml:"Prefix"` + KeyMarker string `xml:"KeyMarker"` + VersionIDMarker string `xml:"VersionIdMarker"` + NextKeyMarker string `xml:"NextKeyMarker,omitempty"` + NextVersionIDMarker string `xml:"NextVersionIdMarker,omitempty"` + MaxKeys int `xml:"MaxKeys"` + IsTruncated bool `xml:"IsTruncated"` + Entries []versionXML `xml:""` +} + +// versionXML is a Version or, by its XMLName, a DeleteMarker, so the two interleave in order. +type versionXML struct { + XMLName xml.Name + Key string `xml:"Key"` + VersionID string `xml:"VersionId"` + IsLatest bool `xml:"IsLatest"` + LastModified string `xml:"LastModified"` + ETag string `xml:"ETag,omitempty"` + Size *int64 `xml:"Size,omitempty"` + StorageClass string `xml:"StorageClass,omitempty"` + Owner ownerXML `xml:"Owner"` +} + +func (s *Server) listObjectVersions(c *call) (*reply, *apiError) { + limit := maxKeys + if v := c.query.Get("max-keys"); v != "" { + n, err := strconv.Atoi(v) + if err != nil || n < 0 { + return nil, invalidArgument("Provided max-keys not an integer or within integer range") + } + limit = min(n, maxKeys) + } + prefix, keyMarker, idMarker := c.query.Get("prefix"), c.query.Get("key-marker"), c.query.Get("version-id-marker") + var markerSeq uint64 + if idMarker != "" { + if keyMarker == "" { + return nil, invalidArgument("A version-id marker cannot be specified without a key marker.") + } + seq, ok := validVersionID(idMarker) + if !ok { + return nil, invalidArgument("Invalid version id specified") + } + markerSeq = seq + } + s.mu.Lock() + defer s.mu.Unlock() + b, err := s.lookup(c) + if err != nil { + return nil, err + } + if idMarker == nullVersion { + markerSeq = b.nullSeqs[keyMarker] + } + out := listVersionsXML{Name: c.bucket, Prefix: prefix, KeyMarker: keyMarker, VersionIDMarker: idMarker, MaxKeys: limit} + var last *versionXML + for _, key := range slices.Sorted(maps.Keys(b.objects)) { + if !strings.HasPrefix(key, prefix) || key < keyMarker || key == keyMarker && idMarker == "" { + continue + } + for i, v := range b.objects[key] { + // A key's versions are newest first, so those after the marker have lower sequence numbers. + if key == keyMarker && v.seq >= markerSeq { + continue + } + if len(out.Entries) == limit { + out.IsTruncated = true + if last != nil { + out.NextKeyMarker, out.NextVersionIDMarker = last.Key, last.VersionID + } + return &reply{body: out}, nil + } + out.Entries = append(out.Entries, entryXML(v.export(key, i == 0))) + last = &out.Entries[len(out.Entries)-1] + } + } + return &reply{body: out}, nil +} + +func entryXML(v ObjectVersion) versionXML { + e := versionXML{ + XMLName: xml.Name{Local: "Version"}, Key: v.Key, VersionID: v.VersionID, IsLatest: v.IsLatest, + LastModified: v.LastModified.Format(timeFormat), Owner: ownerXML{ID: ownerID}, + } + if v.DeleteMarker { + e.XMLName.Local = "DeleteMarker" + return e + } + e.ETag, e.Size, e.StorageClass = v.ETag, &v.Size, "STANDARD" + return e +} + +// ObjectVersions returns the named bucket's versions and delete markers in ListObjectVersions order. +func (s *Server) ObjectVersions(bucketName string) []ObjectVersion { + s.mu.Lock() + defer s.mu.Unlock() + b, ok := s.buckets[bucketName] + if !ok { + return nil + } + var out []ObjectVersion + for _, key := range slices.Sorted(maps.Keys(b.objects)) { + for i, v := range b.objects[key] { + out = append(out, v.export(key, i == 0)) + } + } + return out +} + +// PutObject writes data to key in the named bucket, as PutObject does, and returns the version it made. +func (s *Server) PutObject(bucketName, key string, data []byte) (ObjectVersion, error) { + if key == "" { + return ObjectVersion{}, errors.New("squall: an object needs a key") + } + if len(key) > maxKeyBytes { + return ObjectVersion{}, keyTooLong() + } + s.mu.Lock() + defer s.mu.Unlock() + b, ok := s.buckets[bucketName] + if !ok { + return ObjectVersion{}, noSuchBucket(bucketName) + } + return s.storeObject(b, key, data).export(key, true), nil +} diff --git a/aws/s3/server.go b/aws/s3/server.go new file mode 100644 index 0000000..9238296 --- /dev/null +++ b/aws/s3/server.go @@ -0,0 +1,422 @@ +// Package s3 is a stand-in for the Amazon S3 API, for testing code that +// calls it through the official SDKs. +// +// [Server] is an [http.Handler] that keeps general purpose buckets and their +// object versions in memory and answers the S3 REST API with path-style +// addressing, in S3's XML. Serve it with [net/http/httptest] and point a +// client at it with the SDK's base endpoint setting, or the +// AWS_ENDPOINT_URL_S3 or AWS_ENDPOINT_URL environment variable, and any +// static credentials. The SDK addresses an IP endpoint such as httptest's by +// path; for a host name, set the client's UsePathStyle. A [fault.Injector] +// makes chosen calls fail, including after they took effect, so tests can +// check that code which retries converges. +// +// It covers buckets (CreateBucket, HeadBucket, ListBuckets, DeleteBucket), +// their tags (PutBucketTagging, GetBucketTagging, DeleteBucketTagging), +// versioning (PutBucketVersioning, GetBucketVersioning), policy +// (PutBucketPolicy) and CORS rules (PutBucketCors), and the objects in them +// (PutObject, DeleteObject, DeleteObjects, ListObjectVersions), with the real +// API's error codes. Other operations answer 404 with the code +// NotImplemented. Where it differs from S3: +// +// - Signatures are not checked. The region comes from the credential scope +// of the SigV4 Authorization header (or the X-Amz-Credential query +// parameter); a request with neither fails with AccessDenied. There is +// one account. Bucket names are global, as in S3: a call on a bucket in +// another region than the request's fails with 301 PermanentRedirect and +// the bucket's region in the x-amz-bucket-region header, which the SDK +// does not follow. +// - [Server.PutForeignBucket] holds a name for another account: creating +// it fails with BucketAlreadyExists, HeadBucket with 403 and the bucket's +// region, and every other call on it with AccessDenied. ListBuckets +// lists only the account's own. Other access denials are a fault rule +// away, such as fault.Reject(403, "AccessDenied"). +// - CreateBucket takes a LocationConstraint that must name the request's +// region, and none for us-east-1, where naming us-east-1 fails with +// InvalidLocationConstraint. Any other mismatch fails with +// IllegalLocationConstraintException. Creating a bucket the account +// already owns succeeds in us-east-1 and fails with +// BucketAlreadyOwnedByYou elsewhere. Object ownership is always bucket +// owner enforced: an ACL other than private fails, and object lock and +// directory buckets are not supported. +// - ListBuckets returns the account's buckets ordered by name, in every +// region, and filters by prefix and bucket-region, which must be the +// request's region. As in S3, a request with any of its parameters pages, +// 10,000 buckets at a time unless max-buckets says otherwise, and reports +// each bucket's region; one with none returns every bucket without regions. +// - Objects record their size and ETag, the hex MD5 of their data, not the +// data itself, and keys are up to 1024 bytes. Objects are up to 16 MiB +// and other bodies up to 1 MiB. Checksums and Content-MD5 are not +// verified, and bodies sent with aws-chunked encoding are decoded without +// checking their chunk signatures. Version IDs are 32 hex digits, and +// "null" for an object written while versioning is off or suspended. +// Multipart uploads, copies, storage classes, object tags, lifecycle and +// replication are not supported. S3 documents in-progress multipart +// uploads as blocking DeleteBucket only for directory buckets, so for a +// general purpose bucket only versions and delete markers do. +// - DeleteBucket fails with BucketNotEmpty while any object version or +// delete marker remains. ListObjectVersions orders versions by key and +// then newest first, and its markers are positions, so a page may follow +// one whose versions were deleted; it takes no delimiter or +// encoding-type. +// - Tags follow S3's rules: up to 50, keys of 1 to 128 characters, values +// of up to 256, Unicode letters, numbers, separators and "+-=._:/@", no +// duplicates and no "aws:" prefix. PutBucketTagging with an empty tag set removes +// the tags, so GetBucketTagging fails with NoSuchTagSet, as it does for +// a bucket never tagged. Bucket policies must be JSON and are recorded, +// not enforced; CORS rules are checked for an allowed method and origin +// and recorded. +// - Query parameters the server does not support are rejected with 400 +// InvalidArgument rather than ignored, as are the +// x-amz-expected-bucket-owner and x-amz-mfa headers. +// +// Fault method names are "s3." and the operation, such as "s3.CreateBucket". +// A call's resource is the bucket's name, "bucket/key" for a call on one +// object, and empty for ListBuckets. S3 has no long-running operations, so +// [fault.ModeFailOperation] fails a call as [fault.ModeReject] does. A +// fault's Reason is the error code, except that the fault package's generic +// reasons become InternalError, ServiceUnavailable (for 503) and SlowDown +// (always 503). +// +// A lost response aborts the handler with [http.ErrAbortHandler] after +// sending the response header, so the [http.Server] serving it closes the +// connection. Calling ServeHTTP directly propagates that panic. A HEAD's +// header is its whole answer, so a lost one instead ends the connection +// inside its status line. +package s3 + +import ( + "bytes" + "encoding/xml" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "slices" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/parallelworks/squall/fault" + "github.com/parallelworks/squall/internal/stub" +) + +const ( + namespace = "http://s3.amazonaws.com/doc/2006-03-01/" + timeFormat = "2006-01-02T15:04:05.000Z" + + maxBody = 1 << 20 + maxObject = 16 << 20 + + // ownerID is the account's canonical user ID. + ownerID = "75aa57f09aa0c8caeab4f8c24e99d10f8e7faeebf76c078efc7c6caea54ba06a" +) + +// Server is a stand-in for the Amazon S3 API. Create one with [New]. It is safe for concurrent use. +type Server struct { + faults *fault.Injector + now func() time.Time + requests atomic.Uint64 + + ops map[string]operation + + mu sync.Mutex + buckets map[string]*bucket + foreign map[string]string + nextVersion uint64 +} + +// Option configures a [Server]. +type Option func(*Server) + +// WithFaults makes the server ask in which calls to fail. +func WithFaults(in *fault.Injector) Option { + return func(s *Server) { s.faults = in } +} + +// WithClock sets the clock for creation and modification times. The default is time.Now. +func WithClock(now func() time.Time) Option { + return func(s *Server) { s.now = now } +} + +// New returns an empty server. +func New(opts ...Option) *Server { + s := &Server{now: time.Now, buckets: map[string]*bucket{}, foreign: map[string]string{}} + for _, opt := range opts { + opt(s) + } + s.ops = s.operations() + return s +} + +// call is one parsed request. +type call struct { + r *http.Request + region string + bucket string + key string + query url.Values + body []byte +} + +func (c *call) resource() string { + if c.key != "" { + return c.bucket + "/" + c.key + } + return c.bucket +} + +// reply is a successful answer: a status, headers, and an XML body or none. +type reply struct { + status int + header map[string]string + body any +} + +type operation struct { + name string + // params are the query parameters the operation takes, besides its subresource. + params []string + handle func(c *call) (*reply, *apiError) +} + +// subresources are the query parameters that select an S3 operation. +var subresources = []string{ + "accelerate", "acl", "analytics", "attributes", "cors", "delete", "encryption", "intelligent-tiering", + "inventory", "legal-hold", "lifecycle", "list-type", "location", "logging", "metadataConfiguration", + "metadataTable", "metrics", "notification", "object-lock", "ownershipControls", "partNumber", "policy", + "policyStatus", "publicAccessBlock", "renameObject", "replication", "requestPayment", "restore", "retention", + "select", "session", "tagging", "torrent", "uploadId", "uploads", "versioning", "versions", "website", +} + +// operations maps a method, scope and subresource to its operation; an empty one is not implemented. +func (s *Server) operations() map[string]operation { + return map[string]operation{ + "GET service": {"ListBuckets", listBucketsParams, s.listBuckets}, + "PUT bucket": {"CreateBucket", nil, s.createBucket}, + "HEAD bucket": {"HeadBucket", nil, s.headBucket}, + "DELETE bucket": {"DeleteBucket", nil, s.deleteBucket}, + "GET bucket?tagging": {"GetBucketTagging", nil, s.getBucketTagging}, + "PUT bucket?tagging": {"PutBucketTagging", nil, s.putBucketTagging}, + "DELETE bucket?tagging": {"DeleteBucketTagging", nil, s.deleteBucketTagging}, + "GET bucket?versioning": {"GetBucketVersioning", nil, s.getBucketVersioning}, + "PUT bucket?versioning": {"PutBucketVersioning", nil, s.putBucketVersioning}, + "PUT bucket?policy": {"PutBucketPolicy", nil, s.putBucketPolicy}, + "PUT bucket?cors": {"PutBucketCors", nil, s.putBucketCors}, + "GET bucket?versions": {"ListObjectVersions", []string{"prefix", "key-marker", "version-id-marker", "max-keys"}, s.listObjectVersions}, + "POST bucket?delete": {"DeleteObjects", nil, s.deleteObjects}, + "PUT object": {"PutObject", nil, s.putObject}, + "DELETE object": {"DeleteObject", []string{"versionId"}, s.deleteObject}, + "HEAD service": {}, + "PUT service": {}, + "DELETE service": {}, + "POST service": {}, + "GET bucket?location": {}, + "GET bucket?list-type": {}, + "GET object": {}, + "HEAD object": {}, + "POST object?uploads": {}, + "GET bucket?uploads": {}, + "PUT object?partNumber": {}, + "DELETE object?uploadId": {}, + } +} + +// operation picks the operation for c by its HTTP method, scope and subresource. +func (s *Server) operation(method string, c *call) (operation, *apiError) { + sub := "" + for _, name := range subresources { + if c.query.Has(name) { + sub = name + break + } + } + scope := "object" + switch { + case c.bucket == "": + scope = "service" + case c.key == "": + scope = "bucket" + } + key := method + " " + scope + if sub != "" { + key += "?" + sub + } + op, ok := s.ops[key] + if !ok || op.name == "" { + what := key + if c.bucket != "" { + what += " " + c.resource() + } + return operation{}, &apiError{Status: http.StatusNotFound, Code: "NotImplemented", Message: "squall: " + what + " is not implemented"} + } + for name := range c.query { + presign := len(name) > 6 && strings.EqualFold(name[:6], "X-Amz-") + if name != sub && name != "x-id" && !presign && !slices.Contains(op.params, name) { + return operation{}, invalidArgument("squall: %s does not support the query parameter %s", op.name, name) + } + } + return op, nil +} + +// ServeHTTP answers an S3 REST call. +func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { + requestID := s.requestID() + w.Header().Set("x-amz-request-id", requestID) + w.Header().Set("x-amz-id-2", "squall/"+requestID) + op, c, err := s.parse(w, r) + if err != nil { + writeError(w, r, requestID, err) + return + } + f, _ := s.faults.Decide(fault.Call{Method: "s3." + op.name, Resource: c.resource()}) + reject := f.Mode == fault.ModeReject || f.Mode == fault.ModeFailOperation + stub.Serve(w, r, f, reject, "application/xml", func() { writeError(w, r, requestID, faultError(f)) }, func() (func(), bool) { + rep, err := op.handle(c) + if err != nil { + return func() { writeError(w, r, requestID, err) }, false + } + return func() { rep.write(w) }, true + }) +} + +func (s *Server) parse(w http.ResponseWriter, r *http.Request) (operation, *call, *apiError) { + region, err := regionOf(r) + if err != nil { + return operation{}, nil, err + } + bucket, key, _ := strings.Cut(strings.TrimPrefix(r.URL.Path, "/"), "/") + c := &call{r: r, region: region, bucket: bucket, key: key, query: r.URL.Query()} + op, err := s.operation(r.Method, c) + if err != nil { + return operation{}, nil, err + } + for _, h := range []string{"x-amz-expected-bucket-owner", "x-amz-mfa"} { + if r.Header.Get(h) != "" { + return operation{}, nil, invalidArgument("squall: the %s header is not supported", h) + } + } + limit, tooLarge := int64(maxBody), "MaxMessageLengthExceeded" + if op.name == "PutObject" { + limit, tooLarge = maxObject, "EntityTooLarge" + } + c.body, err = readBody(w, r, limit, tooLarge) + if err != nil { + return operation{}, nil, err + } + return op, c, nil +} + +// regionOf reads the region from the SigV4 credential scope without checking the signature. +func regionOf(r *http.Request) (string, *apiError) { + credential := r.URL.Query().Get("X-Amz-Credential") + if h := r.Header.Get("Authorization"); h != "" { + algorithm, fields, _ := strings.Cut(h, " ") + if algorithm != "AWS4-HMAC-SHA256" { + return "", invalidArgument("Unsupported Authorization Type") + } + for field := range strings.SplitSeq(fields, ",") { + if v, ok := strings.CutPrefix(strings.TrimSpace(field), "Credential="); ok { + credential = v + } + } + } + if credential == "" { + return "", accessDenied("") + } + scope := strings.Split(credential, "/") + if len(scope) != 5 || scope[2] == "" || scope[4] != "aws4_request" { + return "", &apiError{ + Status: http.StatusBadRequest, Code: "AuthorizationHeaderMalformed", + Message: fmt.Sprintf("The authorization header is malformed; the Credential is mal-formed; expecting \"/YYYYMMDD/REGION/SERVICE/aws4_request\". (%s)", credential), + } + } + return scope[2], nil +} + +// readBody reads r's body, decoding aws-chunked encoding, up to limit bytes; tooLarge is the code for a longer one. +func readBody(w http.ResponseWriter, r *http.Request, limit int64, tooLarge string) ([]byte, *apiError) { + tooLong := &apiError{Status: http.StatusBadRequest, Code: tooLarge, Message: fmt.Sprintf("squall: bodies are limited to %d bytes", limit)} + body, err := io.ReadAll(http.MaxBytesReader(w, r.Body, limit+limit/8+4096)) + if _, ok := errors.AsType[*http.MaxBytesError](err); ok { + return nil, tooLong + } + if err != nil { + return nil, &apiError{Status: http.StatusBadRequest, Code: "IncompleteBody", Message: "The request body could not be read: " + err.Error()} + } + chunked := strings.Contains(r.Header.Get("Content-Encoding"), "aws-chunked") || + strings.HasPrefix(r.Header.Get("X-Amz-Content-Sha256"), "STREAMING-") + if chunked { + if body, err = decodeChunked(body); err != nil { + return nil, &apiError{Status: http.StatusBadRequest, Code: "IncompleteBody", Message: "The aws-chunked body is malformed: " + err.Error()} + } + } + if int64(len(body)) > limit { + return nil, tooLong + } + return body, nil +} + +// decodeChunked decodes an aws-chunked body, dropping chunk extensions and trailers. +func decodeChunked(b []byte) ([]byte, error) { + var out []byte + for { + line, rest, ok := bytes.Cut(b, []byte("\r\n")) + if !ok { + return nil, io.ErrUnexpectedEOF + } + sizeHex, _, _ := bytes.Cut(line, []byte(";")) + size, err := strconv.ParseUint(string(sizeHex), 16, 32) + if err != nil { + return nil, fmt.Errorf("bad chunk size %q", sizeHex) + } + if size == 0 { + return out, nil + } + if uint64(len(rest)) < size+2 || string(rest[size:size+2]) != "\r\n" { + return nil, io.ErrUnexpectedEOF + } + out = append(out, rest[:size]...) + b = rest[size+2:] + } +} + +func (rep *reply) write(w http.ResponseWriter) { + for k, v := range rep.header { + w.Header().Set(k, v) + } + status := rep.status + if status == 0 { + status = http.StatusOK + } + if rep.body == nil { + w.WriteHeader(status) + return + } + writeXML(w, status, rep.body) +} + +func writeXML(w http.ResponseWriter, status int, body any) { + w.Header().Set("Content-Type", "application/xml") + w.WriteHeader(status) + _, _ = w.Write([]byte(xml.Header)) + // The status is already sent, so a client that went away is all an error here can mean. + _ = xml.NewEncoder(w).Encode(body) +} + +// requestID returns a fresh request ID in the 16 hex digit form S3 uses. +func (s *Server) requestID() string { + return fmt.Sprintf("%016X", s.requests.Add(1)) +} + +// decodeXML reads an XML request body into v. +func decodeXML(body []byte, v any) *apiError { + //nolint:gosec // v is a fixed struct and the body is bounded by maxBody + if err := xml.Unmarshal(body, v); err != nil { + return malformedXML() + } + return nil +} diff --git a/azure/arm/server.go b/azure/arm/server.go index 3d3c021..7dccec1 100644 --- a/azure/arm/server.go +++ b/azure/arm/server.go @@ -123,7 +123,9 @@ // // A lost response aborts the handler with [http.ErrAbortHandler] after // sending the response header, so the [http.Server] serving it closes the -// connection. Calling ServeHTTP directly propagates that panic. +// connection. Calling ServeHTTP directly propagates that panic. A HEAD's +// header is its whole answer, so a lost one instead ends the connection +// inside its status line. package arm import ( diff --git a/gcp/storage/bench_test.go b/gcp/storage/bench_test.go new file mode 100644 index 0000000..7da7994 --- /dev/null +++ b/gcp/storage/bench_test.go @@ -0,0 +1,36 @@ +package storage_test + +import ( + "net/http" + "net/http/httptest" + "strconv" + "strings" + "testing" + + "github.com/parallelworks/squall/gcp/storage" +) + +func BenchmarkUploadListDelete(b *testing.B) { + s := storage.New() + if _, err := s.PutBucket(storage.Bucket{Name: "bench", Project: "p"}); err != nil { + b.Fatal(err) + } + for i := range 1000 { + if _, err := s.PutObject("bench", "key-"+strconv.Itoa(i), []byte("data")); err != nil { + b.Fatal(err) + } + } + do := func(method, target, body string) { + req := httptest.NewRequestWithContext(b.Context(), method, target, strings.NewReader(body)) + rec := httptest.NewRecorder() + s.ServeHTTP(rec, req) + if rec.Code >= http.StatusBadRequest { + b.Fatalf("%s %s = %d %s", method, target, rec.Code, rec.Body) + } + } + for b.Loop() { + do(http.MethodPost, "/upload/storage/v1/b/bench/o?uploadType=media&name=new", "data") + do(http.MethodGet, "/storage/v1/b/bench/o?maxResults=100&prefix=key-1", "") + do(http.MethodDelete, "/storage/v1/b/bench/o/new", "") + } +} diff --git a/gcp/storage/buckets.go b/gcp/storage/buckets.go new file mode 100644 index 0000000..a050065 --- /dev/null +++ b/gcp/storage/buckets.go @@ -0,0 +1,543 @@ +package storage + +import ( + "cmp" + "encoding/base64" + "encoding/binary" + "encoding/json" + "errors" + "fmt" + "hash/fnv" + "maps" + "net/http" + "net/netip" + "slices" + "strings" + "time" + "unicode" + "unicode/utf8" +) + +const ( + maxLabels = 64 + maxLabelLength = 63 +) + +// Bucket is a bucket, in the API's JSON shape. +type Bucket struct { + Kind string `json:"kind"` + ID string `json:"id"` + SelfLink string `json:"selfLink"` + ProjectNumber uint64 `json:"projectNumber,string"` + Name string `json:"name"` + TimeCreated string `json:"timeCreated"` + Updated string `json:"updated"` + Metageneration int64 `json:"metageneration,string"` + Location string `json:"location"` + LocationType string `json:"locationType"` + StorageClass string `json:"storageClass"` + Etag string `json:"etag"` + Labels map[string]string `json:"labels,omitzero"` + Versioning *Versioning `json:"versioning,omitzero"` + CORS json.RawMessage `json:"cors,omitzero"` + CustomPlacementConfig *CustomPlacementConfig `json:"customPlacementConfig,omitzero"` + Encryption *Encryption `json:"encryption,omitzero"` + RPO string `json:"rpo,omitzero"` + + // Project is the ID of the project the bucket is in; it is not in the JSON. + Project string `json:"-"` +} + +// Versioning is a bucket's versioning setting. +type Versioning struct { + Enabled bool `json:"enabled"` +} + +// CustomPlacementConfig names the regions of a configurable dual-region. +type CustomPlacementConfig struct { + DataLocations []string `json:"dataLocations"` +} + +// Encryption is a bucket's default encryption. +type Encryption struct { + DefaultKmsKeyName string `json:"defaultKmsKeyName,omitzero"` +} + +type bucket struct { + Bucket + // objects holds each name's generations, oldest first; the last is live unless it has TimeDeleted. + objects map[string][]*Object +} + +func (b *bucket) clone() Bucket { + c := b.Bucket + c.Labels = maps.Clone(b.Labels) + if b.Versioning != nil { + c.Versioning = new(*b.Versioning) + } + c.CORS = slices.Clone(b.CORS) + if b.CustomPlacementConfig != nil { + c.CustomPlacementConfig = &CustomPlacementConfig{DataLocations: slices.Clone(b.CustomPlacementConfig.DataLocations)} + } + if b.Encryption != nil { + c.Encryption = new(*b.Encryption) + } + return c +} + +func (b *bucket) versioned() bool { return b.Versioning != nil && b.Versioning.Enabled } + +func (b *bucket) empty() bool { return len(b.objects) == 0 } + +// touch records a change to b's metadata. The caller holds s.mu. +func (s *Server) touch(b *bucket) { + b.Metageneration++ + b.Updated = s.now().UTC().Format(timeFormat) + b.Etag = etag(b.Metageneration) +} + +// etag encodes its numbers as tagged varints, so it changes whenever they do. +func etag(numbers ...int64) string { + var buf []byte + for i, n := range numbers { + buf = append(buf, byte(i+1)<<3) + buf = binary.AppendVarint(buf, n) + } + return base64.StdEncoding.EncodeToString(buf) +} + +// projectNumber derives a 12-digit project number from a project ID. +func projectNumber(project string) uint64 { + h := fnv.New64a() + _, _ = h.Write([]byte(project)) + return 100_000_000_000 + h.Sum64()%900_000_000_000 +} + +// validBucketName reports whether name follows the bucket naming rules. +func validBucketName(name string) bool { + if len(name) < 3 || len(name) > 222 || strings.HasPrefix(name, "goog") || strings.Contains(name, "google") || strings.Contains(name, "g00gle") { + return false + } + if _, err := netip.ParseAddr(name); err == nil { + return false + } + for part := range strings.SplitSeq(name, ".") { + if part == "" || len(part) > 63 { + return false + } + } + if len(name) > 63 && !strings.Contains(name, ".") { + return false + } + for _, c := range []byte(name) { + if (c < 'a' || c > 'z') && (c < '0' || c > '9') && c != '-' && c != '_' && c != '.' { + return false + } + } + alnum := func(c byte) bool { return c >= 'a' && c <= 'z' || c >= '0' && c <= '9' } + return alnum(name[0]) && alnum(name[len(name)-1]) +} + +// validLabel reports whether s is a valid label key or, with key false, value. +func validLabel(s string, key bool) bool { + if utf8.RuneCountInString(s) > maxLabelLength || key && s == "" { + return false + } + for i, c := range s { + letter := unicode.IsLetter(c) && !unicode.IsUpper(c) + if !letter && (i == 0 && key || !unicode.IsDigit(c) && c != '_' && c != '-') { + return false + } + } + return true +} + +func validateLabels(labels map[string]string) *apiError { + if len(labels) > maxLabels { + return invalid("Too many labels: %d, more than %d.", len(labels), maxLabels) + } + for k, v := range labels { + if !validLabel(k, true) || !validLabel(v, false) { + return invalid("Invalid argument: label '%s: %s' violates format constraints.", k, v) + } + } + return nil +} + +var storageClasses = []string{"STANDARD", "NEARLINE", "COLDLINE", "ARCHIVE", "MULTI_REGIONAL", "REGIONAL", "DURABLE_REDUCED_AVAILABILITY"} + +func locationType(location string, custom *CustomPlacementConfig) string { + switch { + case custom != nil || slices.Contains([]string{"NAM4", "EUR4", "EUR5", "EUR7", "EUR8", "ASIA1"}, location): + return "dual-region" + case slices.Contains([]string{"US", "EU", "ASIA"}, location): + return "multi-region" + } + return "region" +} + +func notFoundBucket() *apiError { + return &apiError{Status: http.StatusNotFound, Reason: "notFound", Message: "The specified bucket does not exist."} +} + +func forbidden(permission string) *apiError { + return &apiError{ + Status: http.StatusForbidden, Reason: "forbidden", + Message: fmt.Sprintf("The caller does not have %s access to the Google Cloud Storage bucket. Permission '%s' denied on resource (or it may not exist).", permission, permission), + } +} + +// lookup returns the named bucket, or the error a call needing permission fails with. The caller holds s.mu. +func (s *Server) lookup(name, permission string) (*bucket, *apiError) { + if b, ok := s.buckets[name]; ok { + return b, nil + } + if s.foreign[name] { + return nil, forbidden(permission) + } + return nil, notFoundBucket() +} + +// normalCORS returns cors as stored, nil for none, or an error unless it is an array of objects. +func normalCORS(cors json.RawMessage) (json.RawMessage, *apiError) { + var rules []map[string]json.RawMessage + if len(cors) == 0 { + return nil, nil + } + if err := json.Unmarshal(cors, &rules); err != nil || slices.ContainsFunc(rules, func(r map[string]json.RawMessage) bool { return r == nil }) { + return nil, invalid("Invalid argument: cors must be a list of CORS configurations.") + } + if len(rules) == 0 { + return nil, nil + } + return slices.Clone(cors), nil +} + +// bucketInput is what buckets.insert and buckets.patch take. +type bucketInput struct { + Name string `json:"name"` + Location string `json:"location"` + StorageClass string `json:"storageClass"` + Labels map[string]*string `json:"labels"` + Versioning *Versioning `json:"versioning"` + CORS json.RawMessage `json:"cors"` + CustomPlacementConfig *CustomPlacementConfig `json:"customPlacementConfig"` + Encryption *Encryption `json:"encryption"` + RPO string `json:"rpo"` + // Lifecycle is accepted without rules, since the client sends it empty. + Lifecycle *struct { + Rule []json.RawMessage `json:"rule"` + } `json:"lifecycle"` +} + +// check rejects what the server does not model. +func (in *bucketInput) check() *apiError { + if in.Lifecycle != nil && len(in.Lifecycle.Rule) > 0 { + return invalid("squall: lifecycle rules are not supported") + } + return nil +} + +func checkACLs(r *http.Request) *apiError { + for _, p := range []string{"predefinedAcl", "predefinedDefaultObjectAcl"} { + if v := r.URL.Query().Get(p); v != "" && v != "private" && v != "projectPrivate" { + return invalid("squall: %s=%s is not supported", p, v) + } + } + return nil +} + +func (s *Server) insertBucket(r *http.Request) (any, *apiError) { + project := r.URL.Query().Get("project") + if project == "" { + return nil, required("project") + } + if err := checkACLs(r); err != nil { + return nil, err + } + if retention, err := boolParam(r, "enableObjectRetention"); err != nil { + return nil, err + } else if retention { + return nil, invalid("squall: object retention is not supported") + } + body, err := readBody(r, maxBody) + if err != nil { + return nil, err + } + var in bucketInput + if err := unmarshalStrict(body, &in); err != nil { + return nil, err + } + if err := in.check(); err != nil { + return nil, err + } + labels := map[string]string{} + for k, v := range in.Labels { + if v != nil { + labels[k] = *v + } + } + b := Bucket{ + Name: in.Name, Location: in.Location, StorageClass: in.StorageClass, Labels: labels, Versioning: in.Versioning, + CORS: in.CORS, CustomPlacementConfig: in.CustomPlacementConfig, Encryption: in.Encryption, RPO: in.RPO, Project: project, + } + s.mu.Lock() + defer s.mu.Unlock() + stored, err := s.store(b) + if err != nil { + return nil, err + } + return stored.clone(), nil +} + +// store validates b, fills in what it leaves empty and stores it as a new bucket. The caller holds s.mu. +func (s *Server) store(b Bucket) (*bucket, *apiError) { + if b.Name == "" { + return nil, required("resource.name") + } + if !validBucketName(b.Name) { + return nil, invalid("Invalid bucket name: '%s'", b.Name) + } + if err := validateLabels(b.Labels); err != nil { + return nil, err + } + b.StorageClass = strings.ToUpper(cmp.Or(b.StorageClass, "STANDARD")) + if !slices.Contains(storageClasses, b.StorageClass) { + return nil, invalid("Invalid argument: storage class '%s' is not valid.", b.StorageClass) + } + cors, err := normalCORS(b.CORS) + if err != nil { + return nil, err + } + b.CORS = cors + if c := b.CustomPlacementConfig; c != nil && len(c.DataLocations) != 2 { + return nil, invalid("Invalid argument: a custom dual-region takes exactly two data locations.") + } + if s.foreign[b.Name] { + return nil, &apiError{ + Status: http.StatusConflict, Reason: "conflict", + Message: "Sorry, that name is not available. Please try a different one.", + } + } + if _, ok := s.buckets[b.Name]; ok { + return nil, &apiError{ + Status: http.StatusConflict, Reason: "conflict", + Message: "You already own this bucket. Please select another name.", + } + } + now := s.now().UTC().Format(timeFormat) + b.Kind, b.ID, b.SelfLink = "storage#bucket", b.Name, apiBase+"b/"+b.Name + b.ProjectNumber = projectNumber(b.Project) + b.TimeCreated, b.Updated, b.Metageneration, b.Etag = now, now, 1, etag(1) + b.Location = strings.ToUpper(cmp.Or(b.Location, "US")) + b.LocationType = locationType(b.Location, b.CustomPlacementConfig) + if len(b.Labels) == 0 { + b.Labels = nil + } + stored := &bucket{Bucket: b, objects: map[string][]*Object{}} + stored.Bucket = stored.clone() + s.buckets[b.Name] = stored + return stored, nil +} + +func (s *Server) getBucket(r *http.Request) (any, *apiError) { + s.mu.Lock() + defer s.mu.Unlock() + b, err := s.lookup(r.PathValue("bucket"), "storage.buckets.get") + if err != nil { + return nil, err + } + if err := checkMatch(r, "ifMetagenerationMatch", "ifMetagenerationNotMatch", b.Metageneration); err != nil { + return nil, err + } + return b.clone(), nil +} + +func (s *Server) patchBucket(r *http.Request) (any, *apiError) { + if err := checkACLs(r); err != nil { + return nil, err + } + body, err := readBody(r, maxBody) + if err != nil { + return nil, err + } + var in bucketInput + if err := unmarshalStrict(body, &in); err != nil { + return nil, err + } + if err := in.check(); err != nil { + return nil, err + } + if in.Name != "" || in.Location != "" || in.CustomPlacementConfig != nil || in.RPO != "" { + return nil, invalid("squall: a patch may change only labels, versioning, cors, storageClass and encryption") + } + cors, err := normalCORS(in.CORS) + if err != nil { + return nil, err + } + var fields map[string]json.RawMessage + if err := json.Unmarshal(body, &fields); err != nil { + return nil, invalid("Invalid JSON payload received: %v", err) + } + cleared := func(field string) bool { return string(fields[field]) == "null" } + s.mu.Lock() + defer s.mu.Unlock() + b, err := s.lookup(r.PathValue("bucket"), "storage.buckets.update") + if err != nil { + return nil, err + } + if err := checkMatch(r, "ifMetagenerationMatch", "ifMetagenerationNotMatch", b.Metageneration); err != nil { + return nil, err + } + labels := maps.Clone(b.Labels) + if labels == nil || cleared("labels") { + labels = map[string]string{} + } + for k, v := range in.Labels { + if v == nil { + delete(labels, k) + } else { + labels[k] = *v + } + } + if err := validateLabels(labels); err != nil { + return nil, err + } + class := b.StorageClass + if in.StorageClass != "" { + class = strings.ToUpper(in.StorageClass) + if !slices.Contains(storageClasses, class) { + return nil, invalid("Invalid argument: storage class '%s' is not valid.", class) + } + } + if len(labels) == 0 { + labels = nil + } + b.Labels, b.StorageClass = labels, class + if in.Versioning != nil { + b.Versioning = new(*in.Versioning) + } else if cleared("versioning") { + b.Versioning = nil + } + if in.CORS != nil { + b.CORS = cors + } + if in.Encryption != nil { + b.Encryption = new(*in.Encryption) + } else if cleared("encryption") { + b.Encryption = nil + } + s.touch(b) + return b.clone(), nil +} + +func (s *Server) deleteBucket(r *http.Request) (any, *apiError) { + s.mu.Lock() + defer s.mu.Unlock() + name := r.PathValue("bucket") + b, err := s.lookup(name, "storage.buckets.delete") + if err != nil { + return nil, err + } + if err := checkMatch(r, "ifMetagenerationMatch", "ifMetagenerationNotMatch", b.Metageneration); err != nil { + return nil, err + } + if !b.empty() { + return nil, &apiError{Status: http.StatusConflict, Reason: "conflict", Message: "The bucket you tried to delete is not empty."} + } + delete(s.buckets, name) + return nil, nil +} + +type bucketList struct { + Kind string `json:"kind"` + Items []Bucket `json:"items,omitzero"` + NextPageToken string `json:"nextPageToken,omitzero"` +} + +func (s *Server) listBuckets(r *http.Request) (any, *apiError) { + project := r.URL.Query().Get("project") + if project == "" { + return nil, required("project") + } + if _, err := boolParam(r, "returnPartialSuccess"); err != nil { + return nil, err + } + prefix := r.URL.Query().Get("prefix") + s.mu.Lock() + defer s.mu.Unlock() + var all []Bucket + for _, name := range slices.Sorted(maps.Keys(s.buckets)) { + if b := s.buckets[name]; b.Project == project && strings.HasPrefix(name, prefix) { + all = append(all, b.clone()) + } + } + items, next, err := page(r, all, func(b Bucket) string { return b.Name }) + if err != nil { + return nil, err + } + for i := range items { + items[i].Project = "" + } + return bucketList{Kind: "storage#buckets", Items: items, NextPageToken: next}, nil +} + +// Buckets returns every bucket the caller has, ordered by name. +func (s *Server) Buckets() []Bucket { + s.mu.Lock() + defer s.mu.Unlock() + out := make([]Bucket, 0, len(s.buckets)) + for _, name := range slices.Sorted(maps.Keys(s.buckets)) { + out = append(out, s.buckets[name].clone()) + } + return out +} + +// LookupBucket returns the named bucket, if the caller has it. +func (s *Server) LookupBucket(name string) (Bucket, bool) { + s.mu.Lock() + defer s.mu.Unlock() + b, ok := s.buckets[name] + if !ok { + return Bucket{}, false + } + return b.clone(), true +} + +// PutBucket creates b in its Project from the fields buckets.insert takes, and returns it as stored. +func (s *Server) PutBucket(b Bucket) (Bucket, error) { + if b.Project == "" { + return Bucket{}, errors.New("squall: a bucket needs a project") + } + in := Bucket{ + Name: b.Name, Location: b.Location, StorageClass: b.StorageClass, Labels: b.Labels, Versioning: b.Versioning, + CORS: b.CORS, CustomPlacementConfig: b.CustomPlacementConfig, Encryption: b.Encryption, RPO: b.RPO, Project: b.Project, + } + s.mu.Lock() + defer s.mu.Unlock() + stored, err := s.store((&bucket{Bucket: in}).clone()) + if err != nil { + return Bucket{}, err + } + return stored.clone(), nil +} + +// PutForeignBucket holds name for someone the caller cannot reach, so inserting it conflicts and calls on it are forbidden. +func (s *Server) PutForeignBucket(name string) error { + if !validBucketName(name) { + return invalid("Invalid bucket name: '%s'", name) + } + s.mu.Lock() + defer s.mu.Unlock() + if _, ok := s.buckets[name]; ok { + return fmt.Errorf("squall: the caller has bucket %s", name) + } + s.foreign[name] = true + return nil +} + +// timestamp reads the server's clock, and formats it as the API does. +func (s *Server) timestamp() (time.Time, string) { + t := s.now().UTC() + return t, t.Format(timeFormat) +} diff --git a/gcp/storage/example_test.go b/gcp/storage/example_test.go new file mode 100644 index 0000000..4780b5e --- /dev/null +++ b/gcp/storage/example_test.go @@ -0,0 +1,39 @@ +package storage_test + +import ( + "fmt" + "net/http" + "net/http/httptest" + "strings" + + "github.com/parallelworks/squall/fault" + "github.com/parallelworks/squall/gcp/storage" +) + +// The first insert takes effect but its response is lost, so the caller sees +// an error even though the bucket now exists. Code under test must find and +// claim it rather than take it for someone else's. Real tests point the +// official client library at srv.URL + "/storage/v1/" with +// option.WithEndpoint. +func Example() { + faults := fault.New().Add(fault.Rule{Method: "storage.buckets.insert", Nth: 1, Fault: fault.LoseResponse()}) + s := storage.New(storage.WithFaults(faults)) + srv := httptest.NewServer(s) + defer srv.Close() + + resp, err := http.Post(srv.URL+"/storage/v1/b?project=my-project", + "application/json", strings.NewReader(`{"name":"my-bucket","labels":{"owner":"me"}}`)) + if err == nil { + _, err = resp.Body.Read(make([]byte, 1<<10)) + resp.Body.Close() + } + fmt.Println("insert failed:", err != nil) + + b, _ := s.LookupBucket("my-bucket") + fmt.Println("bucket exists:", b.Name, b.Project, b.Labels) + fmt.Println(faults.Injected()[0]) + // Output: + // insert failed: true + // bucket exists: my-bucket my-project map[owner:me] + // #1 storage.buckets.insert b/my-bucket: lose response +} diff --git a/gcp/storage/http_test.go b/gcp/storage/http_test.go new file mode 100644 index 0000000..d41eeaa --- /dev/null +++ b/gcp/storage/http_test.go @@ -0,0 +1,158 @@ +package storage_test + +import ( + "bytes" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/parallelworks/squall/gcp/storage" +) + +func do(t *testing.T, s *storage.Server, method, target, contentType, body string) (int, string) { + t.Helper() + req := httptest.NewRequestWithContext(t.Context(), method, target, strings.NewReader(body)) + if contentType != "" { + req.Header.Set("Content-Type", contentType) + } + rec := httptest.NewRecorder() + s.ServeHTTP(rec, req) + return rec.Code, rec.Body.String() +} + +func TestResponsesUseTheAPIShape(t *testing.T) { + s := storage.New() + code, body := do(t, s, http.MethodPost, "/storage/v1/b?project=p&alt=json&prettyPrint=false", "application/json", `{"name":"shape","location":"us-east1"}`) + for _, want := range []string{ + `"kind":"storage#bucket"`, `"id":"shape"`, `"selfLink":"https://www.googleapis.com/storage/v1/b/shape"`, + `"metageneration":"1"`, `"location":"US-EAST1"`, `"locationType":"region"`, `"projectNumber":"`, + } { + if code != http.StatusOK || !strings.Contains(body, want) { + t.Fatalf("insert = %d %s, want %s", code, body, want) + } + } + code, body = do(t, s, http.MethodPost, "/upload/storage/v1/b/shape/o?uploadType=media&name=a%2Fb", "text/plain", "hello") + for _, want := range []string{`"kind":"storage#object"`, `"name":"a/b"`, `"size":"5"`, `"md5Hash":"XUFAKrxLKna5cZ2REBfFkg=="`, `"crc32c":"mnG7TA=="`, `"contentType":"text/plain"`} { + if code != http.StatusOK || !strings.Contains(body, want) { + t.Fatalf("upload = %d %s, want %s", code, body, want) + } + } + code, body = do(t, s, http.MethodDelete, "/storage/v1/b/shape", "", "") + if code != http.StatusConflict || body != `{"error":{"code":409,"message":"The bucket you tried to delete is not empty.","errors":[{"message":"The bucket you tried to delete is not empty.","domain":"global","reason":"conflict"}]}}` { + t.Fatalf("delete = %d %s", code, body) + } + if code, body = do(t, s, http.MethodDelete, "/storage/v1/b/shape/o/a%2Fb", "", ""); code != http.StatusNoContent || body != "" { + t.Fatalf("delete object = %d %s", code, body) + } +} + +func TestRequestsTheServerCannotRead(t *testing.T) { + s := storage.New() + if code, body := do(t, s, http.MethodPost, "/storage/v1/b?project=p", "application/json", `{"name":"known"}`); code != http.StatusOK { + t.Fatalf("insert = %d %s", code, body) + } + for _, tc := range []struct { + name, method, target, body string + status int + reason string + }{ + {"no project", http.MethodPost, "/storage/v1/b", `{"name":"x-bucket"}`, http.StatusBadRequest, "required"}, + {"an unknown field", http.MethodPost, "/storage/v1/b?project=p", `{"name":"x-bucket","website":{}}`, http.StatusBadRequest, "invalid"}, + {"lifecycle rules", http.MethodPost, "/storage/v1/b?project=p", `{"name":"x-bucket","lifecycle":{"rule":[{}]}}`, http.StatusBadRequest, "invalid"}, + {"malformed JSON", http.MethodPost, "/storage/v1/b?project=p", `{`, http.StatusBadRequest, "invalid"}, + {"data after the JSON", http.MethodPost, "/storage/v1/b?project=p", `{"name":"x-bucket"} {}`, http.StatusBadRequest, "invalid"}, + {"cors that is not a list", http.MethodPost, "/storage/v1/b?project=p", `{"name":"x-bucket","cors":5}`, http.StatusBadRequest, "invalid"}, + {"a patch of cors that is not a list", http.MethodPatch, "/storage/v1/b/known", `{"cors":[null]}`, http.StatusBadRequest, "invalid"}, + {"an ACME challenge name", http.MethodPost, "/upload/storage/v1/b/known/o?uploadType=media&name=.well-known%2Facme-challenge%2Fx", "x", http.StatusBadRequest, "invalid"}, + {"an unknown parameter", http.MethodGet, "/storage/v1/b?project=p&bogus=1", "", http.StatusBadRequest, "invalid"}, + {"another alt", http.MethodGet, "/storage/v1/b/known?alt=media", "", http.StatusBadRequest, "invalid"}, + {"a public ACL", http.MethodPost, "/storage/v1/b?project=p&predefinedAcl=publicRead", `{"name":"x-bucket"}`, http.StatusBadRequest, "invalid"}, + {"a bad page token", http.MethodGet, "/storage/v1/b?project=p&pageToken=%25", "", http.StatusBadRequest, "invalid"}, + {"a resumable upload", http.MethodPost, "/upload/storage/v1/b/known/o?uploadType=resumable", "", http.StatusBadRequest, "invalid"}, + {"an upload without a name", http.MethodPost, "/upload/storage/v1/b/known/o?uploadType=media", "x", http.StatusBadRequest, "required"}, + {"a patch of the location", http.MethodPatch, "/storage/v1/b/known", `{"location":"EU"}`, http.StatusBadRequest, "invalid"}, + {"an unimplemented method", http.MethodGet, "/storage/v1/b/known/iam", "", http.StatusNotFound, "notFound"}, + } { + t.Run(tc.name, func(t *testing.T) { + code, body := do(t, s, tc.method, tc.target, "application/json", tc.body) + if code != tc.status || !strings.Contains(body, `"reason":"`+tc.reason+`"`) { + t.Fatalf("= %d %s, want %d %s", code, body, tc.status, tc.reason) + } + }) + } +} + +func TestOversizedMultipartUploads(t *testing.T) { + s := storage.New() + if code, body := do(t, s, http.MethodPost, "/storage/v1/b?project=p", "application/json", `{"name":"big"}`); code != http.StatusOK { + t.Fatalf("insert = %d %s", code, body) + } + body := "--x\r\nContent-Type: application/json\r\n\r\n{\"name\":\"o\"}\r\n--x\r\nContent-Type: text/plain\r\n\r\n" + + strings.Repeat("x", 18<<20) + "\r\n--x--\r\n" + code, resp := do(t, s, http.MethodPost, "/upload/storage/v1/b/big/o?uploadType=multipart", "multipart/related; boundary=x", body) + if code != http.StatusRequestEntityTooLarge || !strings.Contains(resp, `"reason":"uploadTooLarge"`) { + t.Fatalf("upload = %d %.200s", code, resp) + } +} + +func TestPatchClearsNullFields(t *testing.T) { + s := storage.New() + if _, err := s.PutBucket(storage.Bucket{ + Name: "cleared", Project: "p", Labels: map[string]string{"a": "1", "b": "2"}, + Versioning: &storage.Versioning{Enabled: true}, Encryption: &storage.Encryption{DefaultKmsKeyName: "k"}, + CORS: json.RawMessage(`[{"origin":["*"]}]`), + }); err != nil { + t.Fatal(err) + } + if code, body := do(t, s, http.MethodPatch, "/storage/v1/b/cleared", "application/json", `{"labels":{"a":null}}`); code != http.StatusOK { + t.Fatalf("patch a label = %d %s", code, body) + } + if b, _ := s.LookupBucket("cleared"); len(b.Labels) != 1 || b.Labels["b"] != "2" { + t.Fatalf("labels = %v", b.Labels) + } + code, body := do(t, s, http.MethodPatch, "/storage/v1/b/cleared", "application/json", `{"labels":null,"versioning":null,"encryption":null,"cors":null}`) + if code != http.StatusOK { + t.Fatalf("patch = %d %s", code, body) + } + if b, _ := s.LookupBucket("cleared"); b.Labels != nil || b.Versioning != nil || b.Encryption != nil || b.CORS != nil || b.Metageneration != 3 { + t.Fatalf("bucket = %+v", b) + } +} + +func FuzzServeHTTP(f *testing.F) { + multipartBody := "--x\r\nContent-Type: application/json\r\n\r\n{\"name\":\"o\"}\r\n--x\r\nContent-Type: text/plain\r\n\r\nhi\r\n--x--\r\n" + for _, seed := range []struct{ method, target, contentType, body string }{ + {http.MethodPost, "/storage/v1/b?project=p", "application/json", `{"name":"other","labels":{"a":"b"},"cors":[{"origin":["*"]}]}`}, + {http.MethodPatch, "/storage/v1/b/fuzz?ifMetagenerationMatch=1", "application/json", `{"labels":{"a":null},"versioning":null}`}, + {http.MethodGet, "/storage/v1/b/fuzz/o?versions=true&maxResults=1&pageToken=YQ&startOffset=a&endOffset=z", "", ""}, + {http.MethodPost, "/upload/storage/v1/b/fuzz/o?uploadType=multipart", "multipart/related; boundary=x", multipartBody}, + {http.MethodPost, "/upload/storage/v1/b/fuzz/o?uploadType=media&name=a%2F..%2Fb&ifGenerationMatch=0", "text/plain", "x"}, + {http.MethodDelete, "/storage/v1/b/fuzz/o/k?generation=1", "", ""}, + } { + f.Add(seed.method, seed.target, seed.contentType, []byte(seed.body)) + } + f.Fuzz(func(t *testing.T, method, target, contentType string, body []byte) { + req, err := http.NewRequestWithContext(t.Context(), method, "http://storage.test"+target, bytes.NewReader(body)) + if err != nil { + return + } + req.Header.Set("Content-Type", contentType) + s := storage.New() + if _, err := s.PutBucket(storage.Bucket{Name: "fuzz", Project: "p", Versioning: &storage.Versioning{Enabled: true}}); err != nil { + t.Fatal(err) + } + if _, err := s.PutObject("fuzz", "k", []byte("v")); err != nil { + t.Fatal(err) + } + rec := httptest.NewRecorder() + s.ServeHTTP(rec, req) + if rec.Code < 200 || rec.Code > 599 { + t.Fatalf("%s %s = %d", method, target, rec.Code) + } + if rec.Code != http.StatusNoContent && method != http.MethodHead && !json.Valid(rec.Body.Bytes()) { + t.Fatalf("%s %s = %d %q, not JSON", method, target, rec.Code, rec.Body) + } + }) +} diff --git a/gcp/storage/objects.go b/gcp/storage/objects.go new file mode 100644 index 0000000..a4caafd --- /dev/null +++ b/gcp/storage/objects.go @@ -0,0 +1,350 @@ +package storage + +import ( + "bytes" + "cmp" + "crypto/md5" //nolint:gosec // the API reports MD5 digests; nothing here relies on them for security + "encoding/base64" + "encoding/binary" + "errors" + "fmt" + "hash/crc32" + "io" + "maps" + "mime" + "mime/multipart" + "net/http" + "net/url" + "slices" + "strconv" + "strings" + "unicode/utf8" +) + +// Object is one generation of an object, in the API's JSON shape; a noncurrent one has TimeDeleted. +type Object struct { + Kind string `json:"kind"` + ID string `json:"id"` + SelfLink string `json:"selfLink"` + MediaLink string `json:"mediaLink"` + Name string `json:"name"` + Bucket string `json:"bucket"` + Generation int64 `json:"generation,string"` + Metageneration int64 `json:"metageneration,string"` + ContentType string `json:"contentType,omitzero"` + ContentEncoding string `json:"contentEncoding,omitzero"` + StorageClass string `json:"storageClass"` + Size uint64 `json:"size,string"` + MD5Hash string `json:"md5Hash"` + CRC32C string `json:"crc32c"` + Etag string `json:"etag"` + TimeCreated string `json:"timeCreated"` + Updated string `json:"updated"` + TimeStorageClassUpdated string `json:"timeStorageClassUpdated"` + TimeDeleted string `json:"timeDeleted,omitzero"` + Metadata map[string]string `json:"metadata,omitzero"` +} + +func (o *Object) clone() Object { + c := *o + c.Metadata = maps.Clone(o.Metadata) + return c +} + +func (o *Object) live() bool { return o.TimeDeleted == "" } + +// liveObject returns name's live generation in b, or nil. The caller holds s.mu. +func (b *bucket) liveObject(name string) *Object { + gens := b.objects[name] + if len(gens) == 0 || !gens[len(gens)-1].live() { + return nil + } + return gens[len(gens)-1] +} + +// retire makes name's live generation noncurrent if b is versioned, or removes it. The caller holds s.mu. +func (s *Server) retire(b *bucket, name string) { + live := b.liveObject(name) + if live == nil { + return + } + if b.versioned() { + _, live.TimeDeleted = s.timestamp() + live.Updated = live.TimeDeleted + return + } + b.drop(name, live.Generation) +} + +// drop removes generation gen of name. The caller holds s.mu. +func (b *bucket) drop(name string, gen int64) { + gens := slices.DeleteFunc(b.objects[name], func(o *Object) bool { return o.Generation == gen }) + if len(gens) == 0 { + delete(b.objects, name) + return + } + b.objects[name] = gens +} + +func noSuchObject(bucket, name string) *apiError { + return &apiError{Status: http.StatusNotFound, Reason: "notFound", Message: fmt.Sprintf("No such object: %s/%s", bucket, name)} +} + +func validObjectName(name string) bool { + return len(name) <= 1024 && utf8.ValidString(name) && !strings.ContainsAny(name, "\r\n") && name != "." && name != ".." && + !strings.HasPrefix(name, ".well-known/acme-challenge/") +} + +// objectInput is the metadata an upload takes. +type objectInput struct { + Bucket string `json:"bucket"` + Name string `json:"name"` + ContentType string `json:"contentType"` + ContentEncoding string `json:"contentEncoding"` + Metadata map[string]string `json:"metadata"` + CRC32C string `json:"crc32c"` + MD5Hash string `json:"md5Hash"` +} + +// splitMultipart returns the metadata and media of a multipart/related upload. +func splitMultipart(contentType string, body []byte) (meta, media []byte, err error) { + mediaType, params, err := mime.ParseMediaType(contentType) + if err != nil || mediaType != "multipart/related" || params["boundary"] == "" { + return nil, nil, errors.New("a multipart upload needs a multipart/related body with a boundary") + } + mr := multipart.NewReader(bytes.NewReader(body), params["boundary"]) + var parts [][]byte + for { + p, err := mr.NextPart() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return nil, nil, err + } + b, err := io.ReadAll(p) + if err != nil { + return nil, nil, err + } + parts = append(parts, b) + } + if len(parts) != 2 { + return nil, nil, fmt.Errorf("a multipart upload has 2 parts, not %d", len(parts)) + } + return parts[0], parts[1], nil +} + +func (s *Server) insertObject(r *http.Request) (any, *apiError) { + if err := checkACLs(r); err != nil { + return nil, err + } + body, err := readBody(r, maxObject+maxBody) + if err != nil { + return nil, err + } + var in objectInput + var data []byte + switch t := r.URL.Query().Get("uploadType"); t { + case "multipart": + meta, media, err := splitMultipart(r.Header.Get("Content-Type"), body) + if err != nil { + return nil, invalid("%v", err) + } + if err := unmarshalStrict(meta, &in); err != nil { + return nil, err + } + data = media + if in.Name == "" { + in.Name = r.URL.Query().Get("name") + } + case "media": + in.Name, in.ContentType, data = r.URL.Query().Get("name"), r.Header.Get("Content-Type"), body + in.ContentEncoding = r.URL.Query().Get("contentEncoding") + case "": + return nil, required("uploadType") + default: + return nil, invalid("squall: uploadType=%s is not supported", t) + } + if len(data) > maxObject { + return nil, &apiError{Status: http.StatusRequestEntityTooLarge, Reason: "uploadTooLarge", Message: fmt.Sprintf("squall: objects are limited to %d bytes", maxObject)} + } + bucketName := r.PathValue("bucket") + if in.Bucket != "" && in.Bucket != bucketName { + return nil, invalid("The bucket in the metadata, %s, is not the bucket of the request, %s.", in.Bucket, bucketName) + } + s.mu.Lock() + defer s.mu.Unlock() + o, err := s.writeObject(bucketName, in, data, func(live int64) *apiError { + return checkMatch(r, "ifGenerationMatch", "ifGenerationNotMatch", live) + }) + if err != nil { + return nil, err + } + return o.clone(), nil +} + +// writeObject stores data as in.Name's newest generation once precondition passes. The caller holds s.mu. +func (s *Server) writeObject(bucketName string, in objectInput, data []byte, precondition func(live int64) *apiError) (*Object, *apiError) { + if in.Name == "" { + return nil, required("name") + } + if !validObjectName(in.Name) { + return nil, invalid("The specified object name is not valid.") + } + sum := md5.Sum(data) //nolint:gosec // a reported digest, not a security check + md5Hash := base64.StdEncoding.EncodeToString(sum[:]) + crc := base64.StdEncoding.EncodeToString(binary.BigEndian.AppendUint32(nil, crc32.Checksum(data, crc32.MakeTable(crc32.Castagnoli)))) + if in.CRC32C != "" && in.CRC32C != crc { + return nil, invalid("Provided CRC32C \"%s\" doesn't match calculated CRC32C \"%s\".", in.CRC32C, crc) + } + if in.MD5Hash != "" && in.MD5Hash != md5Hash { + return nil, invalid("Provided MD5 hash \"%s\" doesn't match calculated MD5 hash \"%s\".", in.MD5Hash, md5Hash) + } + b, err := s.lookup(bucketName, "storage.objects.create") + if err != nil { + return nil, err + } + var liveGen int64 + if live := b.liveObject(in.Name); live != nil { + liveGen = live.Generation + } + if precondition != nil { + if err := precondition(liveGen); err != nil { + return nil, err + } + } + s.retire(b, in.Name) + t, now := s.timestamp() + s.lastGeneration = max(s.lastGeneration+1, t.UnixMicro()) + gen := s.lastGeneration + escaped := url.PathEscape(in.Name) + o := &Object{ + Kind: "storage#object", ID: fmt.Sprintf("%s/%s/%d", bucketName, in.Name, gen), + SelfLink: apiBase + "b/" + bucketName + "/o/" + escaped, + MediaLink: downloadBase + "b/" + bucketName + "/o/" + escaped + "?generation=" + strconv.FormatInt(gen, 10) + "&alt=media", + Name: in.Name, Bucket: bucketName, Generation: gen, Metageneration: 1, + ContentType: cmp.Or(in.ContentType, "application/octet-stream"), ContentEncoding: in.ContentEncoding, + StorageClass: b.StorageClass, Size: uint64(len(data)), MD5Hash: md5Hash, CRC32C: crc, Etag: etag(gen, 1), + TimeCreated: now, Updated: now, TimeStorageClassUpdated: now, Metadata: maps.Clone(in.Metadata), + } + b.objects[in.Name] = append(b.objects[in.Name], o) + return o, nil +} + +func (s *Server) deleteObject(r *http.Request) (any, *apiError) { + gen, byGeneration, err := int64Param(r, "generation") + if err != nil { + return nil, err + } + bucketName, name := r.PathValue("bucket"), r.PathValue("object") + s.mu.Lock() + defer s.mu.Unlock() + b, err := s.lookup(bucketName, "storage.objects.delete") + if err != nil { + return nil, err + } + var target *Object + if byGeneration { + for _, o := range b.objects[name] { + if o.Generation == gen { + target = o + } + } + } else { + target = b.liveObject(name) + } + if target == nil { + return nil, noSuchObject(bucketName, name) + } + if err := checkMatch(r, "ifGenerationMatch", "ifGenerationNotMatch", target.Generation); err != nil { + return nil, err + } + if byGeneration { + b.drop(name, gen) + } else { + s.retire(b, name) + } + return nil, nil +} + +type objectList struct { + Kind string `json:"kind"` + Items []Object `json:"items,omitzero"` + NextPageToken string `json:"nextPageToken,omitzero"` +} + +// versionKey orders generations by name and then generation, escaping NUL so a name that extends another sorts after it. +func versionKey(o Object) string { + return strings.ReplaceAll(o.Name, "\x00", "\x00\xff") + fmt.Sprintf("\x00\x00%020d", o.Generation) +} + +func (s *Server) listObjects(r *http.Request) (any, *apiError) { + q := r.URL.Query() + for _, p := range []string{"delimiter", "matchGlob", "filter"} { + if q.Get(p) != "" { + return nil, invalid("squall: objects.list does not support %s", p) + } + } + for _, p := range []string{"includeTrailingDelimiter", "includeFoldersAsPrefixes", "softDeleted"} { + if on, err := boolParam(r, p); err != nil { + return nil, err + } else if on { + return nil, invalid("squall: objects.list does not support %s", p) + } + } + versions, err := boolParam(r, "versions") + if err != nil { + return nil, err + } + prefix, start, end := q.Get("prefix"), q.Get("startOffset"), q.Get("endOffset") + s.mu.Lock() + defer s.mu.Unlock() + b, err := s.lookup(r.PathValue("bucket"), "storage.objects.list") + if err != nil { + return nil, err + } + var all []Object + for _, name := range slices.Sorted(maps.Keys(b.objects)) { + if !strings.HasPrefix(name, prefix) || name < start || end != "" && name >= end { + continue + } + for _, o := range b.objects[name] { + if versions || o.live() { + all = append(all, o.clone()) + } + } + } + items, next, err := page(r, all, versionKey) + if err != nil { + return nil, err + } + return objectList{Kind: "storage#objects", Items: items, NextPageToken: next}, nil +} + +// Objects returns every live and noncurrent generation in the named bucket, ordered by name and generation. +func (s *Server) Objects(bucketName string) []Object { + s.mu.Lock() + defer s.mu.Unlock() + b, ok := s.buckets[bucketName] + if !ok { + return nil + } + var out []Object + for _, name := range slices.Sorted(maps.Keys(b.objects)) { + for _, o := range b.objects[name] { + out = append(out, o.clone()) + } + } + return out +} + +// PutObject writes data to name in the named bucket, as an upload does, and returns the generation it made. +func (s *Server) PutObject(bucketName, name string, data []byte) (Object, error) { + s.mu.Lock() + defer s.mu.Unlock() + o, err := s.writeObject(bucketName, objectInput{Name: name}, data, nil) + if err != nil { + return Object{}, err + } + return o.clone(), nil +} diff --git a/gcp/storage/server.go b/gcp/storage/server.go new file mode 100644 index 0000000..3eea10e --- /dev/null +++ b/gcp/storage/server.go @@ -0,0 +1,434 @@ +// Package storage is a stand-in for the Cloud Storage JSON API, for testing +// code that calls it through the official client libraries. +// +// [Server] is an [http.Handler] that keeps buckets and their object +// generations in memory and answers the JSON API calls the Go client library +// makes. Serve it with [net/http/httptest] and point a client at it with +// option.WithEndpoint(srv.URL + "/storage/v1/") and +// option.WithoutAuthentication(), or by setting STORAGE_EMULATOR_HOST to +// srv.URL. A [fault.Injector] makes chosen calls fail, including after they +// took effect, so tests can check that code which retries converges. +// +// It covers: +// +// - buckets: insert, get, list, patch, delete; +// - objects: insert by multipart or media upload, list, delete. +// +// Other methods answer 404. Where it differs from Cloud Storage: +// +// - There is one caller, which can reach every project. Bucket names are +// global, as in Cloud Storage. [Server.PutForeignBucket] holds a name for +// someone the caller cannot reach: inserting it fails with 409 conflict +// and every other call on it with 403 forbidden. buckets.list lists only +// the project's own. Other denials are a fault rule away, such as +// fault.Reject(403, "forbidden") for "storage.buckets.list". +// - Inserting a bucket the caller already has fails with 409 conflict, as a +// client's retry of a create that took effect does. Bucket names follow +// the naming rules, and labels the label rules; locations are not +// checked against the real list. A location is stored upper-cased, US by +// default; US, EU and ASIA are multi-regions, NAM4, EUR4, EUR5, EUR7, EUR8 +// and ASIA1 and any location with a customPlacementConfig dual-regions, +// and the rest regions. Project numbers are derived from project IDs. +// - Bucket fields other than name, location, storageClass, labels, +// versioning, cors, customPlacementConfig, encryption and rpo are +// rejected with 400, as are ACLs other than private and projectPrivate. +// cors, encryption and rpo are recorded, not enforced. There is no IAM, +// soft delete, retention, lifecycle, autoclass or hierarchical namespace. +// buckets.patch has patch semantics: it merges labels, deleting one set +// to null, and clears labels, versioning, cors or encryption set to null. +// - Objects record their size, MD5 and CRC32C, not their data; a crc32c or +// md5Hash in the upload's metadata must match the data. Resumable +// uploads, reads, copies, composes and object ACLs are not supported. +// Uploads take up to 16 MiB. +// - With versioning on, writing or deleting an object makes its live +// generation noncurrent, and objects.list with versions=true lists those +// too, ordered by name and then generation. Deleting a bucket fails with +// 409 conflict while any generation, live or noncurrent, remains. +// - objects.list takes prefix, startOffset, endOffset, versions, pageToken +// and maxResults; a delimiter, matchGlob, filter, softDeleted, +// includeTrailingDelimiter or includeFoldersAsPrefixes is rejected with +// 400. Page tokens are positions, so a page may follow one whose items +// were deleted. Bucket preconditions are ifMetagenerationMatch and +// ifMetagenerationNotMatch, and object ones ifGenerationMatch and +// ifGenerationNotMatch. The fields and userProject parameters are +// accepted and ignored: responses are always whole. Other parameters +// the server does not know are rejected with 400. +// +// Fault method names are the API's method IDs, such as +// "storage.buckets.insert". A call's resource is its path below +// /storage/v1/, such as "b/my-bucket" or "b/my-bucket/o/my-object"; an +// insert names the bucket or object it creates, and buckets.list names +// "projects/my-project/b". The JSON API has no long-running operations, so +// [fault.ModeFailOperation] fails a call as [fault.ModeReject] does. +// +// A lost response aborts the handler with [http.ErrAbortHandler] after +// sending the response header, so the [http.Server] serving it closes the +// connection. Calling ServeHTTP directly propagates that panic. +package storage + +import ( + "bytes" + "cmp" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "path" + "slices" + "strconv" + "strings" + "sync" + "time" + + "github.com/parallelworks/squall/fault" + "github.com/parallelworks/squall/internal/stub" +) + +const ( + maxBody = 1 << 20 + maxObject = 16 << 20 + maxList = 1000 + + apiBase = "https://www.googleapis.com/storage/v1/" + downloadBase = "https://storage.googleapis.com/download/storage/v1/" + timeFormat = "2006-01-02T15:04:05.000Z" +) + +// Server is a stand-in for the Cloud Storage JSON API. Create one with [New]. It is safe for concurrent use. +type Server struct { + mux *http.ServeMux + faults *fault.Injector + now func() time.Time + + mu sync.Mutex + buckets map[string]*bucket + foreign map[string]bool + lastGeneration int64 +} + +// Option configures a [Server]. +type Option func(*Server) + +// WithFaults makes the server ask in which calls to fail. +func WithFaults(in *fault.Injector) Option { + return func(s *Server) { s.faults = in } +} + +// WithClock sets the clock for times and generations. The default is time.Now. +func WithClock(now func() time.Time) Option { + return func(s *Server) { s.now = now } +} + +// New returns an empty server. +func New(opts ...Option) *Server { + s := &Server{mux: http.NewServeMux(), now: time.Now, buckets: map[string]*bucket{}, foreign: map[string]bool{}} + for _, opt := range opts { + opt(s) + } + s.registerRoutes() + return s +} + +// ServeHTTP answers a Cloud Storage JSON API call. +func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { + // ServeMux would redirect a path that is not clean, which the API never does. + if p := r.URL.EscapedPath(); p != cleanPath(p) { + notImplemented(w, r) + return + } + s.mux.ServeHTTP(w, r) +} + +// cleanPath is p as ServeMux cleans it, keeping a trailing slash. +func cleanPath(p string) string { + clean := path.Clean("/" + p) + if strings.HasSuffix(p, "/") && clean != "/" { + clean += "/" + } + return clean +} + +func notImplemented(w http.ResponseWriter, r *http.Request) { + writeError(w, &apiError{Status: http.StatusNotFound, Reason: "notFound", Message: "squall: " + r.Method + " " + r.URL.Path + " is not implemented"}) +} + +// commonParams are the query parameters every method takes. +var commonParams = []string{"alt", "prettyPrint", "fields", "userProject", "quotaUser"} + +func (s *Server) registerRoutes() { + const ( + buckets = "/storage/v1/b" + bucket = buckets + "/{bucket}" + ) + s.route("POST "+buckets, "storage.buckets.insert", s.insertBucket, insertTarget, + "project", "projection", "predefinedAcl", "predefinedDefaultObjectAcl", "enableObjectRetention") + s.route("GET "+buckets, "storage.buckets.list", s.listBuckets, listTarget, + "project", "projection", "prefix", "pageToken", "maxResults", "returnPartialSuccess") + s.route("GET "+bucket, "storage.buckets.get", s.getBucket, nil, "projection", "ifMetagenerationMatch", "ifMetagenerationNotMatch") + s.route("PATCH "+bucket, "storage.buckets.patch", s.patchBucket, nil, + "projection", "ifMetagenerationMatch", "ifMetagenerationNotMatch", "predefinedAcl", "predefinedDefaultObjectAcl") + s.route("DELETE "+bucket, "storage.buckets.delete", s.deleteBucket, nil, "ifMetagenerationMatch", "ifMetagenerationNotMatch") + s.route("GET "+bucket+"/o", "storage.objects.list", s.listObjects, nil, + "projection", "prefix", "delimiter", "pageToken", "maxResults", "versions", "startOffset", "endOffset", + "includeTrailingDelimiter", "matchGlob", "includeFoldersAsPrefixes", "softDeleted", "filter") + s.route("DELETE "+bucket+"/o/{object...}", "storage.objects.delete", s.deleteObject, objectTarget, + "generation", "ifGenerationMatch", "ifGenerationNotMatch") + s.route("POST /upload"+bucket+"/o", "storage.objects.insert", s.insertObject, uploadTarget, + "uploadType", "name", "projection", "predefinedAcl", "ifGenerationMatch", "ifGenerationNotMatch", "contentEncoding") + // Without this, ServeMux would redirect an object path with no name to the one with an empty name. + s.mux.HandleFunc("DELETE "+bucket+"/o", notImplemented) + s.mux.HandleFunc("/", notImplemented) +} + +// handler answers one method with a body to encode, or nil for an empty 204. +type handler func(r *http.Request) (any, *apiError) + +// route registers h for pattern as method, which takes params; a non-nil target names the call's resource. +func (s *Server) route(pattern, method string, h handler, target func(*http.Request) string, params ...string) { + s.mux.HandleFunc(pattern, func(w http.ResponseWriter, r *http.Request) { + for name := range r.URL.Query() { + if !slices.Contains(params, name) && !slices.Contains(commonParams, name) { + writeError(w, invalid("squall: %s does not take the parameter %s", method, name)) + return + } + } + if alt := r.URL.Query().Get("alt"); alt != "" && alt != "json" { + writeError(w, invalid("squall: alt=%s is not supported", alt)) + return + } + resource := strings.TrimPrefix(r.URL.Path, "/storage/v1/") + if target != nil { + resource = target(r) + } + f, _ := s.faults.Decide(fault.Call{Method: method, Resource: resource}) + reject := f.Mode == fault.ModeReject || f.Mode == fault.ModeFailOperation + stub.Serve(w, r, f, reject, "application/json; charset=UTF-8", func() { writeError(w, faultError(f)) }, func() (func(), bool) { + body, err := h(r) + if err != nil { + return func() { writeError(w, err) }, false + } + return func() { + if body == nil { + w.WriteHeader(http.StatusNoContent) + return + } + writeJSON(w, http.StatusOK, body) + }, true + }) + }) +} + +// insertTarget names the bucket a buckets.insert creates, from its body. +func insertTarget(r *http.Request) string { + var named struct { + Name string `json:"name"` + } + if json.Unmarshal(peekBody(r, maxBody), &named) != nil || named.Name == "" { + return "b" + } + return "b/" + named.Name +} + +func listTarget(r *http.Request) string { + return "projects/" + r.URL.Query().Get("project") + "/b" +} + +func objectTarget(r *http.Request) string { + return "b/" + r.PathValue("bucket") + "/o/" + r.PathValue("object") +} + +// uploadTarget names the object an upload creates, from its name parameter or metadata. +func uploadTarget(r *http.Request) string { + prefix := "b/" + r.PathValue("bucket") + "/o" + if name := r.URL.Query().Get("name"); name != "" { + return prefix + "/" + name + } + if r.URL.Query().Get("uploadType") != "multipart" { + return prefix + } + meta, _, err := splitMultipart(r.Header.Get("Content-Type"), peekBody(r, maxObject+maxBody)) + var named struct { + Name string `json:"name"` + } + if err != nil || json.Unmarshal(meta, &named) != nil || named.Name == "" { + return prefix + } + return prefix + "/" + named.Name +} + +// peekBody reads up to limit bytes of r's body and puts them back, to name a call's resource before it is handled. +func peekBody(r *http.Request, limit int64) []byte { + body, err := io.ReadAll(io.LimitReader(r.Body, limit)) + r.Body = struct { + io.Reader + io.Closer + }{io.MultiReader(bytes.NewReader(body), r.Body), r.Body} + if err != nil { + return nil + } + return body +} + +// apiError is an error in the shape Google APIs answer with. +type apiError struct { + Status int + Reason string + Message string +} + +func (e *apiError) Error() string { return fmt.Sprintf("%d %s: %s", e.Status, e.Reason, e.Message) } + +func invalid(format string, args ...any) *apiError { + return &apiError{Status: http.StatusBadRequest, Reason: "invalid", Message: fmt.Sprintf(format, args...)} +} + +func required(param string) *apiError { + return &apiError{Status: http.StatusBadRequest, Reason: "required", Message: "Required parameter: " + param} +} + +func conditionNotMet() *apiError { + return &apiError{Status: http.StatusPreconditionFailed, Reason: "conditionNotMet", Message: "At least one of the pre-conditions you specified did not hold."} +} + +func faultError(f fault.Fault) *apiError { + status := cmp.Or(f.Status, http.StatusServiceUnavailable) + return &apiError{Status: status, Reason: cmp.Or(f.Reason, "backendError"), Message: fmt.Sprintf("squall: injected fault (%s)", f)} +} + +type errorBody struct { + Error errorDetail `json:"error"` +} + +type errorDetail struct { + Code int `json:"code"` + Message string `json:"message"` + Errors []errorReason `json:"errors"` +} + +type errorReason struct { + Message string `json:"message"` + Domain string `json:"domain"` + Reason string `json:"reason"` +} + +func writeError(w http.ResponseWriter, e *apiError) { + writeJSON(w, e.Status, errorBody{errorDetail{ + Code: e.Status, + Message: e.Message, + Errors: []errorReason{{Message: e.Message, Domain: "global", Reason: e.Reason}}, + }}) +} + +func writeJSON(w http.ResponseWriter, status int, body any) { + b, err := json.Marshal(body) + if err != nil { + panic("squall: encoding a response: " + err.Error()) + } + w.Header().Set("Content-Type", "application/json; charset=UTF-8") + w.WriteHeader(status) + // Once the status is sent, a client that went away is all a write error can mean. + _, _ = w.Write(b) +} + +// unmarshalStrict decodes data into v, rejecting fields v does not have. +func unmarshalStrict(data []byte, v any) *apiError { + dec := json.NewDecoder(bytes.NewReader(data)) + dec.DisallowUnknownFields() + if err := dec.Decode(v); err != nil { + if name, ok := strings.CutPrefix(err.Error(), "json: unknown field "); ok { + return invalid("squall: the field %s is not supported", name) + } + if _, ok := errors.AsType[*json.SyntaxError](err); ok { + return invalid("Invalid JSON payload received: %v", err) + } + return invalid("Invalid value in the request body: %v", err) + } + if _, err := dec.Token(); !errors.Is(err, io.EOF) { + return invalid("Invalid JSON payload received: data after the top-level value") + } + return nil +} + +func readBody(r *http.Request, limit int64) ([]byte, *apiError) { + body, err := io.ReadAll(io.LimitReader(r.Body, limit+1)) + if err != nil { + return nil, invalid("reading the request body: %v", err) + } + if int64(len(body)) > limit { + return nil, &apiError{Status: http.StatusRequestEntityTooLarge, Reason: "uploadTooLarge", Message: fmt.Sprintf("squall: bodies are limited to %d bytes", limit)} + } + return body, nil +} + +// int64Param reads the named query parameter, reporting whether it is set. +func int64Param(r *http.Request, name string) (int64, bool, *apiError) { + v := r.URL.Query().Get(name) + if v == "" { + return 0, false, nil + } + n, err := strconv.ParseInt(v, 10, 64) + if err != nil { + return 0, false, invalid("Invalid value for %s: %s", name, v) + } + return n, true, nil +} + +// boolParam reads the named query parameter, which is false when empty. +func boolParam(r *http.Request, name string) (bool, *apiError) { + switch v := r.URL.Query().Get(name); v { + case "", "false": + return false, nil + case "true": + return true, nil + default: + return false, invalid("Invalid value for %s: %s", name, v) + } +} + +// checkMatch applies a ...Match and ...NotMatch precondition pair to value. +func checkMatch(r *http.Request, match, notMatch string, value int64) *apiError { + if want, ok, err := int64Param(r, match); err != nil { + return err + } else if ok && want != value { + return conditionNotMet() + } + if unwanted, ok, err := int64Param(r, notMatch); err != nil { + return err + } else if ok && unwanted == value { + return conditionNotMet() + } + return nil +} + +// page returns the key-ordered items after the page token, the last key of the page before, up to maxResults. +func page[T any](r *http.Request, items []T, key func(T) string) ([]T, string, *apiError) { + limit := maxList + if n, ok, err := int64Param(r, "maxResults"); err != nil { + return nil, "", err + } else if ok { + if n < 0 { + return nil, "", invalid("Invalid value for maxResults: %d", n) + } + if n > 0 && n < maxList { + limit = int(n) + } + } + if token := r.URL.Query().Get("pageToken"); token != "" { + after, err := base64.RawURLEncoding.DecodeString(token) + if err != nil || len(after) == 0 { + return nil, "", invalid("Invalid page token: %s", token) + } + i, _ := slices.BinarySearchFunc(items, string(after), func(item T, k string) int { return strings.Compare(key(item), k) }) + for i < len(items) && key(items[i]) <= string(after) { + i++ + } + items = items[i:] + } + if len(items) <= limit { + return items, "", nil + } + items = items[:limit] + return items, base64.RawURLEncoding.EncodeToString([]byte(key(items[limit-1]))), nil +} diff --git a/gcp/storage/testdata/fuzz/FuzzServeHTTP/53c1623a98810619 b/gcp/storage/testdata/fuzz/FuzzServeHTTP/53c1623a98810619 new file mode 100644 index 0000000..bc4e3be --- /dev/null +++ b/gcp/storage/testdata/fuzz/FuzzServeHTTP/53c1623a98810619 @@ -0,0 +1,5 @@ +go test fuzz v1 +string("0") +string("0") +string("0") +[]byte("0") diff --git a/gcp/storage/testdata/fuzz/FuzzServeHTTP/a0d84d1a3af5511f b/gcp/storage/testdata/fuzz/FuzzServeHTTP/a0d84d1a3af5511f new file mode 100644 index 0000000..3e9f9d9 --- /dev/null +++ b/gcp/storage/testdata/fuzz/FuzzServeHTTP/a0d84d1a3af5511f @@ -0,0 +1,5 @@ +go test fuzz v1 +string("DELETE") +string("/storage/v1/b/0/o") +string("") +[]byte("") diff --git a/internal/stub/stub.go b/internal/stub/stub.go index e658b5f..0712846 100644 --- a/internal/stub/stub.go +++ b/internal/stub/stub.go @@ -27,6 +27,8 @@ func Serve(w http.ResponseWriter, r *http.Request, f fault.Fault, reject bool, c } answer, ok := run() switch { + case f.Mode == fault.ModeLoseResponse && r.Method == http.MethodHead: + loseHeadResponse(w) case f.Mode == fault.ModeLoseResponse: loseResponse(w, contentType) case !ok: @@ -49,3 +51,14 @@ func loseResponse(w http.ResponseWriter, contentType string) { _ = http.NewResponseController(w).Flush() panic(http.ErrAbortHandler) } + +// loseHeadResponse breaks off inside the status line, since a sent HEAD header would read as an answer. +func loseHeadResponse(w http.ResponseWriter) { + conn, buf, err := http.NewResponseController(w).Hijack() + if err != nil { + panic(http.ErrAbortHandler) + } + defer conn.Close() + _, _ = buf.WriteString("HTTP/1.1 2") + _ = buf.Flush() +} diff --git a/sdktest/aws/ec2/ec2_test.go b/sdktest/aws/ec2/ec2_test.go index 560362f..fe627a7 100644 --- a/sdktest/aws/ec2/ec2_test.go +++ b/sdktest/aws/ec2/ec2_test.go @@ -17,10 +17,10 @@ import ( "github.com/aws/aws-sdk-go-v2/credentials" awsec2 "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/types" - "github.com/aws/smithy-go" "github.com/parallelworks/squall/aws/ec2" "github.com/parallelworks/squall/fault" + "github.com/parallelworks/squall/sdktest/internal/sdkerr" "github.com/parallelworks/squall/sdktest/internal/sdkhttp" ) @@ -56,20 +56,6 @@ func withRetries(attempts int) func(*awsec2.Options) { return func(o *awsec2.Options) { o.RetryMaxAttempts = attempts } } -func code(err error) string { - if e, ok := errors.AsType[smithy.APIError](err); ok { - return e.ErrorCode() - } - return "" -} - -func status(err error) int { - if e, ok := errors.AsType[*awshttp.ResponseError](err); ok { - return e.HTTPStatusCode() - } - return 0 -} - func tagSpec(resource types.ResourceType, kv ...string) []types.TagSpecification { spec := types.TagSpecification{ResourceType: resource} for i := 0; i < len(kv); i += 2 { @@ -173,7 +159,7 @@ func TestAllocateDescribeRelease(t *testing.T) { if got := s.Addresses(); len(got) != 0 { t.Fatalf("addresses after release = %v", got) } - if _, err := describeAddresses(t, c, &awsec2.DescribeAddressesInput{AllocationIds: []string{id}}); code(err) != "InvalidAllocationID.NotFound" { + if _, err := describeAddresses(t, c, &awsec2.DescribeAddressesInput{AllocationIds: []string{id}}); sdkerr.AWSCode(err) != "InvalidAllocationID.NotFound" { t.Fatalf("describe a released address = %v", err) } } @@ -249,7 +235,7 @@ func TestAddressErrors(t *testing.T) { } { t.Run(tc.name, func(t *testing.T) { err := tc.call() - if code(err) != tc.code || status(err) != http.StatusBadRequest { + if sdkerr.AWSCode(err) != tc.code || sdkerr.AWSStatus(err) != http.StatusBadRequest { t.Fatalf("err = %v, want %s with HTTP 400", err, tc.code) } }) @@ -264,7 +250,7 @@ func TestAddressLimit(t *testing.T) { c := newClient(t, s) allocate(t, c) allocate(t, c) - if _, err := c.AllocateAddress(t.Context(), &awsec2.AllocateAddressInput{}); code(err) != "AddressLimitExceeded" { + if _, err := c.AllocateAddress(t.Context(), &awsec2.AllocateAddressInput{}); sdkerr.AWSCode(err) != "AddressLimitExceeded" { t.Fatalf("a third allocation = %v, want AddressLimitExceeded", err) } other := clientAt(serve(t, s), "eu-west-1", withRetries(1)) @@ -415,7 +401,7 @@ func TestInterfaceErrors(t *testing.T) { }, "InvalidParameterValue"}, } { t.Run(tc.name, func(t *testing.T) { - if err := tc.call(); code(err) != tc.code || status(err) != http.StatusBadRequest { + if err := tc.call(); sdkerr.AWSCode(err) != tc.code || sdkerr.AWSStatus(err) != http.StatusBadRequest { t.Fatalf("err = %v, want %s with HTTP 400", err, tc.code) } }) @@ -530,7 +516,7 @@ func TestInterfaceFilters(t *testing.T) { t.Errorf("filters %v = %v, %v; want %v", describe(tc.filters), names, err, tc.want) } } - if _, err := describeInterfaces(t, c, &awsec2.DescribeNetworkInterfacesInput{Filters: []types.Filter{filter("vpc-id", "vpc-1")}}); code(err) != "InvalidParameterValue" { + if _, err := describeInterfaces(t, c, &awsec2.DescribeNetworkInterfacesInput{Filters: []types.Filter{filter("vpc-id", "vpc-1")}}); sdkerr.AWSCode(err) != "InvalidParameterValue" { t.Errorf("an unsupported filter = %v, want InvalidParameterValue", err) } } @@ -573,7 +559,7 @@ func TestClientToken(t *testing.T) { t.Fatalf("repeating the token = %+v, %v; want the first interface", again, err) } in.Description = aws.String("different") - if _, err := c.CreateNetworkInterface(t.Context(), in); code(err) != "IdempotentParameterMismatch" { + if _, err := c.CreateNetworkInterface(t.Context(), in); sdkerr.AWSCode(err) != "IdempotentParameterMismatch" { t.Fatalf("the token with other parameters = %v, want IdempotentParameterMismatch", err) } west := clientAt(serve(t, s), "us-west-2", withRetries(1)) @@ -591,7 +577,7 @@ func TestClientTokenReplaysAcrossALostResponse(t *testing.T) { s := ec2.New(ec2.WithFaults(faults)) c := newClient(t, s) in := &awsec2.CreateNetworkInterfaceInput{SubnetId: aws.String(subnet), ClientToken: aws.String("token-1")} - if _, err := c.CreateNetworkInterface(t.Context(), in); err == nil || code(err) != "" { + if _, err := c.CreateNetworkInterface(t.Context(), in); err == nil || sdkerr.AWSCode(err) != "" { t.Fatalf("the first create = %v, want a transport error", err) } out, err := c.CreateNetworkInterface(t.Context(), in) @@ -651,9 +637,9 @@ func TestFaults(t *testing.T) { switch { case tc.code == "" && err != nil: t.Fatalf("allocate = %v, want success", err) - case tc.code == "-" && (err == nil || code(err) != ""): + case tc.code == "-" && (err == nil || sdkerr.AWSCode(err) != ""): t.Fatalf("allocate = %v, want a transport error", err) - case tc.code != "" && tc.code != "-" && (code(err) != tc.code || status(err) != tc.status): + case tc.code != "" && tc.code != "-" && (sdkerr.AWSCode(err) != tc.code || sdkerr.AWSStatus(err) != tc.status): t.Fatalf("allocate = %v, want %s with HTTP %d", err, tc.code, tc.status) } if got := len(s.Addresses()); (got == 1) != tc.stored { @@ -679,7 +665,7 @@ func TestAFaultRuleCanTargetOneResource(t *testing.T) { if _, err := c.ReleaseAddress(t.Context(), &awsec2.ReleaseAddressInput{AllocationId: aws.String(keep)}); err != nil { t.Fatalf("releasing another address: %v", err) } - if _, err := c.ReleaseAddress(t.Context(), &awsec2.ReleaseAddressInput{AllocationId: aws.String(target)}); code(err) != "Unavailable" { + if _, err := c.ReleaseAddress(t.Context(), &awsec2.ReleaseAddressInput{AllocationId: aws.String(target)}); sdkerr.AWSCode(err) != "Unavailable" { t.Fatalf("releasing the targeted address = %v, want Unavailable", err) } } @@ -688,7 +674,7 @@ func TestDryRun(t *testing.T) { s := ec2.New() c := newClient(t, s) _, err := c.AllocateAddress(t.Context(), &awsec2.AllocateAddressInput{DryRun: aws.Bool(true)}) - if code(err) != "DryRunOperation" || status(err) != http.StatusPreconditionFailed { + if sdkerr.AWSCode(err) != "DryRunOperation" || sdkerr.AWSStatus(err) != http.StatusPreconditionFailed { t.Fatalf("a dry run = %v, want DryRunOperation with HTTP 412", err) } if got := s.Addresses(); len(got) != 0 { @@ -701,7 +687,7 @@ func TestRegionsAreSeparate(t *testing.T) { url := serve(t, s) east, west := clientAt(url, "us-east-1", withRetries(1)), clientAt(url, "us-west-2", withRetries(1)) id := aws.ToString(allocate(t, east).AllocationId) - if _, err := describeAddresses(t, west, &awsec2.DescribeAddressesInput{AllocationIds: []string{id}}); code(err) != "InvalidAllocationID.NotFound" { + if _, err := describeAddresses(t, west, &awsec2.DescribeAddressesInput{AllocationIds: []string{id}}); sdkerr.AWSCode(err) != "InvalidAllocationID.NotFound" { t.Fatalf("describing it from another region = %v", err) } westIP := aws.ToString(allocate(t, west).PublicIp) @@ -757,7 +743,7 @@ func TestPutHelpersSetUpState(t *testing.T) { if err != nil || aws.ToString(got[0].InstanceId) != "i-1" || aws.ToString(got[0].AssociationId) != a.AssociationID { t.Fatalf("describe = %+v, %v", got, err) } - if _, err := c.DeleteNetworkInterface(t.Context(), &awsec2.DeleteNetworkInterfaceInput{NetworkInterfaceId: aws.String(ni.NetworkInterfaceID)}); code(err) != "InvalidNetworkInterface.InUse" { + if _, err := c.DeleteNetworkInterface(t.Context(), &awsec2.DeleteNetworkInterfaceInput{NetworkInterfaceId: aws.String(ni.NetworkInterfaceID)}); sdkerr.AWSCode(err) != "InvalidNetworkInterface.InUse" { t.Fatalf("deleting an interface put in use = %v", err) } for _, bad := range []ec2.Address{ diff --git a/sdktest/aws/ec2/instances_test.go b/sdktest/aws/ec2/instances_test.go index e93220d..e6c5d18 100644 --- a/sdktest/aws/ec2/instances_test.go +++ b/sdktest/aws/ec2/instances_test.go @@ -13,6 +13,7 @@ import ( "github.com/parallelworks/squall/aws/ec2" "github.com/parallelworks/squall/fault" + "github.com/parallelworks/squall/sdktest/internal/sdkerr" ) func putImage(t *testing.T, s *ec2.Server, img ec2.Image) string { @@ -234,7 +235,7 @@ func TestInstanceWaiters(t *testing.T) { if err := s.ForgetInstance(region, id); err != nil { t.Fatal(err) } - if _, err := describeInstances(t, c, in); code(err) != "InvalidInstanceID.NotFound" { + if _, err := describeInstances(t, c, in); sdkerr.AWSCode(err) != "InvalidInstanceID.NotFound" { t.Fatalf("describing a forgotten instance = %v", err) } } @@ -248,7 +249,7 @@ func TestSettle(t *testing.T) { t.Fatalf("state = %s, want pending until Settle", got.State.Name) } } - if _, err := c.StopInstances(t.Context(), &awsec2.StopInstancesInput{InstanceIds: []string{id}}); code(err) != "IncorrectInstanceState" { + if _, err := c.StopInstances(t.Context(), &awsec2.StopInstancesInput{InstanceIds: []string{id}}); sdkerr.AWSCode(err) != "IncorrectInstanceState" { t.Fatalf("stopping a pending instance = %v", err) } if err := s.ForgetInstance(region, id); err == nil { @@ -322,7 +323,7 @@ func TestInstanceStateErrors(t *testing.T) { }, "InvalidParameterCombination"}, } { t.Run(tc.name, func(t *testing.T) { - if err := tc.call(); code(err) != tc.code { + if err := tc.call(); sdkerr.AWSCode(err) != tc.code { t.Fatalf("= %v, want %s", err, tc.code) } }) @@ -345,7 +346,7 @@ func TestInstanceStateErrors(t *testing.T) { for bad, want := range map[string]string{"t4g.micro": "InvalidParameterValue", "x9.huge": "InvalidInstanceAttributeValue"} { if _, err := c.ModifyInstanceAttribute(t.Context(), &awsec2.ModifyInstanceAttributeInput{ InstanceId: aws.String(stopping), InstanceType: &types.AttributeValue{Value: aws.String(bad)}, - }); code(err) != want { + }); sdkerr.AWSCode(err) != want { t.Errorf("changing the type to %s = %v, want %s", bad, err, want) } } @@ -399,7 +400,7 @@ func TestRunInstancesErrors(t *testing.T) { } in := runInput(image) tc.change(in) - if _, err := c.RunInstances(t.Context(), in); code(err) != tc.code { + if _, err := c.RunInstances(t.Context(), in); sdkerr.AWSCode(err) != tc.code { t.Fatalf("= %v, want %s", err, tc.code) } if got := s.Instances(); len(got) != 0 { @@ -431,7 +432,7 @@ func TestRunInstancesClientToken(t *testing.T) { t.Fatalf("client token = %q", aws.ToString(inst.ClientToken)) } in.InstanceType = types.InstanceTypeT3Small - if _, err := c.RunInstances(t.Context(), in); code(err) != "IdempotentParameterMismatch" { + if _, err := c.RunInstances(t.Context(), in); sdkerr.AWSCode(err) != "IdempotentParameterMismatch" { t.Fatalf("the token with other parameters = %v, want IdempotentParameterMismatch", err) } if got := len(s.Instances()); got != 1 { @@ -449,7 +450,7 @@ func TestRunInstancesReplaysAcrossALostResponse(t *testing.T) { c := newClient(t, s) in := runInput(putImage(t, s, ec2.Image{})) in.ClientToken = aws.String("launch-1") - if _, err := c.RunInstances(t.Context(), in); err == nil || code(err) != "" { + if _, err := c.RunInstances(t.Context(), in); err == nil || sdkerr.AWSCode(err) != "" { t.Fatalf("the first run = %v, want a transport error", err) } out, err := c.RunInstances(t.Context(), in) @@ -668,7 +669,7 @@ func TestAssociateAddressWithInstanceErrors(t *testing.T) { {"no target", &awsec2.AssociateAddressInput{AllocationId: alloc}, "MissingParameter"}, {"two targets", &awsec2.AssociateAddressInput{AllocationId: alloc, InstanceId: aws.String(id), NetworkInterfaceId: aws.String("eni-1")}, "InvalidParameterCombination"}, } { - if _, err := c.AssociateAddress(t.Context(), tc.in); code(err) != tc.code { + if _, err := c.AssociateAddress(t.Context(), tc.in); sdkerr.AWSCode(err) != tc.code { t.Errorf("%s = %v, want %s", tc.name, err, tc.code) } } @@ -721,10 +722,10 @@ func TestImages(t *testing.T) { len(img.BlockDeviceMappings) != 2 || aws.ToInt32(img.BlockDeviceMappings[0].Ebs.VolumeSize) != 20 || img.State != types.ImageStateAvailable { t.Fatalf("image = %+v", img) } - if _, err := describeImages(&awsec2.DescribeImagesInput{ImageIds: []string{mine, "ami-0000000000000dead"}}); code(err) != "InvalidAMIID.NotFound" { + if _, err := describeImages(&awsec2.DescribeImagesInput{ImageIds: []string{mine, "ami-0000000000000dead"}}); sdkerr.AWSCode(err) != "InvalidAMIID.NotFound" { t.Fatalf("an unknown image = %v", err) } - if _, err := describeImages(&awsec2.DescribeImagesInput{ImageIds: []string{"image-1"}}); code(err) != "InvalidAMIID.Malformed" { + if _, err := describeImages(&awsec2.DescribeImagesInput{ImageIds: []string{"image-1"}}); sdkerr.AWSCode(err) != "InvalidAMIID.Malformed" { t.Fatalf("a malformed image ID = %v", err) } @@ -777,7 +778,7 @@ func TestInstanceTypes(t *testing.T) { if err != nil || len(arm.InstanceTypes) != 2 { t.Fatalf("arm types = %+v, %v", arm, err) } - if _, err := c.DescribeInstanceTypes(t.Context(), &awsec2.DescribeInstanceTypesInput{InstanceTypes: []types.InstanceType{"x9.huge"}}); code(err) != "InvalidInstanceType" { + if _, err := c.DescribeInstanceTypes(t.Context(), &awsec2.DescribeInstanceTypesInput{InstanceTypes: []types.InstanceType{"x9.huge"}}); sdkerr.AWSCode(err) != "InvalidInstanceType" { t.Fatalf("an unknown type = %v", err) } var all []types.InstanceType @@ -878,7 +879,7 @@ func TestPlacementGroups(t *testing.T) { return err }, "InvalidPlacementGroup.InUse"}, } { - if err := tc.call(); code(err) != tc.code { + if err := tc.call(); sdkerr.AWSCode(err) != tc.code { t.Errorf("%s = %v, want %s", tc.name, err, tc.code) } } @@ -924,7 +925,7 @@ func TestVolumes(t *testing.T) { if err != nil || aws.ToString(again.VolumeId) != vol { t.Fatalf("repeating the token = %+v, %v", again, err) } - if _, err := c.CreateVolume(t.Context(), &awsec2.CreateVolumeInput{AvailabilityZone: aws.String(region + "a"), Size: aws.Int32(5), ClientToken: aws.String("vol-1")}); code(err) != "IdempotentParameterMismatch" { + if _, err := c.CreateVolume(t.Context(), &awsec2.CreateVolumeInput{AvailabilityZone: aws.String(region + "a"), Size: aws.Int32(5), ClientToken: aws.String("vol-1")}); sdkerr.AWSCode(err) != "IdempotentParameterMismatch" { t.Fatalf("the token with other parameters = %v", err) } waitFast := func(o *awsec2.VolumeAvailableWaiterOptions) { @@ -934,7 +935,7 @@ func TestVolumes(t *testing.T) { t.Fatalf("waiting for the volume: %v", err) } id := ids(run(t, c, runInput(putImage(t, s, ec2.Image{}))))[0] - if _, err := c.AttachVolume(t.Context(), &awsec2.AttachVolumeInput{Device: aws.String("/dev/sdf"), InstanceId: aws.String(id), VolumeId: aws.String(vol)}); code(err) != "IncorrectState" { + if _, err := c.AttachVolume(t.Context(), &awsec2.AttachVolumeInput{Device: aws.String("/dev/sdf"), InstanceId: aws.String(id), VolumeId: aws.String(vol)}); sdkerr.AWSCode(err) != "IncorrectState" { t.Fatalf("attaching to a pending instance = %v", err) } s.Settle() @@ -1000,7 +1001,7 @@ func TestVolumes(t *testing.T) { return err }, "MissingParameter"}, } { - if err := tc.call(); code(err) != tc.code { + if err := tc.call(); sdkerr.AWSCode(err) != tc.code { t.Errorf("%s = %v, want %s", tc.name, err, tc.code) } } @@ -1094,7 +1095,7 @@ func TestCreateAndDeleteTags(t *testing.T) { {"snapshot-1", "InvalidID"}, } { _, err := c.CreateTags(t.Context(), &awsec2.CreateTagsInput{Resources: []string{id, tc.resource}, Tags: []types.Tag{{Key: aws.String("x"), Value: aws.String("y")}}}) - if code(err) != tc.code { + if sdkerr.AWSCode(err) != tc.code { t.Errorf("tagging %s = %v, want %s", tc.resource, err, tc.code) } } @@ -1114,7 +1115,7 @@ func TestInstanceFaults(t *testing.T) { fault.Rule{Method: "ec2.TerminateInstances", Resource: region + "/" + target, Fault: fault.Unavailable()}, fault.Rule{Method: "ec2.DescribeInstances", Nth: 1, Fault: fault.Unavailable()}, ) - if _, err := describeInstances(t, c, &awsec2.DescribeInstancesInput{}); code(err) != "Unavailable" { + if _, err := describeInstances(t, c, &awsec2.DescribeInstancesInput{}); sdkerr.AWSCode(err) != "Unavailable" { t.Fatalf("the first describe = %v, want Unavailable", err) } if got, _ := s.LookupInstance(region, keep); got.State != "pending" { @@ -1123,7 +1124,7 @@ func TestInstanceFaults(t *testing.T) { if _, err := c.TerminateInstances(t.Context(), &awsec2.TerminateInstancesInput{InstanceIds: []string{keep}}); err != nil { t.Fatalf("terminating another instance: %v", err) } - if _, err := c.TerminateInstances(t.Context(), &awsec2.TerminateInstancesInput{InstanceIds: []string{target}}); code(err) != "Unavailable" { + if _, err := c.TerminateInstances(t.Context(), &awsec2.TerminateInstancesInput{InstanceIds: []string{target}}); sdkerr.AWSCode(err) != "Unavailable" { t.Fatalf("terminating the targeted instance = %v, want Unavailable", err) } } @@ -1146,7 +1147,7 @@ func TestVolumeIopsAndThroughputAreCheckedNotAdjusted(t *testing.T) { t.Run(tc.name, func(t *testing.T) { in := tc.in in.AvailabilityZone, in.Size = zone, aws.Int32(100) - if _, err := c.CreateVolume(t.Context(), &in); code(err) != tc.want { + if _, err := c.CreateVolume(t.Context(), &in); sdkerr.AWSCode(err) != tc.want { t.Fatalf("CreateVolume = %v, want %s", err, tc.want) } }) @@ -1159,7 +1160,7 @@ func TestVolumeIopsAndThroughputAreCheckedNotAdjusted(t *testing.T) { image := putImage(t, s, ec2.Image{}) in := runInput(image) in.BlockDeviceMappings = []types.BlockDeviceMapping{{DeviceName: aws.String("/dev/xvda"), Ebs: &types.EbsBlockDevice{VolumeType: types.VolumeTypeGp2, Iops: aws.Int32(500)}}} - if _, err := c.RunInstances(t.Context(), in); code(err) != "InvalidParameterCombination" { + if _, err := c.RunInstances(t.Context(), in); sdkerr.AWSCode(err) != "InvalidParameterCombination" { t.Fatalf("IOPS on a gp2 root = %v", err) } in.BlockDeviceMappings = []types.BlockDeviceMapping{{DeviceName: aws.String("/dev/xvda"), Ebs: &types.EbsBlockDevice{VolumeType: types.VolumeTypeGp2, VolumeSize: aws.Int32(400)}}} diff --git a/sdktest/aws/ec2/launchoptions_test.go b/sdktest/aws/ec2/launchoptions_test.go index b48351a..8a79efa 100644 --- a/sdktest/aws/ec2/launchoptions_test.go +++ b/sdktest/aws/ec2/launchoptions_test.go @@ -9,6 +9,7 @@ import ( "github.com/aws/aws-sdk-go-v2/service/ec2/types" "github.com/parallelworks/squall/aws/ec2" + "github.com/parallelworks/squall/sdktest/internal/sdkerr" ) func withLaunchOptions(in *awsec2.RunInstancesInput) { @@ -132,7 +133,7 @@ func TestCPUOptionsFollowTheInstanceType(t *testing.T) { t.Fatalf("m7a.large CPU options = %+v, want 2 cores, 1 thread", o) } } - if _, err := launch(&types.CpuOptionsRequest{CoreCount: aws.Int32(1), ThreadsPerCore: aws.Int32(2)}); code(err) != "InvalidParameterValue" { + if _, err := launch(&types.CpuOptionsRequest{CoreCount: aws.Int32(1), ThreadsPerCore: aws.Int32(2)}); sdkerr.AWSCode(err) != "InvalidParameterValue" { t.Fatalf("2 threads per core on m7a.large = %v, want InvalidParameterValue", err) } } @@ -190,7 +191,7 @@ func TestRunInstancesLaunchOptionErrors(t *testing.T) { in := runInput(image) withLaunchOptions(in) tc.change(in) - if _, err := c.RunInstances(t.Context(), in); code(err) != tc.code { + if _, err := c.RunInstances(t.Context(), in); sdkerr.AWSCode(err) != tc.code { t.Fatalf("= %v, want %s", err, tc.code) } if got := s.Instances(); len(got) != 0 { @@ -221,7 +222,7 @@ func TestRunInstancesLaunchOptionsAreIdempotent(t *testing.T) { t.Fatalf("the same launch again = %v, want %v", ids(again), ids(first)) } tc.change(in) - if _, err := c.RunInstances(t.Context(), in); code(err) != "IdempotentParameterMismatch" { + if _, err := c.RunInstances(t.Context(), in); sdkerr.AWSCode(err) != "IdempotentParameterMismatch" { t.Fatalf("the token with other %s = %v, want IdempotentParameterMismatch", tc.name, err) } if got := len(s.Instances()); got != 1 { @@ -273,7 +274,7 @@ func TestLaunchOptionEdges(t *testing.T) { _, err := c.RunInstances(t.Context(), in) return err } - if err := attach("efa"); code(err) != "InvalidParameterCombination" { + if err := attach("efa"); sdkerr.AWSCode(err) != "InvalidParameterCombination" { t.Fatalf("efa on a plain interface = %v", err) } if err := attach("interface"); err != nil { @@ -283,7 +284,7 @@ func TestLaunchOptionEdges(t *testing.T) { in = runInput(image) in.SubnetId = nil in.NetworkInterfaces = []types.InstanceNetworkInterfaceSpecification{{DeviceIndex: aws.Int32(0), SubnetId: aws.String(subnet), InterfaceType: aws.String("efa-only")}} - if _, err := c.RunInstances(t.Context(), in); code(err) != "InvalidParameterValue" || !strings.Contains(err.Error(), "primary network interface") { + if _, err := c.RunInstances(t.Context(), in); sdkerr.AWSCode(err) != "InvalidParameterValue" || !strings.Contains(err.Error(), "primary network interface") { t.Fatalf("efa-only primary interface = %v", err) } } diff --git a/sdktest/aws/ec2/snapshots_test.go b/sdktest/aws/ec2/snapshots_test.go index 24f1961..727de08 100644 --- a/sdktest/aws/ec2/snapshots_test.go +++ b/sdktest/aws/ec2/snapshots_test.go @@ -11,6 +11,7 @@ import ( "github.com/parallelworks/squall/aws/ec2" "github.com/parallelworks/squall/fault" + "github.com/parallelworks/squall/sdktest/internal/sdkerr" ) func createVolume(t *testing.T, c *awsec2.Client, size int32) string { @@ -192,7 +193,7 @@ func TestSnapshots(t *testing.T) { return err }, "InvalidParameterValue"}, } { - if err := tc.call(); code(err) != tc.code { + if err := tc.call(); sdkerr.AWSCode(err) != tc.code { t.Errorf("%s = %v, want %s", tc.name, err, tc.code) } } @@ -200,13 +201,13 @@ func TestSnapshots(t *testing.T) { if _, err := c.DeregisterImage(t.Context(), &awsec2.DeregisterImageInput{ImageId: aws.String(image)}); err != nil { t.Fatal(err) } - if _, err := c.DeregisterImage(t.Context(), &awsec2.DeregisterImageInput{ImageId: aws.String(image)}); code(err) != "InvalidAMIID.NotFound" { + if _, err := c.DeregisterImage(t.Context(), &awsec2.DeregisterImageInput{ImageId: aws.String(image)}); sdkerr.AWSCode(err) != "InvalidAMIID.NotFound" { t.Fatalf("deregistering it again = %v, want InvalidAMIID.NotFound", err) } if _, err := c.DeleteSnapshot(t.Context(), &awsec2.DeleteSnapshotInput{SnapshotId: aws.String(snap)}); err != nil { t.Fatalf("deleting the snapshot once its image is gone: %v", err) } - if _, err := c.DeleteSnapshot(t.Context(), &awsec2.DeleteSnapshotInput{SnapshotId: aws.String(snap)}); code(err) != "InvalidSnapshot.NotFound" { + if _, err := c.DeleteSnapshot(t.Context(), &awsec2.DeleteSnapshotInput{SnapshotId: aws.String(snap)}); sdkerr.AWSCode(err) != "InvalidSnapshot.NotFound" { t.Fatalf("deleting it again = %v, want InvalidSnapshot.NotFound", err) } if got := s.Snapshots(); len(got) != 1 || got[0].SnapshotID != other { @@ -272,10 +273,10 @@ func TestSnapshotFaults(t *testing.T) { t.Fatalf("snapshots after a lost response = %v, %v; want the one taken", got, err) } snap := got[0] - if _, err := c.RegisterImage(t.Context(), registerInput("golden-1", snap)); code(err) != "InternalError" { + if _, err := c.RegisterImage(t.Context(), registerInput("golden-1", snap)); sdkerr.AWSCode(err) != "InternalError" { t.Fatalf("a register that fails after taking effect = %v, want InternalError", err) } - if _, err := c.RegisterImage(t.Context(), registerInput("golden-1", snap)); code(err) != "InvalidAMIName.Duplicate" { + if _, err := c.RegisterImage(t.Context(), registerInput("golden-1", snap)); sdkerr.AWSCode(err) != "InvalidAMIName.Duplicate" { t.Fatalf("registering again = %v, want InvalidAMIName.Duplicate", err) } if imgs := s.Images(); len(imgs) != 1 { @@ -284,7 +285,7 @@ func TestSnapshotFaults(t *testing.T) { if _, err := c.DeregisterImage(t.Context(), &awsec2.DeregisterImageInput{ImageId: aws.String(s.Images()[0].ImageID)}); err != nil { t.Fatal(err) } - if _, err := c.DeleteSnapshot(t.Context(), &awsec2.DeleteSnapshotInput{SnapshotId: aws.String(snap)}); code(err) != "Unavailable" { + if _, err := c.DeleteSnapshot(t.Context(), &awsec2.DeleteSnapshotInput{SnapshotId: aws.String(snap)}); sdkerr.AWSCode(err) != "Unavailable" { t.Fatalf("a rejected delete = %v, want Unavailable", err) } if len(s.Snapshots()) != 1 { @@ -305,7 +306,7 @@ func TestAFaultRuleCanTargetOneSnapshot(t *testing.T) { if _, err := c.DeleteSnapshot(t.Context(), &awsec2.DeleteSnapshotInput{SnapshotId: aws.String(keep)}); err != nil { t.Fatalf("deleting another snapshot: %v", err) } - if _, err := c.DeleteSnapshot(t.Context(), &awsec2.DeleteSnapshotInput{SnapshotId: aws.String(target)}); code(err) != "Unavailable" { + if _, err := c.DeleteSnapshot(t.Context(), &awsec2.DeleteSnapshotInput{SnapshotId: aws.String(target)}); sdkerr.AWSCode(err) != "Unavailable" { t.Fatalf("deleting the targeted snapshot = %v, want Unavailable", err) } } diff --git a/sdktest/aws/s3/example_test.go b/sdktest/aws/s3/example_test.go new file mode 100644 index 0000000..6125d50 --- /dev/null +++ b/sdktest/aws/s3/example_test.go @@ -0,0 +1,52 @@ +package s3_test + +import ( + "context" + "fmt" + "net/http/httptest" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/credentials" + awss3 "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + + "github.com/parallelworks/squall/aws/s3" + "github.com/parallelworks/squall/fault" +) + +// The first create takes effect but its response is lost. With retries off +// the caller sees an error, and creating it again outside us-east-1 fails +// with BucketAlreadyOwnedByYou, so code under test must recognize its own +// bucket, by its tags, rather than give up on a taken name. +func Example() { + ctx := context.Background() + faults := fault.New().Add(fault.Rule{Method: "s3.CreateBucket", Nth: 1, Fault: fault.LoseResponse()}) + s := s3.New(s3.WithFaults(faults)) + srv := httptest.NewServer(s) + defer srv.Close() + + client := awss3.New(awss3.Options{ + Region: "us-west-2", + BaseEndpoint: aws.String(srv.URL), + Credentials: credentials.NewStaticCredentialsProvider("AKID", "SECRET", ""), + RetryMaxAttempts: 1, + }) + in := &awss3.CreateBucketInput{ + Bucket: aws.String("my-bucket"), + CreateBucketConfiguration: &types.CreateBucketConfiguration{LocationConstraint: "us-west-2"}, + } + _, err := client.CreateBucket(ctx, in) + fmt.Println("create failed:", err != nil) + _, err = client.CreateBucket(ctx, in) + fmt.Println("create again:", err != nil) + + for _, b := range s.Buckets() { + fmt.Println("bucket exists:", b.Name, b.Region) + } + fmt.Println(faults.Injected()[0]) + // Output: + // create failed: true + // create again: true + // bucket exists: my-bucket us-west-2 + // #1 s3.CreateBucket my-bucket: lose response +} diff --git a/sdktest/aws/s3/s3_test.go b/sdktest/aws/s3/s3_test.go new file mode 100644 index 0000000..9b300b3 --- /dev/null +++ b/sdktest/aws/s3/s3_test.go @@ -0,0 +1,675 @@ +package s3_test + +import ( + "bytes" + "errors" + "io" + "net/http" + "net/http/httptest" + "slices" + "strings" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http" + "github.com/aws/aws-sdk-go-v2/credentials" + awss3 "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + + "github.com/parallelworks/squall/aws/s3" + "github.com/parallelworks/squall/fault" + "github.com/parallelworks/squall/sdktest/internal/sdkerr" + "github.com/parallelworks/squall/sdktest/internal/sdkhttp" +) + +func serve(t *testing.T, s *s3.Server) string { + t.Helper() + srv := httptest.NewServer(s) + t.Cleanup(srv.Close) + return srv.URL +} + +// clientAt returns a client for url in region that does not retry, so each call is one request. +func clientAt(url, region string, opts ...func(*awss3.Options)) *awss3.Client { + return awss3.New(awss3.Options{ + Region: region, + BaseEndpoint: aws.String(url), + Credentials: credentials.NewStaticCredentialsProvider("AKID", "SECRET", ""), + HTTPClient: sdkhttp.CopyBodies(awshttp.NewBuildableClient()), + RetryMaxAttempts: 1, + // config.LoadDefaultConfig sets this, and s3.New alone does not. + RequestChecksumCalculation: aws.RequestChecksumCalculationWhenSupported, + }, opts...) +} + +func newClient(t *testing.T, s *s3.Server, region string) *awss3.Client { + t.Helper() + return clientAt(serve(t, s), region) +} + +func create(t *testing.T, c *awss3.Client, name string) { + t.Helper() + in := &awss3.CreateBucketInput{Bucket: aws.String(name), ACL: types.BucketCannedACLPrivate} + if region := c.Options().Region; region != "us-east-1" { + in.CreateBucketConfiguration = &types.CreateBucketConfiguration{LocationConstraint: types.BucketLocationConstraint(region)} + } + if _, err := c.CreateBucket(t.Context(), in); err != nil { + t.Fatalf("create %s: %v", name, err) + } +} + +func tag(t *testing.T, c *awss3.Client, name string, kv ...string) error { + t.Helper() + set := []types.Tag{} + for i := 0; i < len(kv); i += 2 { + set = append(set, types.Tag{Key: aws.String(kv[i]), Value: aws.String(kv[i+1])}) + } + _, err := c.PutBucketTagging(t.Context(), &awss3.PutBucketTaggingInput{Bucket: aws.String(name), Tagging: &types.Tagging{TagSet: set}}) + return err +} + +func TestCreateTagAndRead(t *testing.T) { + s := s3.New() + url := serve(t, s) + east, west := clientAt(url, "us-east-1"), clientAt(url, "us-west-2") + create(t, east, "east-bucket") + create(t, west, "west-bucket") + if err := tag(t, west, "west-bucket", "Owner", "abc", "Name", "data"); err != nil { + t.Fatalf("tag: %v", err) + } + + head, err := west.HeadBucket(t.Context(), &awss3.HeadBucketInput{Bucket: aws.String("west-bucket")}) + if err != nil || aws.ToString(head.BucketRegion) != "us-west-2" { + t.Fatalf("head = %+v, %v", head, err) + } + tags, err := west.GetBucketTagging(t.Context(), &awss3.GetBucketTaggingInput{Bucket: aws.String("west-bucket")}) + if err != nil { + t.Fatalf("get tags: %v", err) + } + got := map[string]string{} + for _, tg := range tags.TagSet { + got[aws.ToString(tg.Key)] = aws.ToString(tg.Value) + } + if len(got) != 2 || got["Owner"] != "abc" || got["Name"] != "data" { + t.Fatalf("tags = %v", got) + } + if _, err := east.GetBucketTagging(t.Context(), &awss3.GetBucketTaggingInput{Bucket: aws.String("east-bucket")}); sdkerr.AWSCode(err) != "NoSuchTagSet" || sdkerr.AWSStatus(err) != http.StatusNotFound { + t.Fatalf("untagged get = %v", err) + } + if err := tag(t, west, "west-bucket"); err != nil { + t.Fatalf("empty tag set: %v", err) + } + if _, err := west.GetBucketTagging(t.Context(), &awss3.GetBucketTaggingInput{Bucket: aws.String("west-bucket")}); sdkerr.AWSCode(err) != "NoSuchTagSet" { + t.Fatalf("get after an empty tag set = %v", err) + } + if b, ok := s.LookupBucket("west-bucket"); !ok || b.Region != "us-west-2" || b.Tags != nil || b.CreationDate.IsZero() { + t.Fatalf("lookup = %+v, %v", b, ok) + } +} + +func TestCreateBucketErrors(t *testing.T) { + s := s3.New() + if err := s.PutForeignBucket("taken-name", "us-east-1"); err != nil { + t.Fatal(err) + } + url := serve(t, s) + east, west := clientAt(url, "us-east-1"), clientAt(url, "us-west-2") + create(t, east, "mine-east") + create(t, west, "mine-west") + for _, tc := range []struct { + name string + client *awss3.Client + in *awss3.CreateBucketInput + status int + code string + }{ + {"another account's name", east, &awss3.CreateBucketInput{Bucket: aws.String("taken-name")}, http.StatusConflict, "BucketAlreadyExists"}, + {"an owned name outside us-east-1", west, &awss3.CreateBucketInput{ + Bucket: aws.String("mine-west"), + CreateBucketConfiguration: &types.CreateBucketConfiguration{LocationConstraint: "us-west-2"}, + }, http.StatusConflict, "BucketAlreadyOwnedByYou"}, + {"an invalid name", east, &awss3.CreateBucketInput{Bucket: aws.String("Bad_Name")}, http.StatusBadRequest, "InvalidBucketName"}, + {"no constraint outside us-east-1", west, &awss3.CreateBucketInput{Bucket: aws.String("new-west")}, http.StatusBadRequest, "IllegalLocationConstraintException"}, + {"a constraint for another region", west, &awss3.CreateBucketInput{ + Bucket: aws.String("new-west"), + CreateBucketConfiguration: &types.CreateBucketConfiguration{LocationConstraint: "eu-west-1"}, + }, http.StatusBadRequest, "IllegalLocationConstraintException"}, + {"us-east-1 as a constraint", east, &awss3.CreateBucketInput{ + Bucket: aws.String("new-east"), + CreateBucketConfiguration: &types.CreateBucketConfiguration{LocationConstraint: "us-east-1"}, + }, http.StatusBadRequest, "InvalidLocationConstraint"}, + {"a public ACL", east, &awss3.CreateBucketInput{Bucket: aws.String("new-east"), ACL: types.BucketCannedACLPublicRead}, http.StatusBadRequest, "InvalidBucketAclWithObjectOwnership"}, + } { + t.Run(tc.name, func(t *testing.T) { + _, err := tc.client.CreateBucket(t.Context(), tc.in) + if sdkerr.AWSStatus(err) != tc.status || sdkerr.AWSCode(err) != tc.code { + t.Fatalf("create = %v, want %d %s", err, tc.status, tc.code) + } + }) + } + if _, err := east.CreateBucket(t.Context(), &awss3.CreateBucketInput{Bucket: aws.String("mine-east")}); err != nil { + t.Fatalf("creating an owned us-east-1 bucket again: %v", err) + } + if got := len(s.Buckets()); got != 2 { + t.Fatalf("buckets = %d", got) + } +} + +func TestBucketsElsewhereAndOtherAccounts(t *testing.T) { + s := s3.New() + if err := s.PutForeignBucket("theirs", "us-west-2"); err != nil { + t.Fatal(err) + } + url := serve(t, s) + east, west := clientAt(url, "us-east-1"), clientAt(url, "us-west-2") + create(t, east, "mine") + + for _, tc := range []struct { + name, bucket string + client *awss3.Client + head string + other string + }{ + {"a missing bucket", "nobody", east, "NotFound", "NoSuchBucket"}, + {"another account's bucket", "theirs", west, "Forbidden", "AccessDenied"}, + {"another account's bucket elsewhere", "theirs", east, "MovedPermanently", "PermanentRedirect"}, + {"the account's bucket elsewhere", "mine", west, "MovedPermanently", "PermanentRedirect"}, + } { + t.Run(tc.name, func(t *testing.T) { + _, err := tc.client.HeadBucket(t.Context(), &awss3.HeadBucketInput{Bucket: aws.String(tc.bucket)}) + if sdkerr.AWSCode(err) != tc.head { + t.Fatalf("head = %v, want %s", err, tc.head) + } + bucket := aws.String(tc.bucket) + for name, call := range map[string]func() error{ + "tagging": func() error { + _, err := tc.client.GetBucketTagging(t.Context(), &awss3.GetBucketTaggingInput{Bucket: bucket}) + return err + }, + "versions": func() error { + _, err := tc.client.ListObjectVersions(t.Context(), &awss3.ListObjectVersionsInput{Bucket: bucket}) + return err + }, + "delete": func() error { + _, err := tc.client.DeleteBucket(t.Context(), &awss3.DeleteBucketInput{Bucket: bucket}) + return err + }, + } { + if err := call(); sdkerr.AWSCode(err) != tc.other { + t.Errorf("%s = %v, want %s", name, err, tc.other) + } + } + }) + } + _, err := west.HeadBucket(t.Context(), &awss3.HeadBucketInput{Bucket: aws.String("theirs")}) + if re, ok := errors.AsType[*awshttp.ResponseError](err); !ok || re.HTTPStatusCode() != http.StatusForbidden || + re.Response.Header.Get("x-amz-bucket-region") != "us-west-2" { + t.Fatalf("denied head = %v", err) + } + _, err = west.GetBucketTagging(t.Context(), &awss3.GetBucketTaggingInput{Bucket: aws.String("mine")}) + if re, ok := errors.AsType[*awshttp.ResponseError](err); !ok || re.HTTPStatusCode() != http.StatusMovedPermanently || + re.Response.Header.Get("x-amz-bucket-region") != "us-east-1" { + t.Fatalf("redirect = %v", err) + } + if _, ok := s.LookupBucket("mine"); !ok { + t.Fatal("a redirected delete removed the bucket") + } +} + +func TestListBuckets(t *testing.T) { + s := s3.New() + url := serve(t, s) + east, west := clientAt(url, "us-east-1"), clientAt(url, "us-west-2") + for _, name := range []string{"app-a", "app-b", "logs"} { + create(t, east, name) + } + create(t, west, "app-west") + if err := s.PutForeignBucket("app-foreign", "us-east-1"); err != nil { + t.Fatal(err) + } + + names := func(in *awss3.ListBucketsInput) ([]string, []string, int) { + t.Helper() + var names, regions []string + pages := 0 + for p := awss3.NewListBucketsPaginator(west, in); p.HasMorePages(); { + out, err := p.NextPage(t.Context()) + if err != nil { + t.Fatalf("list: %v", err) + } + pages++ + for _, b := range out.Buckets { + names = append(names, aws.ToString(b.Name)) + regions = append(regions, aws.ToString(b.BucketRegion)) + if b.CreationDate == nil { + t.Fatalf("bucket %s has no creation date", aws.ToString(b.Name)) + } + } + } + return names, regions, pages + } + if got, regions, pages := names(&awss3.ListBucketsInput{}); !slices.Equal(got, []string{"app-a", "app-b", "app-west", "logs"}) || pages != 1 || regions[0] != "" { + t.Fatalf("all = %v %v in %d pages", got, regions, pages) + } + got, regions, pages := names(&awss3.ListBucketsInput{MaxBuckets: aws.Int32(1)}) + if !slices.Equal(got, []string{"app-a", "app-b", "app-west", "logs"}) || pages != 4 || + !slices.Equal(regions, []string{"us-east-1", "us-east-1", "us-west-2", "us-east-1"}) { + t.Fatalf("paged = %v %v in %d pages", got, regions, pages) + } + if got, regions, _ := names(&awss3.ListBucketsInput{Prefix: aws.String("app-")}); !slices.Equal(got, []string{"app-a", "app-b", "app-west"}) || + !slices.Equal(regions, []string{"us-east-1", "us-east-1", "us-west-2"}) { + t.Fatalf("prefixed = %v %v", got, regions) + } + if got, _, _ := names(&awss3.ListBucketsInput{Prefix: aws.String("app-"), BucketRegion: aws.String("us-west-2")}); !slices.Equal(got, []string{"app-west"}) { + t.Fatalf("filtered = %v", got) + } + if _, err := west.ListBuckets(t.Context(), &awss3.ListBucketsInput{BucketRegion: aws.String("us-east-1")}); sdkerr.AWSCode(err) != "InvalidArgument" { + t.Fatalf("another region's buckets = %v", err) + } + if out, err := east.ListBuckets(t.Context(), &awss3.ListBucketsInput{BucketRegion: aws.String("us-east-1")}); err != nil || len(out.Buckets) != 3 { + t.Fatalf("us-east-1 buckets = %+v, %v", out, err) + } + if _, err := west.ListBuckets(t.Context(), &awss3.ListBucketsInput{MaxBuckets: aws.Int32(0)}); sdkerr.AWSCode(err) != "InvalidArgument" { + t.Fatalf("max-buckets 0 = %v", err) + } +} + +func TestTagRules(t *testing.T) { + s := s3.New() + c := newClient(t, s, "us-east-1") + create(t, c, "tagged") + many := make([]string, 0, 102) + for i := range 51 { + many = append(many, strings.Repeat("k", i+1), "v") + } + for name, kv := range map[string][]string{ + "an empty key": {"", "v"}, + "a long key": {strings.Repeat("k", 129), "v"}, + "a long value": {"k", strings.Repeat("v", 257)}, + "a duplicate key": {"k", "1", "k", "2"}, + "an aws: prefix": {"aws:owner", "v"}, + "a disallowed rune": {"k", "a,b"}, + "too many tags": many, + } { + if err := tag(t, c, "tagged", kv...); sdkerr.AWSCode(err) != "InvalidTag" { + t.Errorf("%s = %v, want InvalidTag", name, err) + } + } + if err := tag(t, c, "tagged", "Owner Name", "a+b=c.d_e:f/g@h", "empty", ""); err != nil { + t.Fatalf("valid tags: %v", err) + } + if b, _ := s.LookupBucket("tagged"); len(b.Tags) != 2 || b.Tags["empty"] != "" { + t.Fatalf("tags = %v", b.Tags) + } + if _, err := c.DeleteBucketTagging(t.Context(), &awss3.DeleteBucketTaggingInput{Bucket: aws.String("tagged")}); err != nil { + t.Fatal(err) + } + if b, _ := s.LookupBucket("tagged"); b.Tags != nil { + t.Fatalf("tags after delete = %v", b.Tags) + } +} + +// empty removes every version and delete marker, deleting each page before asking for the next. +func empty(t *testing.T, c *awss3.Client, bucket string, pageSize int32) int { + t.Helper() + removed := 0 + p := awss3.NewListObjectVersionsPaginator(c, &awss3.ListObjectVersionsInput{Bucket: aws.String(bucket), MaxKeys: aws.Int32(pageSize)}) + for p.HasMorePages() { + page, err := p.NextPage(t.Context()) + if err != nil { + t.Fatalf("list versions: %v", err) + } + var ids []types.ObjectIdentifier + for _, v := range page.Versions { + ids = append(ids, types.ObjectIdentifier{Key: v.Key, VersionId: v.VersionId}) + } + for _, m := range page.DeleteMarkers { + ids = append(ids, types.ObjectIdentifier{Key: m.Key, VersionId: m.VersionId}) + } + if len(ids) == 0 { + continue + } + out, err := c.DeleteObjects(t.Context(), &awss3.DeleteObjectsInput{Bucket: aws.String(bucket), Delete: &types.Delete{Objects: ids}}) + if err != nil || len(out.Errors) > 0 { + t.Fatalf("delete objects = %+v, %v", out, err) + } + removed += len(out.Deleted) + } + return removed +} + +func put(t *testing.T, c *awss3.Client, bucket, key string, body io.Reader) *awss3.PutObjectOutput { + t.Helper() + out, err := c.PutObject(t.Context(), &awss3.PutObjectInput{Bucket: aws.String(bucket), Key: aws.String(key), Body: body}) + if err != nil { + t.Fatalf("put %s: %v", key, err) + } + return out +} + +func TestEmptyAVersionedBucketThenDeleteIt(t *testing.T) { + s := s3.New() + c := newClient(t, s, "us-east-1") + create(t, c, "versioned") + if _, err := c.PutBucketVersioning(t.Context(), &awss3.PutBucketVersioningInput{ + Bucket: aws.String("versioned"), + VersioningConfiguration: &types.VersioningConfiguration{Status: types.BucketVersioningStatusEnabled}, + }); err != nil { + t.Fatal(err) + } + first := put(t, c, "versioned", "dir/a.txt", strings.NewReader("one")) + put(t, c, "versioned", "dir/a.txt", bytes.NewReader([]byte("two"))) + put(t, c, "versioned", "b.txt", strings.NewReader("a third body")) + if aws.ToString(first.ETag) != `"f97c5d29941bfb1b2fdab0874906ab82"` || len(aws.ToString(first.VersionId)) != 32 { + t.Fatalf("put = etag %s version %s", aws.ToString(first.ETag), aws.ToString(first.VersionId)) + } + del, err := c.DeleteObject(t.Context(), &awss3.DeleteObjectInput{Bucket: aws.String("versioned"), Key: aws.String("b.txt")}) + if err != nil || !aws.ToBool(del.DeleteMarker) || aws.ToString(del.VersionId) == "" { + t.Fatalf("delete = %+v, %v", del, err) + } + + versions := s.ObjectVersions("versioned") + if len(versions) != 4 || versions[0].Key != "b.txt" || !versions[0].DeleteMarker || !versions[0].IsLatest || + versions[1].Size != int64(len("a third body")) || versions[1].IsLatest || versions[3].VersionID != aws.ToString(first.VersionId) { + t.Fatalf("versions = %+v", versions) + } + if _, err := c.DeleteBucket(t.Context(), &awss3.DeleteBucketInput{Bucket: aws.String("versioned")}); sdkerr.AWSCode(err) != "BucketNotEmpty" || sdkerr.AWSStatus(err) != http.StatusConflict { + t.Fatalf("delete a bucket with versions = %v", err) + } + if n := empty(t, c, "versioned", 1); n != 4 { + t.Fatalf("emptied %d versions, want 4", n) + } + if _, err := c.DeleteBucket(t.Context(), &awss3.DeleteBucketInput{Bucket: aws.String("versioned")}); err != nil { + t.Fatalf("delete the empty bucket: %v", err) + } + if _, err := c.DeleteBucket(t.Context(), &awss3.DeleteBucketInput{Bucket: aws.String("versioned")}); sdkerr.AWSCode(err) != "NoSuchBucket" || sdkerr.AWSStatus(err) != http.StatusNotFound { + t.Fatalf("delete again = %v", err) + } +} + +func TestUnversionedBucketsHoldNullVersions(t *testing.T) { + s := s3.New() + c := newClient(t, s, "us-east-1") + create(t, c, "plain") + for range 3 { + put(t, c, "plain", "k", strings.NewReader("data")) + } + put(t, c, "plain", "other", strings.NewReader("")) + out, err := c.ListObjectVersions(t.Context(), &awss3.ListObjectVersionsInput{Bucket: aws.String("plain")}) + if err != nil || len(out.Versions) != 2 || aws.ToString(out.Versions[0].VersionId) != "null" || !aws.ToBool(out.Versions[0].IsLatest) || + aws.ToString(out.Versions[0].Key) != "k" || aws.ToInt64(out.Versions[0].Size) != 4 || len(out.DeleteMarkers) != 0 { + t.Fatalf("versions = %+v, %v", out, err) + } + if _, err := c.DeleteObject(t.Context(), &awss3.DeleteObjectInput{Bucket: aws.String("plain"), Key: aws.String("k")}); err != nil { + t.Fatal(err) + } + if got := s.ObjectVersions("plain"); len(got) != 1 || got[0].Key != "other" { + t.Fatalf("after delete = %+v", got) + } + if n := empty(t, c, "plain", 1000); n != 1 { + t.Fatalf("emptied %d", n) + } + v, err := c.GetBucketVersioning(t.Context(), &awss3.GetBucketVersioningInput{Bucket: aws.String("plain")}) + if err != nil || v.Status != "" { + t.Fatalf("versioning = %+v, %v", v, err) + } +} + +func TestSuspendedVersioningReplacesTheNullVersion(t *testing.T) { + s := s3.New() + if _, err := s.PutBucket(s3.Bucket{Name: "suspended", Region: "us-east-1", Versioning: "Enabled"}); err != nil { + t.Fatal(err) + } + c := newClient(t, s, "us-east-1") + put(t, c, "suspended", "k", strings.NewReader("versioned")) + if _, err := c.PutBucketVersioning(t.Context(), &awss3.PutBucketVersioningInput{ + Bucket: aws.String("suspended"), + VersioningConfiguration: &types.VersioningConfiguration{Status: types.BucketVersioningStatusSuspended}, + }); err != nil { + t.Fatal(err) + } + put(t, c, "suspended", "k", strings.NewReader("null 1")) + put(t, c, "suspended", "k", strings.NewReader("null 2")) + if _, err := c.DeleteObject(t.Context(), &awss3.DeleteObjectInput{Bucket: aws.String("suspended"), Key: aws.String("k")}); err != nil { + t.Fatal(err) + } + got := s.ObjectVersions("suspended") + if len(got) != 2 || !got[0].DeleteMarker || got[0].VersionID != "null" || got[1].VersionID == "null" || got[1].DeleteMarker { + t.Fatalf("versions = %+v", got) + } + v, err := c.GetBucketVersioning(t.Context(), &awss3.GetBucketVersioningInput{Bucket: aws.String("suspended")}) + if err != nil || v.Status != types.BucketVersioningStatusSuspended { + t.Fatalf("versioning = %+v, %v", v, err) + } +} + +func TestAPageAfterADeletedNullVersion(t *testing.T) { + s := s3.New() + if _, err := s.PutBucket(s3.Bucket{Name: "mixed", Region: "us-east-1", Versioning: "Enabled"}); err != nil { + t.Fatal(err) + } + for range 2 { + if _, err := s.PutObject("mixed", "k", []byte("versioned")); err != nil { + t.Fatal(err) + } + } + if _, err := s.PutBucket(s3.Bucket{Name: "mixed", Region: "us-east-1", Versioning: "Suspended"}); err != nil { + t.Fatal(err) + } + if _, err := s.PutObject("mixed", "k", []byte("null")); err != nil { + t.Fatal(err) + } + c := newClient(t, s, "us-east-1") + if n := empty(t, c, "mixed", 1); n != 3 { + t.Fatalf("emptied %d versions page by page, want 3", n) + } +} + +func TestListObjectVersionsPagesAndRejects(t *testing.T) { + s := s3.New() + if _, err := s.PutBucket(s3.Bucket{Name: "pages", Region: "us-east-1", Versioning: "Enabled"}); err != nil { + t.Fatal(err) + } + for _, key := range []string{"a", "a", "a", "b", "c/1", "c/2"} { + if _, err := s.PutObject("pages", key, []byte(key)); err != nil { + t.Fatal(err) + } + } + c := newClient(t, s, "us-east-1") + var keys []string + for p := awss3.NewListObjectVersionsPaginator(c, &awss3.ListObjectVersionsInput{Bucket: aws.String("pages"), MaxKeys: aws.Int32(2)}); p.HasMorePages(); { + page, err := p.NextPage(t.Context()) + if err != nil { + t.Fatal(err) + } + for _, v := range page.Versions { + keys = append(keys, aws.ToString(v.Key)) + } + } + if !slices.Equal(keys, []string{"a", "a", "a", "b", "c/1", "c/2"}) { + t.Fatalf("keys = %v", keys) + } + out, err := c.ListObjectVersions(t.Context(), &awss3.ListObjectVersionsInput{Bucket: aws.String("pages"), Prefix: aws.String("c/")}) + if err != nil || len(out.Versions) != 2 { + t.Fatalf("prefix = %+v, %v", out, err) + } + if _, err := c.ListObjectVersions(t.Context(), &awss3.ListObjectVersionsInput{Bucket: aws.String("pages"), Delimiter: aws.String("/")}); sdkerr.AWSCode(err) != "InvalidArgument" { + t.Fatalf("delimiter = %v", err) + } + if _, err := c.ListObjectVersions(t.Context(), &awss3.ListObjectVersionsInput{Bucket: aws.String("pages"), VersionIdMarker: aws.String("null")}); sdkerr.AWSCode(err) != "InvalidArgument" { + t.Fatalf("version marker without key marker = %v", err) + } + if _, err := c.DeleteObject(t.Context(), &awss3.DeleteObjectInput{Bucket: aws.String("pages"), Key: aws.String("a"), VersionId: aws.String("bogus")}); sdkerr.AWSCode(err) != "InvalidArgument" { + t.Fatalf("delete a malformed version = %v", err) + } + if _, err := c.PutObject(t.Context(), &awss3.PutObjectInput{Bucket: aws.String("pages"), Key: aws.String(strings.Repeat("k", 1025)), Body: strings.NewReader("x")}); sdkerr.AWSCode(err) != "KeyTooLongError" { + t.Fatalf("a long key = %v", err) + } +} + +func TestPolicyCORSAndUnimplemented(t *testing.T) { + s := s3.New() + c := newClient(t, s, "us-east-1") + create(t, c, "web") + policy := `{"Version":"2012-10-17","Statement":[]}` + if _, err := c.PutBucketPolicy(t.Context(), &awss3.PutBucketPolicyInput{Bucket: aws.String("web"), Policy: aws.String(policy)}); err != nil { + t.Fatal(err) + } + if _, err := c.PutBucketPolicy(t.Context(), &awss3.PutBucketPolicyInput{Bucket: aws.String("web"), Policy: aws.String("deny all")}); sdkerr.AWSCode(err) != "MalformedPolicy" { + t.Fatalf("bad policy = %v", err) + } + rule := types.CORSRule{AllowedMethods: []string{"GET", "PUT"}, AllowedOrigins: []string{"*"}, MaxAgeSeconds: aws.Int32(3600)} + if _, err := c.PutBucketCors(t.Context(), &awss3.PutBucketCorsInput{Bucket: aws.String("web"), CORSConfiguration: &types.CORSConfiguration{CORSRules: []types.CORSRule{rule}}}); err != nil { + t.Fatal(err) + } + rule.AllowedMethods = []string{"PATCH"} + if _, err := c.PutBucketCors(t.Context(), &awss3.PutBucketCorsInput{Bucket: aws.String("web"), CORSConfiguration: &types.CORSConfiguration{CORSRules: []types.CORSRule{rule}}}); sdkerr.AWSCode(err) != "InvalidRequest" { + t.Fatalf("bad CORS = %v", err) + } + if b, _ := s.LookupBucket("web"); b.Policy != policy || !strings.Contains(b.CORS, "PUT") { + t.Fatalf("bucket = %+v", b) + } + if _, err := c.GetObject(t.Context(), &awss3.GetObjectInput{Bucket: aws.String("web"), Key: aws.String("k")}); sdkerr.AWSCode(err) != "NotImplemented" || sdkerr.AWSStatus(err) != http.StatusNotFound { + t.Fatalf("get object = %v", err) + } + if _, err := c.GetBucketLocation(t.Context(), &awss3.GetBucketLocationInput{Bucket: aws.String("web")}); sdkerr.AWSCode(err) != "NotImplemented" { + t.Fatalf("get location = %v", err) + } +} + +func TestFaults(t *testing.T) { + faults := fault.New().Add( + fault.Rule{Method: "s3.CreateBucket", Nth: 1, Fault: fault.LoseResponse()}, + fault.Rule{Method: "s3.HeadBucket", Nth: 1, Fault: fault.LoseResponse()}, + fault.Rule{Method: "s3.GetBucketTagging", Resource: "locked", Fault: fault.Reject(http.StatusForbidden, "AccessDenied")}, + fault.Rule{Method: "s3.DeleteObjects", Nth: 1, Fault: fault.FailAfter(http.StatusInternalServerError)}, + fault.Rule{Method: "s3.ListBuckets", Nth: 1, Fault: fault.RateLimited()}, + ) + s := s3.New(s3.WithFaults(faults)) + c := newClient(t, s, "eu-west-1") + in := &awss3.CreateBucketInput{Bucket: aws.String("lost"), CreateBucketConfiguration: &types.CreateBucketConfiguration{LocationConstraint: "eu-west-1"}} + if _, err := c.CreateBucket(t.Context(), in); err == nil { + t.Fatal("a lost create succeeded") + } + if _, ok := s.LookupBucket("lost"); !ok { + t.Fatal("a lost create did not take effect") + } + if _, err := c.CreateBucket(t.Context(), in); sdkerr.AWSCode(err) != "BucketAlreadyOwnedByYou" { + t.Fatalf("retried create = %v", err) + } + if _, err := c.HeadBucket(t.Context(), &awss3.HeadBucketInput{Bucket: aws.String("nothing")}); err == nil || sdkerr.AWSCode(err) == "NotFound" { + t.Fatalf("a lost head = %v, want a transport error", err) + } + + create(t, c, "locked") + if _, err := c.GetBucketTagging(t.Context(), &awss3.GetBucketTaggingInput{Bucket: aws.String("locked")}); sdkerr.AWSCode(err) != "AccessDenied" || sdkerr.AWSStatus(err) != http.StatusForbidden { + t.Fatalf("denied tagging = %v", err) + } + if _, err := s.PutObject("locked", "k", []byte("x")); err != nil { + t.Fatal(err) + } + _, err := c.DeleteObjects(t.Context(), &awss3.DeleteObjectsInput{Bucket: aws.String("locked"), Delete: &types.Delete{Objects: []types.ObjectIdentifier{{Key: aws.String("k")}}}}) + if sdkerr.AWSCode(err) != "InternalError" || len(s.ObjectVersions("locked")) != 0 { + t.Fatalf("delete objects failing after = %v, left %v", err, s.ObjectVersions("locked")) + } + if _, err := c.ListBuckets(t.Context(), &awss3.ListBucketsInput{}); sdkerr.AWSCode(err) != "SlowDown" || sdkerr.AWSStatus(err) != http.StatusServiceUnavailable { + t.Fatalf("throttled list = %v", err) + } + if got := faults.Injected(); len(got) != 5 || got[0].Call.Resource != "lost" || got[3].Call.Resource != "locked" { + t.Fatalf("injected = %v", got) + } +} + +// sweep empties and deletes every bucket, stopping at the first error. +func sweep(t *testing.T, c *awss3.Client) error { + t.Helper() + for p := awss3.NewListBucketsPaginator(c, &awss3.ListBucketsInput{}); p.HasMorePages(); { + page, err := p.NextPage(t.Context()) + if err != nil { + return err + } + for _, b := range page.Buckets { + versions, err := c.ListObjectVersions(t.Context(), &awss3.ListObjectVersionsInput{Bucket: b.Name}) + if err != nil { + return err + } + var ids []types.ObjectIdentifier + for _, v := range versions.Versions { + ids = append(ids, types.ObjectIdentifier{Key: v.Key, VersionId: v.VersionId}) + } + if len(ids) > 0 { + if _, err := c.DeleteObjects(t.Context(), &awss3.DeleteObjectsInput{Bucket: b.Name, Delete: &types.Delete{Objects: ids}}); err != nil { + return err + } + } + if _, err := c.DeleteBucket(t.Context(), &awss3.DeleteBucketInput{Bucket: b.Name}); err != nil && sdkerr.AWSCode(err) != "NoSuchBucket" { + return err + } + } + } + return nil +} + +func TestSweepsConvergeThroughFaults(t *testing.T) { + injected := 0 + for seed := range uint64(20) { + faults := fault.New(fault.Seed(seed)).Random(0.3, fault.LoseResponse(), fault.Unavailable(), fault.FailAfter(http.StatusInternalServerError)) + s := s3.New(s3.WithFaults(faults)) + c := newClient(t, s, "us-east-1") + for _, key := range []string{"a", "b", "c"} { + if _, err := s.PutBucket(s3.Bucket{Name: "bucket-" + key, Region: "us-east-1", Tags: map[string]string{"Owner": "s1"}}); err != nil { + t.Fatal(err) + } + if _, err := s.PutObject("bucket-"+key, key, []byte(key)); err != nil { + t.Fatal(err) + } + } + for range 100 { + if sweep(t, c) == nil { + break + } + } + injected += len(faults.Injected()) + if got := s.Buckets(); len(got) != 0 { + t.Fatalf("seed %d left %v after %v", seed, got, faults.Injected()) + } + } + if injected == 0 { + t.Fatal("no seed injected a fault") + } +} + +func TestChunkedUploadsOverTLS(t *testing.T) { + s := s3.New() + srv := httptest.NewTLSServer(s) + t.Cleanup(srv.Close) + tlsConfig := srv.Client().Transport.(*http.Transport).TLSClientConfig + httpClient := awshttp.NewBuildableClient().WithTransportOptions(func(tr *http.Transport) { tr.TLSClientConfig = tlsConfig }) + c := clientAt(srv.URL, "us-east-1", func(o *awss3.Options) { o.HTTPClient = httpClient }) + create(t, c, "chunked") + body := strings.Repeat("0123456789", 10_000) + var sent http.Header + _, err := c.PutObject(t.Context(), &awss3.PutObjectInput{Bucket: aws.String("chunked"), Key: aws.String("big"), Body: strings.NewReader(body)}, + func(o *awss3.Options) { + o.HTTPClient = headerRecorder{httpClient, &sent} + }) + if err != nil { + t.Fatalf("put: %v", err) + } + if !strings.Contains(sent.Get("Content-Encoding"), "aws-chunked") { + t.Fatalf("the SDK sent %v, not an aws-chunked body", sent) + } + if got := s.ObjectVersions("chunked"); len(got) != 1 || got[0].Size != int64(len(body)) { + t.Fatalf("versions = %+v", got) + } +} + +type headerRecorder struct { + next sdkhttp.Doer + sent *http.Header +} + +func (h headerRecorder) Do(req *http.Request) (*http.Response, error) { + *h.sent = req.Header.Clone() + return h.next.Do(req) +} diff --git a/sdktest/gcp/compute/addresses_test.go b/sdktest/gcp/compute/addresses_test.go index 012c199..5c0952a 100644 --- a/sdktest/gcp/compute/addresses_test.go +++ b/sdktest/gcp/compute/addresses_test.go @@ -12,12 +12,12 @@ import ( gcompute "cloud.google.com/go/compute/apiv1" "cloud.google.com/go/compute/apiv1/computepb" - "google.golang.org/api/googleapi" "google.golang.org/api/iterator" "google.golang.org/api/option" "github.com/parallelworks/squall/fault" "github.com/parallelworks/squall/gcp/compute" + "github.com/parallelworks/squall/sdktest/internal/sdkerr" ) const ( @@ -51,19 +51,6 @@ func get(t *testing.T, c *gcompute.AddressesClient, name string) (*computepb.Add return c.Get(t.Context(), &computepb.GetAddressRequest{Project: project, Region: region, Address: name}) } -func apiCode(err error) int { - if e, ok := errors.AsType[*googleapi.Error](err); ok { - return e.Code - } - if e, ok := errors.AsType[interface { - error - HTTPCode() int - }](err); ok { - return e.HTTPCode() - } - return 0 -} - func TestInsertThenGet(t *testing.T) { s := compute.New() c := newClient(t, s) @@ -107,7 +94,7 @@ func TestInsertErrors(t *testing.T) { {"a label value too long", &computepb.Address{Name: new("ip-6"), Labels: map[string]string{"owner": strings.Repeat("a", 64)}}, http.StatusBadRequest}, } { t.Run(tc.name, func(t *testing.T) { - if err := insert(t, c, tc.address); apiCode(err) != tc.code { + if err := insert(t, c, tc.address); sdkerr.GoogleCode(err) != tc.code { t.Fatalf("insert = %v, want HTTP %d", err, tc.code) } }) @@ -119,11 +106,11 @@ func TestInsertErrors(t *testing.T) { func TestGetOrDeleteAMissingAddressIsNotFound(t *testing.T) { c := newClient(t, compute.New()) - if _, err := get(t, c, "missing"); apiCode(err) != http.StatusNotFound { + if _, err := get(t, c, "missing"); sdkerr.GoogleCode(err) != http.StatusNotFound { t.Fatalf("get = %v", err) } _, err := c.Delete(t.Context(), &computepb.DeleteAddressRequest{Project: project, Region: region, Address: "missing"}) - if apiCode(err) != http.StatusNotFound { + if sdkerr.GoogleCode(err) != http.StatusNotFound { t.Fatalf("delete = %v", err) } } @@ -174,7 +161,7 @@ func TestSetLabelsChecksTheFingerprint(t *testing.T) { if after.GetLabels()["session-id"] != "x" || after.GetLabelFingerprint() == before.GetLabelFingerprint() { t.Fatalf("after setLabels = %v", after) } - if err := setLabels(before.GetLabelFingerprint()); apiCode(err) != http.StatusPreconditionFailed { + if err := setLabels(before.GetLabelFingerprint()); sdkerr.GoogleCode(err) != http.StatusPreconditionFailed { t.Fatalf("setLabels with a stale fingerprint = %v", err) } @@ -183,7 +170,7 @@ func TestSetLabelsChecksTheFingerprint(t *testing.T) { if _, err := s.PutAddress(project, region, current); err != nil { t.Fatal(err) } - if err := setLabels(after.GetLabelFingerprint()); apiCode(err) != http.StatusPreconditionFailed { + if err := setLabels(after.GetLabelFingerprint()); sdkerr.GoogleCode(err) != http.StatusPreconditionFailed { t.Fatalf("setLabels after PutAddress changed the labels = %v", err) } } @@ -233,14 +220,14 @@ func TestListFiltersAndPages(t *testing.T) { t.Errorf("list %q = %v, %v; want %s", tc.filter, names, err, tc.want) } } - if _, err := list("creationTimestamp > 0", 0); apiCode(err) != http.StatusBadRequest { + if _, err := list("creationTimestamp > 0", 0); sdkerr.GoogleCode(err) != http.StatusBadRequest { t.Errorf("an unsupported filter = %v, want HTTP 400", err) } - if _, err := list("", 501); apiCode(err) != http.StatusBadRequest { + if _, err := list("", 501); sdkerr.GoogleCode(err) != http.StatusBadRequest { t.Errorf("maxResults over 500 = %v, want HTTP 400", err) } it := c.List(t.Context(), &computepb.ListAddressesRequest{Project: project, Region: region, OrderBy: new("creationTimestamp desc")}) - if _, err := it.Next(); apiCode(err) != http.StatusBadRequest { + if _, err := it.Next(); sdkerr.GoogleCode(err) != http.StatusBadRequest { t.Errorf("an unsupported orderBy = %v, want HTTP 400", err) } } @@ -256,7 +243,7 @@ func TestPutAddressSetsUpState(t *testing.T) { if err != nil || got.GetLabels()["session-id"] != "someone-else" || got.GetAddress() != held.Address { t.Fatalf("get = %v, %v", got, err) } - if err := insert(t, c, &computepb.Address{Name: new("held")}); apiCode(err) != http.StatusConflict { + if err := insert(t, c, &computepb.Address{Name: new("held")}); sdkerr.GoogleCode(err) != http.StatusConflict { t.Fatalf("insert over it = %v", err) } for _, bad := range []compute.Address{ @@ -308,9 +295,9 @@ func TestFaults(t *testing.T) { switch { case tc.code == 0 && err != nil: t.Fatalf("insert = %v, want success", err) - case tc.code == -1 && (err == nil || apiCode(err) != 0): + case tc.code == -1 && (err == nil || sdkerr.GoogleCode(err) != 0): t.Fatalf("insert = %v, want a transport error", err) - case tc.code > 0 && apiCode(err) != tc.code: + case tc.code > 0 && sdkerr.GoogleCode(err) != tc.code: t.Fatalf("insert = %v, want HTTP %d", err, tc.code) } if _, ok := s.LookupAddress(project, region, "ip-1"); ok != tc.stored { @@ -349,7 +336,7 @@ func TestLostResponseReachesTheCaller(t *testing.T) { if _, err := get(t, c, "ip-1"); err != nil { t.Fatal(err) } - if err := tc.call(t, c); err == nil || apiCode(err) != 0 { + if err := tc.call(t, c); err == nil || sdkerr.GoogleCode(err) != 0 { t.Fatalf("call = %v, want a transport error", err) } }) @@ -360,7 +347,7 @@ func TestOperationFailedOnAReadFailsTheRead(t *testing.T) { faults := fault.New().Add(fault.Rule{Method: "compute.regionOperations.get", Nth: 1, Fault: fault.FailOperation(http.StatusForbidden)}) s := compute.New(compute.WithFaults(faults)) c := newClient(t, s) - if err := insert(t, c, &computepb.Address{Name: new("ip-1")}); apiCode(err) != http.StatusForbidden { + if err := insert(t, c, &computepb.Address{Name: new("ip-1")}); sdkerr.GoogleCode(err) != http.StatusForbidden { t.Fatalf("insert = %v, want the operation poll to fail with HTTP 403", err) } if _, ok := s.LookupAddress(project, region, "ip-1"); !ok { @@ -398,7 +385,7 @@ func TestRequestIDReplaysTheFirstOperation(t *testing.T) { t.Fatalf("addresses = %v, want none", got) } ins.RequestId = new("not-a-uuid") - if _, err := c.Insert(t.Context(), ins); apiCode(err) != http.StatusBadRequest { + if _, err := c.Insert(t.Context(), ins); sdkerr.GoogleCode(err) != http.StatusBadRequest { t.Fatalf("an insert with an invalid request ID = %v, want HTTP 400", err) } } @@ -414,7 +401,7 @@ func reserve(t *testing.T, c *gcompute.AddressesClient, name, owner string) erro return nil } last = err - if apiCode(err) == http.StatusConflict { + if sdkerr.GoogleCode(err) == http.StatusConflict { got, err := get(t, c, name) if err != nil { last = err diff --git a/sdktest/gcp/compute/instances_test.go b/sdktest/gcp/compute/instances_test.go index 7de48ed..c5b79b2 100644 --- a/sdktest/gcp/compute/instances_test.go +++ b/sdktest/gcp/compute/instances_test.go @@ -20,6 +20,7 @@ import ( "github.com/parallelworks/squall/fault" "github.com/parallelworks/squall/gcp/compute" + "github.com/parallelworks/squall/sdktest/internal/sdkerr" ) const zone = "us-east1-b" @@ -258,12 +259,12 @@ func TestBulkInsertNamesFromAPattern(t *testing.T) { } bad := pattern(2) bad.BulkInsertInstanceResourceResource.NamePattern = new("vm-##-##") - if err := c.bulkInsert(t, bad); apiCode(err) != http.StatusBadRequest { + if err := c.bulkInsert(t, bad); sdkerr.GoogleCode(err) != http.StatusBadRequest { t.Errorf("a pattern with two runs = %v, want HTTP 400", err) } none := pattern(3) none.BulkInsertInstanceResourceResource.NamePattern = nil - if err := c.bulkInsert(t, none); apiCode(err) != http.StatusBadRequest { + if err := c.bulkInsert(t, none); sdkerr.GoogleCode(err) != http.StatusBadRequest { t.Errorf("a count with neither names nor a pattern = %v, want HTTP 400", err) } } @@ -272,7 +273,7 @@ func TestBulkInsertRetriedAfterALostResponse(t *testing.T) { faults := fault.New().Add(fault.Rule{Method: "compute.instances.bulkInsert", Nth: 1, Fault: fault.LoseResponse()}) c := setup(t, compute.WithFaults(faults)) req := bulkRequest("s", "node-1", "node-2") - if _, err := c.instances.BulkInsert(t.Context(), req); err == nil || apiCode(err) != 0 { + if _, err := c.instances.BulkInsert(t.Context(), req); err == nil || sdkerr.GoogleCode(err) != 0 { t.Fatalf("first bulkInsert = %v, want a transport error", err) } if got := len(c.server.Instances()); got != 2 { @@ -280,7 +281,7 @@ func TestBulkInsertRetriedAfterALostResponse(t *testing.T) { } // Without a request ID the retry is a new request, and the names are taken. err := c.bulkInsert(t, req) - if apiCode(err) != http.StatusConflict || !strings.Contains(err.Error(), "CONFLICT") || !strings.Contains(err.Error(), "RESOURCE_ALREADY_EXISTS") || + if sdkerr.GoogleCode(err) != http.StatusConflict || !strings.Contains(err.Error(), "CONFLICT") || !strings.Contains(err.Error(), "RESOURCE_ALREADY_EXISTS") || !strings.Contains(err.Error(), "VM_MIN_COUNT_NOT_REACHED") || !strings.Contains(err.Error(), "already exists") { t.Fatalf("retry = %v, want a 409 operation error naming the existing instances", err) } @@ -335,7 +336,7 @@ func TestInstanceInsertErrors(t *testing.T) { t.Run(tc.name, func(t *testing.T) { inst := single("vm-new", "s") tc.change(inst) - if err := c.insert(t, inst); apiCode(err) != tc.code { + if err := c.insert(t, inst); sdkerr.GoogleCode(err) != tc.code { t.Fatalf("insert = %v, want HTTP %d", err, tc.code) } }) @@ -344,13 +345,13 @@ func TestInstanceInsertErrors(t *testing.T) { t.Fatalf("%d instances, want only the first", got) } _, err := c.instances.Insert(t.Context(), &computepb.InsertInstanceRequest{Project: project, Zone: "us-east1-z", InstanceResource: single("vm", "s")}) - if apiCode(err) != http.StatusBadRequest || !strings.Contains(err.Error(), "Unknown zone") { + if sdkerr.GoogleCode(err) != http.StatusBadRequest || !strings.Contains(err.Error(), "Unknown zone") { t.Errorf("insert in an unknown zone = %v", err) } _, err = c.instances.Insert(t.Context(), &computepb.InsertInstanceRequest{ Project: project, Zone: zone, SourceInstanceTemplate: new("global/instanceTemplates/t"), InstanceResource: single("vm", "s"), }) - if apiCode(err) != http.StatusBadRequest { + if sdkerr.GoogleCode(err) != http.StatusBadRequest { t.Errorf("insert from a template = %v, want HTTP 400 rather than the template ignored", err) } } @@ -375,11 +376,11 @@ func TestGetAndDeleteByID(t *testing.T) { if insts, err := c.list(t, "id="+id, 0); err != nil || len(insts) != 0 { t.Fatalf("list by ID after delete = %s, %v", names(insts), err) } - if _, err := c.get(t, "vm-1"); apiCode(err) != http.StatusNotFound { + if _, err := c.get(t, "vm-1"); sdkerr.GoogleCode(err) != http.StatusNotFound { t.Errorf("get after delete = %v", err) } _, err = c.instances.Delete(t.Context(), &computepb.DeleteInstanceRequest{Project: project, Zone: zone, Instance: id}) - if apiCode(err) != http.StatusNotFound { + if sdkerr.GoogleCode(err) != http.StatusNotFound { t.Errorf("deleting it again = %v, want HTTP 404", err) } } @@ -401,7 +402,7 @@ func TestDeleteHonorsAutoDelete(t *testing.T) { t.Fatalf("disks = %v", d) } _, err := c.disks.Delete(t.Context(), &computepb.DeleteDiskRequest{Project: project, Zone: zone, Disk: "vm-1-data"}) - if apiCode(err) != http.StatusBadRequest || apiReason(err) != "resourceInUseByAnotherResource" { + if sdkerr.GoogleCode(err) != http.StatusBadRequest || apiReason(err) != "resourceInUseByAnotherResource" { t.Fatalf("deleting an attached disk = %v", err) } op, err := c.instances.Delete(t.Context(), &computepb.DeleteInstanceRequest{Project: project, Zone: zone, Instance: "vm-1"}) @@ -461,7 +462,7 @@ func TestStopStartAndSetMachineType(t *testing.T) { } return op.Wait(t.Context()) } - if err := setType("zones/" + zone + "/machineTypes/n2-standard-8"); apiCode(err) != http.StatusBadRequest || !strings.Contains(err.Error(), "RESOURCE_NOT_READY") { + if err := setType("zones/" + zone + "/machineTypes/n2-standard-8"); sdkerr.GoogleCode(err) != http.StatusBadRequest || !strings.Contains(err.Error(), "RESOURCE_NOT_READY") { t.Fatalf("setMachineType while running = %v, want an operation that fails with RESOURCE_NOT_READY", err) } stop, err := c.instances.Stop(t.Context(), &computepb.StopInstanceRequest{Project: project, Zone: zone, Instance: "vm-1"}) @@ -523,7 +524,7 @@ func TestSetLabelsChecksTheInstanceFingerprint(t *testing.T) { if after.GetLabels()["stopped-by"] != "me" || after.GetLabelFingerprint() == inst.GetLabelFingerprint() { t.Fatalf("labels = %v", after.GetLabels()) } - if err := setLabels(inst.GetLabelFingerprint(), nil); apiCode(err) != http.StatusPreconditionFailed { + if err := setLabels(inst.GetLabelFingerprint(), nil); sdkerr.GoogleCode(err) != http.StatusPreconditionFailed { t.Fatalf("a stale fingerprint = %v, want HTTP 412", err) } } @@ -549,10 +550,10 @@ func TestAddAccessConfigClaimsAReservedAddress(t *testing.T) { } return op.Wait(t.Context()) } - if err := add("nic0", "100.64.9.9"); apiCode(err) != http.StatusBadRequest { + if err := add("nic0", "100.64.9.9"); sdkerr.GoogleCode(err) != http.StatusBadRequest { t.Fatalf("an address nobody reserved = %v, want HTTP 400", err) } - if err := add("nic1", reserved.Address); apiCode(err) != http.StatusBadRequest { + if err := add("nic1", reserved.Address); sdkerr.GoogleCode(err) != http.StatusBadRequest { t.Fatalf("a missing interface = %v, want HTTP 400", err) } if err := add("nic0", reserved.Address); err != nil { @@ -565,7 +566,7 @@ func TestAddAccessConfigClaimsAReservedAddress(t *testing.T) { if a, _ := c.server.LookupAddress(project, region, "ip-1"); a.Status != "IN_USE" || len(a.Users) != 1 { t.Fatalf("address = %+v", a) } - if err := add("nic0", reserved.Address); apiCode(err) != http.StatusBadRequest || apiReason(err) != "badRequest" { + if err := add("nic0", reserved.Address); sdkerr.GoogleCode(err) != http.StatusBadRequest || apiReason(err) != "badRequest" { t.Fatalf("a second access config = %v, want HTTP 400 badRequest", err) } deleteInstance := func(name string) { @@ -599,7 +600,7 @@ func TestAddAccessConfigClaimsAReservedAddress(t *testing.T) { if got, _ := c.get(t, "vm-2"); got.GetNetworkInterfaces()[0].GetAccessConfigs()[0].GetNatIP() != reserved.Address { t.Fatalf("access config after the address went = %v", got.GetNetworkInterfaces()[0].GetAccessConfigs()) } - if err := insert(t, c.addresses, &computepb.Address{Name: new("ip-2"), Address: new(reserved.Address)}); apiCode(err) != http.StatusBadRequest { + if err := insert(t, c.addresses, &computepb.Address{Name: new("ip-2"), Address: new(reserved.Address)}); sdkerr.GoogleCode(err) != http.StatusBadRequest { t.Fatalf("reserving the IP the instance still holds = %v, want HTTP 400", err) } deleteInstance("vm-2") @@ -645,7 +646,7 @@ func TestAttachAndDetachDisk(t *testing.T) { if err != nil { t.Fatalf("attaching a disk in use = %v, want the call accepted and its operation failed", err) } - if err := again.Wait(t.Context()); apiCode(err) != http.StatusBadRequest || !strings.Contains(err.Error(), "RESOURCE_IN_USE_BY_ANOTHER_RESOURCE") { + if err := again.Wait(t.Context()); sdkerr.GoogleCode(err) != http.StatusBadRequest || !strings.Contains(err.Error(), "RESOURCE_IN_USE_BY_ANOTHER_RESOURCE") { t.Fatalf("attaching a disk in use = %v, want an operation that fails with RESOURCE_IN_USE_BY_ANOTHER_RESOURCE", err) } detach := func(device string) error { @@ -658,7 +659,7 @@ func TestAttachAndDetachDisk(t *testing.T) { if err := detach("data-1"); err != nil { t.Fatal(err) } - if err := detach("data-1"); apiCode(err) != http.StatusBadRequest || !strings.Contains(err.Error(), "INVALID_USAGE") || + if err := detach("data-1"); sdkerr.GoogleCode(err) != http.StatusBadRequest || !strings.Contains(err.Error(), "INVALID_USAGE") || !strings.Contains(err.Error(), "No attached disk found with device name 'data-1'") { t.Fatalf("detaching it again = %v, want an operation that fails with INVALID_USAGE", err) } @@ -699,7 +700,7 @@ func TestListFiltersAndPagesInstances(t *testing.T) { } } for _, bad := range []string{"cpuPlatform = x", "name eq a AND name eq b", "name = vm-*"} { - if _, err := c.list(t, bad, 0); apiCode(err) != http.StatusBadRequest { + if _, err := c.list(t, bad, 0); sdkerr.GoogleCode(err) != http.StatusBadRequest { t.Errorf("list %q = %v, want HTTP 400", bad, err) } } @@ -767,10 +768,10 @@ func TestZonesList(t *testing.T) { t.Errorf("all zones = %v", got) } _, err := c.instances.Get(t.Context(), &computepb.GetInstanceRequest{Project: project, Zone: "us-east1-d", Instance: "vm"}) - if apiCode(err) != http.StatusBadRequest { + if sdkerr.GoogleCode(err) != http.StatusBadRequest { t.Errorf("a zone WithZones removed = %v, want HTTP 400", err) } - if _, err := c.zones.Get(t.Context(), &computepb.GetZoneRequest{Project: project, Zone: "us-east1-d"}); apiCode(err) != http.StatusNotFound { + if _, err := c.zones.Get(t.Context(), &computepb.GetZoneRequest{Project: project, Zone: "us-east1-d"}); sdkerr.GoogleCode(err) != http.StatusNotFound { t.Errorf("zones.get of a removed zone = %v, want HTTP 404", err) } } @@ -809,21 +810,21 @@ func TestOperationsSettleAfterReads(t *testing.T) { } // A change waits for the one in progress, which here leaves nothing to change. _, err = c.instances.Stop(t.Context(), &computepb.StopInstanceRequest{Project: project, Zone: zone, Instance: "vm-1"}) - if apiCode(err) != http.StatusNotFound { + if sdkerr.GoogleCode(err) != http.StatusNotFound { t.Fatalf("stop while deleting = %v, want HTTP 404", err) } waited, err := c.zoneOps.Wait(t.Context(), &computepb.WaitZoneOperationRequest{Project: project, Zone: zone, Operation: del.Name()}) if err != nil || waited.GetStatus() != computepb.Operation_DONE { t.Fatalf("zoneOperations.wait = %v, %v", waited, err) } - if _, err := c.get(t, "vm-1"); apiCode(err) != http.StatusNotFound { + if _, err := c.get(t, "vm-1"); sdkerr.GoogleCode(err) != http.StatusNotFound { t.Fatalf("get after the delete finished = %v", err) } got, err := c.zoneOps.Get(t.Context(), &computepb.GetZoneOperationRequest{Project: project, Zone: zone, Operation: del.Name()}) if err != nil || got.GetStatus() != computepb.Operation_DONE || got.GetOperationType() != "delete" || got.GetEndTime() == "" { t.Fatalf("zoneOperations.get = %v, %v", got, err) } - if _, err := c.zoneOps.Get(t.Context(), &computepb.GetZoneOperationRequest{Project: project, Zone: zone, Operation: "operation-1"}); apiCode(err) != http.StatusNotFound { + if _, err := c.zoneOps.Get(t.Context(), &computepb.GetZoneOperationRequest{Project: project, Zone: zone, Operation: "operation-1"}); sdkerr.GoogleCode(err) != http.StatusNotFound { t.Errorf("a missing operation = %v", err) } } @@ -872,13 +873,13 @@ func TestInstanceFaults(t *testing.T) { } return op.Wait(t.Context()) } - if err := del(); apiCode(err) != http.StatusInternalServerError { + if err := del(); sdkerr.GoogleCode(err) != http.StatusInternalServerError { t.Fatalf("delete with a failed operation = %v, want HTTP 500", err) } if _, ok := c.server.LookupInstance(project, zone, "vm-1"); !ok { t.Fatal("a failed delete took effect") } - if err := del(); apiCode(err) != http.StatusForbidden { + if err := del(); sdkerr.GoogleCode(err) != http.StatusForbidden { t.Fatalf("delete whose poll fails = %v, want HTTP 403", err) } if _, ok := c.server.LookupInstance(project, zone, "vm-1"); ok { @@ -898,7 +899,7 @@ func createNodes(t *testing.T, c *clients, session string, nodes ...string) erro return nil } last = err - if apiCode(err) != http.StatusConflict || !strings.Contains(err.Error(), "RESOURCE_ALREADY_EXISTS") { + if sdkerr.GoogleCode(err) != http.StatusConflict || !strings.Contains(err.Error(), "RESOURCE_ALREADY_EXISTS") { continue } filter := fmt.Sprintf("labels.session-id=%s AND (name=%s)", session, strings.Join(nodes, " OR name=")) @@ -963,7 +964,7 @@ func TestAReadThatFinishesADeleteNoLongerSeesIt(t *testing.T) { if got, err := c.get(t, "vm-3"); err != nil || got.GetStatus() != "STOPPING" { t.Fatalf("get while deleting = %v, %v", got, err) } - if _, err := c.get(t, "vm-3"); apiCode(err) != http.StatusNotFound { + if _, err := c.get(t, "vm-3"); sdkerr.GoogleCode(err) != http.StatusNotFound { t.Errorf("the get that finishes the delete = %v, want HTTP 404", err) } } @@ -1039,7 +1040,7 @@ func TestBulkInsertRejectsDiskNamesItWouldCreateTwice(t *testing.T) { req := bulkRequest("s", "vm", "vm-1") props := req.BulkInsertInstanceResourceResource.InstanceProperties props.Disks = append(props.Disks, &computepb.AttachedDisk{AutoDelete: new(true), InitializeParams: &computepb.AttachedDiskInitializeParams{}}) - if err := c.bulkInsert(t, req); apiCode(err) != http.StatusConflict || !strings.Contains(err.Error(), "disks/vm-1' already exists") { + if err := c.bulkInsert(t, req); sdkerr.GoogleCode(err) != http.StatusConflict || !strings.Contains(err.Error(), "disks/vm-1' already exists") { t.Fatalf("bulkInsert = %v, want a 409 operation error naming disk vm-1", err) } if got := len(c.server.Disks()); got != 0 { @@ -1071,7 +1072,7 @@ func TestBulkInsertCreatesTheFreeNamesAboveMinCount(t *testing.T) { req = bulkRequest("s", "node-2", "node-3", "node-4") req.BulkInsertInstanceResourceResource.MinCount = new(int64(2)) - if err := c.bulkInsert(t, req); apiCode(err) != http.StatusConflict || !strings.Contains(err.Error(), "VM_MIN_COUNT_NOT_REACHED") { + if err := c.bulkInsert(t, req); sdkerr.GoogleCode(err) != http.StatusConflict || !strings.Contains(err.Error(), "VM_MIN_COUNT_NOT_REACHED") { t.Fatalf("bulkInsert that can create 1 of minCount 2 = %v, want a 409 operation error", err) } if got := len(c.server.Instances()); got != 3 { diff --git a/sdktest/gcp/compute/resources_test.go b/sdktest/gcp/compute/resources_test.go index c2b90d1..c45ead2 100644 --- a/sdktest/gcp/compute/resources_test.go +++ b/sdktest/gcp/compute/resources_test.go @@ -13,6 +13,7 @@ import ( "github.com/parallelworks/squall/fault" "github.com/parallelworks/squall/gcp/compute" + "github.com/parallelworks/squall/sdktest/internal/sdkerr" ) func (c *clients) listPolicies(t *testing.T, filter string) ([]string, error) { @@ -56,7 +57,7 @@ func TestResourcePolicies(t *testing.T) { t.Fatal(err) } } - if err := c.insertPolicy(t, "u-p-1-a"); apiCode(err) != http.StatusConflict { + if err := c.insertPolicy(t, "u-p-1-a"); sdkerr.GoogleCode(err) != http.StatusConflict { t.Fatalf("inserting a name in use = %v, want HTTP 409", err) } if got, err := c.listPolicies(t, "name eq u-p-1-.*"); err != nil || fmt.Sprint(got) != "[u-p-1-a u-p-1-b]" { @@ -80,13 +81,13 @@ func TestResourcePolicies(t *testing.T) { missing := bulkRequest("s", "node-3") missing.BulkInsertInstanceResourceResource.InstanceProperties.ResourcePolicies = []string{"nope"} err = c.bulkInsert(t, missing) - if apiCode(err) != http.StatusBadRequest || !strings.Contains(err.Error(), + if sdkerr.GoogleCode(err) != http.StatusBadRequest || !strings.Contains(err.Error(), "Invalid value for field 'resource.instanceProperties.resourcePolicies[0]': 'nope'. The referenced resourcePolicy resource cannot be found.") { t.Errorf("a missing policy = %v, want HTTP 400 naming the field", err) } gone := single("vm-3", "s") gone.ResourcePolicies = []string{"regions/us-east1/resourcePolicies/nope"} - if err := c.insert(t, gone); apiCode(err) != http.StatusBadRequest || !strings.Contains(err.Error(), "resource.resourcePolicies[0]") { + if err := c.insert(t, gone); sdkerr.GoogleCode(err) != http.StatusBadRequest || !strings.Contains(err.Error(), "resource.resourcePolicies[0]") { t.Errorf("insert with a missing policy = %v, want HTTP 400 naming the field", err) } @@ -97,13 +98,13 @@ func TestResourcePolicies(t *testing.T) { } return op.Wait(t.Context()) } - if err := del("u-p-1-a"); apiCode(err) != http.StatusBadRequest { + if err := del("u-p-1-a"); sdkerr.GoogleCode(err) != http.StatusBadRequest { t.Fatalf("deleting a policy in use = %v, want HTTP 400", err) } if err := del("u-p-1-b"); err != nil { t.Fatal(err) } - if err := del("u-p-1-b"); apiCode(err) != http.StatusNotFound { + if err := del("u-p-1-b"); sdkerr.GoogleCode(err) != http.StatusNotFound { t.Fatalf("deleting it again = %v, want HTTP 404", err) } if got := c.server.ResourcePolicies(); len(got) != 2 { @@ -168,7 +169,7 @@ func TestInstanceTemplatesAndGroupManagers(t *testing.T) { err = c.bulkInsert(t, &computepb.BulkInsertInstanceRequest{Project: project, Zone: zone, BulkInsertInstanceResourceResource: &computepb.BulkInsertInstanceResource{ Count: &one, NamePattern: new("other-#"), SourceInstanceTemplate: new("global/instanceTemplates/missing"), }}) - if apiCode(err) != http.StatusBadRequest || !strings.Contains(err.Error(), + if sdkerr.GoogleCode(err) != http.StatusBadRequest || !strings.Contains(err.Error(), "Invalid value for field 'resource.sourceInstanceTemplate': 'global/instanceTemplates/missing'. The referenced instanceTemplate resource cannot be found.") { t.Errorf("bulkInsert from a missing template = %v, want HTTP 400 naming the field", err) } @@ -185,10 +186,10 @@ func TestInstanceTemplatesAndGroupManagers(t *testing.T) { t.Fatalf("group managers = %v, %v; want none", pair, err) } bad := c.managers.AggregatedList(t.Context(), &computepb.AggregatedListInstanceGroupManagersRequest{Project: project, Filter: new("targetSize = 1")}) - if _, err := bad.Next(); apiCode(err) != http.StatusBadRequest { + if _, err := bad.Next(); sdkerr.GoogleCode(err) != http.StatusBadRequest { t.Fatalf("an unsupported filter = %v, want HTTP 400", err) } - if _, err := c.managers.Get(t.Context(), &computepb.GetInstanceGroupManagerRequest{Project: project, Zone: zone, InstanceGroupManager: "g"}); apiCode(err) != http.StatusNotFound { + if _, err := c.managers.Get(t.Context(), &computepb.GetInstanceGroupManagerRequest{Project: project, Zone: zone, InstanceGroupManager: "g"}); sdkerr.GoogleCode(err) != http.StatusNotFound { t.Fatalf("get of a group manager = %v, want HTTP 404", err) } } @@ -196,7 +197,7 @@ func TestInstanceTemplatesAndGroupManagers(t *testing.T) { func TestRoutes(t *testing.T) { c := setup(t) _, err := c.routes.Delete(t.Context(), &computepb.DeleteRouteRequest{Project: project, Route: "nat-gateway-s"}) - if apiCode(err) != http.StatusNotFound { + if sdkerr.GoogleCode(err) != http.StatusNotFound { t.Fatalf("deleting a missing route = %v, want HTTP 404", err) } route := &computepb.Route{ @@ -215,7 +216,7 @@ func TestRoutes(t *testing.T) { if err := insert(); err != nil { t.Fatal(err) } - if err := insert(); apiCode(err) != http.StatusConflict { + if err := insert(); sdkerr.GoogleCode(err) != http.StatusConflict { t.Fatalf("inserting it again = %v, want HTTP 409", err) } got, err := c.routes.Get(t.Context(), &computepb.GetRouteRequest{Project: project, Route: "nat-gateway-s"}) @@ -223,7 +224,7 @@ func TestRoutes(t *testing.T) { t.Fatalf("get = %v, %v", got, err) } route.Name, route.DestRange = new("bad"), new("10.0.0.1/8") - if err := insert(); apiCode(err) != http.StatusBadRequest { + if err := insert(); sdkerr.GoogleCode(err) != http.StatusBadRequest { t.Fatalf("a range with host bits = %v, want HTTP 400", err) } op, err := c.routes.Delete(t.Context(), &computepb.DeleteRouteRequest{Project: project, Route: "nat-gateway-s"}) @@ -259,7 +260,7 @@ func TestDisks(t *testing.T) { {"an unknown image", &computepb.Disk{Name: new("d-3"), SourceImage: new("projects/debian-cloud/global/images/missing")}, http.StatusBadRequest}, {"a type in another zone", &computepb.Disk{Name: new("d-4"), Type: new("zones/us-east1-c/diskTypes/pd-ssd")}, http.StatusBadRequest}, } { - if err := insert(tc.disk); apiCode(err) != tc.code { + if err := insert(tc.disk); sdkerr.GoogleCode(err) != tc.code { t.Errorf("insert with %s = %v, want HTTP %d", tc.name, err, tc.code) } } @@ -269,7 +270,7 @@ func TestDisks(t *testing.T) { if err := insert(&computepb.Disk{Name: new("d-5"), SourceSnapshot: new("global/snapshots/snap-1"), Labels: map[string]string{"session-id": "s"}}); err != nil { t.Fatal(err) } - if err := insert(&computepb.Disk{Name: new("lost")}); err == nil || apiCode(err) != 0 { + if err := insert(&computepb.Disk{Name: new("lost")}); err == nil || sdkerr.GoogleCode(err) != 0 { t.Fatalf("insert whose response is lost = %v", err) } it := c.disks.List(t.Context(), &computepb.ListDisksRequest{Project: project, Zone: zone, Filter: new("sizeGb = 40 OR name = lost")}) @@ -290,7 +291,7 @@ func TestDisks(t *testing.T) { if got := c.aggregatedDisks(t, `labels.session-id="s"`); fmt.Sprint(got) != "[zones/us-east1-b/d-5]" { t.Errorf("aggregated = %v", got) } - if _, err := c.disks.Get(t.Context(), &computepb.GetDiskRequest{Project: project, Zone: zone, Disk: "missing"}); apiCode(err) != http.StatusNotFound { + if _, err := c.disks.Get(t.Context(), &computepb.GetDiskRequest{Project: project, Zone: zone, Disk: "missing"}); sdkerr.GoogleCode(err) != http.StatusNotFound { t.Errorf("get of a missing disk = %v", err) } } @@ -309,7 +310,7 @@ func TestDeletingADiskAgainFinishesTheFirstDelete(t *testing.T) { if _, err := c.disks.Delete(t.Context(), del); err != nil { t.Fatal(err) } - if _, err := c.disks.Delete(t.Context(), del); apiCode(err) != http.StatusNotFound { + if _, err := c.disks.Delete(t.Context(), del); sdkerr.GoogleCode(err) != http.StatusNotFound { t.Fatalf("deleting a disk being deleted = %v, want HTTP 404 once the first delete finishes", err) } insert() diff --git a/sdktest/gcp/compute/snapshots_test.go b/sdktest/gcp/compute/snapshots_test.go index 2bb5641..dbeae76 100644 --- a/sdktest/gcp/compute/snapshots_test.go +++ b/sdktest/gcp/compute/snapshots_test.go @@ -12,6 +12,7 @@ import ( "github.com/parallelworks/squall/fault" "github.com/parallelworks/squall/gcp/compute" + "github.com/parallelworks/squall/sdktest/internal/sdkerr" ) func (c *clients) insertDisk(t *testing.T, d *computepb.Disk) { @@ -91,7 +92,7 @@ func TestSnapshots(t *testing.T) { {"an invalid name", &computepb.Snapshot{Name: new("Data"), SourceDisk: new(disk)}, http.StatusBadRequest}, {"an unsupported field", &computepb.Snapshot{Name: new("data-3"), SourceDisk: new(disk), SnapshotType: new("ARCHIVE")}, http.StatusBadRequest}, } { - if err := c.insertSnapshot(t, tc.snap); apiCode(err) != tc.code { + if err := c.insertSnapshot(t, tc.snap); sdkerr.GoogleCode(err) != tc.code { t.Errorf("insert with %s = %v, want HTTP %d", tc.name, err, tc.code) } } @@ -105,7 +106,7 @@ func TestSnapshots(t *testing.T) { t.Errorf("list %q = %v, %v; want %s", filter, got, err, want) } } - if _, err := c.listSnapshots(t, "storageBytes = 1"); apiCode(err) != http.StatusBadRequest { + if _, err := c.listSnapshots(t, "storageBytes = 1"); sdkerr.GoogleCode(err) != http.StatusBadRequest { t.Errorf("an unsupported filter = %v, want HTTP 400", err) } @@ -123,10 +124,10 @@ func TestSnapshots(t *testing.T) { if err := c.deleteSnapshot(t, "data-1"); err != nil { t.Fatal(err) } - if _, err := c.snapshots.Get(t.Context(), &computepb.GetSnapshotRequest{Project: project, Snapshot: "data-1"}); apiCode(err) != http.StatusNotFound { + if _, err := c.snapshots.Get(t.Context(), &computepb.GetSnapshotRequest{Project: project, Snapshot: "data-1"}); sdkerr.GoogleCode(err) != http.StatusNotFound { t.Fatalf("get of a deleted snapshot = %v, want HTTP 404", err) } - if err := c.deleteSnapshot(t, "data-1"); apiCode(err) != http.StatusNotFound { + if err := c.deleteSnapshot(t, "data-1"); sdkerr.GoogleCode(err) != http.StatusNotFound { t.Fatalf("deleting it again = %v, want HTTP 404", err) } if got := c.server.Snapshots(); len(got) != 1 || got[0].Name != "data-2" || fmt.Sprint(got[0].StorageLocations) != "[us-east1]" { @@ -141,7 +142,7 @@ func TestSnapshotsSettle(t *testing.T) { } if _, err := c.snapshots.Insert(t.Context(), &computepb.InsertSnapshotRequest{Project: project, SnapshotResource: &computepb.Snapshot{ Name: new("data-1"), SourceDisk: new("zones/us-east1-b/disks/data"), - }}); apiCode(err) != http.StatusBadRequest || !strings.Contains(err.Error(), "is not ready") { + }}); sdkerr.GoogleCode(err) != http.StatusBadRequest || !strings.Contains(err.Error(), "is not ready") { t.Fatalf("a snapshot of a disk being created = %v, want resourceNotReady", err) } c.server.Settle() @@ -160,7 +161,7 @@ func TestSnapshotsSettle(t *testing.T) { _, err = c.disks.Insert(t.Context(), &computepb.InsertDiskRequest{Project: project, Zone: zone, DiskResource: &computepb.Disk{ Name: new("restored"), SourceSnapshot: new("global/snapshots/data-1"), }}) - if apiCode(err) != http.StatusBadRequest || !strings.Contains(err.Error(), "is not ready") { + if sdkerr.GoogleCode(err) != http.StatusBadRequest || !strings.Contains(err.Error(), "is not ready") { t.Fatalf("a disk from a snapshot being created = %v, want resourceNotReady", err) } c.server.Settle() @@ -188,7 +189,7 @@ func TestSnapshotFaults(t *testing.T) { c := setup(t, compute.WithFaults(faults)) c.insertDisk(t, &computepb.Disk{Name: new("data")}) source := new("zones/us-east1-b/disks/data") - if err := c.insertSnapshot(t, &computepb.Snapshot{Name: new("lost"), SourceDisk: source}); err == nil || apiCode(err) != 0 { + if err := c.insertSnapshot(t, &computepb.Snapshot{Name: new("lost"), SourceDisk: source}); err == nil || sdkerr.GoogleCode(err) != 0 { t.Fatalf("insert whose response is lost = %v", err) } if _, ok := c.server.LookupSnapshot(project, "lost"); !ok { diff --git a/sdktest/gcp/storage/example_test.go b/sdktest/gcp/storage/example_test.go new file mode 100644 index 0000000..82e0599 --- /dev/null +++ b/sdktest/gcp/storage/example_test.go @@ -0,0 +1,46 @@ +package storage_test + +import ( + "context" + "errors" + "fmt" + "net/http/httptest" + + gcs "cloud.google.com/go/storage" + "google.golang.org/api/googleapi" + "google.golang.org/api/option" + + "github.com/parallelworks/squall/fault" + "github.com/parallelworks/squall/gcp/storage" +) + +// The first insert takes effect but its response is lost. The client +// retries it, and the retry fails with 409 because the first one made the +// bucket, so code under test must check the bucket's labels before it takes +// the name for someone else's. +func Example() { + ctx := context.Background() + faults := fault.New().Add(fault.Rule{Method: "storage.buckets.insert", Nth: 1, Fault: fault.LoseResponse()}) + s := storage.New(storage.WithFaults(faults)) + srv := httptest.NewServer(s) + defer srv.Close() + + client, err := gcs.NewClient(ctx, option.WithEndpoint(srv.URL+"/storage/v1/"), option.WithoutAuthentication()) + if err != nil { + panic(err) + } + defer client.Close() + + err = client.Bucket("my-bucket").Create(ctx, "my-project", &gcs.BucketAttrs{Labels: map[string]string{"owner": "me"}}) + if e, ok := errors.AsType[*googleapi.Error](err); ok { + fmt.Println("create failed:", e.Code) + } + + attrs, _ := client.Bucket("my-bucket").Attrs(ctx) + fmt.Println("bucket exists:", attrs.Name, attrs.Labels["owner"]) + fmt.Println(faults.Injected()[0]) + // Output: + // create failed: 409 + // bucket exists: my-bucket me + // #1 storage.buckets.insert b/my-bucket: lose response +} diff --git a/sdktest/gcp/storage/storage_test.go b/sdktest/gcp/storage/storage_test.go new file mode 100644 index 0000000..860cfd3 --- /dev/null +++ b/sdktest/gcp/storage/storage_test.go @@ -0,0 +1,486 @@ +package storage_test + +import ( + "errors" + "fmt" + "net/http" + "net/http/httptest" + "slices" + "strings" + "testing" + "time" + + gcs "cloud.google.com/go/storage" + "github.com/googleapis/gax-go/v2" + "google.golang.org/api/googleapi" + "google.golang.org/api/iterator" + "google.golang.org/api/option" + + "github.com/parallelworks/squall/fault" + "github.com/parallelworks/squall/gcp/storage" + "github.com/parallelworks/squall/sdktest/internal/sdkerr" +) + +const project = "my-project" + +func serve(t *testing.T, s *storage.Server) string { + t.Helper() + srv := httptest.NewServer(s) + t.Cleanup(srv.Close) + return srv.URL +} + +func clientAt(t *testing.T, url string) *gcs.Client { + t.Helper() + c, err := gcs.NewClient(t.Context(), option.WithEndpoint(url+"/storage/v1/"), option.WithoutAuthentication()) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = c.Close() }) + // The client retries a lost or failed call; a short backoff keeps the tests quick. + c.SetRetry(gcs.WithBackoff(gax.Backoff{Initial: time.Millisecond, Max: 10 * time.Millisecond})) + return c +} + +func newClient(t *testing.T, s *storage.Server) *gcs.Client { + t.Helper() + return clientAt(t, serve(t, s)) +} + +func write(t *testing.T, c *gcs.Client, bucket, name, data string) *gcs.ObjectAttrs { + t.Helper() + w := c.Bucket(bucket).Object(name).NewWriter(t.Context()) + w.ContentType = "text/plain" + if _, err := w.Write([]byte(data)); err != nil { + t.Fatalf("write %s: %v", name, err) + } + if err := w.Close(); err != nil { + t.Fatalf("close %s: %v", name, err) + } + return w.Attrs() +} + +func names(t *testing.T, it *gcs.BucketIterator) []string { + t.Helper() + var out []string + for { + b, err := it.Next() + if errors.Is(err, iterator.Done) { + return out + } + if err != nil { + t.Fatalf("list buckets: %v", err) + } + out = append(out, b.Name) + } +} + +func objects(t *testing.T, it *gcs.ObjectIterator) []*gcs.ObjectAttrs { + t.Helper() + var out []*gcs.ObjectAttrs + for { + o, err := it.Next() + if errors.Is(err, iterator.Done) { + return out + } + if err != nil { + t.Fatalf("list objects: %v", err) + } + out = append(out, o) + } +} + +func TestCreateReadUpdateDelete(t *testing.T) { + s := storage.New() + c := newClient(t, s) + b := c.Bucket("data-bucket") + err := b.Create(t.Context(), project, &gcs.BucketAttrs{ + Location: "us-east1", + Labels: map[string]string{"owner-id": "abc", "owner": "me"}, + }) + if err != nil { + t.Fatalf("create: %v", err) + } + attrs, err := b.Attrs(t.Context()) + if err != nil { + t.Fatalf("attrs: %v", err) + } + if attrs.Name != "data-bucket" || attrs.Location != "US-EAST1" || attrs.LocationType != "region" || attrs.StorageClass != "STANDARD" || + attrs.Labels["owner-id"] != "abc" || attrs.Created.IsZero() || attrs.ProjectNumber == 0 || attrs.MetaGeneration != 1 || attrs.VersioningEnabled { + t.Fatalf("attrs = %+v", attrs) + } + + ua := gcs.BucketAttrsToUpdate{CORS: []gcs.CORS{{Methods: []string{"GET"}, Origins: []string{"*"}, MaxAge: time.Hour}}} + ua.SetLabel("team", "data") + ua.DeleteLabel("owner") + updated, err := b.Update(t.Context(), ua) + if err != nil { + t.Fatalf("update: %v", err) + } + if len(updated.Labels) != 2 || updated.Labels["team"] != "data" || updated.Labels["owner-id"] != "abc" || + len(updated.CORS) != 1 || updated.CORS[0].MaxAge != time.Hour || updated.MetaGeneration != 2 { + t.Fatalf("updated = %+v", updated) + } + if got, ok := s.LookupBucket("data-bucket"); !ok || got.Project != project || !strings.Contains(string(got.CORS), `"maxAgeSeconds":3600`) { + t.Fatalf("lookup = %+v, %v", got, ok) + } + if _, err := b.If(gcs.BucketConditions{MetagenerationMatch: 1}).Update(t.Context(), ua); sdkerr.GoogleCode(err) != http.StatusPreconditionFailed { + t.Fatalf("a stale update = %v", err) + } + + if err := b.Delete(t.Context()); err != nil { + t.Fatalf("delete: %v", err) + } + if _, err := b.Attrs(t.Context()); !errors.Is(err, gcs.ErrBucketNotExist) { + t.Fatalf("attrs after delete = %v", err) + } + if err := b.Delete(t.Context()); sdkerr.GoogleCode(err) != http.StatusNotFound { + t.Fatalf("delete again = %v", err) + } +} + +func TestPlacements(t *testing.T) { + s := storage.New() + c := newClient(t, s) + for name, tc := range map[string]struct { + attrs gcs.BucketAttrs + location, lt string + }{ + "default": {gcs.BucketAttrs{}, "US", "multi-region"}, + "multi-region": {gcs.BucketAttrs{Location: "eu"}, "EU", "multi-region"}, + "dual-region": {gcs.BucketAttrs{Location: "nam4"}, "NAM4", "dual-region"}, + "custom": {gcs.BucketAttrs{ + Location: "US", CustomPlacementConfig: &gcs.CustomPlacementConfig{DataLocations: []string{"US-EAST1", "US-WEST1"}}, + }, "US", "dual-region"}, + } { + if err := c.Bucket("placed-"+name).Create(t.Context(), project, &tc.attrs); err != nil { + t.Fatalf("%s: %v", name, err) + } + attrs, err := c.Bucket("placed-" + name).Attrs(t.Context()) + if err != nil || attrs.Location != tc.location || attrs.LocationType != tc.lt { + t.Errorf("%s = %+v, %v", name, attrs, err) + } + } + attrs, _ := c.Bucket("placed-custom").Attrs(t.Context()) + if attrs.CustomPlacementConfig == nil || !slices.Equal(attrs.CustomPlacementConfig.DataLocations, []string{"US-EAST1", "US-WEST1"}) { + t.Fatalf("custom placement = %+v", attrs.CustomPlacementConfig) + } +} + +func TestBucketErrors(t *testing.T) { + s := storage.New() + if err := s.PutForeignBucket("taken-name"); err != nil { + t.Fatal(err) + } + c := newClient(t, s) + if err := c.Bucket("mine").Create(t.Context(), project, nil); err != nil { + t.Fatal(err) + } + for _, tc := range []struct { + name string + err error + code int + exact error + }{ + {"a name the caller has", c.Bucket("mine").Create(t.Context(), "other-project", nil), http.StatusConflict, nil}, + {"a name someone else has", c.Bucket("taken-name").Create(t.Context(), project, nil), http.StatusConflict, nil}, + {"an invalid name", c.Bucket("Bad_Name").Create(t.Context(), project, nil), http.StatusBadRequest, nil}, + {"a name with google in it", c.Bucket("my-google-bucket").Create(t.Context(), project, nil), http.StatusBadRequest, nil}, + {"an invalid label", c.Bucket("labeled").Create(t.Context(), project, &gcs.BucketAttrs{Labels: map[string]string{"Upper": "x"}}), http.StatusBadRequest, nil}, + {"reading someone else's bucket", func() error { _, err := c.Bucket("taken-name").Attrs(t.Context()); return err }(), http.StatusForbidden, nil}, + {"deleting someone else's bucket", c.Bucket("taken-name").Delete(t.Context()), http.StatusForbidden, nil}, + {"reading a missing bucket", func() error { _, err := c.Bucket("nobody").Attrs(t.Context()); return err }(), 0, gcs.ErrBucketNotExist}, + {"listing someone else's objects", func() error { _, err := c.Bucket("taken-name").Objects(t.Context(), nil).Next(); return err }(), http.StatusForbidden, nil}, + {"an unimplemented method", func() error { _, err := c.Bucket("mine").IAM().Policy(t.Context()); return err }(), http.StatusNotFound, nil}, + } { + if tc.exact != nil && !errors.Is(tc.err, tc.exact) || tc.exact == nil && sdkerr.GoogleCode(tc.err) != tc.code { + t.Errorf("%s = %v, want %d %v", tc.name, tc.err, tc.code, tc.exact) + } + } + if e, ok := errors.AsType[*googleapi.Error](c.Bucket("taken-name").Create(t.Context(), project, nil)); !ok || + len(e.Errors) != 1 || e.Errors[0].Reason != "conflict" || !strings.Contains(e.Message, "not available") { + t.Fatalf("conflict = %+v", e) + } + if got := s.Buckets(); len(got) != 1 || got[0].Project != project { + t.Fatalf("buckets = %+v", got) + } +} + +func TestListBuckets(t *testing.T) { + s := storage.New() + c := newClient(t, s) + for _, name := range []string{"app-c", "app-a", "logs", "app-b"} { + if err := c.Bucket(name).Create(t.Context(), project, nil); err != nil { + t.Fatal(err) + } + } + if err := c.Bucket("elsewhere").Create(t.Context(), "other-project", nil); err != nil { + t.Fatal(err) + } + if err := s.PutForeignBucket("app-foreign"); err != nil { + t.Fatal(err) + } + if got := names(t, c.Buckets(t.Context(), project)); !slices.Equal(got, []string{"app-a", "app-b", "app-c", "logs"}) { + t.Fatalf("all = %v", got) + } + it := c.Buckets(t.Context(), project) + it.Prefix = "app-" + it.PageInfo().MaxSize = 1 + if got := names(t, it); !slices.Equal(got, []string{"app-a", "app-b", "app-c"}) { + t.Fatalf("paged = %v", got) + } + attrs, err := c.Buckets(t.Context(), "other-project").Next() + if err != nil || attrs.Name != "elsewhere" || attrs.Location != "US" { + t.Fatalf("other project = %+v, %v", attrs, err) + } +} + +func TestEmptyThenDelete(t *testing.T) { + s := storage.New() + c := newClient(t, s) + b := c.Bucket("full") + if err := b.Create(t.Context(), project, &gcs.BucketAttrs{Labels: map[string]string{"owner-id": "abc"}}); err != nil { + t.Fatal(err) + } + for i := range 5 { + write(t, c, "full", fmt.Sprintf("dir/file-%d.txt", i), strings.Repeat("x", i)) + } + if err := b.Delete(t.Context()); sdkerr.GoogleCode(err) != http.StatusConflict { + t.Fatalf("delete a full bucket = %v", err) + } + it := b.Objects(t.Context(), nil) + it.PageInfo().MaxSize = 2 + deleted := 0 + for { + o, err := it.Next() + if errors.Is(err, iterator.Done) { + break + } + if err != nil { + t.Fatal(err) + } + if err := b.Object(o.Name).Delete(t.Context()); err != nil { + t.Fatalf("delete %s: %v", o.Name, err) + } + deleted++ + } + if deleted != 5 { + t.Fatalf("deleted %d objects while paging, want 5", deleted) + } + if err := b.Object("dir/file-0.txt").Delete(t.Context()); !errors.Is(err, gcs.ErrObjectNotExist) { + t.Fatalf("delete a deleted object = %v", err) + } + if err := b.Delete(t.Context()); err != nil { + t.Fatalf("delete the empty bucket: %v", err) + } +} + +func TestVersionedBucketsKeepNoncurrentGenerations(t *testing.T) { + s := storage.New() + c := newClient(t, s) + b := c.Bucket("versioned") + if err := b.Create(t.Context(), project, &gcs.BucketAttrs{VersioningEnabled: true}); err != nil { + t.Fatal(err) + } + first := write(t, c, "versioned", "k", "one") + second := write(t, c, "versioned", "k", "two") + if second.Generation <= first.Generation || first.Size != 3 || first.ContentType != "text/plain" || len(first.MD5) != 16 || first.CRC32C == 0 { + t.Fatalf("generations = %+v then %+v", first, second) + } + if err := b.Object("k").Delete(t.Context()); err != nil { + t.Fatal(err) + } + if live := objects(t, b.Objects(t.Context(), nil)); len(live) != 0 { + t.Fatalf("live = %+v", live) + } + all := objects(t, b.Objects(t.Context(), &gcs.Query{Versions: true})) + if len(all) != 2 || all[0].Generation != first.Generation || all[0].Deleted.IsZero() || all[1].Deleted.IsZero() { + t.Fatalf("versions = %+v", all) + } + if err := b.Delete(t.Context()); sdkerr.GoogleCode(err) != http.StatusConflict { + t.Fatalf("delete with noncurrent generations = %v", err) + } + for _, o := range all { + if err := b.Object(o.Name).Generation(o.Generation).Delete(t.Context()); err != nil { + t.Fatal(err) + } + } + if err := b.Delete(t.Context()); err != nil { + t.Fatalf("delete the emptied bucket: %v", err) + } +} + +func TestPagesOfVersionsFollowNameOrder(t *testing.T) { + s := storage.New() + if _, err := s.PutBucket(storage.Bucket{Name: "ordered", Project: project, Versioning: &storage.Versioning{Enabled: true}}); err != nil { + t.Fatal(err) + } + for _, name := range []string{"a", "a\x00", "a", "a\x00b"} { + if _, err := s.PutObject("ordered", name, []byte(name)); err != nil { + t.Fatal(err) + } + } + it := newClient(t, s).Bucket("ordered").Objects(t.Context(), &gcs.Query{Versions: true}) + it.PageInfo().MaxSize = 1 + var got []string + for _, o := range objects(t, it) { + got = append(got, o.Name) + } + if !slices.Equal(got, []string{"a", "a", "a\x00", "a\x00b"}) { + t.Fatalf("names = %q", got) + } +} + +func TestUploadPreconditionsAndChecksums(t *testing.T) { + s := storage.New() + c := newClient(t, s) + if err := c.Bucket("uploads").Create(t.Context(), project, nil); err != nil { + t.Fatal(err) + } + write(t, c, "uploads", "a/b c.txt", "hello") + w := c.Bucket("uploads").Object("a/b c.txt").If(gcs.Conditions{DoesNotExist: true}).NewWriter(t.Context()) + _, _ = w.Write([]byte("again")) + if err := w.Close(); sdkerr.GoogleCode(err) != http.StatusPreconditionFailed { + t.Fatalf("a create-only write over an object = %v", err) + } + w = c.Bucket("uploads").Object("checked").NewWriter(t.Context()) + w.SendCRC32C, w.CRC32C = true, 12345 + _, _ = w.Write([]byte("data")) + if err := w.Close(); sdkerr.GoogleCode(err) != http.StatusBadRequest { + t.Fatalf("a write with the wrong CRC32C = %v", err) + } + got := s.Objects("uploads") + if len(got) != 1 || got[0].Name != "a/b c.txt" || got[0].Size != 5 { + t.Fatalf("objects = %+v", got) + } + if err := c.Bucket("uploads").Object("a/b c.txt").Delete(t.Context()); err != nil { + t.Fatalf("delete a name with a slash and a space: %v", err) + } + if _, err := s.PutObject("uploads", "seeded", []byte("x")); err != nil { + t.Fatal(err) + } + if listed := objects(t, c.Bucket("uploads").Objects(t.Context(), &gcs.Query{Prefix: "see"})); len(listed) != 1 || listed[0].Name != "seeded" { + t.Fatalf("listed = %+v", listed) + } + if _, err := c.Bucket("uploads").Objects(t.Context(), &gcs.Query{Delimiter: "/"}).Next(); sdkerr.GoogleCode(err) != http.StatusBadRequest { + t.Fatalf("a delimiter = %v", err) + } +} + +func TestEmulatorHost(t *testing.T) { + s := storage.New() + t.Setenv("STORAGE_EMULATOR_HOST", serve(t, s)) + c, err := gcs.NewClient(t.Context()) + if err != nil { + t.Fatal(err) + } + defer c.Close() + if err := c.Bucket("emulated").Create(t.Context(), project, nil); err != nil { + t.Fatal(err) + } + write(t, c, "emulated", "k", "v") + if got := s.Objects("emulated"); len(got) != 1 { + t.Fatalf("objects = %+v", got) + } +} + +func TestFaults(t *testing.T) { + faults := fault.New().Add( + fault.Rule{Method: "storage.buckets.insert", Resource: "b/lost", Nth: 1, Fault: fault.LoseResponse()}, + fault.Rule{Method: "storage.buckets.list", Resource: "projects/locked/b", Fault: fault.Reject(http.StatusForbidden, "forbidden")}, + fault.Rule{Method: "storage.buckets.delete", Nth: 1, Fault: fault.FailAfter(http.StatusInternalServerError)}, + fault.Rule{Method: "storage.objects.insert", Resource: "b/lost/o/dir/denied", Fault: fault.Reject(http.StatusForbidden, "forbidden")}, + ) + s := storage.New(storage.WithFaults(faults)) + c := newClient(t, s) + err := c.Bucket("lost").Create(t.Context(), project, nil) + if sdkerr.GoogleCode(err) != http.StatusConflict { + t.Fatalf("a create whose response was lost = %v, want the retry's conflict", err) + } + if _, ok := s.LookupBucket("lost"); !ok { + t.Fatal("the lost create did not take effect") + } + if _, err := c.Buckets(t.Context(), "locked").Next(); sdkerr.GoogleCode(err) != http.StatusForbidden { + t.Fatalf("a denied list = %v", err) + } + w := c.Bucket("lost").Object("dir/denied").NewWriter(t.Context()) + _, _ = w.Write([]byte("x")) + if err := w.Close(); sdkerr.GoogleCode(err) != http.StatusForbidden { + t.Fatalf("a denied upload = %v", err) + } + if err := c.Bucket("lost").Delete(t.Context()); sdkerr.GoogleCode(err) != http.StatusNotFound { + t.Fatalf("a delete failing after it took effect = %v, want the retry's 404", err) + } + if got := s.Buckets(); len(got) != 0 { + t.Fatalf("buckets = %+v", got) + } + if got := faults.Injected(); len(got) != 4 || got[0].Call.Resource != "b/lost" { + t.Fatalf("injected = %v", got) + } +} + +func TestSweepsConvergeThroughFaults(t *testing.T) { + injected := 0 + for seed := range uint64(20) { + faults := fault.New(fault.Seed(seed)).Random(0.3, fault.LoseResponse(), fault.Unavailable(), fault.FailAfter(http.StatusInternalServerError)) + s := storage.New(storage.WithFaults(faults)) + c := newClient(t, s) + for _, name := range []string{"sweep-a", "sweep-b"} { + if _, err := s.PutBucket(storage.Bucket{Name: name, Project: project, Labels: map[string]string{"owner-id": "s1"}}); err != nil { + t.Fatal(err) + } + for _, o := range []string{"x", "y"} { + if _, err := s.PutObject(name, o, []byte(o)); err != nil { + t.Fatal(err) + } + } + } + for range 100 { + if sweep(t, c) == nil { + break + } + } + injected += len(faults.Injected()) + if got := s.Buckets(); len(got) != 0 { + t.Fatalf("seed %d left %+v after %v", seed, got, faults.Injected()) + } + } + if injected == 0 { + t.Fatal("no seed injected a fault") + } +} + +// sweep empties and deletes every bucket labeled for the owner, stopping at the first error. +func sweep(t *testing.T, c *gcs.Client) error { + t.Helper() + it := c.Buckets(t.Context(), project) + for { + attrs, err := it.Next() + if errors.Is(err, iterator.Done) { + return nil + } + if err != nil { + return err + } + if attrs.Labels["owner-id"] != "s1" { + continue + } + b := c.Bucket(attrs.Name) + objs := b.Objects(t.Context(), nil) + for { + o, err := objs.Next() + if errors.Is(err, iterator.Done) { + break + } + if err != nil { + return err + } + if err := b.Object(o.Name).Delete(t.Context()); err != nil && !errors.Is(err, gcs.ErrObjectNotExist) { + return err + } + } + if err := b.Delete(t.Context()); err != nil && sdkerr.GoogleCode(err) != http.StatusNotFound { + return err + } + } +} diff --git a/sdktest/go.mod b/sdktest/go.mod index 24bb33a..4b568f0 100644 --- a/sdktest/go.mod +++ b/sdktest/go.mod @@ -4,6 +4,7 @@ go 1.27.0 require ( cloud.google.com/go/compute v1.71.0 + cloud.google.com/go/storage v1.69.0 github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.3 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v8 v8.4.0 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v9 v9.0.0 @@ -13,43 +14,67 @@ require ( github.com/aws/aws-sdk-go-v2/config v1.33.9 github.com/aws/aws-sdk-go-v2/credentials v1.20.9 github.com/aws/aws-sdk-go-v2/service/ec2 v1.338.4 + github.com/aws/aws-sdk-go-v2/service/s3 v1.114.3 github.com/aws/smithy-go v1.28.5 + github.com/googleapis/gax-go/v2 v2.26.2 github.com/parallelworks/squall v0.0.0 google.golang.org/api v0.301.0 ) require ( + cel.dev/expr v0.25.2 // indirect + cloud.google.com/go v0.123.0 // indirect cloud.google.com/go/auth v0.24.1-0.20261001053825-dbc26066f70a // indirect cloud.google.com/go/auth/oauth2adapt v0.3.0 // indirect cloud.google.com/go/compute/metadata v0.10.0 // indirect + cloud.google.com/go/iam v1.12.0 // indirect + cloud.google.com/go/monitoring v1.30.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.13.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.35.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.57.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.57.0 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.22 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.6 // indirect github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.6 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.6 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.21 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.7 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.6 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.6 // indirect github.com/aws/aws-sdk-go-v2/service/signin v1.10.4 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.38.4 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.4 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.51.4 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 // indirect + github.com/envoyproxy/go-control-plane/envoy v1.37.0 // indirect + github.com/envoyproxy/protoc-gen-validate v1.3.3 // indirect github.com/felixge/httpsnoop v1.1.0 // indirect + github.com/go-jose/go-jose/v4 v4.1.4 // indirect github.com/go-logr/logr v1.4.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/google/s2a-go v0.1.11 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.22 // indirect - github.com/googleapis/gax-go/v2 v2.26.2 // indirect + github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect + github.com/spiffe/go-spiffe/v2 v2.8.1 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/contrib/detectors/gcp v1.45.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.68.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 // indirect go.opentelemetry.io/otel v1.45.0 // indirect go.opentelemetry.io/otel/metric v1.45.0 // indirect + go.opentelemetry.io/otel/sdk v1.45.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.45.0 // indirect go.opentelemetry.io/otel/trace v1.45.0 // indirect golang.org/x/crypto v0.57.0 // indirect golang.org/x/net v0.59.0 // indirect golang.org/x/oauth2 v0.37.0 // indirect + golang.org/x/sync v0.23.0 // indirect golang.org/x/sys v0.48.0 // indirect golang.org/x/text v0.42.0 // indirect + golang.org/x/time v0.16.0 // indirect google.golang.org/genproto v0.0.0-20260715232425-e75dac1f907d // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260715232425-e75dac1f907d // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260928230214-8a89bd6388cc // indirect diff --git a/sdktest/go.sum b/sdktest/go.sum index 466e2d6..5351ba0 100644 --- a/sdktest/go.sum +++ b/sdktest/go.sum @@ -1,3 +1,5 @@ +cel.dev/expr v0.25.2 h1:K6j46C81hXtZQfuX60cVWQFBJahKSE2gfRbNuvr5bFs= +cel.dev/expr v0.25.2/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= cloud.google.com/go v0.123.0 h1:2NAUJwPR47q+E35uaJeYoNhuNEM9kM8SjgRgdeOJUSE= cloud.google.com/go v0.123.0/go.mod h1:xBoMV08QcqUGuPW65Qfm1o9Y4zKZBpGS+7bImXLTAZU= cloud.google.com/go/auth v0.24.1-0.20261001053825-dbc26066f70a h1:jW1ZLFy9oa5ai8LOI40z6s+DL1cO/L1KSm9hVGnibn8= @@ -8,6 +10,18 @@ cloud.google.com/go/compute v1.71.0 h1:Ht+KeLPl6Bw1i+N360H3ZEizl8xL034XREArasIgU cloud.google.com/go/compute v1.71.0/go.mod h1:L8Z2360/cq2L/Z3q1o/D/jTFkzbgKLbgERpOm4/cepE= cloud.google.com/go/compute/metadata v0.10.0 h1:pyKMUQSwchgkIBBJGdILqQbs/BNJXqwSA7Ej6LAvvtY= cloud.google.com/go/compute/metadata v0.10.0/go.mod h1:rGFHRrIif570kSibjFTMbt6/4/tzgJWFGI/HVol4GIk= +cloud.google.com/go/iam v1.12.0 h1:Aki3bX9aHUDKPHfnRJfDcTdVedvy6quGBQcTqx3DRXk= +cloud.google.com/go/iam v1.12.0/go.mod h1:FEZ4lXpADAC2AIpQY7LANNjjwyQ2jK439CI2VaD+sLY= +cloud.google.com/go/logging v1.19.0 h1:NCqhdVUg3wQ8Cobdf16FDSuTGi3+6+hdSBHrY5TsR6Q= +cloud.google.com/go/logging v1.19.0/go.mod h1:i40NZCHC9Gqvod4yE+yQfDWwlgwW/SrshkkGibCHxcA= +cloud.google.com/go/longrunning v1.2.0 h1:WjYH3YHBGCxGJP9M4dWGHBfXr/cFIjMkNgWcJj7/iMM= +cloud.google.com/go/longrunning v1.2.0/go.mod h1:5KMQALFGOCtFoi2xSOA1u3H7WKlhmckgiyFw7+LGQp0= +cloud.google.com/go/monitoring v1.30.0 h1:r/d+JUbyKmJ8b07iznuKfzVzrIXTWxHQ3lBRm3x2LlY= +cloud.google.com/go/monitoring v1.30.0/go.mod h1:htlUR0QWVMrjFzZmN4LGnMAve9xB/eduwjmINxVZ8RM= +cloud.google.com/go/storage v1.69.0 h1:jAAMC1411HEh78nKsU0Zns+eFj3TnhjAWIhg5Ud/XBM= +cloud.google.com/go/storage v1.69.0/go.mod h1:PELYsxTYm2peE4mwLEC1+mS1dA/kUSRUxNv56rOy44g= +cloud.google.com/go/trace v1.16.0 h1:GmQovzFc5F0CNfl0VLgL64aoTtu7xsM0YajW2GlG9+E= +cloud.google.com/go/trace v1.16.0/go.mod h1:r+bdAn16dKLSV1G2D5v3e58IlQlizfxWrUfjx7kM7X0= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.3 h1:qgnWTiWs1G7AqcKZegFvkJD1bmXJcaDU3uC0ELSgZ5U= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.3/go.mod h1:XtxD48QPInqoj+GMlcRlXHEYsoOxFnP6w2JgIZ9rDS4= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.1 h1:u93s+zU2JD62im61Bm5CZIc1ZrOJaIAWEg0WOrMVkEo= @@ -30,8 +44,18 @@ github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources/v3 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources/v3 v3.0.1/go.mod h1:8h8yhzh9o+0HeSIhUxYny+rEQajScrfIpNktvgYG3Q8= github.com/AzureAD/microsoft-authentication-library-for-go v1.8.0 h1:Nljr4q1GRA/5vCrMONS+g4u4LRHNgOXVSh3O43J2CnI= github.com/AzureAD/microsoft-authentication-library-for-go v1.8.0/go.mod h1:Y33QHnf0FfdVewFFISOGe20mkZbxX4H839o955/PoeI= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.35.0 h1:bN1gA3of5bXtbnLsRPrwfmbbe7A5UWFlcTHseujLnpc= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.35.0/go.mod h1:Yj5vHEz/aAepZGliRJsA6uvHAVAQyEwajq9ORCHPxzM= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.57.0 h1:jLdiS1vO+XJFyDSWRHBx56r4s/NNtcl5J6KyCcWUX/w= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.57.0/go.mod h1:8lmpHY+1VRoteiOwyrQMDt1YGXOrFKCz+1wJW7n3ODY= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.57.0 h1:cSjUzZ7KU8hicTgzaSv9NmSyM9fTVK3y5lsBUl3wOis= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.57.0/go.mod h1:dzcEjy1WJ0Q4u9twNR3LcLhNoYMRCrMCMafpxa0TjPQ= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.57.0 h1:RoO5+d7uCmDqovLrHCr2/BuViUXvdcrNxyNM1pN9dDQ= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.57.0/go.mod h1:YqwkQPrWSC7+byyc1VlKbWLBF5JsW5IoL6xUkemYSXk= github.com/aws/aws-sdk-go-v2 v1.47.3 h1:HRoT/MkNi3bjO5xF4dCzLzvVmkvRGHj3/cLINDnJejk= github.com/aws/aws-sdk-go-v2 v1.47.3/go.mod h1:xSl+OSI5ZUw6bSMupWc218zE5Pl+65MCCu3ySWFY99A= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.22 h1:oiEmPz/YMMeh/JvAE8xSxNCIsvWE6lpFe4ZPlyFykQI= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.22/go.mod h1:QWUDuRi9PFjBHIU2iH1Pkldbe7zqLfXJC4wWLbF8nuQ= github.com/aws/aws-sdk-go-v2/config v1.33.9 h1:Hv6bcQUJVnmuWaKjqD6Ot8sE75NrWsWV20oGT5aWle0= github.com/aws/aws-sdk-go-v2/config v1.33.9/go.mod h1:ivowmrZNfBQLcg4QkaItgi5wae4Bzot9RzIdbY1hfTg= github.com/aws/aws-sdk-go-v2/credentials v1.20.9 h1:SNO55PnWhaDafsOcv16lCA0kdoFMl/YlegWoVsU02s0= @@ -48,8 +72,14 @@ github.com/aws/aws-sdk-go-v2/service/ec2 v1.338.4 h1:1PZfVMz36He601uRm1PiKyDcsEK github.com/aws/aws-sdk-go-v2/service/ec2 v1.338.4/go.mod h1:RttUwBa++DWLLpzTAQdcQzqI9uBF9cfRI/ek8gXzLR4= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.21 h1:e6YW5uME30TkrElR+4/xcY7NB6CCj2dcv7vtmGXhfhQ= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.21/go.mod h1:obpHAMP693+f3/A8PkLhIqfK1C+NWWn8zXzqD3OH520= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.7 h1:CwvFlKw0uUScfgOMflXSRZa+ayXgOf6wk7lhyMemQYU= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.7/go.mod h1:4SbD8BB+p+QEnG0aX6DoWwrtk9SufyYXRsUv0nve8LE= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.6 h1:LHHdobjo5le1sjJdolitV7uVvYg3wkaeM1Ju2fDgeZQ= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.6/go.mod h1:CBGB0evAiaMD3Lk4wPO7LQCcCZLuhzL5Qf/wfEUHQ5w= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.6 h1:mYViDr2azcgxOpRrObZOl631bvavqILoE/L+Kxgbwys= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.6/go.mod h1:OXVdAo9jNhggB+hJRGOyy7WolGTKB6Rg7IIulDeSzPM= +github.com/aws/aws-sdk-go-v2/service/s3 v1.114.3 h1:NyeuyXkZL476uoAowNActm4XJO1QPKf7aassgXHpiqA= +github.com/aws/aws-sdk-go-v2/service/s3 v1.114.3/go.mod h1:GfoBeKMDJecgRUsGf/0RGdx0NJAh81ObTd922h8pQys= github.com/aws/aws-sdk-go-v2/service/signin v1.10.4 h1:tvpDjuUvgKURf3Yk4hOaGbZ4QdY+070OWcA5d6LUv2A= github.com/aws/aws-sdk-go-v2/service/signin v1.10.4/go.mod h1:3ibVKg5Ng82Zby8++jELTD2nHxiPk2DJRLEeW/LKp8M= github.com/aws/aws-sdk-go-v2/service/sso v1.38.4 h1:ChCtagixSFOxltv0o6E3NT8F7ksvZtZGPwUBynO5Nmk= @@ -62,8 +92,20 @@ github.com/aws/smithy-go v1.28.5 h1:OqbqKtzY9NUnr2yAe9+hiTdFb20tNaK/lI0PMOGXX9k= github.com/aws/smithy-go v1.28.5/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 h1:aBangftG7EVZoUb69Os8IaYg++6uMOdKK83QtkkvJik= +github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2/go.mod h1:qwXFYgsP6T7XnJtbKlf1HP8AjxZZyzxMmc+Lq5GjlU4= +github.com/envoyproxy/go-control-plane v0.14.0 h1:hbG2kr4RuFj222B6+7T83thSPqLjwBIfQawTkC++2HA= +github.com/envoyproxy/go-control-plane v0.14.0/go.mod h1:NcS5X47pLl/hfqxU70yPwL9ZMkUlwlKxtAohpi2wBEU= +github.com/envoyproxy/go-control-plane/envoy v1.37.0 h1:u3riX6BoYRfF4Dr7dwSOroNfdSbEPe9Yyl09/B6wBrQ= +github.com/envoyproxy/go-control-plane/envoy v1.37.0/go.mod h1:DReE9MMrmecPy+YvQOAOHNYMALuowAnbjjEMkkWOi6A= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0 h1:/G9QYbddjL25KvtKTv3an9lx6VBE2cnb8wp1vEGNYGI= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4= +github.com/envoyproxy/protoc-gen-validate v1.3.3 h1:MVQghNeW+LZcmXe7SY1V36Z+WFMDjpqGAGacLe2T0ds= +github.com/envoyproxy/protoc-gen-validate v1.3.3/go.mod h1:TsndJ/ngyIdQRhMcVVGDDHINPLWB7C82oDArY51KfB0= github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeORc= github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE= +github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA= +github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8= github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -75,6 +117,8 @@ github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= +github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= github.com/google/s2a-go v0.1.11 h1:7v1V1TraNGP63WptsAkNqr2w8jgm4vM6A6qx6Lsw9iM= github.com/google/s2a-go v0.1.11/go.mod h1:pz4tyvwXvJLLbyrkh6FW1eS2zPUXMaTmyNhYtyP2tNw= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= @@ -87,16 +131,28 @@ github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= +github.com/spiffe/go-spiffe/v2 v2.8.1 h1:eXZMLsu+3MLEPJyGJkolqtVrteZfQdUpOWj6LTiDl/E= +github.com/spiffe/go-spiffe/v2 v2.8.1/go.mod h1:47Q0Q9/AqGha8QLHp+kxpH4Wca7X7EnOtlIJy3mxZ3U= github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/detectors/gcp v1.45.0 h1:9jR0ZPRok9ryaOQ2Wx8rg5F7Aon59mxrqbVI60/vlBk= +go.opentelemetry.io/contrib/detectors/gcp v1.45.0/go.mod h1:VSme3o2fvSg5bVg0dRzyHaj4Z5EVhG+g2Fde6LKzmQA= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.68.0 h1:0Qx7VGBacMm9ZENQ7TnNObTYI4ShC+lHI16seduaxZo= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.68.0/go.mod h1:Sje3i3MjSPKTSPvVWCaL8ugBzJwik3u4smCjUeuupqg= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 h1:8tvICD4vSTOOsNrsI4Ljf6C+6UKvpTEH5XY3JMoyPoo= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0/go.mod h1:z9+yiacE0IHRqM4qFfkbt/JYlmYXgss8GY/jXoNuPJI= go.opentelemetry.io/otel v1.45.0 h1:pdrWmLHofpubmArBv1LgFSv1Z0Ie/ppdZzu+kUN5EeU= go.opentelemetry.io/otel v1.45.0/go.mod h1:XZxIqPapzEYnhNSScF5DIqXhm/rYi0FzCe2XddAwZfQ= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.45.0 h1:dm9iyzn6tioYZtwqaiBSU0TSI8Yu/8dTIbfG0+B49DY= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.45.0/go.mod h1:xAvxYjYK28qvt+yu4BYZ/zMmAjwMXINXD6JiMyeB8iI= go.opentelemetry.io/otel/metric v1.45.0 h1:7Eg1uH7CJ5cXv9is6tnBe1FI6rj1nwUdbFypRm3br/M= go.opentelemetry.io/otel/metric v1.45.0/go.mod h1:HAPbm1nd3p1PmFH7v2dR+6BjXxw+Lq4a2+pndMAm08s= +go.opentelemetry.io/otel/metric/x v0.67.0 h1:PcicCNZFkZ4bXfSooXdo3WN7RBOVOtjVdo1wD358Uns= +go.opentelemetry.io/otel/metric/x v0.67.0/go.mod h1:FBjCWZe6wgcqxcMtjdGiClDKXb2YxxXii0CXftE4QtI= go.opentelemetry.io/otel/sdk v1.45.0 h1:4VVSMgQ83dUgW2aoX5f6JgLvHwIvzcuLnF9lUdCSpCw= go.opentelemetry.io/otel/sdk v1.45.0/go.mod h1:Sr40LgXV7DsKMMJMKOhUWOgMWTfAaqvm2kF0g7ilwuA= go.opentelemetry.io/otel/sdk/metric v1.45.0 h1:oVFszMfyj1Am6s24Vtc7wBb8BKLcwepJjNEYILuiE3o= @@ -117,6 +173,8 @@ golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= +golang.org/x/time v0.16.0 h1:vMb6ptszcQMkcwiRTAuNNU50gom6++Q/6gY2hDM6VDE= +golang.org/x/time v0.16.0/go.mod h1:rVKOqvZeKvrDKTQiAHJ7wmwP0RzleSphoEA9RcdLA0s= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= google.golang.org/api v0.301.0 h1:rTvI8szs17R7b09ggb34Bv4XrmFaFl9uCBncgxskqtw= diff --git a/sdktest/internal/sdkerr/sdkerr.go b/sdktest/internal/sdkerr/sdkerr.go new file mode 100644 index 0000000..969e41e --- /dev/null +++ b/sdktest/internal/sdkerr/sdkerr.go @@ -0,0 +1,40 @@ +// Package sdkerr reads the codes the client libraries' errors carry. +package sdkerr + +import ( + "errors" + + awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http" + "github.com/aws/smithy-go" + "google.golang.org/api/googleapi" +) + +// AWSCode returns the error code of an AWS API error, or "". +func AWSCode(err error) string { + if e, ok := errors.AsType[smithy.APIError](err); ok { + return e.ErrorCode() + } + return "" +} + +// AWSStatus returns the HTTP status of an AWS response error, or 0. +func AWSStatus(err error) int { + if e, ok := errors.AsType[*awshttp.ResponseError](err); ok { + return e.HTTPStatusCode() + } + return 0 +} + +// GoogleCode returns the HTTP status of a Google API error, or 0. +func GoogleCode(err error) int { + if e, ok := errors.AsType[*googleapi.Error](err); ok { + return e.Code + } + if e, ok := errors.AsType[interface { + error + HTTPCode() int + }](err); ok { + return e.HTTPCode() + } + return 0 +}