From 56aecd6348fa88e538e0cc9aa3fb20cc46d36cd4 Mon Sep 17 00:00:00 2001 From: mitchell Date: Mon, 21 Sep 2026 09:48:59 -0400 Subject: [PATCH 1/6] Update version.txt --- version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.txt b/version.txt index 7352cc55c9..b1641ea4ae 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.48.1-RC4 +0.48.2-RC1 From e15cf90cc5b2732c5ca71a52146e2eb4f50a99b1 Mon Sep 17 00:00:00 2001 From: Marc Gutman <43051639+icanhasmath@users.noreply.github.com> Date: Fri, 23 Oct 2026 11:13:37 -0500 Subject: [PATCH 2/6] 0.48.2-RC1: #3832, #3858, #3857 on top of 0.48.1-RC4 (#3863) * Speed up file installation on Windows (#3832) * Resolve link paths once per tree instead of once per file smartlink.Link resolved both src and dest via ResolveUniquePath for every entry it recursed into. On Windows ResolveUniquePath calls GetLongPathName, which is a syscall, so installing a runtime with many files performed two extra syscalls per file - a major contributor to slow Windows installs. The tree root is already resolved by LinkContents (and now by the public Link), and descendant paths are built by joining an already-resolved parent with real (long) entry names, so they need no further resolution. Split the recursion into an unexported link() worker that assumes resolved inputs and does not re-resolve, keeping resolution only for the rare symlink-to-file case where the pre-existing behavior links to the dereferenced target. Also treat an "already exists" error from linkFile the same as the existing TargetExists skip, so concurrent deploys of a shared file degrade to the documented "first one wins" behavior rather than failing. Co-Authored-By: Claude Opus 4.8 (1M context) * Deploy linked artifacts concurrently instead of serializing them DeployViaLink held the depot-wide fsMutex for the entire link operation, so even though the install worker pool runs several artifacts at once, the actual file linking was serialized to one artifact at a time. On Windows, where each link is an individual (relatively slow) syscall, this made the install phase far slower than it needed to be. Linking touches no shared in-memory depot state (Exists guards its own map via mapMutex and depotPath is immutable after construction), directory creation is idempotent, and smartlink now treats an already-existing target as a skip, so it is safe to run without the lock. Drop it from DeployViaLink so linking parallelizes across the worker pool. DeployViaCopy keeps the lock: two artifacts copying the same file would race on os.Create and could corrupt the destination. That path is only used for artifacts needing file transforms, when hard links are unsupported, or in portable mode, so it remains the less common case. Co-Authored-By: Claude Opus 4.8 (1M context) * Hold a read lock while linking so copies can't write through links Linking without the depot lock let a concurrent copy deployment write through a link created moments earlier: copyFiles checks the destination is free, a link to another artifact's depot file appears there, and os.Create then truncates and overwrites that depot file, corrupting the shared cache. TestConcurrentCopyAndLinkKeepDepotIntact reproduces it in every run without the lock. Make fsMutex an RWMutex: DeployViaLink holds it for reading, so links still run concurrently, while copies, Put and Undeploy hold it for writing. Also fix the non-constant format string in DeployViaCopy, which made go test's vet step fail the whole package. Add smartlink tests for an existing target, concurrent links to the same destination, and the Link refactor's handling of directories and symlinks. Co-Authored-By: Claude Opus 5.5 * Skip per-file metadata calls when linking link() checked each file with IsDir, IsSymlink, MkdirUnlessExists on its parent and TargetExists before linking it, plus another IsDir inside the Windows linkFile. Each is a file open on Windows. Instead: - take the entry type from the directory listing (os.ReadDir already has it), so only symlinks need a Stat; - rely on the parent directory existing, since the recursion creates it (Link creates it for its single entry); - drop the TargetExists pre-check and treat the link call's "already exists" error as the skip, which already covered concurrent links; - check for a directory only on the symlink-to-directory path, which is the only one that can pass one to linkFile. A regular file now costs one syscall: the link itself. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 4.8 (1M context) (cherry picked from commit 8f473b3c147e845f8787af85416b3845d746b11a) * Remediate CVE-2026-63209 in klauspost/compress (#3858) Bump github.com/klauspost/compress (indirect, via mholt/archives) from v1.18.0 to v1.18.7, the first patched release for GHSA-259r-337f-4rfw (integer overflow and out-of-bounds write in s2). Re-vendored. Co-authored-by: Claude Opus 5.5 (cherry picked from commit b09f7ee4d8b82ef4233add98618ae53d244c3e79) * Support dependencies for encrypted private ingredients (#3857) * Support dependencies for encrypted private ingredients `state publish --build` can now declare and edit the dependencies of an encrypted private ingredient: - Read [project].dependencies from pyproject.toml and publish them as runtime dependencies on language/python packages, with names normalized by the inventory API. Version ranges carry over (~= and ==X.* are expanded). OS and Python version markers become conditions (kernel/Windows|Linux|Darwin, language/python); unsupported markers are skipped with a notice. The wheel's METADATA records them as Requires-Dist. - Read the --meta file before building, so a meta file's name, version and description feed the wheel (flags > meta file > pyproject.toml). - Add `state export ingredient-meta /[@version]`, which writes an existing ingredient's meta file for use with --edit --meta. Fix --edit along the way: keep the meta file's dependencies and authors instead of overwriting them with the previous revision's, keep inherited dependency types and conditions, add the private-builder dependency only once, and reject author/description changes on an existing version up front instead of failing in the API. InventoryRequirementsToString no longer panics on requirements without a version. Add a readme walking through the publish and edit workflow. Co-Authored-By: Claude Opus 5.5 * Address review: keep inherited deps, edit the meta file's version - When every pyproject.toml dependency is skipped, keep the dependencies already in place (on --edit, those inherited from the previous revision) instead of replacing them with an empty list. - Let a meta file's version select the version to publish or edit, like --version (which still takes precedence). An exported meta file passed back with `state publish --edit --meta` without --build was editing the latest version instead of the exported one. Add tests for both, for the up-front author/description rejection on --edit, for InventoryRequirementsToString with requirements missing a version or comparator, and for inherited conditions that carry requirements. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 (cherry picked from commit 44e750fbbb25b53664b80601e49780ff7a623508) * Add 0.48.2 changelog. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 4.8 (1M context) --- changelog.md | 29 + cmd/state/internal/cmdtree/cmdtree.go | 1 + cmd/state/internal/cmdtree/export.go | 24 + go.mod | 2 +- go.sum | 4 +- internal/python/wheel/distinfo.go | 3 + internal/python/wheel/metadata.go | 31 +- internal/python/wheel/metadata_test.go | 12 + internal/python/wheel/wheel.go | 2 + internal/python/wheel/wheel_test.go | 4 +- internal/runners/export/ingredientmeta.go | 40 + internal/runners/publish/build.go | 53 +- internal/runners/publish/build_test.go | 20 + internal/runners/publish/meta.go | 56 + internal/runners/publish/publish.go | 100 +- internal/runners/publish/publish_test.go | 110 ++ internal/runners/publish/pyproject.go | 253 +++ internal/runners/publish/pyproject_test.go | 94 ++ internal/runners/publish/readme.md | 230 +++ internal/smartlink/smartlink.go | 87 +- internal/smartlink/smartlink_test.go | 107 ++ internal/smartlink/smartlink_windows.go | 7 +- pkg/platform/model/inventory.go | 24 +- pkg/platform/model/version_constraints.go | 10 +- .../model/version_constraints_test.go | 39 + pkg/runtime/depot.go | 20 +- pkg/runtime/depot_test.go | 70 + test/integration/publish_int_test.go | 8 +- .../klauspost/compress/.gitattributes | 1 + .../klauspost/compress/.goreleaser.yml | 11 +- .../github.com/klauspost/compress/README.md | 1371 +++++++++-------- .../klauspost/compress/flate/deflate.go | 53 +- .../klauspost/compress/flate/dict_decoder.go | 5 +- .../klauspost/compress/flate/fast_encoder.go | 49 +- .../compress/flate/huffman_bit_writer.go | 49 +- .../klauspost/compress/flate/huffman_code.go | 4 +- .../klauspost/compress/flate/inflate.go | 4 +- .../klauspost/compress/flate/level5.go | 5 +- .../klauspost/compress/flate/regmask_other.go | 1 - .../klauspost/compress/flate/stateless.go | 24 +- .../klauspost/compress/fse/README.md | 156 +- .../klauspost/compress/fse/bitwriter.go | 2 +- .../klauspost/compress/fse/compress.go | 2 +- .../klauspost/compress/huff0/README.md | 178 +-- .../klauspost/compress/huff0/bitwriter.go | 2 +- .../klauspost/compress/huff0/compress.go | 6 +- .../klauspost/compress/huff0/decompress.go | 14 +- .../compress/huff0/decompress_amd64.go | 8 +- .../compress/huff0/decompress_generic.go | 1 - .../klauspost/compress/huff0/huff0.go | 4 +- .../internal/cpuinfo/cpuinfo_amd64.go | 1 - .../compress/internal/le/unsafe_disabled.go | 4 +- .../compress/internal/le/unsafe_enabled.go | 9 +- .../compress/internal/snapref/decode.go | 2 +- .../compress/internal/snapref/encode.go | 4 +- .../klauspost/compress/s2/README.md | 14 + .../klauspost/compress/s2/decode_amd64.s | 2 +- .../klauspost/compress/s2/decode_other.go | 1 - .../github.com/klauspost/compress/s2/dict.go | 6 +- .../klauspost/compress/s2/encode.go | 8 +- .../klauspost/compress/s2/encode_all.go | 5 +- .../klauspost/compress/s2/encode_amd64.go | 1 - .../klauspost/compress/s2/encode_best.go | 33 +- .../klauspost/compress/s2/encode_better.go | 47 +- .../klauspost/compress/s2/encode_go.go | 1 - .../klauspost/compress/s2/hashtable_pool.go | 65 + .../github.com/klauspost/compress/s2/index.go | 2 +- .../klauspost/compress/s2/reader.go | 2 +- .../klauspost/compress/s2/writer.go | 2 +- .../klauspost/compress/zip/reader.go | 11 +- .../klauspost/compress/zlib/reader.go | 2 +- .../klauspost/compress/zstd/bitwriter.go | 2 +- .../klauspost/compress/zstd/blockdec.go | 6 +- .../klauspost/compress/zstd/blockenc.go | 1 + .../klauspost/compress/zstd/decoder.go | 36 +- .../compress/zstd/decoder_options.go | 60 +- .../klauspost/compress/zstd/dict.go | 20 +- .../klauspost/compress/zstd/enc_base.go | 12 +- .../klauspost/compress/zstd/enc_best.go | 37 +- .../klauspost/compress/zstd/enc_better.go | 44 +- .../klauspost/compress/zstd/enc_dfast.go | 38 +- .../klauspost/compress/zstd/enc_fast.go | 36 +- .../klauspost/compress/zstd/encoder.go | 29 + .../compress/zstd/encoder_options.go | 45 +- .../klauspost/compress/zstd/framedec.go | 5 +- .../compress/zstd/fse_decoder_amd64.go | 1 - .../compress/zstd/fse_decoder_generic.go | 1 - .../klauspost/compress/zstd/fse_encoder.go | 2 +- .../zstd/internal/xxhash/xxhash_other.go | 1 - .../klauspost/compress/zstd/matchlen_amd64.go | 1 - .../compress/zstd/matchlen_generic.go | 1 - .../klauspost/compress/zstd/seqdec.go | 5 +- .../klauspost/compress/zstd/seqdec_amd64.go | 11 +- .../klauspost/compress/zstd/seqdec_generic.go | 1 - .../klauspost/compress/zstd/simple_go124.go | 56 + .../klauspost/compress/zstd/snappy.go | 2 +- .../github.com/klauspost/compress/zstd/zip.go | 2 +- .../klauspost/compress/zstd/zstd.go | 4 +- vendor/modules.txt | 4 +- 99 files changed, 2726 insertions(+), 1349 deletions(-) create mode 100644 internal/runners/export/ingredientmeta.go create mode 100644 internal/runners/publish/meta.go create mode 100644 internal/runners/publish/publish_test.go create mode 100644 internal/runners/publish/pyproject.go create mode 100644 internal/runners/publish/pyproject_test.go create mode 100644 internal/runners/publish/readme.md create mode 100644 pkg/platform/model/version_constraints_test.go create mode 100644 pkg/runtime/depot_test.go create mode 100644 vendor/github.com/klauspost/compress/s2/hashtable_pool.go create mode 100644 vendor/github.com/klauspost/compress/zstd/simple_go124.go diff --git a/changelog.md b/changelog.md index dca5b57c95..3a1258d236 100644 --- a/changelog.md +++ b/changelog.md @@ -6,6 +6,35 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## 0.48.2 + +### Added + +- `state publish --build` now reads `[project].dependencies` from `pyproject.toml` and publishes them as runtime + dependencies of the encrypted private ingredient. Version ranges carry over, and OS and Python version markers become + dependency conditions (unsupported markers are skipped with a notice). +- Added `state export ingredient-meta /[@version]`, which writes an existing ingredient's meta file for + use with `state publish --edit --meta`. + +### Changed + +- Installing runtimes on Windows is significantly faster: link paths are resolved once per tree instead of once per + file, and artifacts are linked concurrently. +- A `--meta` file's name, version, and description now feed the built wheel (flags take precedence over the meta file, + which takes precedence over `pyproject.toml`). + +### Fixed + +- `state publish --edit` no longer overwrites the meta file's dependencies and authors with the previous revision's, + keeps inherited dependency types and conditions, and rejects author or description changes on an existing version + up front. +- `state publish --edit --meta` with an exported meta file now edits the version named in the file rather than the + latest version. + +### Security + +- Updated `github.com/klauspost/compress` to v1.18.7 to remediate CVE-2026-63209 (GHSA-259r-337f-4rfw). + ## 0.48.1 ### Added diff --git a/cmd/state/internal/cmdtree/cmdtree.go b/cmd/state/internal/cmdtree/cmdtree.go index 369622afe9..1b82bbfb8b 100644 --- a/cmd/state/internal/cmdtree/cmdtree.go +++ b/cmd/state/internal/cmdtree/cmdtree.go @@ -55,6 +55,7 @@ func New(prime *primer.Values, args ...string) *CmdTree { newExportLogCommand(prime), newExportRuntimeCommand(prime), newExportBuildPlanCommand(prime), + newExportIngredientMetaCommand(prime), deptree, ) diff --git a/cmd/state/internal/cmdtree/export.go b/cmd/state/internal/cmdtree/export.go index 3838ad1b9c..b282c169ec 100644 --- a/cmd/state/internal/cmdtree/export.go +++ b/cmd/state/internal/cmdtree/export.go @@ -370,3 +370,27 @@ func newExportDepTreeIngredientsCommand(prime *primer.Values) *captain.Command { cmd.SetUnstable(true) return cmd } + +func newExportIngredientMetaCommand(prime *primer.Values) *captain.Command { + runner := export.NewIngredientMeta(prime) + params := &export.IngredientMetaParams{} + + return captain.NewCommand( + "ingredient-meta", + locale.Tl("export_ingredient_meta_title", "Exporting Ingredient Meta File"), + locale.Tl("export_ingredient_meta_description", "Export the meta file of a published ingredient, to modify and pass to '[ACTIONABLE]state publish --edit --meta[/RESET]'"), + prime, + []*captain.Flag{}, + []*captain.Argument{ + { + Name: "namespace/name[@version]", + Description: locale.Tl("export_ingredient_meta_arg_description", "The ingredient to export, e.g. private//python/mypkg@1.0. Defaults to the latest version."), + Value: ¶ms.Ingredient, + Required: true, + }, + }, + func(_ *captain.Command, _ []string) error { + return runner.Run(params) + }, + ) +} diff --git a/go.mod b/go.mod index 48f2537f5b..a8b2517f1c 100644 --- a/go.mod +++ b/go.mod @@ -113,7 +113,7 @@ require ( github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/hinshun/vt10x v0.0.0-20220301184237-5011da428d02 // indirect - github.com/klauspost/compress v1.18.0 // indirect + github.com/klauspost/compress v1.18.7 // indirect github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/klauspost/pgzip v1.2.6 // indirect github.com/kr/pty v1.1.8 // indirect diff --git a/go.sum b/go.sum index a52f356506..2c6b0ddb99 100644 --- a/go.sum +++ b/go.sum @@ -478,8 +478,8 @@ github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQL github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= -github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klauspost/compress v1.18.7 h1:aUyZsS4kH3QTKurYhAOwAHxllVPnOthb3vPfnF1Ehjw= +github.com/klauspost/compress v1.18.7/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= diff --git a/internal/python/wheel/distinfo.go b/internal/python/wheel/distinfo.go index 14ebdcd5ba..960600658a 100644 --- a/internal/python/wheel/distinfo.go +++ b/internal/python/wheel/distinfo.go @@ -26,6 +26,9 @@ func buildMetadata(meta Metadata) []byte { if meta.Summary != "" { b.WriteString("Summary: " + meta.Summary + "\n") } + for _, req := range meta.Requires { + b.WriteString("Requires-Dist: " + req + "\n") + } return b.Bytes() } diff --git a/internal/python/wheel/metadata.go b/internal/python/wheel/metadata.go index 28804c383c..83c8de47da 100644 --- a/internal/python/wheel/metadata.go +++ b/internal/python/wheel/metadata.go @@ -12,7 +12,8 @@ import ( // ResolveMetadata reads srcDir's pyproject.toml [project] table and applies the // non-empty fields of override on top, producing the metadata to pack with. It -// errors when neither source supplies a name or version. +// errors when neither source supplies a name or version. Requires always comes +// from pyproject.toml's dependencies. func ResolveMetadata(srcDir string, override Metadata) (*Metadata, error) { proj, err := readPyProject(filepath.Join(srcDir, "pyproject.toml")) if err != nil { @@ -20,9 +21,10 @@ func ResolveMetadata(srcDir string, override Metadata) (*Metadata, error) { } meta := Metadata{ - Name: firstNonEmpty(override.Name, proj.Name), - Version: firstNonEmpty(override.Version, proj.Version), - Summary: firstNonEmpty(override.Summary, proj.Description), + Name: firstNonEmpty(override.Name, proj.Name), + Version: firstNonEmpty(override.Version, proj.Version), + Summary: firstNonEmpty(override.Summary, proj.Description), + Requires: proj.Dependencies, } if meta.Name == "" || meta.Version == "" { return nil, ErrMissingMetadata @@ -31,9 +33,10 @@ func ResolveMetadata(srcDir string, override Metadata) (*Metadata, error) { } type pyProject struct { - Name string - Version string - Description string + Name string + Version string + Description string + Dependencies []string } // readPyProject reads the [project] table from a pyproject.toml. A missing file @@ -49,18 +52,20 @@ func readPyProject(path string) (pyProject, error) { var parsed struct { Project struct { - Name string `toml:"name"` - Version string `toml:"version"` - Description string `toml:"description"` + Name string `toml:"name"` + Version string `toml:"version"` + Description string `toml:"description"` + Dependencies []string `toml:"dependencies"` } `toml:"project"` } if err := toml.Unmarshal(data, &parsed); err != nil { return pyProject{}, errs.Wrap(err, "could not parse pyproject.toml") } return pyProject{ - Name: parsed.Project.Name, - Version: parsed.Project.Version, - Description: parsed.Project.Description, + Name: parsed.Project.Name, + Version: parsed.Project.Version, + Description: parsed.Project.Description, + Dependencies: parsed.Project.Dependencies, }, nil } diff --git a/internal/python/wheel/metadata_test.go b/internal/python/wheel/metadata_test.go index 948f4c682c..2147e88ddc 100644 --- a/internal/python/wheel/metadata_test.go +++ b/internal/python/wheel/metadata_test.go @@ -27,6 +27,18 @@ func TestResolveMetadata(t *testing.T) { } }) + t.Run("pyproject dependencies become Requires", func(t *testing.T) { + dir := t.TempDir() + writePyproject(t, dir, "[project]\nname = \"proj\"\nversion = \"3.1\"\ndependencies = [\"requests>=2\", \"click\"]\n") + res, err := ResolveMetadata(dir, Metadata{Name: "override"}) + if err != nil { + t.Fatal(err) + } + if len(res.Requires) != 2 || res.Requires[0] != "requests>=2" || res.Requires[1] != "click" { + t.Errorf("Requires = %q", res.Requires) + } + }) + t.Run("caller overrides pyproject", func(t *testing.T) { dir := t.TempDir() writePyproject(t, dir, "[project]\nname = \"proj\"\nversion = \"3.1\"\n") diff --git a/internal/python/wheel/wheel.go b/internal/python/wheel/wheel.go index 221cde62ff..ebbf02d01c 100644 --- a/internal/python/wheel/wheel.go +++ b/internal/python/wheel/wheel.go @@ -40,6 +40,8 @@ type Metadata struct { Name string Version string Summary string + // Requires holds PEP 508 requirement strings, recorded as Requires-Dist. + Requires []string } // sourceFile is one file destined for the wheel, identified by its slash path diff --git a/internal/python/wheel/wheel_test.go b/internal/python/wheel/wheel_test.go index 391d4cccb7..ca2b1038b7 100644 --- a/internal/python/wheel/wheel_test.go +++ b/internal/python/wheel/wheel_test.go @@ -68,7 +68,7 @@ func TestPackProducesValidWheel(t *testing.T) { }) out := t.TempDir() - wheelPath, err := Pack(src, Metadata{Name: "My.Pkg-Name", Version: "1.0", Summary: "a pkg"}, out) + wheelPath, err := Pack(src, Metadata{Name: "My.Pkg-Name", Version: "1.0", Summary: "a pkg", Requires: []string{"requests>=2"}}, out) if err != nil { t.Fatalf("Pack: %v", err) } @@ -94,7 +94,7 @@ func TestPackProducesValidWheel(t *testing.T) { // METADATA / WHEEL contents. metaFile := string(entries[di+"/METADATA"]) - for _, want := range []string{"Metadata-Version: 2.1", "Name: My.Pkg-Name", "Version: 1.0", "Summary: a pkg"} { + for _, want := range []string{"Metadata-Version: 2.1", "Name: My.Pkg-Name", "Version: 1.0", "Summary: a pkg", "Requires-Dist: requests>=2"} { if !bytes.Contains([]byte(metaFile), []byte(want)) { t.Errorf("METADATA missing %q; got:\n%s", want, metaFile) } diff --git a/internal/runners/export/ingredientmeta.go b/internal/runners/export/ingredientmeta.go new file mode 100644 index 0000000000..2dc33b3dfe --- /dev/null +++ b/internal/runners/export/ingredientmeta.go @@ -0,0 +1,40 @@ +package export + +import ( + "github.com/ActiveState/cli/internal/captain" + "github.com/ActiveState/cli/internal/errs" + "github.com/ActiveState/cli/internal/locale" + "github.com/ActiveState/cli/internal/runners/publish" +) + +type IngredientMetaParams struct { + Ingredient captain.PackageValueNSRequired +} + +type IngredientMeta struct { + prime primeable +} + +func NewIngredientMeta(p primeable) *IngredientMeta { + return &IngredientMeta{p} +} + +func (i *IngredientMeta) Run(params *IngredientMetaParams) error { + auth := i.prime.Auth() + if !auth.Authenticated() { + return locale.NewInputError("err_auth_required") + } + + meta, err := publish.FetchMeta(params.Ingredient.Namespace, params.Ingredient.Name, params.Ingredient.Version, auth) + if err != nil { + return errs.Wrap(err, "Could not fetch ingredient meta data") + } + + b, err := meta.MarshalYaml(false) + if err != nil { + return errs.Wrap(err, "Could not marshal ingredient meta data") + } + + i.prime.Output().Print(string(b)) + return nil +} diff --git a/internal/runners/publish/build.go b/internal/runners/publish/build.go index 93dae93168..3912b05ca7 100644 --- a/internal/runners/publish/build.go +++ b/internal/runners/publish/build.go @@ -1,6 +1,7 @@ package publish import ( + "cmp" "context" "os" "path/filepath" @@ -14,44 +15,51 @@ import ( "github.com/ActiveState/cli/internal/logging" "github.com/ActiveState/cli/internal/python/wheel" "github.com/ActiveState/cli/internal/runbits/orgkey" + "github.com/ActiveState/cli/pkg/platform/api/graphql/request" "github.com/ActiveState/cli/pkg/platform/model" ) // generateEncryptedArtifact validates the --build inputs, fetches and validates // the org key, builds the wrapped, encrypted artifact, and points the publish // flow at it. It fails closed before producing any artifact if the key is -// unavailable, and returns a cleanup function the caller must defer. -func (r *Runner) generateEncryptedArtifact(params *Params) (cleanup func(), rerr error) { +// unavailable, and returns the resolved wheel metadata and a cleanup function +// the caller must defer. Flags take precedence over the meta file, which takes +// precedence over pyproject.toml. +func (r *Runner) generateEncryptedArtifact(params *Params, metaFile request.PublishVariables) (_ *wheel.Metadata, cleanup func(), rerr error) { if params.Filepath != "" { - return nil, locale.NewInputError("err_publish_build_and_file", "The '[ACTIONABLE]--build[/RESET]' flag cannot be combined with a source archive filepath.") + return nil, nil, locale.NewInputError("err_publish_build_and_file", "The '[ACTIONABLE]--build[/RESET]' flag cannot be combined with a source archive filepath.") } if r.project == nil { - return nil, locale.NewInputError("err_publish_build_no_project", "The '[ACTIONABLE]--build[/RESET]' flag requires a project so the organization can be determined.") + return nil, nil, locale.NewInputError("err_publish_build_no_project", "The '[ACTIONABLE]--build[/RESET]' flag requires a project so the organization can be determined.") } if !fileutils.DirExists(params.Build) { - return nil, locale.NewInputError("err_publish_build_dir_not_found", "The '[ACTIONABLE]--build[/RESET]' source directory does not exist: [ACTIONABLE]{{.V0}}[/RESET]", params.Build) + return nil, nil, locale.NewInputError("err_publish_build_dir_not_found", "The '[ACTIONABLE]--build[/RESET]' source directory does not exist: [ACTIONABLE]{{.V0}}[/RESET]", params.Build) } - meta, err := wheel.ResolveMetadata(params.Build, wheel.Metadata{Name: params.Name, Version: params.Version}) + meta, err := wheel.ResolveMetadata(params.Build, wheel.Metadata{ + Name: cmp.Or(params.Name, metaFile.Name), + Version: cmp.Or(params.Version, metaFile.Version), + Summary: cmp.Or(params.Description, metaFile.Description), + }) if err != nil { - return nil, locale.WrapInputError(err, "err_publish_build_metadata", "Could not determine the ingredient name and version: {{.V0}}", errs.JoinMessage(err)) + return nil, nil, locale.WrapInputError(err, "err_publish_build_metadata", "Could not determine the ingredient name and version: {{.V0}}", errs.JoinMessage(err)) } // Fetch and validate the org key before building anything: a private publish // is encrypted-required, so fail closed before any byte could be uploaded. provider := orgkey.New(r.cfg, r.project.Owner()) if !provider.Configured() { - return nil, locale.NewInputError("err_publish_orgkey_unconfigured", "No organization key service is configured, so this private ingredient cannot be encrypted.") + return nil, nil, locale.NewInputError("err_publish_orgkey_unconfigured", "No organization key service is configured, so this private ingredient cannot be encrypted.") } defer provider.Close() key, keyID, err := provider.Key(context.Background()) if err != nil { - return nil, locale.WrapInputError(err, "err_publish_orgkey_unavailable", "Could not obtain the organization key, so nothing was uploaded: {{.V0}}", errs.JoinMessage(err)) + return nil, nil, locale.WrapInputError(err, "err_publish_orgkey_unavailable", "Could not obtain the organization key, so nothing was uploaded: {{.V0}}", errs.JoinMessage(err)) } archivePath, cleanup, err := buildWrappedArtifact(params.Build, *meta, key, keyID) if err != nil { - return nil, errs.Wrap(err, "Could not build encrypted artifact") + return nil, nil, errs.Wrap(err, "Could not build encrypted artifact") } // TODO(ENG-1641): once the platform supports the genesis/timeless publish @@ -65,7 +73,11 @@ func (r *Runner) generateEncryptedArtifact(params *Params) (cleanup func(), rerr if params.Version == "" { params.Version = meta.Version } - return cleanup, nil + // The platform rejects description changes on existing ingredients. + if params.Description == "" && !params.Edit { + params.Description = meta.Summary + } + return meta, cleanup, nil } // requireOrgNamespace ensures ns belongs to the project owner's private org, so @@ -85,6 +97,25 @@ const ( privateBuilderName = "private-builder" ) +// addPrivateBuilderDependency makes a --build publish depend on the +// private-builder. Without a builder dependency the platform falls back to the +// noop-builder, which produces an empty artifact. An --edit revision already +// inherits it from the previous revision, so it is only added when missing. +func addPrivateBuilderDependency(reqVars *request.PublishVariables) { + for _, dep := range reqVars.Dependencies { + if dep.Namespace == privateBuilderNamespace && dep.Name == privateBuilderName { + return + } + } + reqVars.Dependencies = append(reqVars.Dependencies, request.PublishVariableDep{ + Dependency: request.Dependency{ + Name: privateBuilderName, + Namespace: privateBuilderNamespace, + VersionRequirements: ">=0", + }, + }) +} + // buildWrappedArtifact packs srcDir into a wheel under the given metadata, // encrypts it under the org key, and wraps the ciphertext in a tar.gz ready for // upload. It returns the wrapped archive path and a cleanup function the caller diff --git a/internal/runners/publish/build_test.go b/internal/runners/publish/build_test.go index 94ad3b11ca..3bb54f1dc3 100644 --- a/internal/runners/publish/build_test.go +++ b/internal/runners/publish/build_test.go @@ -13,6 +13,7 @@ import ( "github.com/ActiveState/cli/internal/artifactcrypto" "github.com/ActiveState/cli/internal/python/wheel" + "github.com/ActiveState/cli/pkg/platform/api/graphql/request" ) func testKey() []byte { @@ -173,3 +174,22 @@ func equalStrings(a, b []string) bool { } return true } + +func TestAddPrivateBuilderDependency(t *testing.T) { + userDep := request.PublishVariableDep{Dependency: request.Dependency{Name: "requests", Namespace: "language/python"}} + reqVars := request.PublishVariables{Dependencies: []request.PublishVariableDep{userDep}} + + addPrivateBuilderDependency(&reqVars) + if len(reqVars.Dependencies) != 2 { + t.Fatalf("expected user dependency plus private-builder, got %+v", reqVars.Dependencies) + } + if reqVars.Dependencies[0].Dependency != userDep.Dependency { + t.Errorf("user dependency was not preserved: %+v", reqVars.Dependencies[0]) + } + + // An --edit revision inherits the private-builder; it must not be added twice. + addPrivateBuilderDependency(&reqVars) + if len(reqVars.Dependencies) != 2 { + t.Errorf("private-builder was added twice: %+v", reqVars.Dependencies) + } +} diff --git a/internal/runners/publish/meta.go b/internal/runners/publish/meta.go new file mode 100644 index 0000000000..5f157da642 --- /dev/null +++ b/internal/runners/publish/meta.go @@ -0,0 +1,56 @@ +package publish + +import ( + "errors" + + "github.com/ActiveState/cli/internal/errs" + "github.com/ActiveState/cli/internal/locale" + "github.com/ActiveState/cli/pkg/platform/api/graphql/request" + "github.com/ActiveState/cli/pkg/platform/api/inventory/inventory_client/inventory_operations" + "github.com/ActiveState/cli/pkg/platform/authentication" + "github.com/ActiveState/cli/pkg/platform/model" +) + +// FetchMeta returns the meta file contents of an existing ingredient version (the latest one when version is empty), +// in the format the --meta flag accepts. Its dependencies are the complete set, so the file can be modified and +// passed back with --edit to replace them. Authors and description are left out because the platform rejects them +// on a revision of an existing version. +func FetchMeta(namespace, name, version string, auth *authentication.Auth) (*request.PublishVariables, error) { + latestRevisionTime, err := model.FetchLatestRevisionTimeStamp(auth) + if err != nil { + return nil, errs.Wrap(err, "Unable to determine latest revision time") + } + + var ingredient *ParentIngredient + if version != "" { + i, err := model.GetIngredientByNameAndVersion(namespace, name, version, &latestRevisionTime, auth) + if err != nil { + var notFound *inventory_operations.GetNamespaceIngredientVersionNotFound + if errors.As(err, ¬Found) { + return nil, locale.NewInputError("err_ingredient_meta_version_not_found", + "Could not find version '[ACTIONABLE]{{.V0}}[/RESET]' of ingredient '[ACTIONABLE]{{.V1}}/{{.V2}}[/RESET]'.", + version, namespace, name) + } + return nil, errs.Wrap(err, "could not get ingredient version") + } + ingredient = &ParentIngredient{*i.IngredientID, *i.IngredientVersionID, *i.Version, i.Dependencies} + } else { + ingredients, err := model.SearchIngredientsStrict(namespace, name, true, false, &latestRevisionTime, auth) + var errSearch404 *model.ErrSearch404 + if err != nil && !errors.As(err, &errSearch404) { + return nil, locale.WrapError(err, "err_uploadingredient_search", "Could not search for ingredient") + } + if len(ingredients) == 0 { + return nil, locale.NewInputError("err_ingredient_meta_not_found", + "Could not find ingredient '[ACTIONABLE]{{.V0}}/{{.V1}}[/RESET]'.", namespace, name) + } + i := ingredients[0].LatestVersion + ingredient = &ParentIngredient{*i.IngredientID, *i.IngredientVersionID, *i.Version, i.Dependencies} + } + + meta := &request.PublishVariables{Name: name, Namespace: namespace} + if err := prepareEditRequest(ingredient, meta, true, auth); err != nil { + return nil, errs.Wrap(err, "Could not read ingredient meta data") + } + return meta, nil +} diff --git a/internal/runners/publish/publish.go b/internal/runners/publish/publish.go index a0eaf03353..4ca7e621fa 100644 --- a/internal/runners/publish/publish.go +++ b/internal/runners/publish/publish.go @@ -1,6 +1,7 @@ package publish import ( + "cmp" "errors" "path/filepath" "regexp" @@ -18,6 +19,7 @@ import ( "github.com/ActiveState/cli/internal/output" "github.com/ActiveState/cli/internal/primer" "github.com/ActiveState/cli/internal/prompt" + "github.com/ActiveState/cli/internal/python/wheel" "github.com/ActiveState/cli/internal/rtutils/ptr" "github.com/ActiveState/cli/pkg/platform/api/graphql/request" "github.com/ActiveState/cli/pkg/platform/api/inventory/inventory_client/inventory_operations" @@ -92,12 +94,37 @@ func (r *Runner) Run(params *Params) error { return locale.NewInputError("err_auth_required") } + reqVars := request.PublishVariables{} + var wheelMeta *wheel.Metadata + + // Pass input from meta file + if params.MetaFilepath != "" { + if !fileutils.TargetExists(params.MetaFilepath) { + return locale.NewInputError("err_uploadingredient_metafile_not_found", "Meta file not found: {{.V0}}", params.MetaFilepath) + } + + b, err := fileutils.ReadFile(params.MetaFilepath) + if err != nil { + return locale.WrapExternalError(err, "err_uploadingredient_file_read", "Could not read file: {{.V0}}", params.MetaFilepath) + } + + if err := yaml.Unmarshal(b, &reqVars); err != nil { + return locale.WrapExternalError(err, "err_uploadingredient_file_read", "Failed to unmarshal meta file. Error received: {{.V0}}", err.Error()) + } + } + metaFileHasDeps := len(reqVars.Dependencies) > 0 + + // A meta file's version selects the version to publish or edit, just like --version (which takes precedence). + // Without this, an exported meta file passed back with --edit would edit the latest version instead of its own. + params.Version = cmp.Or(params.Version, reqVars.Version) + if params.Build != "" { - cleanup, err := r.generateEncryptedArtifact(params) // note: this function also mutates params + meta, cleanup, err := r.generateEncryptedArtifact(params, reqVars) // note: this function also mutates params if err != nil { return errs.Wrap(err, "Could not build private ingredient") } defer cleanup() // remove the temporary build directory + wheelMeta = meta } if params.Filepath != "" { @@ -113,24 +140,6 @@ func (r *Runner) Run(params *Params) error { return locale.NewInputError("err_uploadingredient_file_required", "You have to supply the source archive unless editing.") } - reqVars := request.PublishVariables{} - - // Pass input from meta file - if params.MetaFilepath != "" { - if !fileutils.TargetExists(params.MetaFilepath) { - return locale.NewInputError("err_uploadingredient_metafile_not_found", "Meta file not found: {{.V0}}", params.MetaFilepath) - } - - b, err := fileutils.ReadFile(params.MetaFilepath) - if err != nil { - return locale.WrapExternalError(err, "err_uploadingredient_file_read", "Could not read file: {{.V0}}", params.MetaFilepath) - } - - if err := yaml.Unmarshal(b, &reqVars); err != nil { - return locale.WrapExternalError(err, "err_uploadingredient_file_read", "Failed to unmarshal meta file. Error received: {{.V0}}", err.Error()) - } - } - // Namespace if params.Namespace != "" { reqVars.Namespace = params.Namespace @@ -218,21 +227,24 @@ func (r *Runner) Run(params *Params) error { } } + // pyproject.toml dependencies apply unless the meta file lists its own; + // --depend flags still take precedence over both. + if wheelMeta != nil && !metaFileHasDeps { + if err := r.applyPyprojectDependencies(&reqVars, wheelMeta.Requires); err != nil { + return errs.Wrap(err, "Could not read pyproject.toml dependencies") + } + } + if err := prepareRequestFromParams(&reqVars, params, isRevision); err != nil { return errs.Wrap(err, "Could not prepare request from params") } + if err := checkEditKeepsIdentity(&reqVars, params.Edit, isRevision); err != nil { + return err + } + if params.Build != "" { - // A --build publish must be ingested by the private-builder. Without a - // builder dependency the platform falls back to the noop-builder, which - // produces an empty artifact. - reqVars.Dependencies = append(reqVars.Dependencies, request.PublishVariableDep{ - Dependency: request.Dependency{ - Name: privateBuilderName, - Namespace: privateBuilderNamespace, - VersionRequirements: ">=0", - }, - }) + addPrivateBuilderDependency(&reqVars) } if params.Editor { @@ -325,6 +337,16 @@ Do you want to publish this ingredient? var versionRegexp = regexp.MustCompile(`\d+\.\d+(\.\d+)?`) +// checkEditKeepsIdentity rejects author and description changes on a revision of an existing version, which the +// platform does not allow; it would otherwise fail the publish with an unclear API error. +func checkEditKeepsIdentity(r *request.PublishVariables, edit, isRevision bool) error { + if edit && isRevision && (len(r.Authors) > 0 || r.Description != "") { + return locale.NewInputError("err_uploadingredient_edit_identity", + "Authors and description cannot be changed when editing an existing version. Remove them from the meta file and flags.") + } + return nil +} + func prepareRequestFromParams(r *request.PublishVariables, params *Params, isRevision bool) error { if params.Version != "" { r.Version = params.Version @@ -417,11 +439,12 @@ func prepareRequestFromParams(r *request.PublishVariables, params *Params, isRev // prepareEditRequest inherits meta data from the previous ingredient revision if it exists. This should really happen // on the API, but at the time of implementation we did this client side as the API side requires significant refactorings -// to enable this behavior. +// to enable this behavior. Authors and dependencies already supplied by a meta file are kept rather than inherited. +// The version is always the edited ingredient's: an edit never changes it. func prepareEditRequest(ingredient *ParentIngredient, r *request.PublishVariables, isRevision bool, auth *authentication.Auth) error { r.Version = ingredient.Version - if !isRevision { + if !isRevision && len(r.Authors) == 0 { authors, err := model.FetchAuthors(&ingredient.IngredientID, &ingredient.IngredientVersionID, auth) if err != nil { return locale.WrapError(err, "err_uploadingredient_fetch_authors", "Could not fetch authors for ingredient") @@ -442,16 +465,25 @@ func prepareEditRequest(ingredient *ParentIngredient, r *request.PublishVariable } } - if len(ingredient.Dependencies) > 0 { + if len(ingredient.Dependencies) > 0 && len(r.Dependencies) == 0 { r.Dependencies = []request.PublishVariableDep{} for _, dep := range ingredient.Dependencies { + conditions := []request.Dependency{} + for _, c := range dep.Conditions { + conditions = append(conditions, request.Dependency{ + Name: ptr.From(c.Feature, ""), + Namespace: ptr.From(c.Namespace, ""), + VersionRequirements: model.InventoryRequirementsToString(c.Requirements, true), + }) + } r.Dependencies = append( r.Dependencies, - request.PublishVariableDep{request.Dependency{ + request.PublishVariableDep{Dependency: request.Dependency{ Name: ptr.From(dep.Feature, ""), Namespace: ptr.From(dep.Namespace, ""), VersionRequirements: model.InventoryRequirementsToString(dep.Requirements, true), - }, []request.Dependency{}}, + Type: request.DependencyType(dep.Type), + }, Conditions: conditions}, ) } } diff --git a/internal/runners/publish/publish_test.go b/internal/runners/publish/publish_test.go new file mode 100644 index 0000000000..14940dfdba --- /dev/null +++ b/internal/runners/publish/publish_test.go @@ -0,0 +1,110 @@ +package publish + +import ( + "reflect" + "testing" + + "github.com/ActiveState/cli/internal/rtutils/ptr" + "github.com/ActiveState/cli/internal/testhelpers/outputhelper" + "github.com/ActiveState/cli/pkg/platform/api/graphql/request" + "github.com/ActiveState/cli/pkg/platform/api/inventory/inventory_models" +) + +func TestPrepareEditRequestDependencies(t *testing.T) { + ingredient := &ParentIngredient{ + Version: "1.0.0", + Dependencies: []inventory_models.Dependency{{ + Feature: ptr.To("python"), + Namespace: ptr.To("language"), + Type: inventory_models.DependencyTypeRuntime, + Conditions: []*inventory_models.DependencyCondition{ + {Condition: inventory_models.Condition{ + Feature: ptr.To("windows"), + Namespace: ptr.To("kernel"), + }}, + {Condition: inventory_models.Condition{ + Feature: ptr.To("python"), + Namespace: ptr.To("language"), + Requirements: inventory_models.Requirements{ + {Comparator: ptr.To(inventory_models.RequirementComparatorLt), Version: ptr.To("3.11")}, + }, + }}, + }, + }}, + } + + t.Run("inherits when none supplied", func(t *testing.T) { + r := &request.PublishVariables{Version: "9.9.9"} + if err := prepareEditRequest(ingredient, r, true, nil); err != nil { + t.Fatal(err) + } + if len(r.Dependencies) != 1 || r.Dependencies[0].Name != "python" || r.Dependencies[0].Type != request.DependencyTypeRuntime { + t.Errorf("dependencies not inherited: %+v", r.Dependencies) + } + want := []request.Dependency{ + {Name: "windows", Namespace: "kernel"}, + {Name: "python", Namespace: "language", VersionRequirements: "<3.11"}, + } + if c := r.Dependencies[0].Conditions; !reflect.DeepEqual(c, want) { + t.Errorf("conditions not inherited: got %+v, want %+v", c, want) + } + if r.Version != "1.0.0" { + t.Errorf("version = %q, an edit must keep the ingredient's version", r.Version) + } + }) + + t.Run("meta file dependencies are kept", func(t *testing.T) { + metaDep := request.PublishVariableDep{Dependency: request.Dependency{Name: "six", Namespace: "language/python"}} + r := &request.PublishVariables{Dependencies: []request.PublishVariableDep{metaDep}} + if err := prepareEditRequest(ingredient, r, true, nil); err != nil { + t.Fatal(err) + } + if len(r.Dependencies) != 1 || r.Dependencies[0].Dependency != metaDep.Dependency { + t.Errorf("meta file dependencies were overwritten: %+v", r.Dependencies) + } + }) +} + +// TestApplyPyprojectDependencies covers --edit with a pyproject.toml whose dependencies are all skipped: the +// dependencies inherited from the previous revision must be kept rather than replaced by an empty list. +func TestApplyPyprojectDependencies(t *testing.T) { + inherited := []request.PublishVariableDep{{Dependency: request.Dependency{Name: "six", Namespace: "language/python"}}} + r := &Runner{out: outputhelper.NewCatcher()} + + for _, requires := range [][]string{nil, {`colorama; platform_machine == "x86_64"`}} { + reqVars := &request.PublishVariables{Dependencies: inherited} + if err := r.applyPyprojectDependencies(reqVars, requires); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(reqVars.Dependencies, inherited) { + t.Errorf("requires %q: inherited dependencies were replaced: %+v", requires, reqVars.Dependencies) + } + } +} + +func TestCheckEditKeepsIdentity(t *testing.T) { + withAuthor := &request.PublishVariables{Authors: []request.PublishVariableAuthor{{Name: "a", Email: "a@example.com"}}} + withDescription := &request.PublishVariables{Description: "changed"} + plain := &request.PublishVariables{} + + tests := []struct { + name string + vars *request.PublishVariables + edit, isRevision bool + wantErr bool + }{ + {"author on revision", withAuthor, true, true, true}, + {"description on revision", withDescription, true, true, true}, + {"no identity change on revision", plain, true, true, false}, + {"author on new version via edit", withAuthor, true, false, false}, + {"description on new ingredient", withDescription, false, false, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := checkEditKeepsIdentity(tt.vars, tt.edit, tt.isRevision) + if (err != nil) != tt.wantErr { + t.Errorf("err = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} diff --git a/internal/runners/publish/pyproject.go b/internal/runners/publish/pyproject.go new file mode 100644 index 0000000000..8dd58dc1f8 --- /dev/null +++ b/internal/runners/publish/pyproject.go @@ -0,0 +1,253 @@ +package publish + +import ( + "cmp" + "regexp" + "strconv" + "strings" + + "github.com/ActiveState/cli/internal/errs" + "github.com/ActiveState/cli/internal/locale" + "github.com/ActiveState/cli/pkg/platform/api/graphql/request" + "github.com/ActiveState/cli/pkg/platform/model" +) + +// pythonRequirement is a PEP 508 requirement translated into the platform's +// name, version-requirements, and conditions form. +type pythonRequirement struct { + Name string + VersionRequirements string + Conditions []request.Dependency +} + +var ( + // requirementRegexp matches `name[extras] specifiers`, where the specifiers + // may be wrapped in parentheses (the legacy `name (>=1.0)` form). + requirementRegexp = regexp.MustCompile(`^([A-Za-z0-9](?:[A-Za-z0-9._-]*[A-Za-z0-9])?)\s*(?:\[[^\]]*\])?\s*\(?([^()]*)\)?$`) + specifierRegexp = regexp.MustCompile(`^(~=|===|==|!=|<=|>=|<|>)\s*(\S+)$`) + markerRegexp = regexp.MustCompile(`^(\w+)\s*(===|==|!=|<=|>=|~=|<|>)\s*["']([^"']*)["']$`) +) + +// markerKernels maps the platform markers we support to the platform's kernel names. +var markerKernels = map[string]map[string]string{ + "sys_platform": {"win32": "Windows", "linux": "Linux", "darwin": "Darwin"}, + "platform_system": {"Windows": "Windows", "Linux": "Linux", "Darwin": "Darwin"}, + "os_name": {"nt": "Windows"}, +} + +// parsePythonRequirement translates a PEP 508 requirement string. Operating +// system and Python version markers become conditions. Other markers, direct URL +// references, and the != and === operators have no platform equivalent and are +// rejected, so the caller can tell the user to add them with --depend instead. +// Extras are dropped. +func parsePythonRequirement(req string) (*pythonRequirement, error) { + req, marker, _ := strings.Cut(strings.TrimSpace(req), ";") + req = strings.TrimSpace(req) + conditions, err := translateMarker(strings.TrimSpace(marker)) + if err != nil { + return nil, err + } + if strings.Contains(req, "@") { + return nil, errs.New("direct URL references are not supported") + } + m := requirementRegexp.FindStringSubmatch(req) + if m == nil { + return nil, errs.New("not a valid requirement") + } + + var constraints []string + for _, spec := range strings.Split(m[2], ",") { + spec = strings.TrimSpace(spec) + if spec == "" { + continue + } + sm := specifierRegexp.FindStringSubmatch(spec) + if sm == nil { + return nil, errs.New("invalid version specifier %q", spec) + } + translated, err := translateSpecifier(sm[1], sm[2]) + if err != nil { + return nil, err + } + constraints = append(constraints, translated...) + } + + return &pythonRequirement{Name: m[1], VersionRequirements: strings.Join(constraints, ","), Conditions: conditions}, nil +} + +// translateMarker turns an environment marker made of `and`-joined operating +// system and Python version clauses into platform conditions, which the +// platform also requires all of. Python version clauses are merged into a +// single condition. +func translateMarker(marker string) ([]request.Dependency, error) { + if marker == "" { + return nil, nil + } + if strings.Contains(marker, " or ") || strings.ContainsAny(marker, "()") { + return nil, errs.New("only 'and' is supported in environment markers") + } + + var conditions []request.Dependency + var pythonReqs []string + for _, clause := range strings.Split(marker, " and ") { + clause = strings.TrimSpace(clause) + m := markerRegexp.FindStringSubmatch(clause) + if m == nil { + return nil, errs.New("unsupported environment marker '%s'", clause) + } + variable, op, value := m[1], m[2], m[3] + + switch variable { + case "python_version", "python_full_version": + reqs, err := translatePythonMarker(variable, op, value) + if err != nil { + return nil, err + } + pythonReqs = append(pythonReqs, reqs...) + case "extra": + return nil, errs.New("optional dependencies (extras) are not published") + default: + kernels, ok := markerKernels[variable] + if !ok { + return nil, errs.New("unsupported environment marker '%s'", clause) + } + kernel, ok := kernels[value] + if !ok || op != "==" { + return nil, errs.New("unsupported environment marker '%s'", clause) + } + conditions = append(conditions, request.Dependency{Name: kernel, Namespace: "kernel", VersionRequirements: ">=0"}) + } + } + + if len(pythonReqs) > 0 { + conditions = append(conditions, request.Dependency{ + Name: "python", + Namespace: model.NewNamespaceLanguage().String(), + VersionRequirements: strings.Join(pythonReqs, ","), + }) + } + return conditions, nil +} + +// translatePythonMarker turns a Python version marker clause into version +// constraints. python_version holds only major.minor, so comparisons against it +// are widened to cover every patch release: python_version > "3.9" excludes +// 3.9.18, so it becomes >=3.10. +func translatePythonMarker(variable, op, version string) ([]string, error) { + if op == "!=" || op == "===" || op == "~=" || strings.Contains(version, "*") { + return nil, errs.New("unsupported environment marker '%s %s \"%s\"'", variable, op, version) + } + if variable == "python_full_version" { + return []string{op + version}, nil + } + + next, err := bumpRelease(strings.Split(version, ".")) + if err != nil { + return nil, err + } + switch op { + case ">": + return []string{">=" + next}, nil + case "<=": + return []string{"<" + next}, nil + case "==": + return []string{">=" + version, "<" + next}, nil + } + return []string{op + version}, nil // >= and < already align with release boundaries +} + +// translateSpecifier turns one PEP 440 specifier into platform constraints, +// expanding the compatible-release (~=) and prefix-match (==X.*) forms into +// explicit ranges. +func translateSpecifier(op, version string) ([]string, error) { + switch op { + case ">=", "<=", ">", "<": + return []string{op + version}, nil + case "==": + if prefix, ok := strings.CutSuffix(version, ".*"); ok { + upper, err := bumpRelease(strings.Split(prefix, ".")) + if err != nil { + return nil, err + } + return []string{">=" + prefix, "<" + upper}, nil + } + return []string{"==" + version}, nil + case "~=": + parts := strings.Split(version, ".") + if len(parts) < 2 { + return nil, errs.New("~= requires at least two version components: %q", version) + } + upper, err := bumpRelease(parts[:len(parts)-1]) + if err != nil { + return nil, err + } + return []string{">=" + version, "<" + upper}, nil + } + return nil, errs.New("the %s operator is not supported", op) +} + +// bumpRelease increments the last release component: [1 4] -> "1.5". +func bumpRelease(parts []string) (string, error) { + last, err := strconv.Atoi(parts[len(parts)-1]) + if err != nil { + return "", errs.New("cannot compute an upper bound for version %q", strings.Join(parts, ".")) + } + bumped := append(append([]string{}, parts[:len(parts)-1]...), strconv.Itoa(last+1)) + return strings.Join(bumped, "."), nil +} + +// applyPyprojectDependencies replaces reqVars' dependencies with those translated from pyproject.toml. When none +// remain (there are none, or every one was skipped), the existing dependencies are kept: for --edit those are the +// ones inherited from the previous revision. +func (r *Runner) applyPyprojectDependencies(reqVars *request.PublishVariables, requires []string) error { + deps, err := r.pyprojectDependencies(requires) + if err != nil { + return err + } + if len(deps) > 0 { + reqVars.Dependencies = deps + } + return nil +} + +// pyprojectDependencies translates pyproject.toml dependencies into runtime +// dependencies on the platform's Python packages. Requirements that cannot be +// translated are skipped with a notice so the user can add them with --depend. +func (r *Runner) pyprojectDependencies(requires []string) ([]request.PublishVariableDep, error) { + var reqs []*pythonRequirement + var names []string + for _, raw := range requires { + req, err := parsePythonRequirement(raw) + if err != nil { + r.out.Notice(locale.Tl("publish_pyproject_dep_skipped", + "Skipping pyproject.toml dependency '[ACTIONABLE]{{.V0}}[/RESET]' ({{.V1}}). Add it with '[ACTIONABLE]--depend[/RESET]' if it is required.", + raw, err.Error())) + continue + } + reqs = append(reqs, req) + names = append(names, req.Name) + } + if len(reqs) == 0 { + return nil, nil + } + + ns := model.NewNamespacePackage("python") + normalized, err := model.FetchNormalizedNames(ns, names, r.auth) + if err != nil { + return nil, locale.WrapError(err, "err_publish_pyproject_normalize", "Could not resolve the pyproject.toml dependency names") + } + + deps := make([]request.PublishVariableDep, 0, len(reqs)) + for _, req := range reqs { + deps = append(deps, request.PublishVariableDep{ + Dependency: request.Dependency{ + Name: cmp.Or(normalized[req.Name], req.Name), + Namespace: ns.String(), + VersionRequirements: req.VersionRequirements, + Type: request.DependencyTypeRuntime, + }, + Conditions: req.Conditions, + }) + } + return deps, nil +} diff --git a/internal/runners/publish/pyproject_test.go b/internal/runners/publish/pyproject_test.go new file mode 100644 index 0000000000..3e7248fa41 --- /dev/null +++ b/internal/runners/publish/pyproject_test.go @@ -0,0 +1,94 @@ +package publish + +import ( + "reflect" + "testing" + + "github.com/ActiveState/cli/pkg/platform/api/graphql/request" +) + +func TestParsePythonRequirement(t *testing.T) { + valid := []struct { + in, name, versionReqs string + }{ + {"requests", "requests", ""}, + {"requests>=2.0", "requests", ">=2.0"}, + {"requests >= 2.0, < 3", "requests", ">=2.0,<3"}, + {"requests[security,socks]>=2", "requests", ">=2"}, + {"Typing_Extensions (>=4.0)", "Typing_Extensions", ">=4.0"}, + {"django==4.2.7", "django", "==4.2.7"}, + {"django==4.2.*", "django", ">=4.2,<4.3"}, + {"numpy~=1.24", "numpy", ">=1.24,<2"}, + {"numpy~=1.24.2", "numpy", ">=1.24.2,<1.25"}, + } + for _, tt := range valid { + req, err := parsePythonRequirement(tt.in) + if err != nil { + t.Errorf("%q: unexpected error: %v", tt.in, err) + continue + } + if req.Name != tt.name || req.VersionRequirements != tt.versionReqs { + t.Errorf("%q: got (%q, %q), want (%q, %q)", tt.in, req.Name, req.VersionRequirements, tt.name, tt.versionReqs) + } + } + + invalid := []string{ + `colorama; sys_platform == "win32" or sys_platform == "cygwin"`, + `colorama; (sys_platform == "win32")`, + `pywin32; sys_platform != "win32"`, + `uvloop; platform_machine == "x86_64"`, + `pytest; extra == "test"`, + `tomli; python_version != "3.10"`, + "mypkg @ https://example.com/mypkg-1.0.tar.gz", + "requests!=2.1", + "requests===2.1", + "numpy~=1", + "django==4.x.*", + ">=1.0", + } + for _, in := range invalid { + if req, err := parsePythonRequirement(in); err == nil { + t.Errorf("%q: expected an error, got %+v", in, req) + } + } +} + +func TestParsePythonRequirementMarkers(t *testing.T) { + windows := request.Dependency{Name: "Windows", Namespace: "kernel", VersionRequirements: ">=0"} + linux := request.Dependency{Name: "Linux", Namespace: "kernel", VersionRequirements: ">=0"} + python := func(reqs string) request.Dependency { + return request.Dependency{Name: "python", Namespace: "language", VersionRequirements: reqs} + } + + tests := []struct { + in string + want []request.Dependency + }{ + {`pywin32>=300; sys_platform == "win32"`, []request.Dependency{windows}}, + {`colorama ; platform_system == 'Windows'`, []request.Dependency{windows}}, + {`pywin32; os_name == "nt"`, []request.Dependency{windows}}, + {`secretstorage; sys_platform == "linux"`, []request.Dependency{linux}}, + {`tomli; python_version < "3.11"`, []request.Dependency{python("<3.11")}}, + {`tomli; python_version >= "3.8"`, []request.Dependency{python(">=3.8")}}, + {`tomli; python_version > "3.9"`, []request.Dependency{python(">=3.10")}}, + {`tomli; python_version <= "3.9"`, []request.Dependency{python("<3.10")}}, + {`tomli; python_version == "3.9"`, []request.Dependency{python(">=3.9,<3.10")}}, + {`tomli; python_full_version > "3.9.1"`, []request.Dependency{python(">3.9.1")}}, + {`tomli; python_version >= "3.8" and python_version < "3.11"`, []request.Dependency{python(">=3.8,<3.11")}}, + {`pywin32; sys_platform == "win32" and python_version > "3.9"`, []request.Dependency{windows, python(">=3.10")}}, + } + for _, tt := range tests { + req, err := parsePythonRequirement(tt.in) + if err != nil { + t.Errorf("%q: unexpected error: %v", tt.in, err) + continue + } + if !reflect.DeepEqual(req.Conditions, tt.want) { + t.Errorf("%q: conditions = %+v, want %+v", tt.in, req.Conditions, tt.want) + } + } + + if req, err := parsePythonRequirement("six>=1"); err != nil || req.Conditions != nil { + t.Errorf("unconditional requirement: got %+v, %v", req, err) + } +} diff --git a/internal/runners/publish/readme.md b/internal/runners/publish/readme.md new file mode 100644 index 0000000000..1653291e1c --- /dev/null +++ b/internal/runners/publish/readme.md @@ -0,0 +1,230 @@ +# Publishing Encrypted Private Python Ingredients + +This guide walks through publishing a private Python package to the ActiveState Platform with `state publish --build`, +declaring its dependencies, and changing those dependencies later. + +With `--build`, the State Tool packs your source directory into a wheel and encrypts it with your organization's key +before uploading it. The platform only sees the ingredient's name, namespace, version and dependencies, not your source +code. Projects in your organization decrypt and install it into their Python runtime's `site-packages`. + +## Before You Start + +- You are logged in: `state auth`. +- You are in a checkout of a project owned by your organization (it has an `activestate.yaml`). The project determines + which organization's key encrypts the package. +- Your organization's key service is configured (`state config set privateingredient.key_service_url `, plus the + `privateingredient.*` credential settings your administrator gives you). Without it nothing is uploaded. + +Throughout this guide, `` is your organization name and `mypkg` is the package being published. + +## 1. Lay Out the Source + +Point `--build` at a directory whose children are the importable packages. A `pyproject.toml` beside them is optional, +but it lets the State Tool read the name, version, description and dependencies, so you don't have to pass them as flags. + +``` +mypkg-src/ +├── pyproject.toml +└── mypkg/ + ├── __init__.py + └── core.py +``` + +Only pure-Python packages are supported: compiled files (`.so`, `.pyd`, `.dll`, `.dylib`) are rejected. `__pycache__` +directories, `*.pyc` files and version-control directories are left out of the wheel. + +## 2. Declare Dependencies + +List them in `pyproject.toml` the same way you would for pip: + +```toml +[project] +name = "mypkg" +version = "1.0.0" +description = "Internal helpers for the data team" +dependencies = [ + "requests>=2.31,<3", + "six~=1.16", + 'pywin32>=300; sys_platform == "win32"', + 'tomli>=1; python_version < "3.11"', +] +``` + +Each entry becomes a runtime dependency on the platform's Python package of that name. Version ranges carry over, with +`~=1.16` becoming `>=1.16,<2` and `==4.2.*` becoming `>=4.2,<4.3`. + +Platform and Python version markers become **conditions**, so the dependency is only installed where it applies: + +| Marker | Installed only on | +|-------------------------------------------------------------------------------|--------------------------| +| `sys_platform == "win32"`, `platform_system == "Windows"`, `os_name == "nt"` | Windows | +| `sys_platform == "linux"`, `platform_system == "Linux"` | Linux | +| `sys_platform == "darwin"`, `platform_system == "Darwin"` | macOS | +| `python_version < "3.11"` (also `<=`, `>`, `>=`, `==`) | matching Python versions | +| `python_full_version >= "3.9.2"` | matching Python versions | + +Several of these can be combined with `and`, e.g. `sys_platform == "win32" and python_version >= "3.10"`. + +Anything else is skipped with a notice when you publish: markers using `or`, parentheses or `!=`, other markers such as +`platform_machine`, optional dependencies (`extra == "..."`), and direct URL references (`pkg @ https://...`). Add a +skipped dependency by hand with `--depend` or a meta file (see step 5) if it is required. + +## 3. Publish + +From the project checkout: + +``` +state publish --build ./mypkg-src --namespace private//python +``` + +The namespace must be under `private/`, the organization that owns the project. Before uploading, the State Tool +shows the ingredient it is about to publish and asks for confirmation: + +``` +Skipping pyproject.toml dependency 'colorama; platform_machine == "x86_64"' (unsupported environment marker +'platform_machine == "x86_64"'). Add it with '--depend' if it is required. + +Prepared the following ingredient: + +name: mypkg +namespace: private//python +version: 1.0.0 +description: Internal helpers for the data team +dependencies: + - dependency: + name: requests + namespace: language/python + versionRequirements: '>=2.31,<3' + type: runtime + - dependency: + name: pywin32 + namespace: language/python + versionRequirements: '>=300' + type: runtime + conditions: + - name: Windows + namespace: kernel + versionRequirements: '>=0' + ... + - dependency: + name: private-builder + namespace: builder + versionRequirements: '>=0' + +Do you want to publish this ingredient? (Y/n) +``` + +Check the dependency list here: it is exactly what will be published. The `private-builder` dependency is always added; +it is what decrypts and installs the package, so leave it in place. + +Flags override `pyproject.toml`: `--name`, `--version` and `--description` replace the values in it, and any +`--depend`, `--depend-runtime`, `--depend-build` or `--depend-test` flag replaces its whole dependency list. + +## 4. Install and Check + +In any project of the same organization: + +``` +state install private//python:mypkg --ts now +state exec python3 -- -c "import mypkg" +``` + +`--ts now` makes the solver pick up the version you just published. The install summary lists the dependencies that +were pulled in, and conditional dependencies only appear on matching platforms. + +The machine installing the package needs the same key service settings as the one that published it, because the +package is decrypted during installation. + +## 5. Change the Dependencies of a Published Version + +To change the dependencies of a version that is already published, export its meta file, edit it, and publish it back +as a new revision with `--edit`. + +**Export the current meta file:** + +``` +state export ingredient-meta private//python/mypkg@1.0.0 > meta.yaml +``` + +Leave out `@1.0.0` to export the latest version. The file holds the ingredient's name, namespace, version and its +complete dependency list: + +```yaml +name: mypkg +namespace: private//python +version: 1.0.0 +description: "" +dependencies: + - dependency: + name: private-builder + namespace: builder + versionRequirements: '>=0' + type: build + conditions: [] + - dependency: + name: requests + namespace: language/python + versionRequirements: '>=2.31,<3' + type: runtime + conditions: [] + ... +``` + +**Edit the `dependencies` list.** Change version ranges, remove entries, or add new ones. Each entry nests its fields +under `dependency:`, with optional `conditions:` beside it: + +```yaml + - dependency: + name: urllib3 + namespace: language/python + versionRequirements: '>=2' + type: runtime + conditions: + - name: Linux + namespace: kernel + versionRequirements: '>=0' +``` + +For conditions, the kernel names are `Windows`, `Linux` and `Darwin` (namespace `kernel`). A Python version condition +is `name: python`, `namespace: language`, with a requirement such as `<3.11`. + +**Publish the revision:** + +``` +state publish --edit --build ./mypkg-src --meta meta.yaml +``` + +Things to know about `--edit`: + +- **The meta file replaces the whole dependency list.** Anything you delete from it is removed from the ingredient. + Dependencies in `pyproject.toml` are not used when the meta file lists its own. If neither the meta file nor + `pyproject.toml` lists any, the previous revision's dependencies are kept. +- **Name, namespace and version identify what you are editing; they cannot be changed.** To publish under a new + version, see step 6. +- **Authors and description cannot be changed** on an existing version. The exported file leaves authors out and the + description empty. If you fill the description in, the publish is refused with "Authors and description cannot be + changed when editing an existing version." +- The `language/python` dependencies the platform adds by itself come back automatically, so it does no harm if they + are missing from your file. + +Run the export again afterwards to confirm the change. + +## 6. Publish a New Version + +Bump the version in `pyproject.toml` (or pass `--version`) and publish as in step 3, without `--edit`. A new version +takes its dependencies from `pyproject.toml`, flags and `--meta` as usual and does not inherit them from the previous +version. + +Publishing a version that already exists without `--edit` is refused, so an existing version can't be overwritten by +accident. + +## Troubleshooting + +| Message | What to do | +|-------------------------------------------------------------------------|------------------------------------------------------------------------------------------------| +| `Could not determine the ingredient name and version` | Add `name` and `version` to `pyproject.toml`, or pass `--name` and `--version`. | +| `The '--build' flag requires a namespace under 'private/'` | Use a namespace under the organization that owns the project you are in. | +| `org key does not belong to this project's organization` | The key service returned another organization's key. Check that you are in the right project. | +| `No organization key service is configured` | Configure the `privateingredient.*` settings (see Before You Start). | +| `Ingredient ... already exists. To edit an existing ingredient ...` | Pass `--edit` to change an existing version, or bump the version to publish a new one. | +| `Skipping pyproject.toml dependency ...` | The marker is not supported. Add the dependency with `--depend` or in a meta file. | diff --git a/internal/smartlink/smartlink.go b/internal/smartlink/smartlink.go index c101abde5e..afe0591fae 100644 --- a/internal/smartlink/smartlink.go +++ b/internal/smartlink/smartlink.go @@ -1,8 +1,10 @@ package smartlink import ( + "io/fs" "os" "path/filepath" + "runtime" "github.com/ActiveState/cli/internal/errs" "github.com/ActiveState/cli/internal/fileutils" @@ -28,8 +30,12 @@ func LinkContents(src, dest string) error { if err != nil { return errs.Wrap(err, "Reading dir %s failed", src) } + // src and dest are already resolved above, so recurse via link() which does not + // re-resolve each entry. On Windows resolving a path is a syscall (GetLongPathName), + // so resolving once per tree instead of once per file is a large speed-up when + // installing runtimes that contain many files. for _, entry := range entries { - if err := Link(filepath.Join(src, entry.Name()), filepath.Join(dest, entry.Name())); err != nil { + if err := link(filepath.Join(src, entry.Name()), filepath.Join(dest, entry.Name()), entry.Type()); err != nil { return errs.Wrap(err, "Link failed") } } @@ -40,25 +46,65 @@ func LinkContents(src, dest string) error { // Link creates a link from src to target. MS decided to support Symlinks but only if you opt into developer mode (go figure), // which we cannot reasonably force on our users. So on Windows we will instead create dirs and hardlinks. func Link(src, dest string) error { - originalSrc := src + resolvedDest, err := fileutils.ResolveUniquePath(dest) + if err != nil { + return errs.Wrap(err, "Could not resolve dest path") + } - var err error - src, dest, err = resolvePaths(src, dest) + // Resolve the parent of src rather than src itself and rejoin the base name. ResolveUniquePath + // dereferences symlinks, but link() needs to see whether the leaf is itself a symlink, so we must + // not dereference it here. For a non-symlink leaf this is equivalent to resolving src directly. + srcParent, err := fileutils.ResolveUniquePath(filepath.Dir(src)) if err != nil { - return errs.Wrap(err, "Could not resolve src and dest paths") + return errs.Wrap(err, "Could not resolve src path") } + src = filepath.Join(srcParent, filepath.Base(src)) + + info, err := os.Lstat(src) + if err != nil { + return errs.Wrap(err, "Could not stat src path") + } + destDir := filepath.Dir(resolvedDest) + if err := fileutils.MkdirUnlessExists(destDir); err != nil { + return errs.Wrap(err, "could not create directory %s", destDir) + } + + return link(src, resolvedDest, info.Mode().Type()) +} - if fileutils.IsDir(src) { - if fileutils.IsSymlink(originalSrc) { - // If the original src is a symlink, the resolved src is no longer a symlink and could point +// link recursively links src into dest. Unlike Link, it assumes src and dest are already resolved to +// unique paths and does NOT re-resolve them for every entry it visits. Path resolution is a syscall +// per path on Windows (GetLongPathName), so resolving once per tree rather than once per file makes +// installing runtimes with many files dramatically faster. Descendant paths are constructed by joining +// already-resolved parents with real (long) entry names, so they need no further resolution. +// +// typ is src's type as reported by Lstat (or by the directory listing it came from), and dest's parent +// directory must already exist. Together these mean a regular file costs a single syscall: the link itself. +func link(src, dest string, typ fs.FileMode) error { + if typ&fs.ModeSymlink != 0 { + if fileutils.IsDir(src) { + // If src is a symlink, the resolved src is no longer a symlink and could point // to a parent directory, resulting in a recursive directory structure. - // Avoid any potential problems by simply linking the original link to the target. + // Avoid any potential problems by simply linking the symlink to the target. // Links to directories are okay on Linux and macOS, but will fail on Windows. // If we ever get here on Windows, the artifact being deployed is bad and there's nothing we // can do about it except receive the report from Rollbar and report it internally. - return linkFile(originalSrc, dest) + if runtime.GOOS == "windows" { + return errs.New("src is a directory, not a file: %s", src) + } + return linkFile(src, dest) } + // A symlink whose target is a file: link to the resolved target to preserve pre-existing behavior. + resolvedSrc, err := fileutils.ResolveUniquePath(src) + if err != nil { + return errs.Wrap(err, "could not resolve src path %s", src) + } + src = resolvedSrc + typ = 0 + } + + if typ.IsDir() { if err := fileutils.Mkdir(dest); err != nil { return errs.Wrap(err, "could not create directory %s", dest) } @@ -67,26 +113,21 @@ func Link(src, dest string) error { return errs.Wrap(err, "could not read directory %s", src) } for _, entry := range entries { - if err := Link(filepath.Join(src, entry.Name()), filepath.Join(dest, entry.Name())); err != nil { + if err := link(filepath.Join(src, entry.Name()), filepath.Join(dest, entry.Name()), entry.Type()); err != nil { return errs.Wrap(err, "sub link failed") } } return nil } - destDir := filepath.Dir(dest) - if err := fileutils.MkdirUnlessExists(destDir); err != nil { - return errs.Wrap(err, "could not create directory %s", destDir) - } - - // Multiple artifacts can supply the same file. We do not have a better solution for this at the moment other than - // favouring the first one encountered. - if fileutils.TargetExists(dest) { - logging.Warning("Skipping linking '%s' to '%s' as it already exists", src, dest) - return nil - } - if err := linkFile(src, dest); err != nil { + // Multiple artifacts can supply the same file. We do not have a better solution for this at the moment other + // than favouring the first one encountered. Artifacts may be linked concurrently (the caller must not run + // copies at the same time; see depot.DeployViaLink), so whichever links the file first wins. + if os.IsExist(err) { + logging.Warning("Skipping linking '%s' to '%s' as it already exists", src, dest) + return nil + } return errs.Wrap(err, "could not link %s to %s", src, dest) } return nil diff --git a/internal/smartlink/smartlink_test.go b/internal/smartlink/smartlink_test.go index 50a772254f..af21226b4d 100644 --- a/internal/smartlink/smartlink_test.go +++ b/internal/smartlink/smartlink_test.go @@ -1,9 +1,11 @@ package smartlink import ( + "fmt" "os" "path/filepath" "runtime" + "sync" "testing" "github.com/ActiveState/cli/internal/fileutils" @@ -73,3 +75,108 @@ func TestLinkContentsWithCircularLink(t *testing.T) { } require.Equal(t, target, srcCircular) } + +func TestLinkSkipsExistingTarget(t *testing.T) { + srcDir, destDir := t.TempDir(), t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(srcDir, "file.txt"), []byte("new"), 0644)) + require.NoError(t, os.WriteFile(filepath.Join(destDir, "file.txt"), []byte("existing"), 0644)) + + require.NoError(t, Link(filepath.Join(srcDir, "file.txt"), filepath.Join(destDir, "file.txt"))) + + content, err := os.ReadFile(filepath.Join(destDir, "file.txt")) + require.NoError(t, err) + require.Equal(t, "existing", string(content), "an existing target must be left intact") +} + +// TestLinkContentsConcurrentSameDest links several sources that provide the same files into one +// destination at once, as concurrent artifact deployments do. Each file must be linked exactly once +// (first one wins) without any of the deployments failing. +func TestLinkContentsConcurrentSameDest(t *testing.T) { + const sources, files = 8, 50 + destDir := t.TempDir() + srcDirs := make([]string, sources) + for i := range srcDirs { + srcDirs[i] = t.TempDir() + for f := 0; f < files; f++ { + path := filepath.Join(srcDirs[i], "sub", fmt.Sprintf("f%02d.txt", f)) + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0755)) + require.NoError(t, os.WriteFile(path, []byte(fmt.Sprintf("src%d", i)), 0644)) + } + } + + var wg sync.WaitGroup + errs := make([]error, sources) + for i, src := range srcDirs { + wg.Add(1) + go func() { + defer wg.Done() + errs[i] = LinkContents(src, destDir) + }() + } + wg.Wait() + for i, err := range errs { + require.NoError(t, err, "deployment %d failed", i) + } + + for f := 0; f < files; f++ { + content, err := os.ReadFile(filepath.Join(destDir, "sub", fmt.Sprintf("f%02d.txt", f))) + require.NoError(t, err) + require.Regexp(t, `^src\d$`, string(content)) + } + for i, src := range srcDirs { + content, err := os.ReadFile(filepath.Join(src, "sub", "f00.txt")) + require.NoError(t, err) + require.Equal(t, fmt.Sprintf("src%d", i), string(content), "a source file was modified") + } +} + +// TestLinkContentsEquivalence pins down how each kind of entry is linked: regular files and the +// contents of regular directories link to the file in src, a symlink to a file links to the file it +// points at, and a symlink to a directory is linked as the symlink itself (see Link). +func TestLinkContentsEquivalence(t *testing.T) { + srcDir, destDir := t.TempDir(), t.TempDir() + srcDir, err := fileutils.ResolveUniquePath(srcDir) // resolve $TMPDIR symlinks on macOS + require.NoError(t, err) + + // src/ + // ├── top.txt + // ├── dir/nested/deep.txt + // ├── file-link -> dir/nested/deep.txt + // └── dir-link -> dir/nested + require.NoError(t, os.WriteFile(filepath.Join(srcDir, "top.txt"), []byte("top"), 0644)) + require.NoError(t, os.MkdirAll(filepath.Join(srcDir, "dir", "nested"), 0755)) + deep := filepath.Join(srcDir, "dir", "nested", "deep.txt") + require.NoError(t, os.WriteFile(deep, []byte("deep"), 0644)) + withSymlinks := runtime.GOOS != "windows" // creating symlinks needs privileges on Windows + if withSymlinks { + require.NoError(t, os.Symlink(deep, filepath.Join(srcDir, "file-link"))) + require.NoError(t, os.Symlink(filepath.Join(srcDir, "dir", "nested"), filepath.Join(srcDir, "dir-link"))) + } + + require.NoError(t, LinkContents(srcDir, destDir)) + + sameFile := func(a, b string) { + t.Helper() + ai, err := os.Stat(a) + require.NoError(t, err) + bi, err := os.Stat(b) + require.NoError(t, err) + require.True(t, os.SameFile(ai, bi), "%s should be linked to %s", a, b) + } + sameFile(filepath.Join(destDir, "top.txt"), filepath.Join(srcDir, "top.txt")) + sameFile(filepath.Join(destDir, "dir", "nested", "deep.txt"), deep) + require.True(t, fileutils.IsDir(filepath.Join(destDir, "dir", "nested"))) + require.False(t, fileutils.IsSymlink(filepath.Join(destDir, "dir")), "regular directories are recreated, not linked") + + if withSymlinks { + // A symlink to a file links to the file it points at. + target, err := os.Readlink(filepath.Join(destDir, "file-link")) + require.NoError(t, err) + require.Equal(t, deep, target) + + // A symlink to a directory is linked as the symlink itself, not followed. + target, err = os.Readlink(filepath.Join(destDir, "dir-link")) + require.NoError(t, err) + require.Equal(t, filepath.Join(srcDir, "dir-link"), target) + } +} diff --git a/internal/smartlink/smartlink_windows.go b/internal/smartlink/smartlink_windows.go index 1c86110cf7..9b2fccf424 100644 --- a/internal/smartlink/smartlink_windows.go +++ b/internal/smartlink/smartlink_windows.go @@ -2,16 +2,11 @@ package smartlink import ( "os" - - "github.com/ActiveState/cli/internal/errs" - "github.com/ActiveState/cli/internal/fileutils" ) // linkFile will create a symlink from src to dest, and falls back on a hardlink if no symlink is available. // This is a workaround for the fact that Windows does not support symlinks without admin privileges. +// src must not be a directory: hard links to directories are not supported. func linkFile(src, dest string) error { - if fileutils.IsDir(src) { - return errs.New("src is a directory, not a file: %s", src) - } return os.Link(src, dest) } diff --git a/pkg/platform/model/inventory.go b/pkg/platform/model/inventory.go index 2ecfbfffde..81ea987052 100644 --- a/pkg/platform/model/inventory.go +++ b/pkg/platform/model/inventory.go @@ -650,19 +650,33 @@ func FetchLatestRevisionTimeStamp(auth *authentication.Auth) (time.Time, error) } func FetchNormalizedName(namespace Namespace, name string, auth *authentication.Auth) (string, error) { + normalized, err := FetchNormalizedNames(namespace, []string{name}, auth) + if err != nil { + return "", errs.Wrap(err, "NormalizeName failed") + } + if _, ok := normalized[name]; !ok { + return "", errs.New("Normalized name for %s not found", name) + } + return normalized[name], nil +} + +// FetchNormalizedNames normalizes names within namespace in a single request, +// returning a map from each requested name to its normalized form. +func FetchNormalizedNames(namespace Namespace, names []string, auth *authentication.Auth) (map[string]string, error) { client := inventory.Get(auth) params := inventory_operations.NewNormalizeNamesParams() params.SetNamespace(namespace.String()) - params.SetNames(&inventory_models.UnnormalizedNames{Names: []string{name}}) + params.SetNames(&inventory_models.UnnormalizedNames{Names: names}) params.SetHTTPClient(api.NewHTTPClient()) res, err := client.NormalizeNames(params, auth.ClientAuth()) if err != nil { - return "", errs.Wrap(err, "NormalizeName failed") + return nil, errs.Wrap(err, "NormalizeNames failed") } - if len(res.Payload.NormalizedNames) == 0 { - return "", errs.New("Normalized name for %s not found", name) + normalized := make(map[string]string, len(res.Payload.NormalizedNames)) + for _, n := range res.Payload.NormalizedNames { + normalized[*n.RequestedAs] = *n.Normalized } - return *res.Payload.NormalizedNames[0].Normalized, nil + return normalized, nil } func FilterCurrentPlatform(hostPlatform string, platforms []strfmt.UUID, preferredLibcVersion string) (strfmt.UUID, error) { diff --git a/pkg/platform/model/version_constraints.go b/pkg/platform/model/version_constraints.go index 62fa9ce5b7..0348cc7605 100644 --- a/pkg/platform/model/version_constraints.go +++ b/pkg/platform/model/version_constraints.go @@ -21,9 +21,13 @@ func InventoryRequirementsToString(requirements inventory_models.Requirements, f return "" } - constraints := make([]*versionConstraints, len(requirements)) - for i, req := range requirements { - constraints[i] = &versionConstraints{*req.Comparator, *req.Version} + constraints := make([]*versionConstraints, 0, len(requirements)) + for _, req := range requirements { + // A requirement without a version (e.g. on a condition) places no constraint. + if req.Comparator == nil || req.Version == nil { + continue + } + constraints = append(constraints, &versionConstraints{*req.Comparator, *req.Version}) } return versionConstraintsToString(constraints, forAPI) } diff --git a/pkg/platform/model/version_constraints_test.go b/pkg/platform/model/version_constraints_test.go new file mode 100644 index 0000000000..0666f63a10 --- /dev/null +++ b/pkg/platform/model/version_constraints_test.go @@ -0,0 +1,39 @@ +package model + +import ( + "testing" + + "github.com/ActiveState/cli/internal/rtutils/ptr" + "github.com/ActiveState/cli/pkg/platform/api/inventory/inventory_models" +) + +func TestInventoryRequirementsToString(t *testing.T) { + gte := inventory_models.RequirementComparatorGte + lt := inventory_models.RequirementComparatorLt + + tests := []struct { + name string + reqs inventory_models.Requirements + want string + }{ + {"nil", nil, ""}, + {"range", inventory_models.Requirements{ + {Comparator: >e, Version: ptr.To("1.16")}, + {Comparator: <, Version: ptr.To("2")}, + }, ">=1.16,<2"}, + // Some platform ingredients carry condition requirements without a version (e.g. click's + // language/alternative-built-language condition); they place no constraint and must not panic. + {"missing version is skipped", inventory_models.Requirements{ + {Comparator: >e}, + {Comparator: <, Version: ptr.To("3.11")}, + }, "<3.11"}, + {"missing comparator is skipped", inventory_models.Requirements{{Version: ptr.To("1.0")}}, ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := InventoryRequirementsToString(tt.reqs, true); got != tt.want { + t.Errorf("got %q, want %q", got, tt.want) + } + }) + } +} diff --git a/pkg/runtime/depot.go b/pkg/runtime/depot.go index a558400442..1d45c82de2 100644 --- a/pkg/runtime/depot.go +++ b/pkg/runtime/depot.go @@ -77,7 +77,9 @@ type depot struct { config depotConfig depotPath string artifacts map[strfmt.UUID]struct{} - fsMutex sync.Mutex + // fsMutex guards writes to the depot and to runtime directories. Linking holds it for reading, so artifacts can + // be linked concurrently; everything that writes file contents (copying, Put, Undeploy) holds it for writing. + fsMutex sync.RWMutex mapMutex sync.Mutex cacheSize int64 } @@ -214,9 +216,15 @@ func (d *depot) MarkPrivate(id strfmt.UUID) error { // DeployViaLink will take an artifact from the depot and link it to the target path. // It should return deployment info to be used for tracking the artifact. +// +// It holds d.fsMutex for reading, so multiple artifacts can be linked into the runtime concurrently (the +// install worker pool runs several at once). Concurrent links are safe with each other: creating a link never +// writes through an existing file, directory creation is idempotent, and smartlink skips a target that already +// exists, so a file provided by several artifacts is linked once. Copies must not run at the same time, because +// a copy would write through a link created concurrently into another artifact's depot files. func (d *depot) DeployViaLink(id strfmt.UUID, relativeSrc, absoluteDest string) (*deployment, error) { - d.fsMutex.Lock() - defer d.fsMutex.Unlock() + d.fsMutex.RLock() + defer d.fsMutex.RUnlock() if exists, _ := d.Exists(id); !exists { return nil, errs.New("artifact not found in depot") @@ -262,6 +270,10 @@ func (d *depot) DeployViaLink(id strfmt.UUID, relativeSrc, absoluteDest string) // DeployViaCopy will take an artifact from the depot and copy it to the target path. // It should return deployment info to be used for tracking the artifact. +// +// It holds d.fsMutex for writing. Copying writes file contents, so it must not overlap with another copy or +// with a link: if a link to another artifact's depot file appeared at a destination after the copy checked +// that it was free, the copy would write through that link and overwrite the other artifact's depot files. func (d *depot) DeployViaCopy(id strfmt.UUID, relativeSrc, absoluteDest string) (*deployment, error) { d.fsMutex.Lock() defer d.fsMutex.Unlock() @@ -293,7 +305,7 @@ func (d *depot) DeployViaCopy(id strfmt.UUID, relativeSrc, absoluteDest string) if err := fileutils.CopyFiles(absoluteSrc, absoluteDest); err != nil { var errExist *fileutils.ErrAlreadyExist if errors.As(err, &errExist) { - logging.Warning("Skipping files that already exist: " + errs.JoinMessage(errExist)) + logging.Warning("Skipping files that already exist: %s", errs.JoinMessage(errExist)) } else { return nil, errs.Wrap(err, "failed to copy artifact") } diff --git a/pkg/runtime/depot_test.go b/pkg/runtime/depot_test.go new file mode 100644 index 0000000000..6409c4c21a --- /dev/null +++ b/pkg/runtime/depot_test.go @@ -0,0 +1,70 @@ +package runtime + +import ( + "fmt" + "os" + "path/filepath" + "sync" + "testing" + + "github.com/go-openapi/strfmt" + "github.com/stretchr/testify/require" +) + +// newTestDepot returns a depot rooted at dir holding the given artifacts, each with files under +// installdir/shared/ whose content is the artifact's ID. +func newTestDepot(t *testing.T, dir string, files int, ids ...strfmt.UUID) *depot { + t.Helper() + d := &depot{ + depotPath: dir, + artifacts: map[strfmt.UUID]struct{}{}, + config: depotConfig{Deployments: map[strfmt.UUID][]deployment{}, Cache: map[strfmt.UUID]*artifactInfo{}}, + } + for _, id := range ids { + d.artifacts[id] = struct{}{} + for f := 0; f < files; f++ { + path := filepath.Join(d.Path(id), "installdir", "shared", fmt.Sprintf("f%03d.txt", f)) + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0755)) + require.NoError(t, os.WriteFile(path, []byte(id), 0644)) + } + } + return d +} + +// TestConcurrentCopyAndLinkKeepDepotIntact deploys two artifacts that provide the same files into one +// runtime at the same time, one by copy and one by link. Linking creates links back into the depot, so +// if the copy wrote through a link the other artifact had just created, it would overwrite that +// artifact's files in the depot, corrupting the cache for every runtime that uses it. +func TestConcurrentCopyAndLinkKeepDepotIntact(t *testing.T) { + const files, iterations = 200, 20 + copied := strfmt.UUID("00000000-0000-0000-0000-00000000000a") + linked := strfmt.UUID("00000000-0000-0000-0000-00000000000b") + root := t.TempDir() + d := newTestDepot(t, filepath.Join(root, "depot"), files, copied, linked) + + for i := 0; i < iterations; i++ { + dest := filepath.Join(root, fmt.Sprintf("runtime%d", i)) + var wg sync.WaitGroup + var copyErr, linkErr error + wg.Add(2) + go func() { + defer wg.Done() + _, copyErr = d.DeployViaCopy(copied, "installdir", dest) + }() + go func() { + defer wg.Done() + _, linkErr = d.DeployViaLink(linked, "installdir", dest) + }() + wg.Wait() + require.NoError(t, copyErr) + require.NoError(t, linkErr) + + for _, id := range []strfmt.UUID{copied, linked} { + for f := 0; f < files; f++ { + content, err := os.ReadFile(filepath.Join(d.Path(id), "installdir", "shared", fmt.Sprintf("f%03d.txt", f))) + require.NoError(t, err) + require.Equal(t, string(id), string(content), "depot file f%03d.txt of %s was overwritten (iteration %d)", f, id, i) + } + } + } +} diff --git a/test/integration/publish_int_test.go b/test/integration/publish_int_test.go index f43ee0315d..d125553263 100644 --- a/test/integration/publish_int_test.go +++ b/test/integration/publish_int_test.go @@ -518,7 +518,13 @@ func (suite *PublishIntegrationTestSuite) TestPublishBuildEncrypted() { suite.Require().NoError(os.MkdirAll(filepath.Join(srcDir, "greeting"), 0755)) suite.Require().NoError(fileutils.WriteFile( filepath.Join(srcDir, "greeting", "__init__.py"), - []byte(fmt.Sprintf("print(%q)\n", sentinel)), + []byte(fmt.Sprintf("import six\nprint(%q)\n", sentinel)), + )) + // A pyproject.toml dependency must be published as an ingredient dependency + // and installed alongside the private ingredient, or the import above fails. + suite.Require().NoError(fileutils.WriteFile( + filepath.Join(srcDir, "pyproject.toml"), + []byte("[project]\ndependencies = [\"six>=1\"]\n"), )) // Create a fresh project under the testing org. `state publish --build` diff --git a/vendor/github.com/klauspost/compress/.gitattributes b/vendor/github.com/klauspost/compress/.gitattributes index 402433593c..57aa6487c5 100644 --- a/vendor/github.com/klauspost/compress/.gitattributes +++ b/vendor/github.com/klauspost/compress/.gitattributes @@ -1,2 +1,3 @@ * -text *.bin -text -diff +*.md text eol=lf diff --git a/vendor/github.com/klauspost/compress/.goreleaser.yml b/vendor/github.com/klauspost/compress/.goreleaser.yml index 4528059ca6..804a201816 100644 --- a/vendor/github.com/klauspost/compress/.goreleaser.yml +++ b/vendor/github.com/klauspost/compress/.goreleaser.yml @@ -31,6 +31,9 @@ builds: - mips64le goarm: - 7 + ignore: + - goos: windows + goarch: arm - id: "s2d" binary: s2d @@ -57,6 +60,9 @@ builds: - mips64le goarm: - 7 + ignore: + - goos: windows + goarch: arm - id: "s2sx" binary: s2sx @@ -84,6 +90,9 @@ builds: - mips64le goarm: - 7 + ignore: + - goos: windows + goarch: arm archives: - @@ -91,7 +100,7 @@ archives: name_template: "s2-{{ .Os }}_{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}" format_overrides: - goos: windows - format: zip + formats: ['zip'] files: - unpack/* - s2/LICENSE diff --git a/vendor/github.com/klauspost/compress/README.md b/vendor/github.com/klauspost/compress/README.md index 244ee19c4b..fb023f2cf2 100644 --- a/vendor/github.com/klauspost/compress/README.md +++ b/vendor/github.com/klauspost/compress/README.md @@ -1,671 +1,700 @@ -# compress - -This package provides various compression algorithms. - -* [zstandard](https://github.com/klauspost/compress/tree/master/zstd#zstd) compression and decompression in pure Go. -* [S2](https://github.com/klauspost/compress/tree/master/s2#s2-compression) is a high performance replacement for Snappy. -* Optimized [deflate](https://godoc.org/github.com/klauspost/compress/flate) packages which can be used as a dropin replacement for [gzip](https://godoc.org/github.com/klauspost/compress/gzip), [zip](https://godoc.org/github.com/klauspost/compress/zip) and [zlib](https://godoc.org/github.com/klauspost/compress/zlib). -* [snappy](https://github.com/klauspost/compress/tree/master/snappy) is a drop-in replacement for `github.com/golang/snappy` offering better compression and concurrent streams. -* [huff0](https://github.com/klauspost/compress/tree/master/huff0) and [FSE](https://github.com/klauspost/compress/tree/master/fse) implementations for raw entropy encoding. -* [gzhttp](https://github.com/klauspost/compress/tree/master/gzhttp) Provides client and server wrappers for handling gzipped requests efficiently. -* [pgzip](https://github.com/klauspost/pgzip) is a separate package that provides a very fast parallel gzip implementation. - -[![Go Reference](https://pkg.go.dev/badge/klauspost/compress.svg)](https://pkg.go.dev/github.com/klauspost/compress?tab=subdirectories) -[![Go](https://github.com/klauspost/compress/actions/workflows/go.yml/badge.svg)](https://github.com/klauspost/compress/actions/workflows/go.yml) -[![Sourcegraph Badge](https://sourcegraph.com/github.com/klauspost/compress/-/badge.svg)](https://sourcegraph.com/github.com/klauspost/compress?badge) - -# package usage - -Use `go get github.com/klauspost/compress@latest` to add it to your project. - -This package will support the current Go version and 2 versions back. - -* Use the `nounsafe` tag to disable all use of the "unsafe" package. -* Use the `noasm` tag to disable all assembly across packages. - -Use the links above for more information on each. - -# changelog - -* Feb 19th, 2025 - [1.18.0](https://github.com/klauspost/compress/releases/tag/v1.18.0) - * Add unsafe little endian loaders https://github.com/klauspost/compress/pull/1036 - * fix: check `r.err != nil` but return a nil value error `err` by @alingse in https://github.com/klauspost/compress/pull/1028 - * flate: Simplify L4-6 loading https://github.com/klauspost/compress/pull/1043 - * flate: Simplify matchlen (remove asm) https://github.com/klauspost/compress/pull/1045 - * s2: Improve small block compression speed w/o asm https://github.com/klauspost/compress/pull/1048 - * flate: Fix matchlen L5+L6 https://github.com/klauspost/compress/pull/1049 - * flate: Cleanup & reduce casts https://github.com/klauspost/compress/pull/1050 - -* Oct 11th, 2024 - [1.17.11](https://github.com/klauspost/compress/releases/tag/v1.17.11) - * zstd: Fix extra CRC written with multiple Close calls https://github.com/klauspost/compress/pull/1017 - * s2: Don't use stack for index tables https://github.com/klauspost/compress/pull/1014 - * gzhttp: No content-type on no body response code by @juliens in https://github.com/klauspost/compress/pull/1011 - * gzhttp: Do not set the content-type when response has no body by @kevinpollet in https://github.com/klauspost/compress/pull/1013 - -* Sep 23rd, 2024 - [1.17.10](https://github.com/klauspost/compress/releases/tag/v1.17.10) - * gzhttp: Add TransportAlwaysDecompress option. https://github.com/klauspost/compress/pull/978 - * gzhttp: Add supported decompress request body by @mirecl in https://github.com/klauspost/compress/pull/1002 - * s2: Add EncodeBuffer buffer recycling callback https://github.com/klauspost/compress/pull/982 - * zstd: Improve memory usage on small streaming encodes https://github.com/klauspost/compress/pull/1007 - * flate: read data written with partial flush by @vajexal in https://github.com/klauspost/compress/pull/996 - -* Jun 12th, 2024 - [1.17.9](https://github.com/klauspost/compress/releases/tag/v1.17.9) - * s2: Reduce ReadFrom temporary allocations https://github.com/klauspost/compress/pull/949 - * flate, zstd: Shave some bytes off amd64 matchLen by @greatroar in https://github.com/klauspost/compress/pull/963 - * Upgrade zip/zlib to 1.22.4 upstream https://github.com/klauspost/compress/pull/970 https://github.com/klauspost/compress/pull/971 - * zstd: BuildDict fails with RLE table https://github.com/klauspost/compress/pull/951 - -* Apr 9th, 2024 - [1.17.8](https://github.com/klauspost/compress/releases/tag/v1.17.8) - * zstd: Reject blocks where reserved values are not 0 https://github.com/klauspost/compress/pull/885 - * zstd: Add RLE detection+encoding https://github.com/klauspost/compress/pull/938 - -* Feb 21st, 2024 - [1.17.7](https://github.com/klauspost/compress/releases/tag/v1.17.7) - * s2: Add AsyncFlush method: Complete the block without flushing by @Jille in https://github.com/klauspost/compress/pull/927 - * s2: Fix literal+repeat exceeds dst crash https://github.com/klauspost/compress/pull/930 - -* Feb 5th, 2024 - [1.17.6](https://github.com/klauspost/compress/releases/tag/v1.17.6) - * zstd: Fix incorrect repeat coding in best mode https://github.com/klauspost/compress/pull/923 - * s2: Fix DecodeConcurrent deadlock on errors https://github.com/klauspost/compress/pull/925 - -* Jan 26th, 2024 - [v1.17.5](https://github.com/klauspost/compress/releases/tag/v1.17.5) - * flate: Fix reset with dictionary on custom window encodes https://github.com/klauspost/compress/pull/912 - * zstd: Add Frame header encoding and stripping https://github.com/klauspost/compress/pull/908 - * zstd: Limit better/best default window to 8MB https://github.com/klauspost/compress/pull/913 - * zstd: Speed improvements by @greatroar in https://github.com/klauspost/compress/pull/896 https://github.com/klauspost/compress/pull/910 - * s2: Fix callbacks for skippable blocks and disallow 0xfe (Padding) by @Jille in https://github.com/klauspost/compress/pull/916 https://github.com/klauspost/compress/pull/917 -https://github.com/klauspost/compress/pull/919 https://github.com/klauspost/compress/pull/918 - -* Dec 1st, 2023 - [v1.17.4](https://github.com/klauspost/compress/releases/tag/v1.17.4) - * huff0: Speed up symbol counting by @greatroar in https://github.com/klauspost/compress/pull/887 - * huff0: Remove byteReader by @greatroar in https://github.com/klauspost/compress/pull/886 - * gzhttp: Allow overriding decompression on transport https://github.com/klauspost/compress/pull/892 - * gzhttp: Clamp compression level https://github.com/klauspost/compress/pull/890 - * gzip: Error out if reserved bits are set https://github.com/klauspost/compress/pull/891 - -* Nov 15th, 2023 - [v1.17.3](https://github.com/klauspost/compress/releases/tag/v1.17.3) - * fse: Fix max header size https://github.com/klauspost/compress/pull/881 - * zstd: Improve better/best compression https://github.com/klauspost/compress/pull/877 - * gzhttp: Fix missing content type on Close https://github.com/klauspost/compress/pull/883 - -* Oct 22nd, 2023 - [v1.17.2](https://github.com/klauspost/compress/releases/tag/v1.17.2) - * zstd: Fix rare *CORRUPTION* output in "best" mode. See https://github.com/klauspost/compress/pull/876 - -* Oct 14th, 2023 - [v1.17.1](https://github.com/klauspost/compress/releases/tag/v1.17.1) - * s2: Fix S2 "best" dictionary wrong encoding https://github.com/klauspost/compress/pull/871 - * flate: Reduce allocations in decompressor and minor code improvements by @fakefloordiv in https://github.com/klauspost/compress/pull/869 - * s2: Fix EstimateBlockSize on 6&7 length input https://github.com/klauspost/compress/pull/867 - -* Sept 19th, 2023 - [v1.17.0](https://github.com/klauspost/compress/releases/tag/v1.17.0) - * Add experimental dictionary builder https://github.com/klauspost/compress/pull/853 - * Add xerial snappy read/writer https://github.com/klauspost/compress/pull/838 - * flate: Add limited window compression https://github.com/klauspost/compress/pull/843 - * s2: Do 2 overlapping match checks https://github.com/klauspost/compress/pull/839 - * flate: Add amd64 assembly matchlen https://github.com/klauspost/compress/pull/837 - * gzip: Copy bufio.Reader on Reset by @thatguystone in https://github.com/klauspost/compress/pull/860 - -
- See changes to v1.16.x - - -* July 1st, 2023 - [v1.16.7](https://github.com/klauspost/compress/releases/tag/v1.16.7) - * zstd: Fix default level first dictionary encode https://github.com/klauspost/compress/pull/829 - * s2: add GetBufferCapacity() method by @GiedriusS in https://github.com/klauspost/compress/pull/832 - -* June 13, 2023 - [v1.16.6](https://github.com/klauspost/compress/releases/tag/v1.16.6) - * zstd: correctly ignore WithEncoderPadding(1) by @ianlancetaylor in https://github.com/klauspost/compress/pull/806 - * zstd: Add amd64 match length assembly https://github.com/klauspost/compress/pull/824 - * gzhttp: Handle informational headers by @rtribotte in https://github.com/klauspost/compress/pull/815 - * s2: Improve Better compression slightly https://github.com/klauspost/compress/pull/663 - -* Apr 16, 2023 - [v1.16.5](https://github.com/klauspost/compress/releases/tag/v1.16.5) - * zstd: readByte needs to use io.ReadFull by @jnoxon in https://github.com/klauspost/compress/pull/802 - * gzip: Fix WriterTo after initial read https://github.com/klauspost/compress/pull/804 - -* Apr 5, 2023 - [v1.16.4](https://github.com/klauspost/compress/releases/tag/v1.16.4) - * zstd: Improve zstd best efficiency by @greatroar and @klauspost in https://github.com/klauspost/compress/pull/784 - * zstd: Respect WithAllLitEntropyCompression https://github.com/klauspost/compress/pull/792 - * zstd: Fix amd64 not always detecting corrupt data https://github.com/klauspost/compress/pull/785 - * zstd: Various minor improvements by @greatroar in https://github.com/klauspost/compress/pull/788 https://github.com/klauspost/compress/pull/794 https://github.com/klauspost/compress/pull/795 - * s2: Fix huge block overflow https://github.com/klauspost/compress/pull/779 - * s2: Allow CustomEncoder fallback https://github.com/klauspost/compress/pull/780 - * gzhttp: Support ResponseWriter Unwrap() in gzhttp handler by @jgimenez in https://github.com/klauspost/compress/pull/799 - -* Mar 13, 2023 - [v1.16.1](https://github.com/klauspost/compress/releases/tag/v1.16.1) - * zstd: Speed up + improve best encoder by @greatroar in https://github.com/klauspost/compress/pull/776 - * gzhttp: Add optional [BREACH mitigation](https://github.com/klauspost/compress/tree/master/gzhttp#breach-mitigation). https://github.com/klauspost/compress/pull/762 https://github.com/klauspost/compress/pull/768 https://github.com/klauspost/compress/pull/769 https://github.com/klauspost/compress/pull/770 https://github.com/klauspost/compress/pull/767 - * s2: Add Intel LZ4s converter https://github.com/klauspost/compress/pull/766 - * zstd: Minor bug fixes https://github.com/klauspost/compress/pull/771 https://github.com/klauspost/compress/pull/772 https://github.com/klauspost/compress/pull/773 - * huff0: Speed up compress1xDo by @greatroar in https://github.com/klauspost/compress/pull/774 - -* Feb 26, 2023 - [v1.16.0](https://github.com/klauspost/compress/releases/tag/v1.16.0) - * s2: Add [Dictionary](https://github.com/klauspost/compress/tree/master/s2#dictionaries) support. https://github.com/klauspost/compress/pull/685 - * s2: Add Compression Size Estimate. https://github.com/klauspost/compress/pull/752 - * s2: Add support for custom stream encoder. https://github.com/klauspost/compress/pull/755 - * s2: Add LZ4 block converter. https://github.com/klauspost/compress/pull/748 - * s2: Support io.ReaderAt in ReadSeeker. https://github.com/klauspost/compress/pull/747 - * s2c/s2sx: Use concurrent decoding. https://github.com/klauspost/compress/pull/746 -
- -
- See changes to v1.15.x - -* Jan 21st, 2023 (v1.15.15) - * deflate: Improve level 7-9 https://github.com/klauspost/compress/pull/739 - * zstd: Add delta encoding support by @greatroar in https://github.com/klauspost/compress/pull/728 - * zstd: Various speed improvements by @greatroar https://github.com/klauspost/compress/pull/741 https://github.com/klauspost/compress/pull/734 https://github.com/klauspost/compress/pull/736 https://github.com/klauspost/compress/pull/744 https://github.com/klauspost/compress/pull/743 https://github.com/klauspost/compress/pull/745 - * gzhttp: Add SuffixETag() and DropETag() options to prevent ETag collisions on compressed responses by @willbicks in https://github.com/klauspost/compress/pull/740 - -* Jan 3rd, 2023 (v1.15.14) - - * flate: Improve speed in big stateless blocks https://github.com/klauspost/compress/pull/718 - * zstd: Minor speed tweaks by @greatroar in https://github.com/klauspost/compress/pull/716 https://github.com/klauspost/compress/pull/720 - * export NoGzipResponseWriter for custom ResponseWriter wrappers by @harshavardhana in https://github.com/klauspost/compress/pull/722 - * s2: Add example for indexing and existing stream https://github.com/klauspost/compress/pull/723 - -* Dec 11, 2022 (v1.15.13) - * zstd: Add [MaxEncodedSize](https://pkg.go.dev/github.com/klauspost/compress@v1.15.13/zstd#Encoder.MaxEncodedSize) to encoder https://github.com/klauspost/compress/pull/691 - * zstd: Various tweaks and improvements https://github.com/klauspost/compress/pull/693 https://github.com/klauspost/compress/pull/695 https://github.com/klauspost/compress/pull/696 https://github.com/klauspost/compress/pull/701 https://github.com/klauspost/compress/pull/702 https://github.com/klauspost/compress/pull/703 https://github.com/klauspost/compress/pull/704 https://github.com/klauspost/compress/pull/705 https://github.com/klauspost/compress/pull/706 https://github.com/klauspost/compress/pull/707 https://github.com/klauspost/compress/pull/708 - -* Oct 26, 2022 (v1.15.12) - - * zstd: Tweak decoder allocs. https://github.com/klauspost/compress/pull/680 - * gzhttp: Always delete `HeaderNoCompression` https://github.com/klauspost/compress/pull/683 - -* Sept 26, 2022 (v1.15.11) - - * flate: Improve level 1-3 compression https://github.com/klauspost/compress/pull/678 - * zstd: Improve "best" compression by @nightwolfz in https://github.com/klauspost/compress/pull/677 - * zstd: Fix+reduce decompression allocations https://github.com/klauspost/compress/pull/668 - * zstd: Fix non-effective noescape tag https://github.com/klauspost/compress/pull/667 - -* Sept 16, 2022 (v1.15.10) - - * zstd: Add [WithDecodeAllCapLimit](https://pkg.go.dev/github.com/klauspost/compress@v1.15.10/zstd#WithDecodeAllCapLimit) https://github.com/klauspost/compress/pull/649 - * Add Go 1.19 - deprecate Go 1.16 https://github.com/klauspost/compress/pull/651 - * flate: Improve level 5+6 compression https://github.com/klauspost/compress/pull/656 - * zstd: Improve "better" compression https://github.com/klauspost/compress/pull/657 - * s2: Improve "best" compression https://github.com/klauspost/compress/pull/658 - * s2: Improve "better" compression. https://github.com/klauspost/compress/pull/635 - * s2: Slightly faster non-assembly decompression https://github.com/klauspost/compress/pull/646 - * Use arrays for constant size copies https://github.com/klauspost/compress/pull/659 - -* July 21, 2022 (v1.15.9) - - * zstd: Fix decoder crash on amd64 (no BMI) on invalid input https://github.com/klauspost/compress/pull/645 - * zstd: Disable decoder extended memory copies (amd64) due to possible crashes https://github.com/klauspost/compress/pull/644 - * zstd: Allow single segments up to "max decoded size" https://github.com/klauspost/compress/pull/643 - -* July 13, 2022 (v1.15.8) - - * gzip: fix stack exhaustion bug in Reader.Read https://github.com/klauspost/compress/pull/641 - * s2: Add Index header trim/restore https://github.com/klauspost/compress/pull/638 - * zstd: Optimize seqdeq amd64 asm by @greatroar in https://github.com/klauspost/compress/pull/636 - * zstd: Improve decoder memcopy https://github.com/klauspost/compress/pull/637 - * huff0: Pass a single bitReader pointer to asm by @greatroar in https://github.com/klauspost/compress/pull/634 - * zstd: Branchless getBits for amd64 w/o BMI2 by @greatroar in https://github.com/klauspost/compress/pull/640 - * gzhttp: Remove header before writing https://github.com/klauspost/compress/pull/639 - -* June 29, 2022 (v1.15.7) - - * s2: Fix absolute forward seeks https://github.com/klauspost/compress/pull/633 - * zip: Merge upstream https://github.com/klauspost/compress/pull/631 - * zip: Re-add zip64 fix https://github.com/klauspost/compress/pull/624 - * zstd: translate fseDecoder.buildDtable into asm by @WojciechMula in https://github.com/klauspost/compress/pull/598 - * flate: Faster histograms https://github.com/klauspost/compress/pull/620 - * deflate: Use compound hcode https://github.com/klauspost/compress/pull/622 - -* June 3, 2022 (v1.15.6) - * s2: Improve coding for long, close matches https://github.com/klauspost/compress/pull/613 - * s2c: Add Snappy/S2 stream recompression https://github.com/klauspost/compress/pull/611 - * zstd: Always use configured block size https://github.com/klauspost/compress/pull/605 - * zstd: Fix incorrect hash table placement for dict encoding in default https://github.com/klauspost/compress/pull/606 - * zstd: Apply default config to ZipDecompressor without options https://github.com/klauspost/compress/pull/608 - * gzhttp: Exclude more common archive formats https://github.com/klauspost/compress/pull/612 - * s2: Add ReaderIgnoreCRC https://github.com/klauspost/compress/pull/609 - * s2: Remove sanity load on index creation https://github.com/klauspost/compress/pull/607 - * snappy: Use dedicated function for scoring https://github.com/klauspost/compress/pull/614 - * s2c+s2d: Use official snappy framed extension https://github.com/klauspost/compress/pull/610 - -* May 25, 2022 (v1.15.5) - * s2: Add concurrent stream decompression https://github.com/klauspost/compress/pull/602 - * s2: Fix final emit oob read crash on amd64 https://github.com/klauspost/compress/pull/601 - * huff0: asm implementation of Decompress1X by @WojciechMula https://github.com/klauspost/compress/pull/596 - * zstd: Use 1 less goroutine for stream decoding https://github.com/klauspost/compress/pull/588 - * zstd: Copy literal in 16 byte blocks when possible https://github.com/klauspost/compress/pull/592 - * zstd: Speed up when WithDecoderLowmem(false) https://github.com/klauspost/compress/pull/599 - * zstd: faster next state update in BMI2 version of decode by @WojciechMula in https://github.com/klauspost/compress/pull/593 - * huff0: Do not check max size when reading table. https://github.com/klauspost/compress/pull/586 - * flate: Inplace hashing for level 7-9 https://github.com/klauspost/compress/pull/590 - - -* May 11, 2022 (v1.15.4) - * huff0: decompress directly into output by @WojciechMula in [#577](https://github.com/klauspost/compress/pull/577) - * inflate: Keep dict on stack [#581](https://github.com/klauspost/compress/pull/581) - * zstd: Faster decoding memcopy in asm [#583](https://github.com/klauspost/compress/pull/583) - * zstd: Fix ignored crc [#580](https://github.com/klauspost/compress/pull/580) - -* May 5, 2022 (v1.15.3) - * zstd: Allow to ignore checksum checking by @WojciechMula [#572](https://github.com/klauspost/compress/pull/572) - * s2: Fix incorrect seek for io.SeekEnd in [#575](https://github.com/klauspost/compress/pull/575) - -* Apr 26, 2022 (v1.15.2) - * zstd: Add x86-64 assembly for decompression on streams and blocks. Contributed by [@WojciechMula](https://github.com/WojciechMula). Typically 2x faster. [#528](https://github.com/klauspost/compress/pull/528) [#531](https://github.com/klauspost/compress/pull/531) [#545](https://github.com/klauspost/compress/pull/545) [#537](https://github.com/klauspost/compress/pull/537) - * zstd: Add options to ZipDecompressor and fixes [#539](https://github.com/klauspost/compress/pull/539) - * s2: Use sorted search for index [#555](https://github.com/klauspost/compress/pull/555) - * Minimum version is Go 1.16, added CI test on 1.18. - -* Mar 11, 2022 (v1.15.1) - * huff0: Add x86 assembly of Decode4X by @WojciechMula in [#512](https://github.com/klauspost/compress/pull/512) - * zstd: Reuse zip decoders in [#514](https://github.com/klauspost/compress/pull/514) - * zstd: Detect extra block data and report as corrupted in [#520](https://github.com/klauspost/compress/pull/520) - * zstd: Handle zero sized frame content size stricter in [#521](https://github.com/klauspost/compress/pull/521) - * zstd: Add stricter block size checks in [#523](https://github.com/klauspost/compress/pull/523) - -* Mar 3, 2022 (v1.15.0) - * zstd: Refactor decoder [#498](https://github.com/klauspost/compress/pull/498) - * zstd: Add stream encoding without goroutines [#505](https://github.com/klauspost/compress/pull/505) - * huff0: Prevent single blocks exceeding 16 bits by @klauspost in[#507](https://github.com/klauspost/compress/pull/507) - * flate: Inline literal emission [#509](https://github.com/klauspost/compress/pull/509) - * gzhttp: Add zstd to transport [#400](https://github.com/klauspost/compress/pull/400) - * gzhttp: Make content-type optional [#510](https://github.com/klauspost/compress/pull/510) - -Both compression and decompression now supports "synchronous" stream operations. This means that whenever "concurrency" is set to 1, they will operate without spawning goroutines. - -Stream decompression is now faster on asynchronous, since the goroutine allocation much more effectively splits the workload. On typical streams this will typically use 2 cores fully for decompression. When a stream has finished decoding no goroutines will be left over, so decoders can now safely be pooled and still be garbage collected. - -While the release has been extensively tested, it is recommended to testing when upgrading. - -
- -
- See changes to v1.14.x - -* Feb 22, 2022 (v1.14.4) - * flate: Fix rare huffman only (-2) corruption. [#503](https://github.com/klauspost/compress/pull/503) - * zip: Update deprecated CreateHeaderRaw to correctly call CreateRaw by @saracen in [#502](https://github.com/klauspost/compress/pull/502) - * zip: don't read data descriptor early by @saracen in [#501](https://github.com/klauspost/compress/pull/501) #501 - * huff0: Use static decompression buffer up to 30% faster [#499](https://github.com/klauspost/compress/pull/499) [#500](https://github.com/klauspost/compress/pull/500) - -* Feb 17, 2022 (v1.14.3) - * flate: Improve fastest levels compression speed ~10% more throughput. [#482](https://github.com/klauspost/compress/pull/482) [#489](https://github.com/klauspost/compress/pull/489) [#490](https://github.com/klauspost/compress/pull/490) [#491](https://github.com/klauspost/compress/pull/491) [#494](https://github.com/klauspost/compress/pull/494) [#478](https://github.com/klauspost/compress/pull/478) - * flate: Faster decompression speed, ~5-10%. [#483](https://github.com/klauspost/compress/pull/483) - * s2: Faster compression with Go v1.18 and amd64 microarch level 3+. [#484](https://github.com/klauspost/compress/pull/484) [#486](https://github.com/klauspost/compress/pull/486) - -* Jan 25, 2022 (v1.14.2) - * zstd: improve header decoder by @dsnet [#476](https://github.com/klauspost/compress/pull/476) - * zstd: Add bigger default blocks [#469](https://github.com/klauspost/compress/pull/469) - * zstd: Remove unused decompression buffer [#470](https://github.com/klauspost/compress/pull/470) - * zstd: Fix logically dead code by @ningmingxiao [#472](https://github.com/klauspost/compress/pull/472) - * flate: Improve level 7-9 [#471](https://github.com/klauspost/compress/pull/471) [#473](https://github.com/klauspost/compress/pull/473) - * zstd: Add noasm tag for xxhash [#475](https://github.com/klauspost/compress/pull/475) - -* Jan 11, 2022 (v1.14.1) - * s2: Add stream index in [#462](https://github.com/klauspost/compress/pull/462) - * flate: Speed and efficiency improvements in [#439](https://github.com/klauspost/compress/pull/439) [#461](https://github.com/klauspost/compress/pull/461) [#455](https://github.com/klauspost/compress/pull/455) [#452](https://github.com/klauspost/compress/pull/452) [#458](https://github.com/klauspost/compress/pull/458) - * zstd: Performance improvement in [#420]( https://github.com/klauspost/compress/pull/420) [#456](https://github.com/klauspost/compress/pull/456) [#437](https://github.com/klauspost/compress/pull/437) [#467](https://github.com/klauspost/compress/pull/467) [#468](https://github.com/klauspost/compress/pull/468) - * zstd: add arm64 xxhash assembly in [#464](https://github.com/klauspost/compress/pull/464) - * Add garbled for binaries for s2 in [#445](https://github.com/klauspost/compress/pull/445) -
- -
- See changes to v1.13.x - -* Aug 30, 2021 (v1.13.5) - * gz/zlib/flate: Alias stdlib errors [#425](https://github.com/klauspost/compress/pull/425) - * s2: Add block support to commandline tools [#413](https://github.com/klauspost/compress/pull/413) - * zstd: pooledZipWriter should return Writers to the same pool [#426](https://github.com/klauspost/compress/pull/426) - * Removed golang/snappy as external dependency for tests [#421](https://github.com/klauspost/compress/pull/421) - -* Aug 12, 2021 (v1.13.4) - * Add [snappy replacement package](https://github.com/klauspost/compress/tree/master/snappy). - * zstd: Fix incorrect encoding in "best" mode [#415](https://github.com/klauspost/compress/pull/415) - -* Aug 3, 2021 (v1.13.3) - * zstd: Improve Best compression [#404](https://github.com/klauspost/compress/pull/404) - * zstd: Fix WriteTo error forwarding [#411](https://github.com/klauspost/compress/pull/411) - * gzhttp: Return http.HandlerFunc instead of http.Handler. Unlikely breaking change. [#406](https://github.com/klauspost/compress/pull/406) - * s2sx: Fix max size error [#399](https://github.com/klauspost/compress/pull/399) - * zstd: Add optional stream content size on reset [#401](https://github.com/klauspost/compress/pull/401) - * zstd: use SpeedBestCompression for level >= 10 [#410](https://github.com/klauspost/compress/pull/410) - -* Jun 14, 2021 (v1.13.1) - * s2: Add full Snappy output support [#396](https://github.com/klauspost/compress/pull/396) - * zstd: Add configurable [Decoder window](https://pkg.go.dev/github.com/klauspost/compress/zstd#WithDecoderMaxWindow) size [#394](https://github.com/klauspost/compress/pull/394) - * gzhttp: Add header to skip compression [#389](https://github.com/klauspost/compress/pull/389) - * s2: Improve speed with bigger output margin [#395](https://github.com/klauspost/compress/pull/395) - -* Jun 3, 2021 (v1.13.0) - * Added [gzhttp](https://github.com/klauspost/compress/tree/master/gzhttp#gzip-handler) which allows wrapping HTTP servers and clients with GZIP compressors. - * zstd: Detect short invalid signatures [#382](https://github.com/klauspost/compress/pull/382) - * zstd: Spawn decoder goroutine only if needed. [#380](https://github.com/klauspost/compress/pull/380) -
- - -
- See changes to v1.12.x - -* May 25, 2021 (v1.12.3) - * deflate: Better/faster Huffman encoding [#374](https://github.com/klauspost/compress/pull/374) - * deflate: Allocate less for history. [#375](https://github.com/klauspost/compress/pull/375) - * zstd: Forward read errors [#373](https://github.com/klauspost/compress/pull/373) - -* Apr 27, 2021 (v1.12.2) - * zstd: Improve better/best compression [#360](https://github.com/klauspost/compress/pull/360) [#364](https://github.com/klauspost/compress/pull/364) [#365](https://github.com/klauspost/compress/pull/365) - * zstd: Add helpers to compress/decompress zstd inside zip files [#363](https://github.com/klauspost/compress/pull/363) - * deflate: Improve level 5+6 compression [#367](https://github.com/klauspost/compress/pull/367) - * s2: Improve better/best compression [#358](https://github.com/klauspost/compress/pull/358) [#359](https://github.com/klauspost/compress/pull/358) - * s2: Load after checking src limit on amd64. [#362](https://github.com/klauspost/compress/pull/362) - * s2sx: Limit max executable size [#368](https://github.com/klauspost/compress/pull/368) - -* Apr 14, 2021 (v1.12.1) - * snappy package removed. Upstream added as dependency. - * s2: Better compression in "best" mode [#353](https://github.com/klauspost/compress/pull/353) - * s2sx: Add stdin input and detect pre-compressed from signature [#352](https://github.com/klauspost/compress/pull/352) - * s2c/s2d: Add http as possible input [#348](https://github.com/klauspost/compress/pull/348) - * s2c/s2d/s2sx: Always truncate when writing files [#352](https://github.com/klauspost/compress/pull/352) - * zstd: Reduce memory usage further when using [WithLowerEncoderMem](https://pkg.go.dev/github.com/klauspost/compress/zstd#WithLowerEncoderMem) [#346](https://github.com/klauspost/compress/pull/346) - * s2: Fix potential problem with amd64 assembly and profilers [#349](https://github.com/klauspost/compress/pull/349) -
- -
- See changes to v1.11.x - -* Mar 26, 2021 (v1.11.13) - * zstd: Big speedup on small dictionary encodes [#344](https://github.com/klauspost/compress/pull/344) [#345](https://github.com/klauspost/compress/pull/345) - * zstd: Add [WithLowerEncoderMem](https://pkg.go.dev/github.com/klauspost/compress/zstd#WithLowerEncoderMem) encoder option [#336](https://github.com/klauspost/compress/pull/336) - * deflate: Improve entropy compression [#338](https://github.com/klauspost/compress/pull/338) - * s2: Clean up and minor performance improvement in best [#341](https://github.com/klauspost/compress/pull/341) - -* Mar 5, 2021 (v1.11.12) - * s2: Add `s2sx` binary that creates [self extracting archives](https://github.com/klauspost/compress/tree/master/s2#s2sx-self-extracting-archives). - * s2: Speed up decompression on non-assembly platforms [#328](https://github.com/klauspost/compress/pull/328) - -* Mar 1, 2021 (v1.11.9) - * s2: Add ARM64 decompression assembly. Around 2x output speed. [#324](https://github.com/klauspost/compress/pull/324) - * s2: Improve "better" speed and efficiency. [#325](https://github.com/klauspost/compress/pull/325) - * s2: Fix binaries. - -* Feb 25, 2021 (v1.11.8) - * s2: Fixed occasional out-of-bounds write on amd64. Upgrade recommended. - * s2: Add AMD64 assembly for better mode. 25-50% faster. [#315](https://github.com/klauspost/compress/pull/315) - * s2: Less upfront decoder allocation. [#322](https://github.com/klauspost/compress/pull/322) - * zstd: Faster "compression" of incompressible data. [#314](https://github.com/klauspost/compress/pull/314) - * zip: Fix zip64 headers. [#313](https://github.com/klauspost/compress/pull/313) - -* Jan 14, 2021 (v1.11.7) - * Use Bytes() interface to get bytes across packages. [#309](https://github.com/klauspost/compress/pull/309) - * s2: Add 'best' compression option. [#310](https://github.com/klauspost/compress/pull/310) - * s2: Add ReaderMaxBlockSize, changes `s2.NewReader` signature to include varargs. [#311](https://github.com/klauspost/compress/pull/311) - * s2: Fix crash on small better buffers. [#308](https://github.com/klauspost/compress/pull/308) - * s2: Clean up decoder. [#312](https://github.com/klauspost/compress/pull/312) - -* Jan 7, 2021 (v1.11.6) - * zstd: Make decoder allocations smaller [#306](https://github.com/klauspost/compress/pull/306) - * zstd: Free Decoder resources when Reset is called with a nil io.Reader [#305](https://github.com/klauspost/compress/pull/305) - -* Dec 20, 2020 (v1.11.4) - * zstd: Add Best compression mode [#304](https://github.com/klauspost/compress/pull/304) - * Add header decoder [#299](https://github.com/klauspost/compress/pull/299) - * s2: Add uncompressed stream option [#297](https://github.com/klauspost/compress/pull/297) - * Simplify/speed up small blocks with known max size. [#300](https://github.com/klauspost/compress/pull/300) - * zstd: Always reset literal dict encoder [#303](https://github.com/klauspost/compress/pull/303) - -* Nov 15, 2020 (v1.11.3) - * inflate: 10-15% faster decompression [#293](https://github.com/klauspost/compress/pull/293) - * zstd: Tweak DecodeAll default allocation [#295](https://github.com/klauspost/compress/pull/295) - -* Oct 11, 2020 (v1.11.2) - * s2: Fix out of bounds read in "better" block compression [#291](https://github.com/klauspost/compress/pull/291) - -* Oct 1, 2020 (v1.11.1) - * zstd: Set allLitEntropy true in default configuration [#286](https://github.com/klauspost/compress/pull/286) - -* Sept 8, 2020 (v1.11.0) - * zstd: Add experimental compression [dictionaries](https://github.com/klauspost/compress/tree/master/zstd#dictionaries) [#281](https://github.com/klauspost/compress/pull/281) - * zstd: Fix mixed Write and ReadFrom calls [#282](https://github.com/klauspost/compress/pull/282) - * inflate/gz: Limit variable shifts, ~5% faster decompression [#274](https://github.com/klauspost/compress/pull/274) -
- -
- See changes to v1.10.x - -* July 8, 2020 (v1.10.11) - * zstd: Fix extra block when compressing with ReadFrom. [#278](https://github.com/klauspost/compress/pull/278) - * huff0: Also populate compression table when reading decoding table. [#275](https://github.com/klauspost/compress/pull/275) - -* June 23, 2020 (v1.10.10) - * zstd: Skip entropy compression in fastest mode when no matches. [#270](https://github.com/klauspost/compress/pull/270) - -* June 16, 2020 (v1.10.9): - * zstd: API change for specifying dictionaries. See [#268](https://github.com/klauspost/compress/pull/268) - * zip: update CreateHeaderRaw to handle zip64 fields. [#266](https://github.com/klauspost/compress/pull/266) - * Fuzzit tests removed. The service has been purchased and is no longer available. - -* June 5, 2020 (v1.10.8): - * 1.15x faster zstd block decompression. [#265](https://github.com/klauspost/compress/pull/265) - -* June 1, 2020 (v1.10.7): - * Added zstd decompression [dictionary support](https://github.com/klauspost/compress/tree/master/zstd#dictionaries) - * Increase zstd decompression speed up to 1.19x. [#259](https://github.com/klauspost/compress/pull/259) - * Remove internal reset call in zstd compression and reduce allocations. [#263](https://github.com/klauspost/compress/pull/263) - -* May 21, 2020: (v1.10.6) - * zstd: Reduce allocations while decoding. [#258](https://github.com/klauspost/compress/pull/258), [#252](https://github.com/klauspost/compress/pull/252) - * zstd: Stricter decompression checks. - -* April 12, 2020: (v1.10.5) - * s2-commands: Flush output when receiving SIGINT. [#239](https://github.com/klauspost/compress/pull/239) - -* Apr 8, 2020: (v1.10.4) - * zstd: Minor/special case optimizations. [#251](https://github.com/klauspost/compress/pull/251), [#250](https://github.com/klauspost/compress/pull/250), [#249](https://github.com/klauspost/compress/pull/249), [#247](https://github.com/klauspost/compress/pull/247) -* Mar 11, 2020: (v1.10.3) - * s2: Use S2 encoder in pure Go mode for Snappy output as well. [#245](https://github.com/klauspost/compress/pull/245) - * s2: Fix pure Go block encoder. [#244](https://github.com/klauspost/compress/pull/244) - * zstd: Added "better compression" mode. [#240](https://github.com/klauspost/compress/pull/240) - * zstd: Improve speed of fastest compression mode by 5-10% [#241](https://github.com/klauspost/compress/pull/241) - * zstd: Skip creating encoders when not needed. [#238](https://github.com/klauspost/compress/pull/238) - -* Feb 27, 2020: (v1.10.2) - * Close to 50% speedup in inflate (gzip/zip decompression). [#236](https://github.com/klauspost/compress/pull/236) [#234](https://github.com/klauspost/compress/pull/234) [#232](https://github.com/klauspost/compress/pull/232) - * Reduce deflate level 1-6 memory usage up to 59%. [#227](https://github.com/klauspost/compress/pull/227) - -* Feb 18, 2020: (v1.10.1) - * Fix zstd crash when resetting multiple times without sending data. [#226](https://github.com/klauspost/compress/pull/226) - * deflate: Fix dictionary use on level 1-6. [#224](https://github.com/klauspost/compress/pull/224) - * Remove deflate writer reference when closing. [#224](https://github.com/klauspost/compress/pull/224) - -* Feb 4, 2020: (v1.10.0) - * Add optional dictionary to [stateless deflate](https://pkg.go.dev/github.com/klauspost/compress/flate?tab=doc#StatelessDeflate). Breaking change, send `nil` for previous behaviour. [#216](https://github.com/klauspost/compress/pull/216) - * Fix buffer overflow on repeated small block deflate. [#218](https://github.com/klauspost/compress/pull/218) - * Allow copying content from an existing ZIP file without decompressing+compressing. [#214](https://github.com/klauspost/compress/pull/214) - * Added [S2](https://github.com/klauspost/compress/tree/master/s2#s2-compression) AMD64 assembler and various optimizations. Stream speed >10GB/s. [#186](https://github.com/klauspost/compress/pull/186) - -
- -
- See changes prior to v1.10.0 - -* Jan 20,2020 (v1.9.8) Optimize gzip/deflate with better size estimates and faster table generation. [#207](https://github.com/klauspost/compress/pull/207) by [luyu6056](https://github.com/luyu6056), [#206](https://github.com/klauspost/compress/pull/206). -* Jan 11, 2020: S2 Encode/Decode will use provided buffer if capacity is big enough. [#204](https://github.com/klauspost/compress/pull/204) -* Jan 5, 2020: (v1.9.7) Fix another zstd regression in v1.9.5 - v1.9.6 removed. -* Jan 4, 2020: (v1.9.6) Regression in v1.9.5 fixed causing corrupt zstd encodes in rare cases. -* Jan 4, 2020: Faster IO in [s2c + s2d commandline tools](https://github.com/klauspost/compress/tree/master/s2#commandline-tools) compression/decompression. [#192](https://github.com/klauspost/compress/pull/192) -* Dec 29, 2019: Removed v1.9.5 since fuzz tests showed a compatibility problem with the reference zstandard decoder. -* Dec 29, 2019: (v1.9.5) zstd: 10-20% faster block compression. [#199](https://github.com/klauspost/compress/pull/199) -* Dec 29, 2019: [zip](https://godoc.org/github.com/klauspost/compress/zip) package updated with latest Go features -* Dec 29, 2019: zstd: Single segment flag condintions tweaked. [#197](https://github.com/klauspost/compress/pull/197) -* Dec 18, 2019: s2: Faster compression when ReadFrom is used. [#198](https://github.com/klauspost/compress/pull/198) -* Dec 10, 2019: s2: Fix repeat length output when just above at 16MB limit. -* Dec 10, 2019: zstd: Add function to get decoder as io.ReadCloser. [#191](https://github.com/klauspost/compress/pull/191) -* Dec 3, 2019: (v1.9.4) S2: limit max repeat length. [#188](https://github.com/klauspost/compress/pull/188) -* Dec 3, 2019: Add [WithNoEntropyCompression](https://godoc.org/github.com/klauspost/compress/zstd#WithNoEntropyCompression) to zstd [#187](https://github.com/klauspost/compress/pull/187) -* Dec 3, 2019: Reduce memory use for tests. Check for leaked goroutines. -* Nov 28, 2019 (v1.9.3) Less allocations in stateless deflate. -* Nov 28, 2019: 5-20% Faster huff0 decode. Impacts zstd as well. [#184](https://github.com/klauspost/compress/pull/184) -* Nov 12, 2019 (v1.9.2) Added [Stateless Compression](#stateless-compression) for gzip/deflate. -* Nov 12, 2019: Fixed zstd decompression of large single blocks. [#180](https://github.com/klauspost/compress/pull/180) -* Nov 11, 2019: Set default [s2c](https://github.com/klauspost/compress/tree/master/s2#commandline-tools) block size to 4MB. -* Nov 11, 2019: Reduce inflate memory use by 1KB. -* Nov 10, 2019: Less allocations in deflate bit writer. -* Nov 10, 2019: Fix inconsistent error returned by zstd decoder. -* Oct 28, 2019 (v1.9.1) ztsd: Fix crash when compressing blocks. [#174](https://github.com/klauspost/compress/pull/174) -* Oct 24, 2019 (v1.9.0) zstd: Fix rare data corruption [#173](https://github.com/klauspost/compress/pull/173) -* Oct 24, 2019 zstd: Fix huff0 out of buffer write [#171](https://github.com/klauspost/compress/pull/171) and always return errors [#172](https://github.com/klauspost/compress/pull/172) -* Oct 10, 2019: Big deflate rewrite, 30-40% faster with better compression [#105](https://github.com/klauspost/compress/pull/105) - -
- -
- See changes prior to v1.9.0 - -* Oct 10, 2019: (v1.8.6) zstd: Allow partial reads to get flushed data. [#169](https://github.com/klauspost/compress/pull/169) -* Oct 3, 2019: Fix inconsistent results on broken zstd streams. -* Sep 25, 2019: Added `-rm` (remove source files) and `-q` (no output except errors) to `s2c` and `s2d` [commands](https://github.com/klauspost/compress/tree/master/s2#commandline-tools) -* Sep 16, 2019: (v1.8.4) Add `s2c` and `s2d` [commandline tools](https://github.com/klauspost/compress/tree/master/s2#commandline-tools). -* Sep 10, 2019: (v1.8.3) Fix s2 decoder [Skip](https://godoc.org/github.com/klauspost/compress/s2#Reader.Skip). -* Sep 7, 2019: zstd: Added [WithWindowSize](https://godoc.org/github.com/klauspost/compress/zstd#WithWindowSize), contributed by [ianwilkes](https://github.com/ianwilkes). -* Sep 5, 2019: (v1.8.2) Add [WithZeroFrames](https://godoc.org/github.com/klauspost/compress/zstd#WithZeroFrames) which adds full zero payload block encoding option. -* Sep 5, 2019: Lazy initialization of zstandard predefined en/decoder tables. -* Aug 26, 2019: (v1.8.1) S2: 1-2% compression increase in "better" compression mode. -* Aug 26, 2019: zstd: Check maximum size of Huffman 1X compressed literals while decoding. -* Aug 24, 2019: (v1.8.0) Added [S2 compression](https://github.com/klauspost/compress/tree/master/s2#s2-compression), a high performance replacement for Snappy. -* Aug 21, 2019: (v1.7.6) Fixed minor issues found by fuzzer. One could lead to zstd not decompressing. -* Aug 18, 2019: Add [fuzzit](https://fuzzit.dev/) continuous fuzzing. -* Aug 14, 2019: zstd: Skip incompressible data 2x faster. [#147](https://github.com/klauspost/compress/pull/147) -* Aug 4, 2019 (v1.7.5): Better literal compression. [#146](https://github.com/klauspost/compress/pull/146) -* Aug 4, 2019: Faster zstd compression. [#143](https://github.com/klauspost/compress/pull/143) [#144](https://github.com/klauspost/compress/pull/144) -* Aug 4, 2019: Faster zstd decompression. [#145](https://github.com/klauspost/compress/pull/145) [#143](https://github.com/klauspost/compress/pull/143) [#142](https://github.com/klauspost/compress/pull/142) -* July 15, 2019 (v1.7.4): Fix double EOF block in rare cases on zstd encoder. -* July 15, 2019 (v1.7.3): Minor speedup/compression increase in default zstd encoder. -* July 14, 2019: zstd decoder: Fix decompression error on multiple uses with mixed content. -* July 7, 2019 (v1.7.2): Snappy update, zstd decoder potential race fix. -* June 17, 2019: zstd decompression bugfix. -* June 17, 2019: fix 32 bit builds. -* June 17, 2019: Easier use in modules (less dependencies). -* June 9, 2019: New stronger "default" [zstd](https://github.com/klauspost/compress/tree/master/zstd#zstd) compression mode. Matches zstd default compression ratio. -* June 5, 2019: 20-40% throughput in [zstandard](https://github.com/klauspost/compress/tree/master/zstd#zstd) compression and better compression. -* June 5, 2019: deflate/gzip compression: Reduce memory usage of lower compression levels. -* June 2, 2019: Added [zstandard](https://github.com/klauspost/compress/tree/master/zstd#zstd) compression! -* May 25, 2019: deflate/gzip: 10% faster bit writer, mostly visible in lower levels. -* Apr 22, 2019: [zstd](https://github.com/klauspost/compress/tree/master/zstd#zstd) decompression added. -* Aug 1, 2018: Added [huff0 README](https://github.com/klauspost/compress/tree/master/huff0#huff0-entropy-compression). -* Jul 8, 2018: Added [Performance Update 2018](#performance-update-2018) below. -* Jun 23, 2018: Merged [Go 1.11 inflate optimizations](https://go-review.googlesource.com/c/go/+/102235). Go 1.9 is now required. Backwards compatible version tagged with [v1.3.0](https://github.com/klauspost/compress/releases/tag/v1.3.0). -* Apr 2, 2018: Added [huff0](https://godoc.org/github.com/klauspost/compress/huff0) en/decoder. Experimental for now, API may change. -* Mar 4, 2018: Added [FSE Entropy](https://godoc.org/github.com/klauspost/compress/fse) en/decoder. Experimental for now, API may change. -* Nov 3, 2017: Add compression [Estimate](https://godoc.org/github.com/klauspost/compress#Estimate) function. -* May 28, 2017: Reduce allocations when resetting decoder. -* Apr 02, 2017: Change back to official crc32, since changes were merged in Go 1.7. -* Jan 14, 2017: Reduce stack pressure due to array copies. See [Issue #18625](https://github.com/golang/go/issues/18625). -* Oct 25, 2016: Level 2-4 have been rewritten and now offers significantly better performance than before. -* Oct 20, 2016: Port zlib changes from Go 1.7 to fix zlib writer issue. Please update. -* Oct 16, 2016: Go 1.7 changes merged. Apples to apples this package is a few percent faster, but has a significantly better balance between speed and compression per level. -* Mar 24, 2016: Always attempt Huffman encoding on level 4-7. This improves base 64 encoded data compression. -* Mar 24, 2016: Small speedup for level 1-3. -* Feb 19, 2016: Faster bit writer, level -2 is 15% faster, level 1 is 4% faster. -* Feb 19, 2016: Handle small payloads faster in level 1-3. -* Feb 19, 2016: Added faster level 2 + 3 compression modes. -* Feb 19, 2016: [Rebalanced compression levels](https://blog.klauspost.com/rebalancing-deflate-compression-levels/), so there is a more even progression in terms of compression. New default level is 5. -* Feb 14, 2016: Snappy: Merge upstream changes. -* Feb 14, 2016: Snappy: Fix aggressive skipping. -* Feb 14, 2016: Snappy: Update benchmark. -* Feb 13, 2016: Deflate: Fixed assembler problem that could lead to sub-optimal compression. -* Feb 12, 2016: Snappy: Added AMD64 SSE 4.2 optimizations to matching, which makes easy to compress material run faster. Typical speedup is around 25%. -* Feb 9, 2016: Added Snappy package fork. This version is 5-7% faster, much more on hard to compress content. -* Jan 30, 2016: Optimize level 1 to 3 by not considering static dictionary or storing uncompressed. ~4-5% speedup. -* Jan 16, 2016: Optimization on deflate level 1,2,3 compression. -* Jan 8 2016: Merge [CL 18317](https://go-review.googlesource.com/#/c/18317): fix reading, writing of zip64 archives. -* Dec 8 2015: Make level 1 and -2 deterministic even if write size differs. -* Dec 8 2015: Split encoding functions, so hashing and matching can potentially be inlined. 1-3% faster on AMD64. 5% faster on other platforms. -* Dec 8 2015: Fixed rare [one byte out-of bounds read](https://github.com/klauspost/compress/issues/20). Please update! -* Nov 23 2015: Optimization on token writer. ~2-4% faster. Contributed by [@dsnet](https://github.com/dsnet). -* Nov 20 2015: Small optimization to bit writer on 64 bit systems. -* Nov 17 2015: Fixed out-of-bound errors if the underlying Writer returned an error. See [#15](https://github.com/klauspost/compress/issues/15). -* Nov 12 2015: Added [io.WriterTo](https://golang.org/pkg/io/#WriterTo) support to gzip/inflate. -* Nov 11 2015: Merged [CL 16669](https://go-review.googlesource.com/#/c/16669/4): archive/zip: enable overriding (de)compressors per file -* Oct 15 2015: Added skipping on uncompressible data. Random data speed up >5x. - -
- -# deflate usage - -The packages are drop-in replacements for standard libraries. Simply replace the import path to use them: - -Typical speed is about 2x of the standard library packages. - -| old import | new import | Documentation | -|------------------|---------------------------------------|-------------------------------------------------------------------------| -| `compress/gzip` | `github.com/klauspost/compress/gzip` | [gzip](https://pkg.go.dev/github.com/klauspost/compress/gzip?tab=doc) | -| `compress/zlib` | `github.com/klauspost/compress/zlib` | [zlib](https://pkg.go.dev/github.com/klauspost/compress/zlib?tab=doc) | -| `archive/zip` | `github.com/klauspost/compress/zip` | [zip](https://pkg.go.dev/github.com/klauspost/compress/zip?tab=doc) | -| `compress/flate` | `github.com/klauspost/compress/flate` | [flate](https://pkg.go.dev/github.com/klauspost/compress/flate?tab=doc) | - -* Optimized [deflate](https://godoc.org/github.com/klauspost/compress/flate) packages which can be used as a dropin replacement for [gzip](https://godoc.org/github.com/klauspost/compress/gzip), [zip](https://godoc.org/github.com/klauspost/compress/zip) and [zlib](https://godoc.org/github.com/klauspost/compress/zlib). - -You may also be interested in [pgzip](https://github.com/klauspost/pgzip), which is a drop in replacement for gzip, which support multithreaded compression on big files and the optimized [crc32](https://github.com/klauspost/crc32) package used by these packages. - -The packages contains the same as the standard library, so you can use the godoc for that: [gzip](http://golang.org/pkg/compress/gzip/), [zip](http://golang.org/pkg/archive/zip/), [zlib](http://golang.org/pkg/compress/zlib/), [flate](http://golang.org/pkg/compress/flate/). - -Currently there is only minor speedup on decompression (mostly CRC32 calculation). - -Memory usage is typically 1MB for a Writer. stdlib is in the same range. -If you expect to have a lot of concurrently allocated Writers consider using -the stateless compress described below. - -For compression performance, see: [this spreadsheet](https://docs.google.com/spreadsheets/d/1nuNE2nPfuINCZJRMt6wFWhKpToF95I47XjSsc-1rbPQ/edit?usp=sharing). - -To disable all assembly add `-tags=noasm`. This works across all packages. - -# Stateless compression - -This package offers stateless compression as a special option for gzip/deflate. -It will do compression but without maintaining any state between Write calls. - -This means there will be no memory kept between Write calls, but compression and speed will be suboptimal. - -This is only relevant in cases where you expect to run many thousands of compressors concurrently, -but with very little activity. This is *not* intended for regular web servers serving individual requests. - -Because of this, the size of actual Write calls will affect output size. - -In gzip, specify level `-3` / `gzip.StatelessCompression` to enable. - -For direct deflate use, NewStatelessWriter and StatelessDeflate are available. See [documentation](https://godoc.org/github.com/klauspost/compress/flate#NewStatelessWriter) - -A `bufio.Writer` can of course be used to control write sizes. For example, to use a 4KB buffer: - -```go - // replace 'ioutil.Discard' with your output. - gzw, err := gzip.NewWriterLevel(ioutil.Discard, gzip.StatelessCompression) - if err != nil { - return err - } - defer gzw.Close() - - w := bufio.NewWriterSize(gzw, 4096) - defer w.Flush() - - // Write to 'w' -``` - -This will only use up to 4KB in memory when the writer is idle. - -Compression is almost always worse than the fastest compression level -and each write will allocate (a little) memory. - - -# Other packages - -Here are other packages of good quality and pure Go (no cgo wrappers or autoconverted code): - -* [github.com/pierrec/lz4](https://github.com/pierrec/lz4) - strong multithreaded LZ4 compression. -* [github.com/cosnicolaou/pbzip2](https://github.com/cosnicolaou/pbzip2) - multithreaded bzip2 decompression. -* [github.com/dsnet/compress](https://github.com/dsnet/compress) - brotli decompression, bzip2 writer. -* [github.com/ronanh/intcomp](https://github.com/ronanh/intcomp) - Integer compression. -* [github.com/spenczar/fpc](https://github.com/spenczar/fpc) - Float compression. -* [github.com/minio/zipindex](https://github.com/minio/zipindex) - External ZIP directory index. -* [github.com/ybirader/pzip](https://github.com/ybirader/pzip) - Fast concurrent zip archiver and extractor. - -# license - -This code is licensed under the same conditions as the original Go code. See LICENSE file. +# compress + +This package provides various compression algorithms. + +* [zstandard](https://github.com/klauspost/compress/tree/master/zstd#zstd) compression and decompression in pure Go. +* [S2](https://github.com/klauspost/compress/tree/master/s2#s2-compression) is a high performance replacement for Snappy. +* Optimized [deflate](https://godoc.org/github.com/klauspost/compress/flate) packages which can be used as a dropin replacement for [gzip](https://godoc.org/github.com/klauspost/compress/gzip), [zip](https://godoc.org/github.com/klauspost/compress/zip) and [zlib](https://godoc.org/github.com/klauspost/compress/zlib). +* [snappy](https://github.com/klauspost/compress/tree/master/snappy) is a drop-in replacement for `github.com/golang/snappy` offering better compression and concurrent streams. +* [huff0](https://github.com/klauspost/compress/tree/master/huff0) and [FSE](https://github.com/klauspost/compress/tree/master/fse) implementations for raw entropy encoding. +* [gzhttp](https://github.com/klauspost/compress/tree/master/gzhttp) Provides client and server wrappers for handling gzipped/zstd HTTP requests efficiently. +* [pgzip](https://github.com/klauspost/pgzip) is a separate package that provides a very fast parallel gzip implementation. + +[![Go Reference](https://pkg.go.dev/badge/klauspost/compress.svg)](https://pkg.go.dev/github.com/klauspost/compress?tab=subdirectories) +[![Go](https://github.com/klauspost/compress/actions/workflows/go.yml/badge.svg)](https://github.com/klauspost/compress/actions/workflows/go.yml) +[![Sourcegraph Badge](https://sourcegraph.com/github.com/klauspost/compress/-/badge.svg)](https://sourcegraph.com/github.com/klauspost/compress?badge) + +# package usage + +Use `go get github.com/klauspost/compress@latest` to add it to your project. + +This package will support the current Go version and 2 versions back. + +* Use the `nounsafe` tag to disable all use of the "unsafe" package. +* Use the `noasm` tag to disable all assembly across packages. + +Use the links above for more information on each. + +# changelog + +* Feb 9th, 2026 [1.18.4](https://github.com/klauspost/compress/releases/tag/v1.18.4) + * gzhttp: Add zstandard to server handler wrapper https://github.com/klauspost/compress/pull/1121 + * zstd: Add ResetWithOptions to encoder/decoder https://github.com/klauspost/compress/pull/1122 + * gzhttp: preserve qvalue when extra parameters follow in Accept-Encoding by @analytically in https://github.com/klauspost/compress/pull/1116 + +* Jan 16th, 2026 [1.18.3](https://github.com/klauspost/compress/releases/tag/v1.18.3) + * Downstream CVE-2025-61728. See [golang/go#77102](https://github.com/golang/go/issues/77102). + +* Dec 1st, 2025 - [1.18.2](https://github.com/klauspost/compress/releases/tag/v1.18.2) + * flate: Fix invalid encoding on level 9 with single value input in https://github.com/klauspost/compress/pull/1115 + * flate: reduce stateless allocations by @RXamzin in https://github.com/klauspost/compress/pull/1106 + +* Oct 20, 2025 - [1.18.1](https://github.com/klauspost/compress/releases/tag/v1.18.1) - RETRACTED + * zstd: Add simple zstd EncodeTo/DecodeTo functions https://github.com/klauspost/compress/pull/1079 + * zstd: Fix incorrect buffer size in dictionary encodes https://github.com/klauspost/compress/pull/1059 + * s2: check for cap, not len of buffer in EncodeBetter/Best by @vdarulis in https://github.com/klauspost/compress/pull/1080 + * zlib: Avoiding extra allocation in zlib.reader.Reset by @travelpolicy in https://github.com/klauspost/compress/pull/1086 + * gzhttp: remove redundant err check in zstdReader by @ryanfowler in https://github.com/klauspost/compress/pull/1090 + * flate: Faster load+store https://github.com/klauspost/compress/pull/1104 + * flate: Simplify matchlen https://github.com/klauspost/compress/pull/1101 + * flate: Use exact sizes for huffman tables https://github.com/klauspost/compress/pull/1103 + +* Feb 19th, 2025 - [1.18.0](https://github.com/klauspost/compress/releases/tag/v1.18.0) + * Add unsafe little endian loaders https://github.com/klauspost/compress/pull/1036 + * fix: check `r.err != nil` but return a nil value error `err` by @alingse in https://github.com/klauspost/compress/pull/1028 + * flate: Simplify L4-6 loading https://github.com/klauspost/compress/pull/1043 + * flate: Simplify matchlen (remove asm) https://github.com/klauspost/compress/pull/1045 + * s2: Improve small block compression speed w/o asm https://github.com/klauspost/compress/pull/1048 + * flate: Fix matchlen L5+L6 https://github.com/klauspost/compress/pull/1049 + * flate: Cleanup & reduce casts https://github.com/klauspost/compress/pull/1050 + +
+ See changes to v1.17.x + +* Oct 11th, 2024 - [1.17.11](https://github.com/klauspost/compress/releases/tag/v1.17.11) + * zstd: Fix extra CRC written with multiple Close calls https://github.com/klauspost/compress/pull/1017 + * s2: Don't use stack for index tables https://github.com/klauspost/compress/pull/1014 + * gzhttp: No content-type on no body response code by @juliens in https://github.com/klauspost/compress/pull/1011 + * gzhttp: Do not set the content-type when response has no body by @kevinpollet in https://github.com/klauspost/compress/pull/1013 + +* Sep 23rd, 2024 - [1.17.10](https://github.com/klauspost/compress/releases/tag/v1.17.10) + * gzhttp: Add TransportAlwaysDecompress option. https://github.com/klauspost/compress/pull/978 + * gzhttp: Add supported decompress request body by @mirecl in https://github.com/klauspost/compress/pull/1002 + * s2: Add EncodeBuffer buffer recycling callback https://github.com/klauspost/compress/pull/982 + * zstd: Improve memory usage on small streaming encodes https://github.com/klauspost/compress/pull/1007 + * flate: read data written with partial flush by @vajexal in https://github.com/klauspost/compress/pull/996 + +* Jun 12th, 2024 - [1.17.9](https://github.com/klauspost/compress/releases/tag/v1.17.9) + * s2: Reduce ReadFrom temporary allocations https://github.com/klauspost/compress/pull/949 + * flate, zstd: Shave some bytes off amd64 matchLen by @greatroar in https://github.com/klauspost/compress/pull/963 + * Upgrade zip/zlib to 1.22.4 upstream https://github.com/klauspost/compress/pull/970 https://github.com/klauspost/compress/pull/971 + * zstd: BuildDict fails with RLE table https://github.com/klauspost/compress/pull/951 + +* Apr 9th, 2024 - [1.17.8](https://github.com/klauspost/compress/releases/tag/v1.17.8) + * zstd: Reject blocks where reserved values are not 0 https://github.com/klauspost/compress/pull/885 + * zstd: Add RLE detection+encoding https://github.com/klauspost/compress/pull/938 + +* Feb 21st, 2024 - [1.17.7](https://github.com/klauspost/compress/releases/tag/v1.17.7) + * s2: Add AsyncFlush method: Complete the block without flushing by @Jille in https://github.com/klauspost/compress/pull/927 + * s2: Fix literal+repeat exceeds dst crash https://github.com/klauspost/compress/pull/930 + +* Feb 5th, 2024 - [1.17.6](https://github.com/klauspost/compress/releases/tag/v1.17.6) + * zstd: Fix incorrect repeat coding in best mode https://github.com/klauspost/compress/pull/923 + * s2: Fix DecodeConcurrent deadlock on errors https://github.com/klauspost/compress/pull/925 + +* Jan 26th, 2024 - [v1.17.5](https://github.com/klauspost/compress/releases/tag/v1.17.5) + * flate: Fix reset with dictionary on custom window encodes https://github.com/klauspost/compress/pull/912 + * zstd: Add Frame header encoding and stripping https://github.com/klauspost/compress/pull/908 + * zstd: Limit better/best default window to 8MB https://github.com/klauspost/compress/pull/913 + * zstd: Speed improvements by @greatroar in https://github.com/klauspost/compress/pull/896 https://github.com/klauspost/compress/pull/910 + * s2: Fix callbacks for skippable blocks and disallow 0xfe (Padding) by @Jille in https://github.com/klauspost/compress/pull/916 https://github.com/klauspost/compress/pull/917 +https://github.com/klauspost/compress/pull/919 https://github.com/klauspost/compress/pull/918 + +* Dec 1st, 2023 - [v1.17.4](https://github.com/klauspost/compress/releases/tag/v1.17.4) + * huff0: Speed up symbol counting by @greatroar in https://github.com/klauspost/compress/pull/887 + * huff0: Remove byteReader by @greatroar in https://github.com/klauspost/compress/pull/886 + * gzhttp: Allow overriding decompression on transport https://github.com/klauspost/compress/pull/892 + * gzhttp: Clamp compression level https://github.com/klauspost/compress/pull/890 + * gzip: Error out if reserved bits are set https://github.com/klauspost/compress/pull/891 + +* Nov 15th, 2023 - [v1.17.3](https://github.com/klauspost/compress/releases/tag/v1.17.3) + * fse: Fix max header size https://github.com/klauspost/compress/pull/881 + * zstd: Improve better/best compression https://github.com/klauspost/compress/pull/877 + * gzhttp: Fix missing content type on Close https://github.com/klauspost/compress/pull/883 + +* Oct 22nd, 2023 - [v1.17.2](https://github.com/klauspost/compress/releases/tag/v1.17.2) + * zstd: Fix rare *CORRUPTION* output in "best" mode. See https://github.com/klauspost/compress/pull/876 + +* Oct 14th, 2023 - [v1.17.1](https://github.com/klauspost/compress/releases/tag/v1.17.1) + * s2: Fix S2 "best" dictionary wrong encoding https://github.com/klauspost/compress/pull/871 + * flate: Reduce allocations in decompressor and minor code improvements by @fakefloordiv in https://github.com/klauspost/compress/pull/869 + * s2: Fix EstimateBlockSize on 6&7 length input https://github.com/klauspost/compress/pull/867 + +* Sept 19th, 2023 - [v1.17.0](https://github.com/klauspost/compress/releases/tag/v1.17.0) + * Add experimental dictionary builder https://github.com/klauspost/compress/pull/853 + * Add xerial snappy read/writer https://github.com/klauspost/compress/pull/838 + * flate: Add limited window compression https://github.com/klauspost/compress/pull/843 + * s2: Do 2 overlapping match checks https://github.com/klauspost/compress/pull/839 + * flate: Add amd64 assembly matchlen https://github.com/klauspost/compress/pull/837 + * gzip: Copy bufio.Reader on Reset by @thatguystone in https://github.com/klauspost/compress/pull/860 + +
+
+ See changes to v1.16.x + + +* July 1st, 2023 - [v1.16.7](https://github.com/klauspost/compress/releases/tag/v1.16.7) + * zstd: Fix default level first dictionary encode https://github.com/klauspost/compress/pull/829 + * s2: add GetBufferCapacity() method by @GiedriusS in https://github.com/klauspost/compress/pull/832 + +* June 13, 2023 - [v1.16.6](https://github.com/klauspost/compress/releases/tag/v1.16.6) + * zstd: correctly ignore WithEncoderPadding(1) by @ianlancetaylor in https://github.com/klauspost/compress/pull/806 + * zstd: Add amd64 match length assembly https://github.com/klauspost/compress/pull/824 + * gzhttp: Handle informational headers by @rtribotte in https://github.com/klauspost/compress/pull/815 + * s2: Improve Better compression slightly https://github.com/klauspost/compress/pull/663 + +* Apr 16, 2023 - [v1.16.5](https://github.com/klauspost/compress/releases/tag/v1.16.5) + * zstd: readByte needs to use io.ReadFull by @jnoxon in https://github.com/klauspost/compress/pull/802 + * gzip: Fix WriterTo after initial read https://github.com/klauspost/compress/pull/804 + +* Apr 5, 2023 - [v1.16.4](https://github.com/klauspost/compress/releases/tag/v1.16.4) + * zstd: Improve zstd best efficiency by @greatroar and @klauspost in https://github.com/klauspost/compress/pull/784 + * zstd: Respect WithAllLitEntropyCompression https://github.com/klauspost/compress/pull/792 + * zstd: Fix amd64 not always detecting corrupt data https://github.com/klauspost/compress/pull/785 + * zstd: Various minor improvements by @greatroar in https://github.com/klauspost/compress/pull/788 https://github.com/klauspost/compress/pull/794 https://github.com/klauspost/compress/pull/795 + * s2: Fix huge block overflow https://github.com/klauspost/compress/pull/779 + * s2: Allow CustomEncoder fallback https://github.com/klauspost/compress/pull/780 + * gzhttp: Support ResponseWriter Unwrap() in gzhttp handler by @jgimenez in https://github.com/klauspost/compress/pull/799 + +* Mar 13, 2023 - [v1.16.1](https://github.com/klauspost/compress/releases/tag/v1.16.1) + * zstd: Speed up + improve best encoder by @greatroar in https://github.com/klauspost/compress/pull/776 + * gzhttp: Add optional [BREACH mitigation](https://github.com/klauspost/compress/tree/master/gzhttp#breach-mitigation). https://github.com/klauspost/compress/pull/762 https://github.com/klauspost/compress/pull/768 https://github.com/klauspost/compress/pull/769 https://github.com/klauspost/compress/pull/770 https://github.com/klauspost/compress/pull/767 + * s2: Add Intel LZ4s converter https://github.com/klauspost/compress/pull/766 + * zstd: Minor bug fixes https://github.com/klauspost/compress/pull/771 https://github.com/klauspost/compress/pull/772 https://github.com/klauspost/compress/pull/773 + * huff0: Speed up compress1xDo by @greatroar in https://github.com/klauspost/compress/pull/774 + +* Feb 26, 2023 - [v1.16.0](https://github.com/klauspost/compress/releases/tag/v1.16.0) + * s2: Add [Dictionary](https://github.com/klauspost/compress/tree/master/s2#dictionaries) support. https://github.com/klauspost/compress/pull/685 + * s2: Add Compression Size Estimate. https://github.com/klauspost/compress/pull/752 + * s2: Add support for custom stream encoder. https://github.com/klauspost/compress/pull/755 + * s2: Add LZ4 block converter. https://github.com/klauspost/compress/pull/748 + * s2: Support io.ReaderAt in ReadSeeker. https://github.com/klauspost/compress/pull/747 + * s2c/s2sx: Use concurrent decoding. https://github.com/klauspost/compress/pull/746 +
+ +
+ See changes to v1.15.x + +* Jan 21st, 2023 (v1.15.15) + * deflate: Improve level 7-9 https://github.com/klauspost/compress/pull/739 + * zstd: Add delta encoding support by @greatroar in https://github.com/klauspost/compress/pull/728 + * zstd: Various speed improvements by @greatroar https://github.com/klauspost/compress/pull/741 https://github.com/klauspost/compress/pull/734 https://github.com/klauspost/compress/pull/736 https://github.com/klauspost/compress/pull/744 https://github.com/klauspost/compress/pull/743 https://github.com/klauspost/compress/pull/745 + * gzhttp: Add SuffixETag() and DropETag() options to prevent ETag collisions on compressed responses by @willbicks in https://github.com/klauspost/compress/pull/740 + +* Jan 3rd, 2023 (v1.15.14) + + * flate: Improve speed in big stateless blocks https://github.com/klauspost/compress/pull/718 + * zstd: Minor speed tweaks by @greatroar in https://github.com/klauspost/compress/pull/716 https://github.com/klauspost/compress/pull/720 + * export NoGzipResponseWriter for custom ResponseWriter wrappers by @harshavardhana in https://github.com/klauspost/compress/pull/722 + * s2: Add example for indexing and existing stream https://github.com/klauspost/compress/pull/723 + +* Dec 11, 2022 (v1.15.13) + * zstd: Add [MaxEncodedSize](https://pkg.go.dev/github.com/klauspost/compress@v1.15.13/zstd#Encoder.MaxEncodedSize) to encoder https://github.com/klauspost/compress/pull/691 + * zstd: Various tweaks and improvements https://github.com/klauspost/compress/pull/693 https://github.com/klauspost/compress/pull/695 https://github.com/klauspost/compress/pull/696 https://github.com/klauspost/compress/pull/701 https://github.com/klauspost/compress/pull/702 https://github.com/klauspost/compress/pull/703 https://github.com/klauspost/compress/pull/704 https://github.com/klauspost/compress/pull/705 https://github.com/klauspost/compress/pull/706 https://github.com/klauspost/compress/pull/707 https://github.com/klauspost/compress/pull/708 + +* Oct 26, 2022 (v1.15.12) + + * zstd: Tweak decoder allocs. https://github.com/klauspost/compress/pull/680 + * gzhttp: Always delete `HeaderNoCompression` https://github.com/klauspost/compress/pull/683 + +* Sept 26, 2022 (v1.15.11) + + * flate: Improve level 1-3 compression https://github.com/klauspost/compress/pull/678 + * zstd: Improve "best" compression by @nightwolfz in https://github.com/klauspost/compress/pull/677 + * zstd: Fix+reduce decompression allocations https://github.com/klauspost/compress/pull/668 + * zstd: Fix non-effective noescape tag https://github.com/klauspost/compress/pull/667 + +* Sept 16, 2022 (v1.15.10) + + * zstd: Add [WithDecodeAllCapLimit](https://pkg.go.dev/github.com/klauspost/compress@v1.15.10/zstd#WithDecodeAllCapLimit) https://github.com/klauspost/compress/pull/649 + * Add Go 1.19 - deprecate Go 1.16 https://github.com/klauspost/compress/pull/651 + * flate: Improve level 5+6 compression https://github.com/klauspost/compress/pull/656 + * zstd: Improve "better" compression https://github.com/klauspost/compress/pull/657 + * s2: Improve "best" compression https://github.com/klauspost/compress/pull/658 + * s2: Improve "better" compression. https://github.com/klauspost/compress/pull/635 + * s2: Slightly faster non-assembly decompression https://github.com/klauspost/compress/pull/646 + * Use arrays for constant size copies https://github.com/klauspost/compress/pull/659 + +* July 21, 2022 (v1.15.9) + + * zstd: Fix decoder crash on amd64 (no BMI) on invalid input https://github.com/klauspost/compress/pull/645 + * zstd: Disable decoder extended memory copies (amd64) due to possible crashes https://github.com/klauspost/compress/pull/644 + * zstd: Allow single segments up to "max decoded size" https://github.com/klauspost/compress/pull/643 + +* July 13, 2022 (v1.15.8) + + * gzip: fix stack exhaustion bug in Reader.Read https://github.com/klauspost/compress/pull/641 + * s2: Add Index header trim/restore https://github.com/klauspost/compress/pull/638 + * zstd: Optimize seqdeq amd64 asm by @greatroar in https://github.com/klauspost/compress/pull/636 + * zstd: Improve decoder memcopy https://github.com/klauspost/compress/pull/637 + * huff0: Pass a single bitReader pointer to asm by @greatroar in https://github.com/klauspost/compress/pull/634 + * zstd: Branchless getBits for amd64 w/o BMI2 by @greatroar in https://github.com/klauspost/compress/pull/640 + * gzhttp: Remove header before writing https://github.com/klauspost/compress/pull/639 + +* June 29, 2022 (v1.15.7) + + * s2: Fix absolute forward seeks https://github.com/klauspost/compress/pull/633 + * zip: Merge upstream https://github.com/klauspost/compress/pull/631 + * zip: Re-add zip64 fix https://github.com/klauspost/compress/pull/624 + * zstd: translate fseDecoder.buildDtable into asm by @WojciechMula in https://github.com/klauspost/compress/pull/598 + * flate: Faster histograms https://github.com/klauspost/compress/pull/620 + * deflate: Use compound hcode https://github.com/klauspost/compress/pull/622 + +* June 3, 2022 (v1.15.6) + * s2: Improve coding for long, close matches https://github.com/klauspost/compress/pull/613 + * s2c: Add Snappy/S2 stream recompression https://github.com/klauspost/compress/pull/611 + * zstd: Always use configured block size https://github.com/klauspost/compress/pull/605 + * zstd: Fix incorrect hash table placement for dict encoding in default https://github.com/klauspost/compress/pull/606 + * zstd: Apply default config to ZipDecompressor without options https://github.com/klauspost/compress/pull/608 + * gzhttp: Exclude more common archive formats https://github.com/klauspost/compress/pull/612 + * s2: Add ReaderIgnoreCRC https://github.com/klauspost/compress/pull/609 + * s2: Remove sanity load on index creation https://github.com/klauspost/compress/pull/607 + * snappy: Use dedicated function for scoring https://github.com/klauspost/compress/pull/614 + * s2c+s2d: Use official snappy framed extension https://github.com/klauspost/compress/pull/610 + +* May 25, 2022 (v1.15.5) + * s2: Add concurrent stream decompression https://github.com/klauspost/compress/pull/602 + * s2: Fix final emit oob read crash on amd64 https://github.com/klauspost/compress/pull/601 + * huff0: asm implementation of Decompress1X by @WojciechMula https://github.com/klauspost/compress/pull/596 + * zstd: Use 1 less goroutine for stream decoding https://github.com/klauspost/compress/pull/588 + * zstd: Copy literal in 16 byte blocks when possible https://github.com/klauspost/compress/pull/592 + * zstd: Speed up when WithDecoderLowmem(false) https://github.com/klauspost/compress/pull/599 + * zstd: faster next state update in BMI2 version of decode by @WojciechMula in https://github.com/klauspost/compress/pull/593 + * huff0: Do not check max size when reading table. https://github.com/klauspost/compress/pull/586 + * flate: Inplace hashing for level 7-9 https://github.com/klauspost/compress/pull/590 + + +* May 11, 2022 (v1.15.4) + * huff0: decompress directly into output by @WojciechMula in [#577](https://github.com/klauspost/compress/pull/577) + * inflate: Keep dict on stack [#581](https://github.com/klauspost/compress/pull/581) + * zstd: Faster decoding memcopy in asm [#583](https://github.com/klauspost/compress/pull/583) + * zstd: Fix ignored crc [#580](https://github.com/klauspost/compress/pull/580) + +* May 5, 2022 (v1.15.3) + * zstd: Allow to ignore checksum checking by @WojciechMula [#572](https://github.com/klauspost/compress/pull/572) + * s2: Fix incorrect seek for io.SeekEnd in [#575](https://github.com/klauspost/compress/pull/575) + +* Apr 26, 2022 (v1.15.2) + * zstd: Add x86-64 assembly for decompression on streams and blocks. Contributed by [@WojciechMula](https://github.com/WojciechMula). Typically 2x faster. [#528](https://github.com/klauspost/compress/pull/528) [#531](https://github.com/klauspost/compress/pull/531) [#545](https://github.com/klauspost/compress/pull/545) [#537](https://github.com/klauspost/compress/pull/537) + * zstd: Add options to ZipDecompressor and fixes [#539](https://github.com/klauspost/compress/pull/539) + * s2: Use sorted search for index [#555](https://github.com/klauspost/compress/pull/555) + * Minimum version is Go 1.16, added CI test on 1.18. + +* Mar 11, 2022 (v1.15.1) + * huff0: Add x86 assembly of Decode4X by @WojciechMula in [#512](https://github.com/klauspost/compress/pull/512) + * zstd: Reuse zip decoders in [#514](https://github.com/klauspost/compress/pull/514) + * zstd: Detect extra block data and report as corrupted in [#520](https://github.com/klauspost/compress/pull/520) + * zstd: Handle zero sized frame content size stricter in [#521](https://github.com/klauspost/compress/pull/521) + * zstd: Add stricter block size checks in [#523](https://github.com/klauspost/compress/pull/523) + +* Mar 3, 2022 (v1.15.0) + * zstd: Refactor decoder [#498](https://github.com/klauspost/compress/pull/498) + * zstd: Add stream encoding without goroutines [#505](https://github.com/klauspost/compress/pull/505) + * huff0: Prevent single blocks exceeding 16 bits by @klauspost in[#507](https://github.com/klauspost/compress/pull/507) + * flate: Inline literal emission [#509](https://github.com/klauspost/compress/pull/509) + * gzhttp: Add zstd to transport [#400](https://github.com/klauspost/compress/pull/400) + * gzhttp: Make content-type optional [#510](https://github.com/klauspost/compress/pull/510) + +Both compression and decompression now supports "synchronous" stream operations. This means that whenever "concurrency" is set to 1, they will operate without spawning goroutines. + +Stream decompression is now faster on asynchronous, since the goroutine allocation much more effectively splits the workload. On typical streams this will typically use 2 cores fully for decompression. When a stream has finished decoding no goroutines will be left over, so decoders can now safely be pooled and still be garbage collected. + +While the release has been extensively tested, it is recommended to testing when upgrading. + +
+ +
+ See changes to v1.14.x + +* Feb 22, 2022 (v1.14.4) + * flate: Fix rare huffman only (-2) corruption. [#503](https://github.com/klauspost/compress/pull/503) + * zip: Update deprecated CreateHeaderRaw to correctly call CreateRaw by @saracen in [#502](https://github.com/klauspost/compress/pull/502) + * zip: don't read data descriptor early by @saracen in [#501](https://github.com/klauspost/compress/pull/501) #501 + * huff0: Use static decompression buffer up to 30% faster [#499](https://github.com/klauspost/compress/pull/499) [#500](https://github.com/klauspost/compress/pull/500) + +* Feb 17, 2022 (v1.14.3) + * flate: Improve fastest levels compression speed ~10% more throughput. [#482](https://github.com/klauspost/compress/pull/482) [#489](https://github.com/klauspost/compress/pull/489) [#490](https://github.com/klauspost/compress/pull/490) [#491](https://github.com/klauspost/compress/pull/491) [#494](https://github.com/klauspost/compress/pull/494) [#478](https://github.com/klauspost/compress/pull/478) + * flate: Faster decompression speed, ~5-10%. [#483](https://github.com/klauspost/compress/pull/483) + * s2: Faster compression with Go v1.18 and amd64 microarch level 3+. [#484](https://github.com/klauspost/compress/pull/484) [#486](https://github.com/klauspost/compress/pull/486) + +* Jan 25, 2022 (v1.14.2) + * zstd: improve header decoder by @dsnet [#476](https://github.com/klauspost/compress/pull/476) + * zstd: Add bigger default blocks [#469](https://github.com/klauspost/compress/pull/469) + * zstd: Remove unused decompression buffer [#470](https://github.com/klauspost/compress/pull/470) + * zstd: Fix logically dead code by @ningmingxiao [#472](https://github.com/klauspost/compress/pull/472) + * flate: Improve level 7-9 [#471](https://github.com/klauspost/compress/pull/471) [#473](https://github.com/klauspost/compress/pull/473) + * zstd: Add noasm tag for xxhash [#475](https://github.com/klauspost/compress/pull/475) + +* Jan 11, 2022 (v1.14.1) + * s2: Add stream index in [#462](https://github.com/klauspost/compress/pull/462) + * flate: Speed and efficiency improvements in [#439](https://github.com/klauspost/compress/pull/439) [#461](https://github.com/klauspost/compress/pull/461) [#455](https://github.com/klauspost/compress/pull/455) [#452](https://github.com/klauspost/compress/pull/452) [#458](https://github.com/klauspost/compress/pull/458) + * zstd: Performance improvement in [#420]( https://github.com/klauspost/compress/pull/420) [#456](https://github.com/klauspost/compress/pull/456) [#437](https://github.com/klauspost/compress/pull/437) [#467](https://github.com/klauspost/compress/pull/467) [#468](https://github.com/klauspost/compress/pull/468) + * zstd: add arm64 xxhash assembly in [#464](https://github.com/klauspost/compress/pull/464) + * Add garbled for binaries for s2 in [#445](https://github.com/klauspost/compress/pull/445) +
+ +
+ See changes to v1.13.x + +* Aug 30, 2021 (v1.13.5) + * gz/zlib/flate: Alias stdlib errors [#425](https://github.com/klauspost/compress/pull/425) + * s2: Add block support to commandline tools [#413](https://github.com/klauspost/compress/pull/413) + * zstd: pooledZipWriter should return Writers to the same pool [#426](https://github.com/klauspost/compress/pull/426) + * Removed golang/snappy as external dependency for tests [#421](https://github.com/klauspost/compress/pull/421) + +* Aug 12, 2021 (v1.13.4) + * Add [snappy replacement package](https://github.com/klauspost/compress/tree/master/snappy). + * zstd: Fix incorrect encoding in "best" mode [#415](https://github.com/klauspost/compress/pull/415) + +* Aug 3, 2021 (v1.13.3) + * zstd: Improve Best compression [#404](https://github.com/klauspost/compress/pull/404) + * zstd: Fix WriteTo error forwarding [#411](https://github.com/klauspost/compress/pull/411) + * gzhttp: Return http.HandlerFunc instead of http.Handler. Unlikely breaking change. [#406](https://github.com/klauspost/compress/pull/406) + * s2sx: Fix max size error [#399](https://github.com/klauspost/compress/pull/399) + * zstd: Add optional stream content size on reset [#401](https://github.com/klauspost/compress/pull/401) + * zstd: use SpeedBestCompression for level >= 10 [#410](https://github.com/klauspost/compress/pull/410) + +* Jun 14, 2021 (v1.13.1) + * s2: Add full Snappy output support [#396](https://github.com/klauspost/compress/pull/396) + * zstd: Add configurable [Decoder window](https://pkg.go.dev/github.com/klauspost/compress/zstd#WithDecoderMaxWindow) size [#394](https://github.com/klauspost/compress/pull/394) + * gzhttp: Add header to skip compression [#389](https://github.com/klauspost/compress/pull/389) + * s2: Improve speed with bigger output margin [#395](https://github.com/klauspost/compress/pull/395) + +* Jun 3, 2021 (v1.13.0) + * Added [gzhttp](https://github.com/klauspost/compress/tree/master/gzhttp#gzip-handler) which allows wrapping HTTP servers and clients with GZIP compressors. + * zstd: Detect short invalid signatures [#382](https://github.com/klauspost/compress/pull/382) + * zstd: Spawn decoder goroutine only if needed. [#380](https://github.com/klauspost/compress/pull/380) +
+ + +
+ See changes to v1.12.x + +* May 25, 2021 (v1.12.3) + * deflate: Better/faster Huffman encoding [#374](https://github.com/klauspost/compress/pull/374) + * deflate: Allocate less for history. [#375](https://github.com/klauspost/compress/pull/375) + * zstd: Forward read errors [#373](https://github.com/klauspost/compress/pull/373) + +* Apr 27, 2021 (v1.12.2) + * zstd: Improve better/best compression [#360](https://github.com/klauspost/compress/pull/360) [#364](https://github.com/klauspost/compress/pull/364) [#365](https://github.com/klauspost/compress/pull/365) + * zstd: Add helpers to compress/decompress zstd inside zip files [#363](https://github.com/klauspost/compress/pull/363) + * deflate: Improve level 5+6 compression [#367](https://github.com/klauspost/compress/pull/367) + * s2: Improve better/best compression [#358](https://github.com/klauspost/compress/pull/358) [#359](https://github.com/klauspost/compress/pull/358) + * s2: Load after checking src limit on amd64. [#362](https://github.com/klauspost/compress/pull/362) + * s2sx: Limit max executable size [#368](https://github.com/klauspost/compress/pull/368) + +* Apr 14, 2021 (v1.12.1) + * snappy package removed. Upstream added as dependency. + * s2: Better compression in "best" mode [#353](https://github.com/klauspost/compress/pull/353) + * s2sx: Add stdin input and detect pre-compressed from signature [#352](https://github.com/klauspost/compress/pull/352) + * s2c/s2d: Add http as possible input [#348](https://github.com/klauspost/compress/pull/348) + * s2c/s2d/s2sx: Always truncate when writing files [#352](https://github.com/klauspost/compress/pull/352) + * zstd: Reduce memory usage further when using [WithLowerEncoderMem](https://pkg.go.dev/github.com/klauspost/compress/zstd#WithLowerEncoderMem) [#346](https://github.com/klauspost/compress/pull/346) + * s2: Fix potential problem with amd64 assembly and profilers [#349](https://github.com/klauspost/compress/pull/349) +
+ +
+ See changes to v1.11.x + +* Mar 26, 2021 (v1.11.13) + * zstd: Big speedup on small dictionary encodes [#344](https://github.com/klauspost/compress/pull/344) [#345](https://github.com/klauspost/compress/pull/345) + * zstd: Add [WithLowerEncoderMem](https://pkg.go.dev/github.com/klauspost/compress/zstd#WithLowerEncoderMem) encoder option [#336](https://github.com/klauspost/compress/pull/336) + * deflate: Improve entropy compression [#338](https://github.com/klauspost/compress/pull/338) + * s2: Clean up and minor performance improvement in best [#341](https://github.com/klauspost/compress/pull/341) + +* Mar 5, 2021 (v1.11.12) + * s2: Add `s2sx` binary that creates [self extracting archives](https://github.com/klauspost/compress/tree/master/s2#s2sx-self-extracting-archives). + * s2: Speed up decompression on non-assembly platforms [#328](https://github.com/klauspost/compress/pull/328) + +* Mar 1, 2021 (v1.11.9) + * s2: Add ARM64 decompression assembly. Around 2x output speed. [#324](https://github.com/klauspost/compress/pull/324) + * s2: Improve "better" speed and efficiency. [#325](https://github.com/klauspost/compress/pull/325) + * s2: Fix binaries. + +* Feb 25, 2021 (v1.11.8) + * s2: Fixed occasional out-of-bounds write on amd64. Upgrade recommended. + * s2: Add AMD64 assembly for better mode. 25-50% faster. [#315](https://github.com/klauspost/compress/pull/315) + * s2: Less upfront decoder allocation. [#322](https://github.com/klauspost/compress/pull/322) + * zstd: Faster "compression" of incompressible data. [#314](https://github.com/klauspost/compress/pull/314) + * zip: Fix zip64 headers. [#313](https://github.com/klauspost/compress/pull/313) + +* Jan 14, 2021 (v1.11.7) + * Use Bytes() interface to get bytes across packages. [#309](https://github.com/klauspost/compress/pull/309) + * s2: Add 'best' compression option. [#310](https://github.com/klauspost/compress/pull/310) + * s2: Add ReaderMaxBlockSize, changes `s2.NewReader` signature to include varargs. [#311](https://github.com/klauspost/compress/pull/311) + * s2: Fix crash on small better buffers. [#308](https://github.com/klauspost/compress/pull/308) + * s2: Clean up decoder. [#312](https://github.com/klauspost/compress/pull/312) + +* Jan 7, 2021 (v1.11.6) + * zstd: Make decoder allocations smaller [#306](https://github.com/klauspost/compress/pull/306) + * zstd: Free Decoder resources when Reset is called with a nil io.Reader [#305](https://github.com/klauspost/compress/pull/305) + +* Dec 20, 2020 (v1.11.4) + * zstd: Add Best compression mode [#304](https://github.com/klauspost/compress/pull/304) + * Add header decoder [#299](https://github.com/klauspost/compress/pull/299) + * s2: Add uncompressed stream option [#297](https://github.com/klauspost/compress/pull/297) + * Simplify/speed up small blocks with known max size. [#300](https://github.com/klauspost/compress/pull/300) + * zstd: Always reset literal dict encoder [#303](https://github.com/klauspost/compress/pull/303) + +* Nov 15, 2020 (v1.11.3) + * inflate: 10-15% faster decompression [#293](https://github.com/klauspost/compress/pull/293) + * zstd: Tweak DecodeAll default allocation [#295](https://github.com/klauspost/compress/pull/295) + +* Oct 11, 2020 (v1.11.2) + * s2: Fix out of bounds read in "better" block compression [#291](https://github.com/klauspost/compress/pull/291) + +* Oct 1, 2020 (v1.11.1) + * zstd: Set allLitEntropy true in default configuration [#286](https://github.com/klauspost/compress/pull/286) + +* Sept 8, 2020 (v1.11.0) + * zstd: Add experimental compression [dictionaries](https://github.com/klauspost/compress/tree/master/zstd#dictionaries) [#281](https://github.com/klauspost/compress/pull/281) + * zstd: Fix mixed Write and ReadFrom calls [#282](https://github.com/klauspost/compress/pull/282) + * inflate/gz: Limit variable shifts, ~5% faster decompression [#274](https://github.com/klauspost/compress/pull/274) +
+ +
+ See changes to v1.10.x + +* July 8, 2020 (v1.10.11) + * zstd: Fix extra block when compressing with ReadFrom. [#278](https://github.com/klauspost/compress/pull/278) + * huff0: Also populate compression table when reading decoding table. [#275](https://github.com/klauspost/compress/pull/275) + +* June 23, 2020 (v1.10.10) + * zstd: Skip entropy compression in fastest mode when no matches. [#270](https://github.com/klauspost/compress/pull/270) + +* June 16, 2020 (v1.10.9): + * zstd: API change for specifying dictionaries. See [#268](https://github.com/klauspost/compress/pull/268) + * zip: update CreateHeaderRaw to handle zip64 fields. [#266](https://github.com/klauspost/compress/pull/266) + * Fuzzit tests removed. The service has been purchased and is no longer available. + +* June 5, 2020 (v1.10.8): + * 1.15x faster zstd block decompression. [#265](https://github.com/klauspost/compress/pull/265) + +* June 1, 2020 (v1.10.7): + * Added zstd decompression [dictionary support](https://github.com/klauspost/compress/tree/master/zstd#dictionaries) + * Increase zstd decompression speed up to 1.19x. [#259](https://github.com/klauspost/compress/pull/259) + * Remove internal reset call in zstd compression and reduce allocations. [#263](https://github.com/klauspost/compress/pull/263) + +* May 21, 2020: (v1.10.6) + * zstd: Reduce allocations while decoding. [#258](https://github.com/klauspost/compress/pull/258), [#252](https://github.com/klauspost/compress/pull/252) + * zstd: Stricter decompression checks. + +* April 12, 2020: (v1.10.5) + * s2-commands: Flush output when receiving SIGINT. [#239](https://github.com/klauspost/compress/pull/239) + +* Apr 8, 2020: (v1.10.4) + * zstd: Minor/special case optimizations. [#251](https://github.com/klauspost/compress/pull/251), [#250](https://github.com/klauspost/compress/pull/250), [#249](https://github.com/klauspost/compress/pull/249), [#247](https://github.com/klauspost/compress/pull/247) +* Mar 11, 2020: (v1.10.3) + * s2: Use S2 encoder in pure Go mode for Snappy output as well. [#245](https://github.com/klauspost/compress/pull/245) + * s2: Fix pure Go block encoder. [#244](https://github.com/klauspost/compress/pull/244) + * zstd: Added "better compression" mode. [#240](https://github.com/klauspost/compress/pull/240) + * zstd: Improve speed of fastest compression mode by 5-10% [#241](https://github.com/klauspost/compress/pull/241) + * zstd: Skip creating encoders when not needed. [#238](https://github.com/klauspost/compress/pull/238) + +* Feb 27, 2020: (v1.10.2) + * Close to 50% speedup in inflate (gzip/zip decompression). [#236](https://github.com/klauspost/compress/pull/236) [#234](https://github.com/klauspost/compress/pull/234) [#232](https://github.com/klauspost/compress/pull/232) + * Reduce deflate level 1-6 memory usage up to 59%. [#227](https://github.com/klauspost/compress/pull/227) + +* Feb 18, 2020: (v1.10.1) + * Fix zstd crash when resetting multiple times without sending data. [#226](https://github.com/klauspost/compress/pull/226) + * deflate: Fix dictionary use on level 1-6. [#224](https://github.com/klauspost/compress/pull/224) + * Remove deflate writer reference when closing. [#224](https://github.com/klauspost/compress/pull/224) + +* Feb 4, 2020: (v1.10.0) + * Add optional dictionary to [stateless deflate](https://pkg.go.dev/github.com/klauspost/compress/flate?tab=doc#StatelessDeflate). Breaking change, send `nil` for previous behaviour. [#216](https://github.com/klauspost/compress/pull/216) + * Fix buffer overflow on repeated small block deflate. [#218](https://github.com/klauspost/compress/pull/218) + * Allow copying content from an existing ZIP file without decompressing+compressing. [#214](https://github.com/klauspost/compress/pull/214) + * Added [S2](https://github.com/klauspost/compress/tree/master/s2#s2-compression) AMD64 assembler and various optimizations. Stream speed >10GB/s. [#186](https://github.com/klauspost/compress/pull/186) + +
+ +
+ See changes prior to v1.10.0 + +* Jan 20,2020 (v1.9.8) Optimize gzip/deflate with better size estimates and faster table generation. [#207](https://github.com/klauspost/compress/pull/207) by [luyu6056](https://github.com/luyu6056), [#206](https://github.com/klauspost/compress/pull/206). +* Jan 11, 2020: S2 Encode/Decode will use provided buffer if capacity is big enough. [#204](https://github.com/klauspost/compress/pull/204) +* Jan 5, 2020: (v1.9.7) Fix another zstd regression in v1.9.5 - v1.9.6 removed. +* Jan 4, 2020: (v1.9.6) Regression in v1.9.5 fixed causing corrupt zstd encodes in rare cases. +* Jan 4, 2020: Faster IO in [s2c + s2d commandline tools](https://github.com/klauspost/compress/tree/master/s2#commandline-tools) compression/decompression. [#192](https://github.com/klauspost/compress/pull/192) +* Dec 29, 2019: Removed v1.9.5 since fuzz tests showed a compatibility problem with the reference zstandard decoder. +* Dec 29, 2019: (v1.9.5) zstd: 10-20% faster block compression. [#199](https://github.com/klauspost/compress/pull/199) +* Dec 29, 2019: [zip](https://godoc.org/github.com/klauspost/compress/zip) package updated with latest Go features +* Dec 29, 2019: zstd: Single segment flag condintions tweaked. [#197](https://github.com/klauspost/compress/pull/197) +* Dec 18, 2019: s2: Faster compression when ReadFrom is used. [#198](https://github.com/klauspost/compress/pull/198) +* Dec 10, 2019: s2: Fix repeat length output when just above at 16MB limit. +* Dec 10, 2019: zstd: Add function to get decoder as io.ReadCloser. [#191](https://github.com/klauspost/compress/pull/191) +* Dec 3, 2019: (v1.9.4) S2: limit max repeat length. [#188](https://github.com/klauspost/compress/pull/188) +* Dec 3, 2019: Add [WithNoEntropyCompression](https://godoc.org/github.com/klauspost/compress/zstd#WithNoEntropyCompression) to zstd [#187](https://github.com/klauspost/compress/pull/187) +* Dec 3, 2019: Reduce memory use for tests. Check for leaked goroutines. +* Nov 28, 2019 (v1.9.3) Less allocations in stateless deflate. +* Nov 28, 2019: 5-20% Faster huff0 decode. Impacts zstd as well. [#184](https://github.com/klauspost/compress/pull/184) +* Nov 12, 2019 (v1.9.2) Added [Stateless Compression](#stateless-compression) for gzip/deflate. +* Nov 12, 2019: Fixed zstd decompression of large single blocks. [#180](https://github.com/klauspost/compress/pull/180) +* Nov 11, 2019: Set default [s2c](https://github.com/klauspost/compress/tree/master/s2#commandline-tools) block size to 4MB. +* Nov 11, 2019: Reduce inflate memory use by 1KB. +* Nov 10, 2019: Less allocations in deflate bit writer. +* Nov 10, 2019: Fix inconsistent error returned by zstd decoder. +* Oct 28, 2019 (v1.9.1) ztsd: Fix crash when compressing blocks. [#174](https://github.com/klauspost/compress/pull/174) +* Oct 24, 2019 (v1.9.0) zstd: Fix rare data corruption [#173](https://github.com/klauspost/compress/pull/173) +* Oct 24, 2019 zstd: Fix huff0 out of buffer write [#171](https://github.com/klauspost/compress/pull/171) and always return errors [#172](https://github.com/klauspost/compress/pull/172) +* Oct 10, 2019: Big deflate rewrite, 30-40% faster with better compression [#105](https://github.com/klauspost/compress/pull/105) + +
+ +
+ See changes prior to v1.9.0 + +* Oct 10, 2019: (v1.8.6) zstd: Allow partial reads to get flushed data. [#169](https://github.com/klauspost/compress/pull/169) +* Oct 3, 2019: Fix inconsistent results on broken zstd streams. +* Sep 25, 2019: Added `-rm` (remove source files) and `-q` (no output except errors) to `s2c` and `s2d` [commands](https://github.com/klauspost/compress/tree/master/s2#commandline-tools) +* Sep 16, 2019: (v1.8.4) Add `s2c` and `s2d` [commandline tools](https://github.com/klauspost/compress/tree/master/s2#commandline-tools). +* Sep 10, 2019: (v1.8.3) Fix s2 decoder [Skip](https://godoc.org/github.com/klauspost/compress/s2#Reader.Skip). +* Sep 7, 2019: zstd: Added [WithWindowSize](https://godoc.org/github.com/klauspost/compress/zstd#WithWindowSize), contributed by [ianwilkes](https://github.com/ianwilkes). +* Sep 5, 2019: (v1.8.2) Add [WithZeroFrames](https://godoc.org/github.com/klauspost/compress/zstd#WithZeroFrames) which adds full zero payload block encoding option. +* Sep 5, 2019: Lazy initialization of zstandard predefined en/decoder tables. +* Aug 26, 2019: (v1.8.1) S2: 1-2% compression increase in "better" compression mode. +* Aug 26, 2019: zstd: Check maximum size of Huffman 1X compressed literals while decoding. +* Aug 24, 2019: (v1.8.0) Added [S2 compression](https://github.com/klauspost/compress/tree/master/s2#s2-compression), a high performance replacement for Snappy. +* Aug 21, 2019: (v1.7.6) Fixed minor issues found by fuzzer. One could lead to zstd not decompressing. +* Aug 18, 2019: Add [fuzzit](https://fuzzit.dev/) continuous fuzzing. +* Aug 14, 2019: zstd: Skip incompressible data 2x faster. [#147](https://github.com/klauspost/compress/pull/147) +* Aug 4, 2019 (v1.7.5): Better literal compression. [#146](https://github.com/klauspost/compress/pull/146) +* Aug 4, 2019: Faster zstd compression. [#143](https://github.com/klauspost/compress/pull/143) [#144](https://github.com/klauspost/compress/pull/144) +* Aug 4, 2019: Faster zstd decompression. [#145](https://github.com/klauspost/compress/pull/145) [#143](https://github.com/klauspost/compress/pull/143) [#142](https://github.com/klauspost/compress/pull/142) +* July 15, 2019 (v1.7.4): Fix double EOF block in rare cases on zstd encoder. +* July 15, 2019 (v1.7.3): Minor speedup/compression increase in default zstd encoder. +* July 14, 2019: zstd decoder: Fix decompression error on multiple uses with mixed content. +* July 7, 2019 (v1.7.2): Snappy update, zstd decoder potential race fix. +* June 17, 2019: zstd decompression bugfix. +* June 17, 2019: fix 32 bit builds. +* June 17, 2019: Easier use in modules (less dependencies). +* June 9, 2019: New stronger "default" [zstd](https://github.com/klauspost/compress/tree/master/zstd#zstd) compression mode. Matches zstd default compression ratio. +* June 5, 2019: 20-40% throughput in [zstandard](https://github.com/klauspost/compress/tree/master/zstd#zstd) compression and better compression. +* June 5, 2019: deflate/gzip compression: Reduce memory usage of lower compression levels. +* June 2, 2019: Added [zstandard](https://github.com/klauspost/compress/tree/master/zstd#zstd) compression! +* May 25, 2019: deflate/gzip: 10% faster bit writer, mostly visible in lower levels. +* Apr 22, 2019: [zstd](https://github.com/klauspost/compress/tree/master/zstd#zstd) decompression added. +* Aug 1, 2018: Added [huff0 README](https://github.com/klauspost/compress/tree/master/huff0#huff0-entropy-compression). +* Jul 8, 2018: Added [Performance Update 2018](#performance-update-2018) below. +* Jun 23, 2018: Merged [Go 1.11 inflate optimizations](https://go-review.googlesource.com/c/go/+/102235). Go 1.9 is now required. Backwards compatible version tagged with [v1.3.0](https://github.com/klauspost/compress/releases/tag/v1.3.0). +* Apr 2, 2018: Added [huff0](https://godoc.org/github.com/klauspost/compress/huff0) en/decoder. Experimental for now, API may change. +* Mar 4, 2018: Added [FSE Entropy](https://godoc.org/github.com/klauspost/compress/fse) en/decoder. Experimental for now, API may change. +* Nov 3, 2017: Add compression [Estimate](https://godoc.org/github.com/klauspost/compress#Estimate) function. +* May 28, 2017: Reduce allocations when resetting decoder. +* Apr 02, 2017: Change back to official crc32, since changes were merged in Go 1.7. +* Jan 14, 2017: Reduce stack pressure due to array copies. See [Issue #18625](https://github.com/golang/go/issues/18625). +* Oct 25, 2016: Level 2-4 have been rewritten and now offers significantly better performance than before. +* Oct 20, 2016: Port zlib changes from Go 1.7 to fix zlib writer issue. Please update. +* Oct 16, 2016: Go 1.7 changes merged. Apples to apples this package is a few percent faster, but has a significantly better balance between speed and compression per level. +* Mar 24, 2016: Always attempt Huffman encoding on level 4-7. This improves base 64 encoded data compression. +* Mar 24, 2016: Small speedup for level 1-3. +* Feb 19, 2016: Faster bit writer, level -2 is 15% faster, level 1 is 4% faster. +* Feb 19, 2016: Handle small payloads faster in level 1-3. +* Feb 19, 2016: Added faster level 2 + 3 compression modes. +* Feb 19, 2016: [Rebalanced compression levels](https://blog.klauspost.com/rebalancing-deflate-compression-levels/), so there is a more even progression in terms of compression. New default level is 5. +* Feb 14, 2016: Snappy: Merge upstream changes. +* Feb 14, 2016: Snappy: Fix aggressive skipping. +* Feb 14, 2016: Snappy: Update benchmark. +* Feb 13, 2016: Deflate: Fixed assembler problem that could lead to sub-optimal compression. +* Feb 12, 2016: Snappy: Added AMD64 SSE 4.2 optimizations to matching, which makes easy to compress material run faster. Typical speedup is around 25%. +* Feb 9, 2016: Added Snappy package fork. This version is 5-7% faster, much more on hard to compress content. +* Jan 30, 2016: Optimize level 1 to 3 by not considering static dictionary or storing uncompressed. ~4-5% speedup. +* Jan 16, 2016: Optimization on deflate level 1,2,3 compression. +* Jan 8 2016: Merge [CL 18317](https://go-review.googlesource.com/#/c/18317): fix reading, writing of zip64 archives. +* Dec 8 2015: Make level 1 and -2 deterministic even if write size differs. +* Dec 8 2015: Split encoding functions, so hashing and matching can potentially be inlined. 1-3% faster on AMD64. 5% faster on other platforms. +* Dec 8 2015: Fixed rare [one byte out-of bounds read](https://github.com/klauspost/compress/issues/20). Please update! +* Nov 23 2015: Optimization on token writer. ~2-4% faster. Contributed by [@dsnet](https://github.com/dsnet). +* Nov 20 2015: Small optimization to bit writer on 64 bit systems. +* Nov 17 2015: Fixed out-of-bound errors if the underlying Writer returned an error. See [#15](https://github.com/klauspost/compress/issues/15). +* Nov 12 2015: Added [io.WriterTo](https://golang.org/pkg/io/#WriterTo) support to gzip/inflate. +* Nov 11 2015: Merged [CL 16669](https://go-review.googlesource.com/#/c/16669/4): archive/zip: enable overriding (de)compressors per file +* Oct 15 2015: Added skipping on uncompressible data. Random data speed up >5x. + +
+ +# deflate usage + +The packages are drop-in replacements for standard library [deflate](https://godoc.org/github.com/klauspost/compress/flate), [gzip](https://godoc.org/github.com/klauspost/compress/gzip), [zip](https://godoc.org/github.com/klauspost/compress/zip), and [zlib](https://godoc.org/github.com/klauspost/compress/zlib). Simply replace the import path to use them: + +Typical speed is about 2x of the standard library packages. + +| old import | new import | Documentation | +|------------------|---------------------------------------|-------------------------------------------------------------------------| +| `compress/gzip` | `github.com/klauspost/compress/gzip` | [gzip](https://pkg.go.dev/github.com/klauspost/compress/gzip?tab=doc) | +| `compress/zlib` | `github.com/klauspost/compress/zlib` | [zlib](https://pkg.go.dev/github.com/klauspost/compress/zlib?tab=doc) | +| `archive/zip` | `github.com/klauspost/compress/zip` | [zip](https://pkg.go.dev/github.com/klauspost/compress/zip?tab=doc) | +| `compress/flate` | `github.com/klauspost/compress/flate` | [flate](https://pkg.go.dev/github.com/klauspost/compress/flate?tab=doc) | + +You may also be interested in [pgzip](https://github.com/klauspost/pgzip), which is a drop-in replacement for gzip, which support multithreaded compression on big files and the optimized [crc32](https://github.com/klauspost/crc32) package used by these packages. + +The packages implement the same API as the standard library, so you can use the original godoc documentation: [gzip](http://golang.org/pkg/compress/gzip/), [zip](http://golang.org/pkg/archive/zip/), [zlib](http://golang.org/pkg/compress/zlib/), [flate](http://golang.org/pkg/compress/flate/). + +Currently there is only minor speedup on decompression (mostly CRC32 calculation). + +Memory usage is typically 1MB for a Writer. stdlib is in the same range. +If you expect to have a lot of concurrently allocated Writers consider using +the stateless compression described below. + +For compression performance, see: [this spreadsheet](https://docs.google.com/spreadsheets/d/1nuNE2nPfuINCZJRMt6wFWhKpToF95I47XjSsc-1rbPQ/edit?usp=sharing). + +To disable all assembly add `-tags=noasm`. This works across all packages. + +# Stateless compression + +This package offers stateless compression as a special option for gzip/deflate. +It will do compression but without maintaining any state between Write calls. + +This means there will be no memory kept between Write calls, but compression and speed will be suboptimal. + +This is only relevant in cases where you expect to run many thousands of compressors concurrently, +but with very little activity. This is *not* intended for regular web servers serving individual requests. + +Because of this, the size of actual Write calls will affect output size. + +In gzip, specify level `-3` / `gzip.StatelessCompression` to enable. + +For direct deflate use, NewStatelessWriter and StatelessDeflate are available. See [documentation](https://godoc.org/github.com/klauspost/compress/flate#NewStatelessWriter) + +A `bufio.Writer` can of course be used to control write sizes. For example, to use a 4KB buffer: + +```go + // replace 'ioutil.Discard' with your output. + gzw, err := gzip.NewWriterLevel(ioutil.Discard, gzip.StatelessCompression) + if err != nil { + return err + } + defer gzw.Close() + + w := bufio.NewWriterSize(gzw, 4096) + defer w.Flush() + + // Write to 'w' +``` + +This will only use up to 4KB in memory when the writer is idle. + +Compression is almost always worse than the fastest compression level +and each write will allocate (a little) memory. + + +# Other packages + +Here are other packages of good quality and pure Go (no cgo wrappers or autoconverted code): + +* [github.com/pierrec/lz4](https://github.com/pierrec/lz4) - strong multithreaded LZ4 compression. +* [github.com/cosnicolaou/pbzip2](https://github.com/cosnicolaou/pbzip2) - multithreaded bzip2 decompression. +* [github.com/dsnet/compress](https://github.com/dsnet/compress) - brotli decompression, bzip2 writer. +* [github.com/ronanh/intcomp](https://github.com/ronanh/intcomp) - Integer compression. +* [github.com/spenczar/fpc](https://github.com/spenczar/fpc) - Float compression. +* [github.com/minio/zipindex](https://github.com/minio/zipindex) - External ZIP directory index. +* [github.com/ybirader/pzip](https://github.com/ybirader/pzip) - Fast concurrent zip archiver and extractor. + +# license + +This code is licensed under the same conditions as the original Go code. See LICENSE file. + + + + + diff --git a/vendor/github.com/klauspost/compress/flate/deflate.go b/vendor/github.com/klauspost/compress/flate/deflate.go index af53fb860c..57d17eeab9 100644 --- a/vendor/github.com/klauspost/compress/flate/deflate.go +++ b/vendor/github.com/klauspost/compress/flate/deflate.go @@ -6,11 +6,12 @@ package flate import ( - "encoding/binary" "errors" "fmt" "io" "math" + + "github.com/klauspost/compress/internal/le" ) const ( @@ -234,12 +235,9 @@ func (d *compressor) fillWindow(b []byte) { // Calculate 256 hashes at the time (more L1 cache hits) loops := (n + 256 - minMatchLength) / 256 - for j := 0; j < loops; j++ { + for j := range loops { startindex := j * 256 - end := startindex + 256 + minMatchLength - 1 - if end > n { - end = n - } + end := min(startindex+256+minMatchLength-1, n) tocheck := d.window[startindex:end] dstSize := len(tocheck) - minMatchLength + 1 @@ -269,18 +267,12 @@ func (d *compressor) fillWindow(b []byte) { // We only look at chainCount possibilities before giving up. // pos = s.index, prevHead = s.chainHead-s.hashOffset, prevLength=minMatchLength-1, lookahead func (d *compressor) findMatch(pos int, prevHead int, lookahead int) (length, offset int, ok bool) { - minMatchLook := maxMatchLength - if lookahead < minMatchLook { - minMatchLook = lookahead - } + minMatchLook := min(lookahead, maxMatchLength) win := d.window[0 : pos+minMatchLook] // We quit when we get a match that's at least nice long - nice := len(win) - pos - if d.nice < nice { - nice = d.nice - } + nice := min(d.nice, len(win)-pos) // If we've got a match that's good enough, only look in 1/4 the chain. tries := d.chain @@ -288,10 +280,7 @@ func (d *compressor) findMatch(pos int, prevHead int, lookahead int) (length, of wEnd := win[pos+length] wPos := win[pos:] - minIndex := pos - windowSize - if minIndex < 0 { - minIndex = 0 - } + minIndex := max(pos-windowSize, 0) offset = 0 if d.chain < 100 { @@ -374,7 +363,7 @@ func (d *compressor) writeStoredBlock(buf []byte) error { // of the supplied slice. // The caller must ensure that len(b) >= 4. func hash4(b []byte) uint32 { - return hash4u(binary.LittleEndian.Uint32(b), hashBits) + return hash4u(le.Load32(b, 0), hashBits) } // hash4 returns the hash of u to fit in a hash table with h bits. @@ -389,7 +378,7 @@ func bulkHash4(b []byte, dst []uint32) { if len(b) < 4 { return } - hb := binary.LittleEndian.Uint32(b) + hb := le.Load32(b, 0) dst[0] = hash4u(hb, hashBits) end := len(b) - 4 + 1 @@ -432,7 +421,9 @@ func (d *compressor) deflateLazy() { d.h = newHuffmanEncoder(maxFlateBlockTokens) } var tmp [256]uint16 - for _, v := range d.window[s.index:d.windowEnd] { + toIndex := d.window[s.index:d.windowEnd] + toIndex = toIndex[:min(len(toIndex), maxFlateBlockTokens)] + for _, v := range toIndex { tmp[v]++ } d.h.generate(tmp[:], 15) @@ -480,10 +471,7 @@ func (d *compressor) deflateLazy() { prevOffset := s.offset s.length = minMatchLength - 1 s.offset = 0 - minIndex := s.index - windowSize - if minIndex < 0 { - minIndex = 0 - } + minIndex := max(s.index-windowSize, 0) if s.chainHead-s.hashOffset >= minIndex && lookahead > prevLength && prevLength < d.lazy { if newLength, newOffset, ok := d.findMatch(s.index, s.chainHead-s.hashOffset, lookahead); ok { @@ -503,10 +491,7 @@ func (d *compressor) deflateLazy() { if prevLength < maxMatchLength-checkOff { prevIndex := s.index - 1 if prevIndex+prevLength < s.maxInsertIndex { - end := lookahead - if lookahead > maxMatchLength+checkOff { - end = maxMatchLength + checkOff - } + end := min(lookahead, maxMatchLength+checkOff) end += prevIndex // Hash at match end. @@ -603,15 +588,9 @@ func (d *compressor) deflateLazy() { // table. newIndex := s.index + prevLength - 1 // Calculate missing hashes - end := newIndex - if end > s.maxInsertIndex { - end = s.maxInsertIndex - } + end := min(newIndex, s.maxInsertIndex) end += minMatchLength - 1 - startindex := s.index + 1 - if startindex > s.maxInsertIndex { - startindex = s.maxInsertIndex - } + startindex := min(s.index+1, s.maxInsertIndex) tocheck := d.window[startindex:end] dstSize := len(tocheck) - minMatchLength + 1 if dstSize > 0 { diff --git a/vendor/github.com/klauspost/compress/flate/dict_decoder.go b/vendor/github.com/klauspost/compress/flate/dict_decoder.go index bb36351a5a..cb855abc4b 100644 --- a/vendor/github.com/klauspost/compress/flate/dict_decoder.go +++ b/vendor/github.com/klauspost/compress/flate/dict_decoder.go @@ -104,10 +104,7 @@ func (dd *dictDecoder) writeCopy(dist, length int) int { dstBase := dd.wrPos dstPos := dstBase srcPos := dstPos - dist - endPos := dstPos + length - if endPos > len(dd.hist) { - endPos = len(dd.hist) - } + endPos := min(dstPos+length, len(dd.hist)) // Copy non-overlapping section after destination position. // diff --git a/vendor/github.com/klauspost/compress/flate/fast_encoder.go b/vendor/github.com/klauspost/compress/flate/fast_encoder.go index 0e8b1630c0..791c9dcbfd 100644 --- a/vendor/github.com/klauspost/compress/flate/fast_encoder.go +++ b/vendor/github.com/klauspost/compress/flate/fast_encoder.go @@ -7,7 +7,6 @@ package flate import ( "fmt" - "math/bits" "github.com/klauspost/compress/internal/le" ) @@ -151,29 +150,9 @@ func (e *fastGen) matchlen(s, t int, src []byte) int32 { panic(fmt.Sprint(s, "-", t, "(", s-t, ") > maxMatchLength (", maxMatchOffset, ")")) } } - s1 := min(s+maxMatchLength-4, len(src)) - left := s1 - s - n := int32(0) - for left >= 8 { - diff := le.Load64(src, s) ^ le.Load64(src, t) - if diff != 0 { - return n + int32(bits.TrailingZeros64(diff)>>3) - } - s += 8 - t += 8 - n += 8 - left -= 8 - } - - a := src[s:s1] + a := src[s:min(s+maxMatchLength-4, len(src))] b := src[t:] - for i := range a { - if a[i] != b[i] { - break - } - n++ - } - return n + return int32(matchLen(a, b)) } // matchlenLong will return the match length between offsets and t in src. @@ -193,29 +172,7 @@ func (e *fastGen) matchlenLong(s, t int, src []byte) int32 { panic(fmt.Sprint(s, "-", t, "(", s-t, ") > maxMatchLength (", maxMatchOffset, ")")) } } - // Extend the match to be as long as possible. - left := len(src) - s - n := int32(0) - for left >= 8 { - diff := le.Load64(src, s) ^ le.Load64(src, t) - if diff != 0 { - return n + int32(bits.TrailingZeros64(diff)>>3) - } - s += 8 - t += 8 - n += 8 - left -= 8 - } - - a := src[s:] - b := src[t:] - for i := range a { - if a[i] != b[i] { - break - } - n++ - } - return n + return int32(matchLen(src[s:], src[t:])) } // Reset the encoding table. diff --git a/vendor/github.com/klauspost/compress/flate/huffman_bit_writer.go b/vendor/github.com/klauspost/compress/flate/huffman_bit_writer.go index afdc8c053a..7151140ccd 100644 --- a/vendor/github.com/klauspost/compress/flate/huffman_bit_writer.go +++ b/vendor/github.com/klauspost/compress/flate/huffman_bit_writer.go @@ -211,7 +211,9 @@ func (w *huffmanBitWriter) flush() { n++ } w.bits = 0 - w.write(w.bytes[:n]) + if n > 0 { + w.write(w.bytes[:n]) + } w.nbytes = 0 } @@ -303,10 +305,7 @@ func (w *huffmanBitWriter) generateCodegen(numLiterals int, numOffsets int, litE w.codegenFreq[size]++ count-- for count >= 3 { - n := 6 - if n > count { - n = count - } + n := min(6, count) codegen[outIndex] = 16 outIndex++ codegen[outIndex] = uint8(n - 3) @@ -316,10 +315,7 @@ func (w *huffmanBitWriter) generateCodegen(numLiterals int, numOffsets int, litE } } else { for count >= 11 { - n := 138 - if n > count { - n = count - } + n := min(138, count) codegen[outIndex] = 18 outIndex++ codegen[outIndex] = uint8(n - 11) @@ -438,8 +434,8 @@ func (w *huffmanBitWriter) writeOutBits() { w.nbits -= 48 n := w.nbytes - // We over-write, but faster... - le.Store64(w.bytes[n:], bits) + // We overwrite, but faster... + le.Store64(w.bytes[:], n, bits) n += 6 if n >= bufferFlushSize { @@ -472,7 +468,7 @@ func (w *huffmanBitWriter) writeDynamicHeader(numLiterals int, numOffsets int, n w.writeBits(int32(numOffsets-1), 5) w.writeBits(int32(numCodegens-4), 4) - for i := 0; i < numCodegens; i++ { + for i := range numCodegens { value := uint(w.codegenEncoding.codes[codegenOrder[i]].len()) w.writeBits(int32(value), 3) } @@ -650,7 +646,7 @@ func (w *huffmanBitWriter) writeBlockDynamic(tokens *tokens, eof bool, input []b w.lastHeader = 0 } - numLiterals, numOffsets := w.indexTokens(tokens, !sync) + numLiterals, numOffsets := w.indexTokens(tokens, true) extraBits := 0 ssize, storable := w.storedSize(input) @@ -785,7 +781,7 @@ func (w *huffmanBitWriter) fillTokens() { // literalFreq and offsetFreq, and generates literalEncoding // and offsetEncoding. // The number of literal and offset tokens is returned. -func (w *huffmanBitWriter) indexTokens(t *tokens, filled bool) (numLiterals, numOffsets int) { +func (w *huffmanBitWriter) indexTokens(t *tokens, alwaysEOB bool) (numLiterals, numOffsets int) { //copy(w.literalFreq[:], t.litHist[:]) *(*[256]uint16)(w.literalFreq[:]) = t.litHist //copy(w.literalFreq[256:], t.extraHist[:]) @@ -795,9 +791,10 @@ func (w *huffmanBitWriter) indexTokens(t *tokens, filled bool) (numLiterals, num if t.n == 0 { return } - if filled { - return maxNumLit, maxNumDist + if alwaysEOB { + w.literalFreq[endBlockMarker] = 1 } + // get the number of literals numLiterals = len(w.literalFreq) for w.literalFreq[numLiterals-1] == 0 { @@ -855,8 +852,7 @@ func (w *huffmanBitWriter) writeTokens(tokens []token, leCodes, oeCodes []hcode) bits |= c.code64() << (nbits & 63) nbits += c.len() if nbits >= 48 { - le.Store64(w.bytes[nbytes:], bits) - //*(*uint64)(unsafe.Pointer(&w.bytes[nbytes])) = bits + le.Store64(w.bytes[:], nbytes, bits) bits >>= 48 nbits -= 48 nbytes += 6 @@ -883,8 +879,7 @@ func (w *huffmanBitWriter) writeTokens(tokens []token, leCodes, oeCodes []hcode) bits |= c.code64() << (nbits & 63) nbits += c.len() if nbits >= 48 { - le.Store64(w.bytes[nbytes:], bits) - //*(*uint64)(unsafe.Pointer(&w.bytes[nbytes])) = bits + le.Store64(w.bytes[:], nbytes, bits) bits >>= 48 nbits -= 48 nbytes += 6 @@ -906,8 +901,7 @@ func (w *huffmanBitWriter) writeTokens(tokens []token, leCodes, oeCodes []hcode) bits |= uint64(extraLength) << (nbits & 63) nbits += extraLengthBits if nbits >= 48 { - le.Store64(w.bytes[nbytes:], bits) - //*(*uint64)(unsafe.Pointer(&w.bytes[nbytes])) = bits + le.Store64(w.bytes[:], nbytes, bits) bits >>= 48 nbits -= 48 nbytes += 6 @@ -932,8 +926,7 @@ func (w *huffmanBitWriter) writeTokens(tokens []token, leCodes, oeCodes []hcode) bits |= c.code64() << (nbits & 63) nbits += c.len() if nbits >= 48 { - le.Store64(w.bytes[nbytes:], bits) - //*(*uint64)(unsafe.Pointer(&w.bytes[nbytes])) = bits + le.Store64(w.bytes[:], nbytes, bits) bits >>= 48 nbits -= 48 nbytes += 6 @@ -954,8 +947,7 @@ func (w *huffmanBitWriter) writeTokens(tokens []token, leCodes, oeCodes []hcode) bits |= uint64((offset-(offsetComb>>8))&matchOffsetOnlyMask) << (nbits & 63) nbits += uint8(offsetComb) if nbits >= 48 { - le.Store64(w.bytes[nbytes:], bits) - //*(*uint64)(unsafe.Pointer(&w.bytes[nbytes])) = bits + le.Store64(w.bytes[:], nbytes, bits) bits >>= 48 nbits -= 48 nbytes += 6 @@ -1108,7 +1100,7 @@ func (w *huffmanBitWriter) writeBlockHuff(eof bool, input []byte, sync bool) { // We must have at least 48 bits free. if nbits >= 8 { n := nbits >> 3 - le.Store64(w.bytes[nbytes:], bits) + le.Store64(w.bytes[:], nbytes, bits) bits >>= (n * 8) & 63 nbits -= n * 8 nbytes += n @@ -1137,8 +1129,7 @@ func (w *huffmanBitWriter) writeBlockHuff(eof bool, input []byte, sync bool) { // Remaining... for _, t := range input { if nbits >= 48 { - le.Store64(w.bytes[nbytes:], bits) - //*(*uint64)(unsafe.Pointer(&w.bytes[nbytes])) = bits + le.Store64(w.bytes[:], nbytes, bits) bits >>= 48 nbits -= 48 nbytes += 6 diff --git a/vendor/github.com/klauspost/compress/flate/huffman_code.go b/vendor/github.com/klauspost/compress/flate/huffman_code.go index be7b58b473..4b312dea3e 100644 --- a/vendor/github.com/klauspost/compress/flate/huffman_code.go +++ b/vendor/github.com/klauspost/compress/flate/huffman_code.go @@ -91,7 +91,7 @@ func generateFixedLiteralEncoding() *huffmanEncoder { h := newHuffmanEncoder(literalCount) codes := h.codes var ch uint16 - for ch = 0; ch < literalCount; ch++ { + for ch = range uint16(literalCount) { var bits uint16 var size uint8 switch { @@ -407,8 +407,8 @@ func histogramSplit(b []byte, h []uint16) { for i, t := range x { v0 := &h[t] v1 := &h[y[i]] - v3 := &h[w[i]] v2 := &h[z[i]] + v3 := &h[w[i]] *v0++ *v1++ *v2++ diff --git a/vendor/github.com/klauspost/compress/flate/inflate.go b/vendor/github.com/klauspost/compress/flate/inflate.go index 0d7b437f1c..6e90126db0 100644 --- a/vendor/github.com/klauspost/compress/flate/inflate.go +++ b/vendor/github.com/klauspost/compress/flate/inflate.go @@ -485,7 +485,7 @@ func (f *decompressor) readHuffman() error { f.nb -= 5 + 5 + 4 // (HCLEN+4)*3 bits: code lengths in the magic codeOrder order. - for i := 0; i < nclen; i++ { + for i := range nclen { for f.nb < 3 { if err := f.moreBits(); err != nil { return err @@ -776,7 +776,7 @@ func fixedHuffmanDecoderInit() { fixedOnce.Do(func() { // These come from the RFC section 3.2.6. var bits [288]int - for i := 0; i < 144; i++ { + for i := range 144 { bits[i] = 8 } for i := 144; i < 256; i++ { diff --git a/vendor/github.com/klauspost/compress/flate/level5.go b/vendor/github.com/klauspost/compress/flate/level5.go index 6e5c21502f..a22ad7d125 100644 --- a/vendor/github.com/klauspost/compress/flate/level5.go +++ b/vendor/github.com/klauspost/compress/flate/level5.go @@ -677,10 +677,7 @@ func (e *fastEncL5Window) matchlen(s, t int32, src []byte) int32 { panic(fmt.Sprint(s, "-", t, "(", s-t, ") > maxMatchLength (", maxMatchOffset, ")")) } } - s1 := int(s) + maxMatchLength - 4 - if s1 > len(src) { - s1 = len(src) - } + s1 := min(int(s)+maxMatchLength-4, len(src)) // Extend the match to be as long as possible. return int32(matchLen(src[s:s1], src[t:])) diff --git a/vendor/github.com/klauspost/compress/flate/regmask_other.go b/vendor/github.com/klauspost/compress/flate/regmask_other.go index 1b7a2cbd79..e62caf711e 100644 --- a/vendor/github.com/klauspost/compress/flate/regmask_other.go +++ b/vendor/github.com/klauspost/compress/flate/regmask_other.go @@ -1,5 +1,4 @@ //go:build !amd64 -// +build !amd64 package flate diff --git a/vendor/github.com/klauspost/compress/flate/stateless.go b/vendor/github.com/klauspost/compress/flate/stateless.go index 13b9b100db..455ed3e2b5 100644 --- a/vendor/github.com/klauspost/compress/flate/stateless.go +++ b/vendor/github.com/klauspost/compress/flate/stateless.go @@ -56,18 +56,24 @@ func NewStatelessWriter(dst io.Writer) io.WriteCloser { // bitWriterPool contains bit writers that can be reused. var bitWriterPool = sync.Pool{ - New: func() interface{} { + New: func() any { return newHuffmanBitWriter(nil) }, } +// tokensPool contains tokens struct objects that can be reused +var tokensPool = sync.Pool{ + New: func() any { + return &tokens{} + }, +} + // StatelessDeflate allows compressing directly to a Writer without retaining state. // When returning everything will be flushed. // Up to 8KB of an optional dictionary can be given which is presumed to precede the block. // Longer dictionaries will be truncated and will still produce valid output. // Sending nil dictionary is perfectly fine. func StatelessDeflate(out io.Writer, in []byte, eof bool, dict []byte) error { - var dst tokens bw := bitWriterPool.Get().(*huffmanBitWriter) bw.reset(out) defer func() { @@ -91,6 +97,12 @@ func StatelessDeflate(out io.Writer, in []byte, eof bool, dict []byte) error { // For subsequent loops, keep shallow dict reference to avoid alloc+copy. var inDict []byte + dst := tokensPool.Get().(*tokens) + dst.Reset() + defer func() { + tokensPool.Put(dst) + }() + for len(in) > 0 { todo := in if len(inDict) > 0 { @@ -113,9 +125,9 @@ func StatelessDeflate(out io.Writer, in []byte, eof bool, dict []byte) error { } // Compress if len(inDict) == 0 { - statelessEnc(&dst, todo, int16(len(dict))) + statelessEnc(dst, todo, int16(len(dict))) } else { - statelessEnc(&dst, inDict[:maxStatelessDict+len(todo)], maxStatelessDict) + statelessEnc(dst, inDict[:maxStatelessDict+len(todo)], maxStatelessDict) } isEof := eof && len(in) == 0 @@ -129,7 +141,7 @@ func StatelessDeflate(out io.Writer, in []byte, eof bool, dict []byte) error { // If we removed less than 1/16th, huffman compress the block. bw.writeBlockHuff(isEof, uncompressed, len(in) == 0) } else { - bw.writeBlockDynamic(&dst, isEof, uncompressed, len(in) == 0) + bw.writeBlockDynamic(dst, isEof, uncompressed, len(in) == 0) } if len(in) > 0 { // Retain a dict if we have more @@ -184,7 +196,7 @@ func statelessEnc(dst *tokens, src []byte, startAt int16) { // Index until startAt if startAt > 0 { cv := load3232(src, 0) - for i := int16(0); i < startAt; i++ { + for i := range startAt { table[hashSL(cv)] = tableEntry{offset: i} cv = (cv >> 8) | (uint32(src[i+4]) << 24) } diff --git a/vendor/github.com/klauspost/compress/fse/README.md b/vendor/github.com/klauspost/compress/fse/README.md index ea7324da67..27d8ed56fc 100644 --- a/vendor/github.com/klauspost/compress/fse/README.md +++ b/vendor/github.com/klauspost/compress/fse/README.md @@ -1,79 +1,79 @@ -# Finite State Entropy - -This package provides Finite State Entropy encoding and decoding. - -Finite State Entropy (also referenced as [tANS](https://en.wikipedia.org/wiki/Asymmetric_numeral_systems#tANS)) -encoding provides a fast near-optimal symbol encoding/decoding -for byte blocks as implemented in [zstandard](https://github.com/facebook/zstd). - -This can be used for compressing input with a lot of similar input values to the smallest number of bytes. -This does not perform any multi-byte [dictionary coding](https://en.wikipedia.org/wiki/Dictionary_coder) as LZ coders, -but it can be used as a secondary step to compressors (like Snappy) that does not do entropy encoding. - -* [Godoc documentation](https://godoc.org/github.com/klauspost/compress/fse) - -## News - - * Feb 2018: First implementation released. Consider this beta software for now. - -# Usage - -This package provides a low level interface that allows to compress single independent blocks. - -Each block is separate, and there is no built in integrity checks. -This means that the caller should keep track of block sizes and also do checksums if needed. - -Compressing a block is done via the [`Compress`](https://godoc.org/github.com/klauspost/compress/fse#Compress) function. -You must provide input and will receive the output and maybe an error. - -These error values can be returned: - -| Error | Description | -|---------------------|-----------------------------------------------------------------------------| -| `` | Everything ok, output is returned | -| `ErrIncompressible` | Returned when input is judged to be too hard to compress | -| `ErrUseRLE` | Returned from the compressor when the input is a single byte value repeated | -| `(error)` | An internal error occurred. | - -As can be seen above there are errors that will be returned even under normal operation so it is important to handle these. - -To reduce allocations you can provide a [`Scratch`](https://godoc.org/github.com/klauspost/compress/fse#Scratch) object -that can be re-used for successive calls. Both compression and decompression accepts a `Scratch` object, and the same -object can be used for both. - -Be aware, that when re-using a `Scratch` object that the *output* buffer is also re-used, so if you are still using this -you must set the `Out` field in the scratch to nil. The same buffer is used for compression and decompression output. - -Decompressing is done by calling the [`Decompress`](https://godoc.org/github.com/klauspost/compress/fse#Decompress) function. -You must provide the output from the compression stage, at exactly the size you got back. If you receive an error back -your input was likely corrupted. - -It is important to note that a successful decoding does *not* mean your output matches your original input. -There are no integrity checks, so relying on errors from the decompressor does not assure your data is valid. - -For more detailed usage, see examples in the [godoc documentation](https://godoc.org/github.com/klauspost/compress/fse#pkg-examples). - -# Performance - -A lot of factors are affecting speed. Block sizes and compressibility of the material are primary factors. -All compression functions are currently only running on the calling goroutine so only one core will be used per block. - -The compressor is significantly faster if symbols are kept as small as possible. The highest byte value of the input -is used to reduce some of the processing, so if all your input is above byte value 64 for instance, it may be -beneficial to transpose all your input values down by 64. - -With moderate block sizes around 64k speed are typically 200MB/s per core for compression and -around 300MB/s decompression speed. - -The same hardware typically does Huffman (deflate) encoding at 125MB/s and decompression at 100MB/s. - -# Plans - -At one point, more internals will be exposed to facilitate more "expert" usage of the components. - -A streaming interface is also likely to be implemented. Likely compatible with [FSE stream format](https://github.com/Cyan4973/FiniteStateEntropy/blob/dev/programs/fileio.c#L261). - -# Contributing - -Contributions are always welcome. Be aware that adding public functions will require good justification and breaking +# Finite State Entropy + +This package provides Finite State Entropy encoding and decoding. + +Finite State Entropy (also referenced as [tANS](https://en.wikipedia.org/wiki/Asymmetric_numeral_systems#tANS)) +encoding provides a fast near-optimal symbol encoding/decoding +for byte blocks as implemented in [zstandard](https://github.com/facebook/zstd). + +This can be used for compressing input with a lot of similar input values to the smallest number of bytes. +This does not perform any multi-byte [dictionary coding](https://en.wikipedia.org/wiki/Dictionary_coder) as LZ coders, +but it can be used as a secondary step to compressors (like Snappy) that does not do entropy encoding. + +* [Godoc documentation](https://godoc.org/github.com/klauspost/compress/fse) + +## News + + * Feb 2018: First implementation released. Consider this beta software for now. + +# Usage + +This package provides a low level interface that allows to compress single independent blocks. + +Each block is separate, and there is no built in integrity checks. +This means that the caller should keep track of block sizes and also do checksums if needed. + +Compressing a block is done via the [`Compress`](https://godoc.org/github.com/klauspost/compress/fse#Compress) function. +You must provide input and will receive the output and maybe an error. + +These error values can be returned: + +| Error | Description | +|---------------------|-----------------------------------------------------------------------------| +| `` | Everything ok, output is returned | +| `ErrIncompressible` | Returned when input is judged to be too hard to compress | +| `ErrUseRLE` | Returned from the compressor when the input is a single byte value repeated | +| `(error)` | An internal error occurred. | + +As can be seen above there are errors that will be returned even under normal operation so it is important to handle these. + +To reduce allocations you can provide a [`Scratch`](https://godoc.org/github.com/klauspost/compress/fse#Scratch) object +that can be re-used for successive calls. Both compression and decompression accepts a `Scratch` object, and the same +object can be used for both. + +Be aware, that when re-using a `Scratch` object that the *output* buffer is also re-used, so if you are still using this +you must set the `Out` field in the scratch to nil. The same buffer is used for compression and decompression output. + +Decompressing is done by calling the [`Decompress`](https://godoc.org/github.com/klauspost/compress/fse#Decompress) function. +You must provide the output from the compression stage, at exactly the size you got back. If you receive an error back +your input was likely corrupted. + +It is important to note that a successful decoding does *not* mean your output matches your original input. +There are no integrity checks, so relying on errors from the decompressor does not assure your data is valid. + +For more detailed usage, see examples in the [godoc documentation](https://godoc.org/github.com/klauspost/compress/fse#pkg-examples). + +# Performance + +A lot of factors are affecting speed. Block sizes and compressibility of the material are primary factors. +All compression functions are currently only running on the calling goroutine so only one core will be used per block. + +The compressor is significantly faster if symbols are kept as small as possible. The highest byte value of the input +is used to reduce some of the processing, so if all your input is above byte value 64 for instance, it may be +beneficial to transpose all your input values down by 64. + +With moderate block sizes around 64k speed are typically 200MB/s per core for compression and +around 300MB/s decompression speed. + +The same hardware typically does Huffman (deflate) encoding at 125MB/s and decompression at 100MB/s. + +# Plans + +At one point, more internals will be exposed to facilitate more "expert" usage of the components. + +A streaming interface is also likely to be implemented. Likely compatible with [FSE stream format](https://github.com/Cyan4973/FiniteStateEntropy/blob/dev/programs/fileio.c#L261). + +# Contributing + +Contributions are always welcome. Be aware that adding public functions will require good justification and breaking changes will likely not be accepted. If in doubt open an issue before writing the PR. \ No newline at end of file diff --git a/vendor/github.com/klauspost/compress/fse/bitwriter.go b/vendor/github.com/klauspost/compress/fse/bitwriter.go index e82fa3bb7b..d58b3fe423 100644 --- a/vendor/github.com/klauspost/compress/fse/bitwriter.go +++ b/vendor/github.com/klauspost/compress/fse/bitwriter.go @@ -143,7 +143,7 @@ func (b *bitWriter) flush32() { // flushAlign will flush remaining full bytes and align to next byte boundary. func (b *bitWriter) flushAlign() { nbBytes := (b.nBits + 7) >> 3 - for i := uint8(0); i < nbBytes; i++ { + for i := range nbBytes { b.out = append(b.out, byte(b.bitContainer>>(i*8))) } b.nBits = 0 diff --git a/vendor/github.com/klauspost/compress/fse/compress.go b/vendor/github.com/klauspost/compress/fse/compress.go index 074018d8f9..8c8baa4fc2 100644 --- a/vendor/github.com/klauspost/compress/fse/compress.go +++ b/vendor/github.com/klauspost/compress/fse/compress.go @@ -396,7 +396,7 @@ func (s *Scratch) buildCTable() error { if v > largeLimit { s.zeroBits = true } - for nbOccurrences := int16(0); nbOccurrences < v; nbOccurrences++ { + for range v { tableSymbol[position] = symbol position = (position + step) & tableMask for position > highThreshold { diff --git a/vendor/github.com/klauspost/compress/huff0/README.md b/vendor/github.com/klauspost/compress/huff0/README.md index 8b6e5c6638..26d5101b36 100644 --- a/vendor/github.com/klauspost/compress/huff0/README.md +++ b/vendor/github.com/klauspost/compress/huff0/README.md @@ -1,89 +1,89 @@ -# Huff0 entropy compression - -This package provides Huff0 encoding and decoding as used in zstd. - -[Huff0](https://github.com/Cyan4973/FiniteStateEntropy#new-generation-entropy-coders), -a Huffman codec designed for modern CPU, featuring OoO (Out of Order) operations on multiple ALU -(Arithmetic Logic Unit), achieving extremely fast compression and decompression speeds. - -This can be used for compressing input with a lot of similar input values to the smallest number of bytes. -This does not perform any multi-byte [dictionary coding](https://en.wikipedia.org/wiki/Dictionary_coder) as LZ coders, -but it can be used as a secondary step to compressors (like Snappy) that does not do entropy encoding. - -* [Godoc documentation](https://godoc.org/github.com/klauspost/compress/huff0) - -## News - -This is used as part of the [zstandard](https://github.com/klauspost/compress/tree/master/zstd#zstd) compression and decompression package. - -This ensures that most functionality is well tested. - -# Usage - -This package provides a low level interface that allows to compress single independent blocks. - -Each block is separate, and there is no built in integrity checks. -This means that the caller should keep track of block sizes and also do checksums if needed. - -Compressing a block is done via the [`Compress1X`](https://godoc.org/github.com/klauspost/compress/huff0#Compress1X) and -[`Compress4X`](https://godoc.org/github.com/klauspost/compress/huff0#Compress4X) functions. -You must provide input and will receive the output and maybe an error. - -These error values can be returned: - -| Error | Description | -|---------------------|-----------------------------------------------------------------------------| -| `` | Everything ok, output is returned | -| `ErrIncompressible` | Returned when input is judged to be too hard to compress | -| `ErrUseRLE` | Returned from the compressor when the input is a single byte value repeated | -| `ErrTooBig` | Returned if the input block exceeds the maximum allowed size (128 Kib) | -| `(error)` | An internal error occurred. | - - -As can be seen above some of there are errors that will be returned even under normal operation so it is important to handle these. - -To reduce allocations you can provide a [`Scratch`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch) object -that can be re-used for successive calls. Both compression and decompression accepts a `Scratch` object, and the same -object can be used for both. - -Be aware, that when re-using a `Scratch` object that the *output* buffer is also re-used, so if you are still using this -you must set the `Out` field in the scratch to nil. The same buffer is used for compression and decompression output. - -The `Scratch` object will retain state that allows to re-use previous tables for encoding and decoding. - -## Tables and re-use - -Huff0 allows for reusing tables from the previous block to save space if that is expected to give better/faster results. - -The Scratch object allows you to set a [`ReusePolicy`](https://godoc.org/github.com/klauspost/compress/huff0#ReusePolicy) -that controls this behaviour. See the documentation for details. This can be altered between each block. - -Do however note that this information is *not* stored in the output block and it is up to the users of the package to -record whether [`ReadTable`](https://godoc.org/github.com/klauspost/compress/huff0#ReadTable) should be called, -based on the boolean reported back from the CompressXX call. - -If you want to store the table separate from the data, you can access them as `OutData` and `OutTable` on the -[`Scratch`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch) object. - -## Decompressing - -The first part of decoding is to initialize the decoding table through [`ReadTable`](https://godoc.org/github.com/klauspost/compress/huff0#ReadTable). -This will initialize the decoding tables. -You can supply the complete block to `ReadTable` and it will return the data part of the block -which can be given to the decompressor. - -Decompressing is done by calling the [`Decompress1X`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch.Decompress1X) -or [`Decompress4X`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch.Decompress4X) function. - -For concurrently decompressing content with a fixed table a stateless [`Decoder`](https://godoc.org/github.com/klauspost/compress/huff0#Decoder) can be requested which will remain correct as long as the scratch is unchanged. The capacity of the provided slice indicates the expected output size. - -You must provide the output from the compression stage, at exactly the size you got back. If you receive an error back -your input was likely corrupted. - -It is important to note that a successful decoding does *not* mean your output matches your original input. -There are no integrity checks, so relying on errors from the decompressor does not assure your data is valid. - -# Contributing - -Contributions are always welcome. Be aware that adding public functions will require good justification and breaking -changes will likely not be accepted. If in doubt open an issue before writing the PR. +# Huff0 entropy compression + +This package provides Huff0 encoding and decoding as used in zstd. + +[Huff0](https://github.com/Cyan4973/FiniteStateEntropy#new-generation-entropy-coders), +a Huffman codec designed for modern CPU, featuring OoO (Out of Order) operations on multiple ALU +(Arithmetic Logic Unit), achieving extremely fast compression and decompression speeds. + +This can be used for compressing input with a lot of similar input values to the smallest number of bytes. +This does not perform any multi-byte [dictionary coding](https://en.wikipedia.org/wiki/Dictionary_coder) as LZ coders, +but it can be used as a secondary step to compressors (like Snappy) that does not do entropy encoding. + +* [Godoc documentation](https://godoc.org/github.com/klauspost/compress/huff0) + +## News + +This is used as part of the [zstandard](https://github.com/klauspost/compress/tree/master/zstd#zstd) compression and decompression package. + +This ensures that most functionality is well tested. + +# Usage + +This package provides a low level interface that allows to compress single independent blocks. + +Each block is separate, and there is no built in integrity checks. +This means that the caller should keep track of block sizes and also do checksums if needed. + +Compressing a block is done via the [`Compress1X`](https://godoc.org/github.com/klauspost/compress/huff0#Compress1X) and +[`Compress4X`](https://godoc.org/github.com/klauspost/compress/huff0#Compress4X) functions. +You must provide input and will receive the output and maybe an error. + +These error values can be returned: + +| Error | Description | +|---------------------|-----------------------------------------------------------------------------| +| `` | Everything ok, output is returned | +| `ErrIncompressible` | Returned when input is judged to be too hard to compress | +| `ErrUseRLE` | Returned from the compressor when the input is a single byte value repeated | +| `ErrTooBig` | Returned if the input block exceeds the maximum allowed size (128 Kib) | +| `(error)` | An internal error occurred. | + + +As can be seen above some of there are errors that will be returned even under normal operation so it is important to handle these. + +To reduce allocations you can provide a [`Scratch`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch) object +that can be re-used for successive calls. Both compression and decompression accepts a `Scratch` object, and the same +object can be used for both. + +Be aware, that when re-using a `Scratch` object that the *output* buffer is also re-used, so if you are still using this +you must set the `Out` field in the scratch to nil. The same buffer is used for compression and decompression output. + +The `Scratch` object will retain state that allows to re-use previous tables for encoding and decoding. + +## Tables and re-use + +Huff0 allows for reusing tables from the previous block to save space if that is expected to give better/faster results. + +The Scratch object allows you to set a [`ReusePolicy`](https://godoc.org/github.com/klauspost/compress/huff0#ReusePolicy) +that controls this behaviour. See the documentation for details. This can be altered between each block. + +Do however note that this information is *not* stored in the output block and it is up to the users of the package to +record whether [`ReadTable`](https://godoc.org/github.com/klauspost/compress/huff0#ReadTable) should be called, +based on the boolean reported back from the CompressXX call. + +If you want to store the table separate from the data, you can access them as `OutData` and `OutTable` on the +[`Scratch`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch) object. + +## Decompressing + +The first part of decoding is to initialize the decoding table through [`ReadTable`](https://godoc.org/github.com/klauspost/compress/huff0#ReadTable). +This will initialize the decoding tables. +You can supply the complete block to `ReadTable` and it will return the data part of the block +which can be given to the decompressor. + +Decompressing is done by calling the [`Decompress1X`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch.Decompress1X) +or [`Decompress4X`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch.Decompress4X) function. + +For concurrently decompressing content with a fixed table a stateless [`Decoder`](https://godoc.org/github.com/klauspost/compress/huff0#Decoder) can be requested which will remain correct as long as the scratch is unchanged. The capacity of the provided slice indicates the expected output size. + +You must provide the output from the compression stage, at exactly the size you got back. If you receive an error back +your input was likely corrupted. + +It is important to note that a successful decoding does *not* mean your output matches your original input. +There are no integrity checks, so relying on errors from the decompressor does not assure your data is valid. + +# Contributing + +Contributions are always welcome. Be aware that adding public functions will require good justification and breaking +changes will likely not be accepted. If in doubt open an issue before writing the PR. diff --git a/vendor/github.com/klauspost/compress/huff0/bitwriter.go b/vendor/github.com/klauspost/compress/huff0/bitwriter.go index 0ebc9aaac7..41db94cded 100644 --- a/vendor/github.com/klauspost/compress/huff0/bitwriter.go +++ b/vendor/github.com/klauspost/compress/huff0/bitwriter.go @@ -85,7 +85,7 @@ func (b *bitWriter) flush32() { // flushAlign will flush remaining full bytes and align to next byte boundary. func (b *bitWriter) flushAlign() { nbBytes := (b.nBits + 7) >> 3 - for i := uint8(0); i < nbBytes; i++ { + for i := range nbBytes { b.out = append(b.out, byte(b.bitContainer>>(i*8))) } b.nBits = 0 diff --git a/vendor/github.com/klauspost/compress/huff0/compress.go b/vendor/github.com/klauspost/compress/huff0/compress.go index 84aa3d12f0..a97cf1b5d3 100644 --- a/vendor/github.com/klauspost/compress/huff0/compress.go +++ b/vendor/github.com/klauspost/compress/huff0/compress.go @@ -276,7 +276,7 @@ func (s *Scratch) compress4X(src []byte) ([]byte, error) { offsetIdx := len(s.Out) s.Out = append(s.Out, sixZeros[:]...) - for i := 0; i < 4; i++ { + for i := range 4 { toDo := src if len(toDo) > segmentSize { toDo = toDo[:segmentSize] @@ -312,7 +312,7 @@ func (s *Scratch) compress4Xp(src []byte) ([]byte, error) { segmentSize := (len(src) + 3) / 4 var wg sync.WaitGroup wg.Add(4) - for i := 0; i < 4; i++ { + for i := range 4 { toDo := src if len(toDo) > segmentSize { toDo = toDo[:segmentSize] @@ -326,7 +326,7 @@ func (s *Scratch) compress4Xp(src []byte) ([]byte, error) { }(i) } wg.Wait() - for i := 0; i < 4; i++ { + for i := range 4 { o := s.tmpOut[i] if len(o) > math.MaxUint16 { // We cannot store the size in the jump table diff --git a/vendor/github.com/klauspost/compress/huff0/decompress.go b/vendor/github.com/klauspost/compress/huff0/decompress.go index 0f56b02d74..7d0efa8818 100644 --- a/vendor/github.com/klauspost/compress/huff0/decompress.go +++ b/vendor/github.com/klauspost/compress/huff0/decompress.go @@ -626,7 +626,7 @@ func (d *Decoder) decompress4X8bit(dst, src []byte) ([]byte, error) { var br [4]bitReaderBytes start := 6 - for i := 0; i < 3; i++ { + for i := range 3 { length := int(src[i*2]) | (int(src[i*2+1]) << 8) if start+length >= len(src) { return nil, errors.New("truncated input (or invalid offset)") @@ -798,10 +798,7 @@ func (d *Decoder) decompress4X8bit(dst, src []byte) ([]byte, error) { remainBytes := dstEvery - (decoded / 4) for i := range br { offset := dstEvery * i - endsAt := offset + remainBytes - if endsAt > len(out) { - endsAt = len(out) - } + endsAt := min(offset+remainBytes, len(out)) br := &br[i] bitsLeft := br.remaining() for bitsLeft > 0 { @@ -864,7 +861,7 @@ func (d *Decoder) decompress4X8bit(dst, src []byte) ([]byte, error) { func (d *Decoder) decompress4X8bitExactly(dst, src []byte) ([]byte, error) { var br [4]bitReaderBytes start := 6 - for i := 0; i < 3; i++ { + for i := range 3 { length := int(src[i*2]) | (int(src[i*2+1]) << 8) if start+length >= len(src) { return nil, errors.New("truncated input (or invalid offset)") @@ -1035,10 +1032,7 @@ func (d *Decoder) decompress4X8bitExactly(dst, src []byte) ([]byte, error) { remainBytes := dstEvery - (decoded / 4) for i := range br { offset := dstEvery * i - endsAt := offset + remainBytes - if endsAt > len(out) { - endsAt = len(out) - } + endsAt := min(offset+remainBytes, len(out)) br := &br[i] bitsLeft := br.remaining() for bitsLeft > 0 { diff --git a/vendor/github.com/klauspost/compress/huff0/decompress_amd64.go b/vendor/github.com/klauspost/compress/huff0/decompress_amd64.go index ba7e8e6b02..2d6ef64be1 100644 --- a/vendor/github.com/klauspost/compress/huff0/decompress_amd64.go +++ b/vendor/github.com/klauspost/compress/huff0/decompress_amd64.go @@ -1,5 +1,4 @@ //go:build amd64 && !appengine && !noasm && gc -// +build amd64,!appengine,!noasm,gc // This file contains the specialisation of Decoder.Decompress4X // and Decoder.Decompress1X that use an asm implementation of thir main loops. @@ -58,7 +57,7 @@ func (d *Decoder) Decompress4X(dst, src []byte) ([]byte, error) { var br [4]bitReaderShifted // Decode "jump table" start := 6 - for i := 0; i < 3; i++ { + for i := range 3 { length := int(src[i*2]) | (int(src[i*2+1]) << 8) if start+length >= len(src) { return nil, errors.New("truncated input (or invalid offset)") @@ -109,10 +108,7 @@ func (d *Decoder) Decompress4X(dst, src []byte) ([]byte, error) { remainBytes := dstEvery - (decoded / 4) for i := range br { offset := dstEvery * i - endsAt := offset + remainBytes - if endsAt > len(out) { - endsAt = len(out) - } + endsAt := min(offset+remainBytes, len(out)) br := &br[i] bitsLeft := br.remaining() for bitsLeft > 0 { diff --git a/vendor/github.com/klauspost/compress/huff0/decompress_generic.go b/vendor/github.com/klauspost/compress/huff0/decompress_generic.go index 908c17de63..6103923222 100644 --- a/vendor/github.com/klauspost/compress/huff0/decompress_generic.go +++ b/vendor/github.com/klauspost/compress/huff0/decompress_generic.go @@ -1,5 +1,4 @@ //go:build !amd64 || appengine || !gc || noasm -// +build !amd64 appengine !gc noasm // This file contains a generic implementation of Decoder.Decompress4X. package huff0 diff --git a/vendor/github.com/klauspost/compress/huff0/huff0.go b/vendor/github.com/klauspost/compress/huff0/huff0.go index 77ecd68e0a..67d9e05b6c 100644 --- a/vendor/github.com/klauspost/compress/huff0/huff0.go +++ b/vendor/github.com/klauspost/compress/huff0/huff0.go @@ -201,7 +201,7 @@ func (c cTable) write(s *Scratch) error { for i := range hist[:16] { hist[i] = 0 } - for n := uint8(0); n < maxSymbolValue; n++ { + for n := range maxSymbolValue { v := bitsToWeight[c[n].nBits] & 15 huffWeight[n] = v hist[v]++ @@ -271,7 +271,7 @@ func (c cTable) estTableSize(s *Scratch) (sz int, err error) { for i := range hist[:16] { hist[i] = 0 } - for n := uint8(0); n < maxSymbolValue; n++ { + for n := range maxSymbolValue { v := bitsToWeight[c[n].nBits] & 15 huffWeight[n] = v hist[v]++ diff --git a/vendor/github.com/klauspost/compress/internal/cpuinfo/cpuinfo_amd64.go b/vendor/github.com/klauspost/compress/internal/cpuinfo/cpuinfo_amd64.go index e802579c4f..b97f9056f4 100644 --- a/vendor/github.com/klauspost/compress/internal/cpuinfo/cpuinfo_amd64.go +++ b/vendor/github.com/klauspost/compress/internal/cpuinfo/cpuinfo_amd64.go @@ -1,5 +1,4 @@ //go:build amd64 && !appengine && !noasm && gc -// +build amd64,!appengine,!noasm,gc package cpuinfo diff --git a/vendor/github.com/klauspost/compress/internal/le/unsafe_disabled.go b/vendor/github.com/klauspost/compress/internal/le/unsafe_disabled.go index 0cfb5c0e27..4f2a0d8c58 100644 --- a/vendor/github.com/klauspost/compress/internal/le/unsafe_disabled.go +++ b/vendor/github.com/klauspost/compress/internal/le/unsafe_disabled.go @@ -37,6 +37,6 @@ func Store32(b []byte, v uint32) { } // Store64 will store v at b. -func Store64(b []byte, v uint64) { - binary.LittleEndian.PutUint64(b, v) +func Store64[I Indexer](b []byte, i I, v uint64) { + binary.LittleEndian.PutUint64(b[i:], v) } diff --git a/vendor/github.com/klauspost/compress/internal/le/unsafe_enabled.go b/vendor/github.com/klauspost/compress/internal/le/unsafe_enabled.go index ada45cd909..218a38bc4a 100644 --- a/vendor/github.com/klauspost/compress/internal/le/unsafe_enabled.go +++ b/vendor/github.com/klauspost/compress/internal/le/unsafe_enabled.go @@ -38,18 +38,15 @@ func Load64[I Indexer](b []byte, i I) uint64 { // Store16 will store v at b. func Store16(b []byte, v uint16) { - //binary.LittleEndian.PutUint16(b, v) *(*uint16)(unsafe.Pointer(unsafe.SliceData(b))) = v } // Store32 will store v at b. func Store32(b []byte, v uint32) { - //binary.LittleEndian.PutUint32(b, v) *(*uint32)(unsafe.Pointer(unsafe.SliceData(b))) = v } -// Store64 will store v at b. -func Store64(b []byte, v uint64) { - //binary.LittleEndian.PutUint64(b, v) - *(*uint64)(unsafe.Pointer(unsafe.SliceData(b))) = v +// Store64 will store v at b[i:]. +func Store64[I Indexer](b []byte, i I, v uint64) { + *(*uint64)(unsafe.Add(unsafe.Pointer(unsafe.SliceData(b)), i)) = v } diff --git a/vendor/github.com/klauspost/compress/internal/snapref/decode.go b/vendor/github.com/klauspost/compress/internal/snapref/decode.go index 40796a49d6..a2c82fcd22 100644 --- a/vendor/github.com/klauspost/compress/internal/snapref/decode.go +++ b/vendor/github.com/klauspost/compress/internal/snapref/decode.go @@ -209,7 +209,7 @@ func (r *Reader) fill() error { if !r.readFull(r.buf[:len(magicBody)], false) { return r.err } - for i := 0; i < len(magicBody); i++ { + for i := range len(magicBody) { if r.buf[i] != magicBody[i] { r.err = ErrCorrupt return r.err diff --git a/vendor/github.com/klauspost/compress/internal/snapref/encode.go b/vendor/github.com/klauspost/compress/internal/snapref/encode.go index 13c6040a5d..860a994167 100644 --- a/vendor/github.com/klauspost/compress/internal/snapref/encode.go +++ b/vendor/github.com/klauspost/compress/internal/snapref/encode.go @@ -20,8 +20,10 @@ import ( func Encode(dst, src []byte) []byte { if n := MaxEncodedLen(len(src)); n < 0 { panic(ErrTooLarge) - } else if len(dst) < n { + } else if cap(dst) < n { dst = make([]byte, n) + } else { + dst = dst[:n] } // The block starts with the varint-encoded length of the decompressed bytes. diff --git a/vendor/github.com/klauspost/compress/s2/README.md b/vendor/github.com/klauspost/compress/s2/README.md index 1d9220cbf5..b0bf59fbbd 100644 --- a/vendor/github.com/klauspost/compress/s2/README.md +++ b/vendor/github.com/klauspost/compress/s2/README.md @@ -1,3 +1,17 @@ +# MinLZ + +I have taken the experiences from this library and created a backwards compatible compression package called MinLZ. + +That package will seamlessly decode S2 content, making the transition from this package fairly trivial. + +There are many improvements to pretty much all aspects of S2 since we have "broken free" of the Snappy format specification. +You can read a writeup on [Design and Improvements over S2](https://gist.github.com/klauspost/a25b66198cdbdf7b5b224f670c894ed5). + +The only aspect not covered is custom dictionary encoding. While I do intend to fix errors in this package, +I do not expect to make significant improvements, since I consider MinLZ a better basis for going forward. + +See https://github.com/minio/minlz for all details. + # S2 Compression S2 is an extension of [Snappy](https://github.com/google/snappy). diff --git a/vendor/github.com/klauspost/compress/s2/decode_amd64.s b/vendor/github.com/klauspost/compress/s2/decode_amd64.s index 9b105e03c5..1216df78f5 100644 --- a/vendor/github.com/klauspost/compress/s2/decode_amd64.s +++ b/vendor/github.com/klauspost/compress/s2/decode_amd64.s @@ -51,7 +51,7 @@ // // The d variable is implicitly R_DST - R_DBASE, and len(dst)-d is R_DEND - R_DST. // The s variable is implicitly R_SRC - R_SBASE, and len(src)-s is R_SEND - R_SRC. -TEXT ·s2Decode(SB), NOSPLIT, $48-56 +TEXT ·s2Decode(SB), NOSPLIT, $56-56 // Initialize R_SRC, R_DST and R_DBASE-R_SEND. MOVQ dst_base+0(FP), R_DBASE MOVQ dst_len+8(FP), R_DLEN diff --git a/vendor/github.com/klauspost/compress/s2/decode_other.go b/vendor/github.com/klauspost/compress/s2/decode_other.go index c99d40b69d..2905ba2774 100644 --- a/vendor/github.com/klauspost/compress/s2/decode_other.go +++ b/vendor/github.com/klauspost/compress/s2/decode_other.go @@ -4,7 +4,6 @@ // license that can be found in the LICENSE file. //go:build (!amd64 && !arm64) || appengine || !gc || noasm -// +build !amd64,!arm64 appengine !gc noasm package s2 diff --git a/vendor/github.com/klauspost/compress/s2/dict.go b/vendor/github.com/klauspost/compress/s2/dict.go index f125ad0963..f8dc652800 100644 --- a/vendor/github.com/klauspost/compress/s2/dict.go +++ b/vendor/github.com/klauspost/compress/s2/dict.go @@ -56,10 +56,12 @@ func NewDict(dict []byte) *Dict { if len(dict) < MinDictSize || len(dict) > MaxDictSize { return nil } - d.repeat = int(r) - if d.repeat > len(dict) { + // Compare as uint64: int(r) would wrap negative for r > MaxInt64, + // slipping past the bounds check and causing an OOB read in encode. + if r > uint64(len(dict)) { return nil } + d.repeat = int(r) return &d } diff --git a/vendor/github.com/klauspost/compress/s2/encode.go b/vendor/github.com/klauspost/compress/s2/encode.go index 20b802270a..330e755716 100644 --- a/vendor/github.com/klauspost/compress/s2/encode.go +++ b/vendor/github.com/klauspost/compress/s2/encode.go @@ -117,8 +117,10 @@ func EstimateBlockSize(src []byte) (d int) { func EncodeBetter(dst, src []byte) []byte { if n := MaxEncodedLen(len(src)); n < 0 { panic(ErrTooLarge) - } else if len(dst) < n { + } else if cap(dst) < n { dst = make([]byte, n) + } else { + dst = dst[:n] } // The block starts with the varint-encoded length of the decompressed bytes. @@ -159,8 +161,10 @@ func EncodeBetter(dst, src []byte) []byte { func EncodeBest(dst, src []byte) []byte { if n := MaxEncodedLen(len(src)); n < 0 { panic(ErrTooLarge) - } else if len(dst) < n { + } else if cap(dst) < n { dst = make([]byte, n) + } else { + dst = dst[:n] } // The block starts with the varint-encoded length of the decompressed bytes. diff --git a/vendor/github.com/klauspost/compress/s2/encode_all.go b/vendor/github.com/klauspost/compress/s2/encode_all.go index a473b64529..9d12c44f38 100644 --- a/vendor/github.com/klauspost/compress/s2/encode_all.go +++ b/vendor/github.com/klauspost/compress/s2/encode_all.go @@ -903,10 +903,7 @@ func encodeBlockDictGo(dst, src []byte, dict *Dict) (d int) { // sLimit is when to stop looking for offset/length copies. The inputMargin // lets us use a fast path for emitLiteral in the main loop, while we are // looking for copies. - sLimit := len(src) - inputMargin - if sLimit > MaxDictSrcOffset-maxAhead { - sLimit = MaxDictSrcOffset - maxAhead - } + sLimit := min(len(src)-inputMargin, MaxDictSrcOffset-maxAhead) // Bail if we can't compress to at least this. dstLimit := len(src) - len(src)>>5 - 5 diff --git a/vendor/github.com/klauspost/compress/s2/encode_amd64.go b/vendor/github.com/klauspost/compress/s2/encode_amd64.go index 7aadd255fe..68d72a41d3 100644 --- a/vendor/github.com/klauspost/compress/s2/encode_amd64.go +++ b/vendor/github.com/klauspost/compress/s2/encode_amd64.go @@ -1,5 +1,4 @@ //go:build !appengine && !noasm && gc -// +build !appengine,!noasm,gc package s2 diff --git a/vendor/github.com/klauspost/compress/s2/encode_best.go b/vendor/github.com/klauspost/compress/s2/encode_best.go index 47bac74234..49f2166ec1 100644 --- a/vendor/github.com/klauspost/compress/s2/encode_best.go +++ b/vendor/github.com/klauspost/compress/s2/encode_best.go @@ -23,12 +23,12 @@ func encodeBlockBest(dst, src []byte, dict *Dict) (d int) { // Initialize the hash tables. const ( // Long hash matches. - lTableBits = 19 - maxLTableSize = 1 << lTableBits + lTableBits = bestLongTableBits + maxLTableSize = bestLongTableSize // Short hash matches. - sTableBits = 16 - maxSTableSize = 1 << sTableBits + sTableBits = bestShortTableBits + maxSTableSize = bestShortTableSize inputMargin = 8 + 2 @@ -42,13 +42,12 @@ func encodeBlockBest(dst, src []byte, dict *Dict) (d int) { if len(src) < minNonLiteralBlockSize { return 0 } - sLimitDict := len(src) - inputMargin - if sLimitDict > MaxDictSrcOffset-inputMargin { - sLimitDict = MaxDictSrcOffset - inputMargin - } + sLimitDict := min(len(src)-inputMargin, MaxDictSrcOffset-inputMargin) - var lTable [maxLTableSize]uint64 - var sTable [maxSTableSize]uint64 + tbl := getBestTables() + lTable := &tbl.lTable + sTable := &tbl.sTable + defer bestTablePool.Put(tbl) // Bail if we can't compress to at least this. dstLimit := len(src) - 5 @@ -459,12 +458,12 @@ func encodeBlockBestSnappy(dst, src []byte) (d int) { // Initialize the hash tables. const ( // Long hash matches. - lTableBits = 19 - maxLTableSize = 1 << lTableBits + lTableBits = bestLongTableBits + maxLTableSize = bestLongTableSize // Short hash matches. - sTableBits = 16 - maxSTableSize = 1 << sTableBits + sTableBits = bestShortTableBits + maxSTableSize = bestShortTableSize inputMargin = 8 + 2 ) @@ -477,8 +476,10 @@ func encodeBlockBestSnappy(dst, src []byte) (d int) { return 0 } - var lTable [maxLTableSize]uint64 - var sTable [maxSTableSize]uint64 + tbl := getBestTables() + lTable := &tbl.lTable + sTable := &tbl.sTable + defer bestTablePool.Put(tbl) // Bail if we can't compress to at least this. dstLimit := len(src) - 5 diff --git a/vendor/github.com/klauspost/compress/s2/encode_better.go b/vendor/github.com/klauspost/compress/s2/encode_better.go index 90ebf89c20..adbc57c4d2 100644 --- a/vendor/github.com/klauspost/compress/s2/encode_better.go +++ b/vendor/github.com/klauspost/compress/s2/encode_better.go @@ -59,16 +59,18 @@ func encodeBlockBetterGo(dst, src []byte) (d int) { // Initialize the hash tables. const ( // Long hash matches. - lTableBits = 17 - maxLTableSize = 1 << lTableBits + lTableBits = betterLongTableBits + maxLTableSize = betterLongTableSize // Short hash matches. - sTableBits = 14 - maxSTableSize = 1 << sTableBits + sTableBits = betterShortTableBits + maxSTableSize = betterShortTableSize ) - var lTable [maxLTableSize]uint32 - var sTable [maxSTableSize]uint32 + tbl := getBetterTables() + lTable := &tbl.lTable + sTable := &tbl.sTable + defer betterTablePool.Put(tbl) // Bail if we can't compress to at least this. dstLimit := len(src) - len(src)>>5 - 6 @@ -317,16 +319,18 @@ func encodeBlockBetterSnappyGo(dst, src []byte) (d int) { // Initialize the hash tables. const ( // Long hash matches. - lTableBits = 16 - maxLTableSize = 1 << lTableBits + lTableBits = betterSnappyLongTableBits + maxLTableSize = betterSnappyLongTableSize // Short hash matches. - sTableBits = 14 - maxSTableSize = 1 << sTableBits + sTableBits = betterShortTableBits + maxSTableSize = betterShortTableSize ) - var lTable [maxLTableSize]uint32 - var sTable [maxSTableSize]uint32 + tbl := getBetterSnappyTables() + lTable := &tbl.lTable + sTable := &tbl.sTable + defer betterSnappyTablePool.Put(tbl) // Bail if we can't compress to at least this. dstLimit := len(src) - len(src)>>5 - 6 @@ -902,30 +906,29 @@ func encodeBlockBetterDict(dst, src []byte, dict *Dict) (d int) { // Initialize the hash tables. const ( // Long hash matches. - lTableBits = 17 - maxLTableSize = 1 << lTableBits + lTableBits = betterLongTableBits + maxLTableSize = betterLongTableSize // Short hash matches. - sTableBits = 14 - maxSTableSize = 1 << sTableBits + sTableBits = betterShortTableBits + maxSTableSize = betterShortTableSize maxAhead = 8 // maximum bytes ahead without checking sLimit debug = false ) - sLimit := len(src) - inputMargin - if sLimit > MaxDictSrcOffset-maxAhead { - sLimit = MaxDictSrcOffset - maxAhead - } + sLimit := min(len(src)-inputMargin, MaxDictSrcOffset-maxAhead) if len(src) < minNonLiteralBlockSize { return 0 } dict.initBetter() - var lTable [maxLTableSize]uint32 - var sTable [maxSTableSize]uint32 + tbl := getBetterTables() + lTable := &tbl.lTable + sTable := &tbl.sTable + defer betterTablePool.Put(tbl) // Bail if we can't compress to at least this. dstLimit := len(src) - len(src)>>5 - 6 diff --git a/vendor/github.com/klauspost/compress/s2/encode_go.go b/vendor/github.com/klauspost/compress/s2/encode_go.go index e25b78445d..5597f3ef2e 100644 --- a/vendor/github.com/klauspost/compress/s2/encode_go.go +++ b/vendor/github.com/klauspost/compress/s2/encode_go.go @@ -1,5 +1,4 @@ //go:build !amd64 || appengine || !gc || noasm -// +build !amd64 appengine !gc noasm package s2 diff --git a/vendor/github.com/klauspost/compress/s2/hashtable_pool.go b/vendor/github.com/klauspost/compress/s2/hashtable_pool.go new file mode 100644 index 0000000000..bc7cabd5c5 --- /dev/null +++ b/vendor/github.com/klauspost/compress/s2/hashtable_pool.go @@ -0,0 +1,65 @@ +package s2 + +import "sync" + +// Table size constants +const ( + betterLongTableBits = 17 + betterLongTableSize = 1 << betterLongTableBits // 131072 + + betterShortTableBits = 14 + betterShortTableSize = 1 << betterShortTableBits // 16384 + + betterSnappyLongTableBits = 16 + betterSnappyLongTableSize = 1 << betterSnappyLongTableBits // 65536 + + bestLongTableBits = 19 + bestLongTableSize = 1 << bestLongTableBits // 524288 + + bestShortTableBits = 16 + bestShortTableSize = 1 << bestShortTableBits // 65536 +) + +type betterTables struct { + lTable [betterLongTableSize]uint32 + sTable [betterShortTableSize]uint32 +} + +var betterTablePool = sync.Pool{New: func() interface{} { return &betterTables{} }} + +// betterSnappyTables holds better-snappy compression hash tables. +type betterSnappyTables struct { + lTable [betterSnappyLongTableSize]uint32 + sTable [betterShortTableSize]uint32 +} + +var betterSnappyTablePool = sync.Pool{New: func() interface{} { return &betterSnappyTables{} }} + +// bestTables holds best compression hash tables. +type bestTables struct { + lTable [bestLongTableSize]uint64 + sTable [bestShortTableSize]uint64 +} + +var bestTablePool = sync.Pool{New: func() interface{} { return &bestTables{} }} + +// getBetterTables gets a zeroed betterTables from the pool. +func getBetterTables() *betterTables { + t := betterTablePool.Get().(*betterTables) + *t = betterTables{} + return t +} + +// getBetterSnappyTables gets a zeroed betterSnappyTables from the pool. +func getBetterSnappyTables() *betterSnappyTables { + t := betterSnappyTablePool.Get().(*betterSnappyTables) + *t = betterSnappyTables{} + return t +} + +// getBestTables gets a zeroed bestTables from the pool. +func getBestTables() *bestTables { + t := bestTablePool.Get().(*bestTables) + *t = bestTables{} + return t +} diff --git a/vendor/github.com/klauspost/compress/s2/index.go b/vendor/github.com/klauspost/compress/s2/index.go index 4229957b96..fb7db25315 100644 --- a/vendor/github.com/klauspost/compress/s2/index.go +++ b/vendor/github.com/klauspost/compress/s2/index.go @@ -72,7 +72,7 @@ func (i *Index) add(compressedOffset, uncompressedOffset int64) error { return fmt.Errorf("internal error: Earlier uncompressed received (%d > %d)", latest.uncompressedOffset, uncompressedOffset) } if latest.compressedOffset > compressedOffset { - return fmt.Errorf("internal error: Earlier compressed received (%d > %d)", latest.uncompressedOffset, uncompressedOffset) + return fmt.Errorf("internal error: Earlier compressed received (%d > %d)", latest.compressedOffset, compressedOffset) } if latest.uncompressedOffset+minIndexDist > uncompressedOffset { // Only add entry if distance is large enough. diff --git a/vendor/github.com/klauspost/compress/s2/reader.go b/vendor/github.com/klauspost/compress/s2/reader.go index 8372d752f9..4d01c4190c 100644 --- a/vendor/github.com/klauspost/compress/s2/reader.go +++ b/vendor/github.com/klauspost/compress/s2/reader.go @@ -1046,7 +1046,7 @@ func (r *Reader) ReadByte() (byte, error) { return c, nil } var tmp [1]byte - for i := 0; i < 10; i++ { + for range 10 { n, err := r.Read(tmp[:]) if err != nil { return 0, err diff --git a/vendor/github.com/klauspost/compress/s2/writer.go b/vendor/github.com/klauspost/compress/s2/writer.go index fd15078f7d..09f1cff3a9 100644 --- a/vendor/github.com/klauspost/compress/s2/writer.go +++ b/vendor/github.com/klauspost/compress/s2/writer.go @@ -47,7 +47,7 @@ func NewWriter(w io.Writer, opts ...WriterOption) *Writer { w2.obufLen = obufHeaderLen + MaxEncodedLen(w2.blockSize) w2.paramsOK = true w2.ibuf = make([]byte, 0, w2.blockSize) - w2.buffers.New = func() interface{} { + w2.buffers.New = func() any { return make([]byte, w2.obufLen) } w2.Reset(w) diff --git a/vendor/github.com/klauspost/compress/zip/reader.go b/vendor/github.com/klauspost/compress/zip/reader.go index c3bcc88383..39c2b4f012 100644 --- a/vendor/github.com/klauspost/compress/zip/reader.go +++ b/vendor/github.com/klauspost/compress/zip/reader.go @@ -902,7 +902,16 @@ func (r *Reader) initFileList() { continue } - for dir := path.Dir(name); dir != "."; dir = path.Dir(dir) { + dir := name + for { + if idx := strings.LastIndex(dir, "/"); idx < 0 { + break + } else { + dir = dir[:idx] + } + if dirs[dir] { + break + } dirs[dir] = true } diff --git a/vendor/github.com/klauspost/compress/zlib/reader.go b/vendor/github.com/klauspost/compress/zlib/reader.go index cb652b9089..587e4d47c4 100644 --- a/vendor/github.com/klauspost/compress/zlib/reader.go +++ b/vendor/github.com/klauspost/compress/zlib/reader.go @@ -132,7 +132,7 @@ func (z *reader) Close() error { } func (z *reader) Reset(r io.Reader, dict []byte) error { - *z = reader{decompressor: z.decompressor} + *z = reader{decompressor: z.decompressor, digest: z.digest} if fr, ok := r.(flate.Reader); ok { z.r = fr } else { diff --git a/vendor/github.com/klauspost/compress/zstd/bitwriter.go b/vendor/github.com/klauspost/compress/zstd/bitwriter.go index 1952f175b0..b22b297e62 100644 --- a/vendor/github.com/klauspost/compress/zstd/bitwriter.go +++ b/vendor/github.com/klauspost/compress/zstd/bitwriter.go @@ -88,7 +88,7 @@ func (b *bitWriter) flush32() { // flushAlign will flush remaining full bytes and align to next byte boundary. func (b *bitWriter) flushAlign() { nbBytes := (b.nBits + 7) >> 3 - for i := uint8(0); i < nbBytes; i++ { + for i := range nbBytes { b.out = append(b.out, byte(b.bitContainer>>(i*8))) } b.nBits = 0 diff --git a/vendor/github.com/klauspost/compress/zstd/blockdec.go b/vendor/github.com/klauspost/compress/zstd/blockdec.go index 0dd742fd2a..2329e996f8 100644 --- a/vendor/github.com/klauspost/compress/zstd/blockdec.go +++ b/vendor/github.com/klauspost/compress/zstd/blockdec.go @@ -54,11 +54,11 @@ const ( ) var ( - huffDecoderPool = sync.Pool{New: func() interface{} { + huffDecoderPool = sync.Pool{New: func() any { return &huff0.Scratch{} }} - fseDecoderPool = sync.Pool{New: func() interface{} { + fseDecoderPool = sync.Pool{New: func() any { return &fseDecoder{} }} ) @@ -553,7 +553,7 @@ func (b *blockDec) prepareSequences(in []byte, hist *history) (err error) { if compMode&3 != 0 { return errors.New("corrupt block: reserved bits not zero") } - for i := uint(0); i < 3; i++ { + for i := range uint(3) { mode := seqCompMode((compMode >> (6 - i*2)) & 3) if debugDecoder { println("Table", tableIndex(i), "is", mode) diff --git a/vendor/github.com/klauspost/compress/zstd/blockenc.go b/vendor/github.com/klauspost/compress/zstd/blockenc.go index fd35ea1480..0e33aea442 100644 --- a/vendor/github.com/klauspost/compress/zstd/blockenc.go +++ b/vendor/github.com/klauspost/compress/zstd/blockenc.go @@ -78,6 +78,7 @@ func (b *blockEnc) initNewEncode() { b.recentOffsets = [3]uint32{1, 4, 8} b.litEnc.Reuse = huff0.ReusePolicyNone b.coders.setPrev(nil, nil, nil) + b.dictLitEnc = nil } // reset will reset the block for a new encode, but in the same stream, diff --git a/vendor/github.com/klauspost/compress/zstd/decoder.go b/vendor/github.com/klauspost/compress/zstd/decoder.go index ea2a19376c..c7e500f02a 100644 --- a/vendor/github.com/klauspost/compress/zstd/decoder.go +++ b/vendor/github.com/klauspost/compress/zstd/decoder.go @@ -39,9 +39,6 @@ type Decoder struct { frame *frameDec - // Custom dictionaries. - dicts map[uint32]*dict - // streamWg is the waitgroup for all streams streamWg sync.WaitGroup } @@ -101,12 +98,10 @@ func NewReader(r io.Reader, opts ...DOption) (*Decoder, error) { d.current.err = ErrDecoderNilInput } - // Transfer option dicts. - d.dicts = make(map[uint32]*dict, len(d.o.dicts)) - for _, dc := range d.o.dicts { - d.dicts[dc.id] = dc + // Initialize dict map if needed. + if d.o.dicts == nil { + d.o.dicts = make(map[uint32]*dict) } - d.o.dicts = nil // Create decoders d.decoders = make(chan *blockDec, d.o.concurrent) @@ -238,6 +233,21 @@ func (d *Decoder) Reset(r io.Reader) error { return nil } +// ResetWithOptions will reset the decoder and apply the given options +// for the next stream or DecodeAll operation. +// Options are applied on top of the existing options. +// Some options cannot be changed on reset and will return an error. +func (d *Decoder) ResetWithOptions(r io.Reader, opts ...DOption) error { + d.o.resetOpt = true + defer func() { d.o.resetOpt = false }() + for _, o := range opts { + if err := o(&d.o); err != nil { + return err + } + } + return d.Reset(r) +} + // drainOutput will drain the output until errEndOfStream is sent. func (d *Decoder) drainOutput() { if d.current.cancel != nil { @@ -373,11 +383,9 @@ func (d *Decoder) DecodeAll(input, dst []byte) ([]byte, error) { if cap(dst) == 0 && !d.o.limitToCap { // Allocate len(input) * 2 by default if nothing is provided // and we didn't get frame content size. - size := len(input) * 2 - // Cap to 1 MB. - if size > 1<<20 { - size = 1 << 20 - } + size := min( + // Cap to 1 MB. + len(input)*2, 1<<20) if uint64(size) > d.o.maxDecodedSize { size = int(d.o.maxDecodedSize) } @@ -932,7 +940,7 @@ decodeStream: } func (d *Decoder) setDict(frame *frameDec) (err error) { - dict, ok := d.dicts[frame.DictionaryID] + dict, ok := d.o.dicts[frame.DictionaryID] if ok { if debugDecoder { println("setting dict", frame.DictionaryID) diff --git a/vendor/github.com/klauspost/compress/zstd/decoder_options.go b/vendor/github.com/klauspost/compress/zstd/decoder_options.go index 774c5f00fe..537627a078 100644 --- a/vendor/github.com/klauspost/compress/zstd/decoder_options.go +++ b/vendor/github.com/klauspost/compress/zstd/decoder_options.go @@ -20,10 +20,11 @@ type decoderOptions struct { concurrent int maxDecodedSize uint64 maxWindowSize uint64 - dicts []*dict + dicts map[uint32]*dict ignoreChecksum bool limitToCap bool decodeBufsBelow int + resetOpt bool } func (o *decoderOptions) setDefault() { @@ -42,8 +43,15 @@ func (o *decoderOptions) setDefault() { // WithDecoderLowmem will set whether to use a lower amount of memory, // but possibly have to allocate more while running. +// Cannot be changed with ResetWithOptions. func WithDecoderLowmem(b bool) DOption { - return func(o *decoderOptions) error { o.lowMem = b; return nil } + return func(o *decoderOptions) error { + if o.resetOpt && b != o.lowMem { + return errors.New("WithDecoderLowmem cannot be changed on Reset") + } + o.lowMem = b + return nil + } } // WithDecoderConcurrency sets the number of created decoders. @@ -53,18 +61,23 @@ func WithDecoderLowmem(b bool) DOption { // inflight blocks. // When decoding streams and setting maximum to 1, // no async decoding will be done. +// The value supplied must be at least 0. // When a value of 0 is provided GOMAXPROCS will be used. // By default this will be set to 4 or GOMAXPROCS, whatever is lower. +// Cannot be changed with ResetWithOptions. func WithDecoderConcurrency(n int) DOption { return func(o *decoderOptions) error { if n < 0 { - return errors.New("concurrency must be at least 1") + return errors.New("concurrency must be at least 0") } + newVal := n if n == 0 { - o.concurrent = runtime.GOMAXPROCS(0) - } else { - o.concurrent = n + newVal = runtime.GOMAXPROCS(0) } + if o.resetOpt && newVal != o.concurrent { + return errors.New("WithDecoderConcurrency cannot be changed on Reset") + } + o.concurrent = newVal return nil } } @@ -73,6 +86,7 @@ func WithDecoderConcurrency(n int) DOption { // non-streaming operations or maximum window size for streaming operations. // This can be used to control memory usage of potentially hostile content. // Maximum is 1 << 63 bytes. Default is 64GiB. +// Can be changed with ResetWithOptions. func WithDecoderMaxMemory(n uint64) DOption { return func(o *decoderOptions) error { if n == 0 { @@ -92,16 +106,20 @@ func WithDecoderMaxMemory(n uint64) DOption { // "zstd --train" from the Zstandard reference implementation. // // If several dictionaries with the same ID are provided, the last one will be used. +// Can be changed with ResetWithOptions. // // [dictionary format]: https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#dictionary-format func WithDecoderDicts(dicts ...[]byte) DOption { return func(o *decoderOptions) error { + if o.dicts == nil { + o.dicts = make(map[uint32]*dict) + } for _, b := range dicts { d, err := loadDict(b) if err != nil { return err } - o.dicts = append(o.dicts, d) + o.dicts[d.id] = d } return nil } @@ -109,12 +127,16 @@ func WithDecoderDicts(dicts ...[]byte) DOption { // WithDecoderDictRaw registers a dictionary that may be used by the decoder. // The slice content can be arbitrary data. +// Can be changed with ResetWithOptions. func WithDecoderDictRaw(id uint32, content []byte) DOption { return func(o *decoderOptions) error { if bits.UintSize > 32 && uint(len(content)) > dictMaxLength { return fmt.Errorf("dictionary of size %d > 2GiB too large", len(content)) } - o.dicts = append(o.dicts, &dict{id: id, content: content, offsets: [3]int{1, 4, 8}}) + if o.dicts == nil { + o.dicts = make(map[uint32]*dict) + } + o.dicts[id] = &dict{id: id, content: content, offsets: [3]int{1, 4, 8}} return nil } } @@ -124,6 +146,7 @@ func WithDecoderDictRaw(id uint32, content []byte) DOption { // The Decoder will likely allocate more memory based on the WithDecoderLowmem setting. // If WithDecoderMaxMemory is set to a lower value, that will be used. // Default is 512MB, Maximum is ~3.75 TB as per zstandard spec. +// Can be changed with ResetWithOptions. func WithDecoderMaxWindow(size uint64) DOption { return func(o *decoderOptions) error { if size < MinWindowSize { @@ -141,6 +164,7 @@ func WithDecoderMaxWindow(size uint64) DOption { // or any size set in WithDecoderMaxMemory. // This can be used to limit decoding to a specific maximum output size. // Disabled by default. +// Can be changed with ResetWithOptions. func WithDecodeAllCapLimit(b bool) DOption { return func(o *decoderOptions) error { o.limitToCap = b @@ -153,17 +177,37 @@ func WithDecodeAllCapLimit(b bool) DOption { // This typically uses less allocations but will have the full decompressed object in memory. // Note that DecodeAllCapLimit will disable this, as well as giving a size of 0 or less. // Default is 128KiB. +// Cannot be changed with ResetWithOptions. func WithDecodeBuffersBelow(size int) DOption { return func(o *decoderOptions) error { + if o.resetOpt && size != o.decodeBufsBelow { + return errors.New("WithDecodeBuffersBelow cannot be changed on Reset") + } o.decodeBufsBelow = size return nil } } // IgnoreChecksum allows to forcibly ignore checksum checking. +// Can be changed with ResetWithOptions. func IgnoreChecksum(b bool) DOption { return func(o *decoderOptions) error { o.ignoreChecksum = b return nil } } + +// WithDecoderDictDelete removes dictionaries by ID. +// If no ids are passed, all dictionaries are deleted. +// Should be used with ResetWithOptions. +func WithDecoderDictDelete(ids ...uint32) DOption { + return func(o *decoderOptions) error { + if len(ids) == 0 { + clear(o.dicts) + } + for _, id := range ids { + delete(o.dicts, id) + } + return nil + } +} diff --git a/vendor/github.com/klauspost/compress/zstd/dict.go b/vendor/github.com/klauspost/compress/zstd/dict.go index b7b83164bc..2ffbfdf379 100644 --- a/vendor/github.com/klauspost/compress/zstd/dict.go +++ b/vendor/github.com/klauspost/compress/zstd/dict.go @@ -194,17 +194,17 @@ func BuildDict(o BuildDictOptions) ([]byte, error) { hist := o.History contents := o.Contents debug := o.DebugOut != nil - println := func(args ...interface{}) { + println := func(args ...any) { if o.DebugOut != nil { fmt.Fprintln(o.DebugOut, args...) } } - printf := func(s string, args ...interface{}) { + printf := func(s string, args ...any) { if o.DebugOut != nil { fmt.Fprintf(o.DebugOut, s, args...) } } - print := func(args ...interface{}) { + print := func(args ...any) { if o.DebugOut != nil { fmt.Fprint(o.DebugOut, args...) } @@ -424,16 +424,10 @@ func BuildDict(o BuildDictOptions) ([]byte, error) { } // Literal table - avgSize := litTotal - if avgSize > huff0.BlockSizeMax/2 { - avgSize = huff0.BlockSizeMax / 2 - } + avgSize := min(litTotal, huff0.BlockSizeMax/2) huffBuff := make([]byte, 0, avgSize) // Target size - div := litTotal / avgSize - if div < 1 { - div = 1 - } + div := max(litTotal/avgSize, 1) if debug { println("Huffman weights:") } @@ -454,7 +448,7 @@ func BuildDict(o BuildDictOptions) ([]byte, error) { huffBuff = append(huffBuff, 255) } scratch := &huff0.Scratch{TableLog: 11} - for tries := 0; tries < 255; tries++ { + for tries := range 255 { scratch = &huff0.Scratch{TableLog: 11} _, _, err = huff0.Compress1X(huffBuff, scratch) if err == nil { @@ -471,7 +465,7 @@ func BuildDict(o BuildDictOptions) ([]byte, error) { // Bail out.... Just generate something huffBuff = append(huffBuff, bytes.Repeat([]byte{255}, 10000)...) - for i := 0; i < 128; i++ { + for i := range 128 { huffBuff = append(huffBuff, byte(i)) } continue diff --git a/vendor/github.com/klauspost/compress/zstd/enc_base.go b/vendor/github.com/klauspost/compress/zstd/enc_base.go index 7d250c67f5..c4de134a7a 100644 --- a/vendor/github.com/klauspost/compress/zstd/enc_base.go +++ b/vendor/github.com/klauspost/compress/zstd/enc_base.go @@ -8,7 +8,7 @@ import ( ) const ( - dictShardBits = 6 + dictShardBits = 7 ) type fastBase struct { @@ -21,7 +21,7 @@ type fastBase struct { crc *xxhash.Digest tmp [8]byte blk *blockEnc - lastDictID uint32 + lastDict *dict lowMem bool } @@ -41,11 +41,9 @@ func (e *fastBase) AppendCRC(dst []byte) []byte { // or a window size small enough to contain the input size, if > 0. func (e *fastBase) WindowSize(size int64) int32 { if size > 0 && size < int64(e.maxMatchOff) { - b := int32(1) << uint(bits.Len(uint(size))) - // Keep minimum window. - if b < 1024 { - b = 1024 - } + b := max( + // Keep minimum window. + int32(1)< tMin && s > nextEmit && src[offset-1] == src[s-1] && l < maxMatchLength { s-- offset-- @@ -382,10 +377,7 @@ encodeLoop: nextEmit = s // Index skipped... - end := s - if s > sLimit+4 { - end = sLimit + 4 - } + end := min(s, sLimit+4) off := index0 + e.cur for index0 < end { cv0 := load6432(src, index0) @@ -444,10 +436,7 @@ encodeLoop: nextEmit = s // Index old s + 1 -> s - 1 or sLimit - end := s - if s > sLimit-4 { - end = sLimit - 4 - } + end := min(s, sLimit-4) off := index0 + e.cur for index0 < end { @@ -490,10 +479,13 @@ func (e *bestFastEncoder) Reset(d *dict, singleBlock bool) { if d == nil { return } + dictChanged := d != e.lastDict // Init or copy dict table - if len(e.dictTable) != len(e.table) || d.id != e.lastDictID { + if len(e.dictTable) != len(e.table) || dictChanged { if len(e.dictTable) != len(e.table) { e.dictTable = make([]prevEntry, len(e.table)) + } else { + clear(e.dictTable) } end := int32(len(d.content)) - 8 + e.maxMatchOff for i := e.maxMatchOff; i < end; i += 4 { @@ -521,13 +513,14 @@ func (e *bestFastEncoder) Reset(d *dict, singleBlock bool) { offset: i + 3, } } - e.lastDictID = d.id } - // Init or copy dict table - if len(e.dictLongTable) != len(e.longTable) || d.id != e.lastDictID { + // Init or copy dict long table + if len(e.dictLongTable) != len(e.longTable) || dictChanged { if len(e.dictLongTable) != len(e.longTable) { e.dictLongTable = make([]prevEntry, len(e.longTable)) + } else { + clear(e.dictLongTable) } if len(d.content) >= 8 { cv := load6432(d.content, 0) @@ -549,8 +542,8 @@ func (e *bestFastEncoder) Reset(d *dict, singleBlock bool) { off++ } } - e.lastDictID = d.id } + e.lastDict = d // Reset table to initial state copy(e.longTable[:], e.dictLongTable) diff --git a/vendor/github.com/klauspost/compress/zstd/enc_better.go b/vendor/github.com/klauspost/compress/zstd/enc_better.go index 84a79fde76..3305f09248 100644 --- a/vendor/github.com/klauspost/compress/zstd/enc_better.go +++ b/vendor/github.com/klauspost/compress/zstd/enc_better.go @@ -190,10 +190,7 @@ encodeLoop: // and have to do special offset treatment. startLimit := nextEmit + 1 - tMin := s - e.maxMatchOff - if tMin < 0 { - tMin = 0 - } + tMin := max(s-e.maxMatchOff, 0) for repIndex > tMin && start > startLimit && src[repIndex-1] == src[start-1] && seq.matchLen < maxMatchLength-zstdMinMatch-1 { repIndex-- start-- @@ -252,10 +249,7 @@ encodeLoop: // and have to do special offset treatment. startLimit := nextEmit + 1 - tMin := s - e.maxMatchOff - if tMin < 0 { - tMin = 0 - } + tMin := max(s-e.maxMatchOff, 0) for repIndex > tMin && start > startLimit && src[repIndex-1] == src[start-1] && seq.matchLen < maxMatchLength-zstdMinMatch-1 { repIndex-- start-- @@ -480,10 +474,7 @@ encodeLoop: l := matched // Extend backwards - tMin := s - e.maxMatchOff - if tMin < 0 { - tMin = 0 - } + tMin := max(s-e.maxMatchOff, 0) for t > tMin && s > nextEmit && src[t-1] == src[s-1] && l < maxMatchLength { s-- t-- @@ -719,10 +710,7 @@ encodeLoop: // and have to do special offset treatment. startLimit := nextEmit + 1 - tMin := s - e.maxMatchOff - if tMin < 0 { - tMin = 0 - } + tMin := max(s-e.maxMatchOff, 0) for repIndex > tMin && start > startLimit && src[repIndex-1] == src[start-1] && seq.matchLen < maxMatchLength-zstdMinMatch-1 { repIndex-- start-- @@ -783,10 +771,7 @@ encodeLoop: // and have to do special offset treatment. startLimit := nextEmit + 1 - tMin := s - e.maxMatchOff - if tMin < 0 { - tMin = 0 - } + tMin := max(s-e.maxMatchOff, 0) for repIndex > tMin && start > startLimit && src[repIndex-1] == src[start-1] && seq.matchLen < maxMatchLength-zstdMinMatch-1 { repIndex-- start-- @@ -1005,10 +990,7 @@ encodeLoop: l := matched // Extend backwards - tMin := s - e.maxMatchOff - if tMin < 0 { - tMin = 0 - } + tMin := max(s-e.maxMatchOff, 0) for t > tMin && s > nextEmit && src[t-1] == src[s-1] && l < maxMatchLength { s-- t-- @@ -1120,10 +1102,13 @@ func (e *betterFastEncoderDict) Reset(d *dict, singleBlock bool) { if d == nil { return } + dictChanged := d != e.lastDict // Init or copy dict table - if len(e.dictTable) != len(e.table) || d.id != e.lastDictID { + if len(e.dictTable) != len(e.table) || dictChanged { if len(e.dictTable) != len(e.table) { e.dictTable = make([]tableEntry, len(e.table)) + } else { + clear(e.dictTable) } end := int32(len(d.content)) - 8 + e.maxMatchOff for i := e.maxMatchOff; i < end; i += 4 { @@ -1151,14 +1136,15 @@ func (e *betterFastEncoderDict) Reset(d *dict, singleBlock bool) { offset: i + 3, } } - e.lastDictID = d.id e.allDirty = true } - // Init or copy dict table - if len(e.dictLongTable) != len(e.longTable) || d.id != e.lastDictID { + // Init or copy dict long table + if len(e.dictLongTable) != len(e.longTable) || dictChanged { if len(e.dictLongTable) != len(e.longTable) { e.dictLongTable = make([]prevEntry, len(e.longTable)) + } else { + clear(e.dictLongTable) } if len(d.content) >= 8 { cv := load6432(d.content, 0) @@ -1180,9 +1166,9 @@ func (e *betterFastEncoderDict) Reset(d *dict, singleBlock bool) { off++ } } - e.lastDictID = d.id e.allDirty = true } + e.lastDict = d // Reset table to initial state { diff --git a/vendor/github.com/klauspost/compress/zstd/enc_dfast.go b/vendor/github.com/klauspost/compress/zstd/enc_dfast.go index d36be7bd8c..2fb6da112b 100644 --- a/vendor/github.com/klauspost/compress/zstd/enc_dfast.go +++ b/vendor/github.com/klauspost/compress/zstd/enc_dfast.go @@ -13,7 +13,7 @@ const ( dFastLongLen = 8 // Bytes used for table hash dLongTableShardCnt = 1 << (dFastLongTableBits - dictShardBits) // Number of shards in the table - dLongTableShardSize = dFastLongTableSize / tableShardCnt // Size of an individual shard + dLongTableShardSize = dFastLongTableSize / dLongTableShardCnt // Size of an individual shard dFastShortTableBits = tableBits // Bits used in the short match table dFastShortTableSize = 1 << dFastShortTableBits // Size of the table @@ -149,10 +149,7 @@ encodeLoop: // and have to do special offset treatment. startLimit := nextEmit + 1 - tMin := s - e.maxMatchOff - if tMin < 0 { - tMin = 0 - } + tMin := max(s-e.maxMatchOff, 0) for repIndex > tMin && start > startLimit && src[repIndex-1] == src[start-1] && seq.matchLen < maxMatchLength-zstdMinMatch-1 { repIndex-- start-- @@ -266,10 +263,7 @@ encodeLoop: l := e.matchlen(s+4, t+4, src) + 4 // Extend backwards - tMin := s - e.maxMatchOff - if tMin < 0 { - tMin = 0 - } + tMin := max(s-e.maxMatchOff, 0) for t > tMin && s > nextEmit && src[t-1] == src[s-1] && l < maxMatchLength { s-- t-- @@ -462,10 +456,7 @@ encodeLoop: // and have to do special offset treatment. startLimit := nextEmit + 1 - tMin := s - e.maxMatchOff - if tMin < 0 { - tMin = 0 - } + tMin := max(s-e.maxMatchOff, 0) for repIndex > tMin && start > startLimit && src[repIndex-1] == src[start-1] { repIndex-- start-- @@ -576,10 +567,7 @@ encodeLoop: l := int32(matchLen(src[s+4:], src[t+4:])) + 4 // Extend backwards - tMin := s - e.maxMatchOff - if tMin < 0 { - tMin = 0 - } + tMin := max(s-e.maxMatchOff, 0) for t > tMin && s > nextEmit && src[t-1] == src[s-1] { s-- t-- @@ -809,10 +797,7 @@ encodeLoop: // and have to do special offset treatment. startLimit := nextEmit + 1 - tMin := s - e.maxMatchOff - if tMin < 0 { - tMin = 0 - } + tMin := max(s-e.maxMatchOff, 0) for repIndex > tMin && start > startLimit && src[repIndex-1] == src[start-1] && seq.matchLen < maxMatchLength-zstdMinMatch-1 { repIndex-- start-- @@ -927,10 +912,7 @@ encodeLoop: l := e.matchlen(s+4, t+4, src) + 4 // Extend backwards - tMin := s - e.maxMatchOff - if tMin < 0 { - tMin = 0 - } + tMin := max(s-e.maxMatchOff, 0) for t > tMin && s > nextEmit && src[t-1] == src[s-1] && l < maxMatchLength { s-- t-- @@ -1058,15 +1040,18 @@ func (e *doubleFastEncoder) Reset(d *dict, singleBlock bool) { // ResetDict will reset and set a dictionary if not nil func (e *doubleFastEncoderDict) Reset(d *dict, singleBlock bool) { allDirty := e.allDirty + dictChanged := d != e.lastDict e.fastEncoderDict.Reset(d, singleBlock) if d == nil { return } // Init or copy dict table - if len(e.dictLongTable) != len(e.longTable) || d.id != e.lastDictID { + if len(e.dictLongTable) != len(e.longTable) || dictChanged { if len(e.dictLongTable) != len(e.longTable) { e.dictLongTable = make([]tableEntry, len(e.longTable)) + } else { + clear(e.dictLongTable) } if len(d.content) >= 8 { cv := load6432(d.content, 0) @@ -1083,7 +1068,6 @@ func (e *doubleFastEncoderDict) Reset(d *dict, singleBlock bool) { } } } - e.lastDictID = d.id allDirty = true } // Reset table to initial state diff --git a/vendor/github.com/klauspost/compress/zstd/enc_fast.go b/vendor/github.com/klauspost/compress/zstd/enc_fast.go index f45a3da7da..5e104f1a48 100644 --- a/vendor/github.com/klauspost/compress/zstd/enc_fast.go +++ b/vendor/github.com/klauspost/compress/zstd/enc_fast.go @@ -143,10 +143,7 @@ encodeLoop: // and have to do special offset treatment. startLimit := nextEmit + 1 - sMin := s - e.maxMatchOff - if sMin < 0 { - sMin = 0 - } + sMin := max(s-e.maxMatchOff, 0) for repIndex > sMin && start > startLimit && src[repIndex-1] == src[start-1] && seq.matchLen < maxMatchLength-zstdMinMatch { repIndex-- start-- @@ -223,10 +220,7 @@ encodeLoop: l := e.matchlen(s+4, t+4, src) + 4 // Extend backwards - tMin := s - e.maxMatchOff - if tMin < 0 { - tMin = 0 - } + tMin := max(s-e.maxMatchOff, 0) for t > tMin && s > nextEmit && src[t-1] == src[s-1] && l < maxMatchLength { s-- t-- @@ -387,10 +381,7 @@ encodeLoop: // and have to do special offset treatment. startLimit := nextEmit + 1 - sMin := s - e.maxMatchOff - if sMin < 0 { - sMin = 0 - } + sMin := max(s-e.maxMatchOff, 0) for repIndex > sMin && start > startLimit && src[repIndex-1] == src[start-1] { repIndex-- start-- @@ -469,10 +460,7 @@ encodeLoop: l := e.matchlen(s+4, t+4, src) + 4 // Extend backwards - tMin := s - e.maxMatchOff - if tMin < 0 { - tMin = 0 - } + tMin := max(s-e.maxMatchOff, 0) for t > tMin && s > nextEmit && src[t-1] == src[s-1] { s-- t-- @@ -655,10 +643,7 @@ encodeLoop: // and have to do special offset treatment. startLimit := nextEmit + 1 - sMin := s - e.maxMatchOff - if sMin < 0 { - sMin = 0 - } + sMin := max(s-e.maxMatchOff, 0) for repIndex > sMin && start > startLimit && src[repIndex-1] == src[start-1] && seq.matchLen < maxMatchLength-zstdMinMatch { repIndex-- start-- @@ -735,10 +720,7 @@ encodeLoop: l := e.matchlen(s+4, t+4, src) + 4 // Extend backwards - tMin := s - e.maxMatchOff - if tMin < 0 { - tMin = 0 - } + tMin := max(s-e.maxMatchOff, 0) for t > tMin && s > nextEmit && src[t-1] == src[s-1] && l < maxMatchLength { s-- t-- @@ -823,9 +805,11 @@ func (e *fastEncoderDict) Reset(d *dict, singleBlock bool) { } // Init or copy dict table - if len(e.dictTable) != len(e.table) || d.id != e.lastDictID { + if len(e.dictTable) != len(e.table) || d != e.lastDict { if len(e.dictTable) != len(e.table) { e.dictTable = make([]tableEntry, len(e.table)) + } else { + clear(e.dictTable) } if true { end := e.maxMatchOff + int32(len(d.content)) - 8 @@ -845,7 +829,7 @@ func (e *fastEncoderDict) Reset(d *dict, singleBlock bool) { } } } - e.lastDictID = d.id + e.lastDict = d e.allDirty = true } diff --git a/vendor/github.com/klauspost/compress/zstd/encoder.go b/vendor/github.com/klauspost/compress/zstd/encoder.go index 8f8223cd3a..0f2a00a003 100644 --- a/vendor/github.com/klauspost/compress/zstd/encoder.go +++ b/vendor/github.com/klauspost/compress/zstd/encoder.go @@ -131,6 +131,29 @@ func (e *Encoder) Reset(w io.Writer) { s.frameContentSize = 0 } +// ResetWithOptions will re-initialize the writer and apply the given options +// as a new, independent stream. +// Options are applied on top of the existing options. +// Some options cannot be changed on reset and will return an error. +func (e *Encoder) ResetWithOptions(w io.Writer, opts ...EOption) error { + e.o.resetOpt = true + defer func() { e.o.resetOpt = false }() + hadDict := e.o.dict != nil + for _, o := range opts { + if err := o(&e.o); err != nil { + return err + } + } + hasDict := e.o.dict != nil + if hadDict != hasDict { + // Dict presence changed — encoder type must be recreated. + e.state.encoder = nil + e.init = sync.Once{} + } + e.Reset(w) + return nil +} + // ResetContentSize will reset and set a content size for the next stream. // If the bytes written does not match the size given an error will be returned // when calling Close(). @@ -432,6 +455,12 @@ func (e *Encoder) Close() error { if s.encoder == nil { return nil } + if s.w == nil { + if len(s.filling) == 0 && !s.headerWritten && !s.eofWritten && s.nInput == 0 { + return nil + } + return errors.New("zstd: encoder has no writer") + } err := e.nextBlock(true) if err != nil { if errors.Is(s.err, ErrEncoderClosed) { diff --git a/vendor/github.com/klauspost/compress/zstd/encoder_options.go b/vendor/github.com/klauspost/compress/zstd/encoder_options.go index 20671dcb91..e217be0a17 100644 --- a/vendor/github.com/klauspost/compress/zstd/encoder_options.go +++ b/vendor/github.com/klauspost/compress/zstd/encoder_options.go @@ -14,6 +14,7 @@ type EOption func(*encoderOptions) error // options retains accumulated state of multiple options. type encoderOptions struct { + resetOpt bool concurrent int level EncoderLevel single *bool @@ -41,6 +42,7 @@ func (o *encoderOptions) setDefault() { level: SpeedDefault, allLitEntropy: false, lowMem: false, + fullZero: true, } } @@ -71,19 +73,28 @@ func (o encoderOptions) encoder() encoder { // WithEncoderCRC will add CRC value to output. // Output will be 4 bytes larger. +// Can be changed with ResetWithOptions. func WithEncoderCRC(b bool) EOption { return func(o *encoderOptions) error { o.crc = b; return nil } } // WithEncoderConcurrency will set the concurrency, // meaning the maximum number of encoders to run concurrently. -// The value supplied must be at least 1. +// The value supplied must be at least 0. +// When a value of 0 is provided GOMAXPROCS will be used. // For streams, setting a value of 1 will disable async compression. // By default this will be set to GOMAXPROCS. +// Cannot be changed with ResetWithOptions. func WithEncoderConcurrency(n int) EOption { return func(o *encoderOptions) error { - if n <= 0 { - return fmt.Errorf("concurrency must be at least 1") + if n < 0 { + return errors.New("concurrency must at least 0") + } + if n == 0 { + n = runtime.GOMAXPROCS(0) + } + if o.resetOpt && n != o.concurrent { + return errors.New("WithEncoderConcurrency cannot be changed on Reset") } o.concurrent = n return nil @@ -95,6 +106,7 @@ func WithEncoderConcurrency(n int) EOption { // A larger value will enable better compression but allocate more memory and, // for above-default values, take considerably longer. // The default value is determined by the compression level and max 8MB. +// Cannot be changed with ResetWithOptions. func WithWindowSize(n int) EOption { return func(o *encoderOptions) error { switch { @@ -105,6 +117,9 @@ func WithWindowSize(n int) EOption { case (n & (n - 1)) != 0: return errors.New("window size must be a power of 2") } + if o.resetOpt && n != o.windowSize { + return errors.New("WithWindowSize cannot be changed on Reset") + } o.windowSize = n o.customWindow = true @@ -122,6 +137,7 @@ func WithWindowSize(n int) EOption { // n must be > 0 and <= 1GB, 1<<30 bytes. // The padded area will be filled with data from crypto/rand.Reader. // If `EncodeAll` is used with data already in the destination, the total size will be multiple of this. +// Can be changed with ResetWithOptions. func WithEncoderPadding(n int) EOption { return func(o *encoderOptions) error { if n <= 0 { @@ -215,12 +231,16 @@ func (e EncoderLevel) String() string { } // WithEncoderLevel specifies a predefined compression level. +// Cannot be changed with ResetWithOptions. func WithEncoderLevel(l EncoderLevel) EOption { return func(o *encoderOptions) error { switch { case l <= speedNotSet || l >= speedLast: return fmt.Errorf("unknown encoder level") } + if o.resetOpt && l != o.level { + return errors.New("WithEncoderLevel cannot be changed on Reset") + } o.level = l if !o.customWindow { switch o.level { @@ -248,6 +268,7 @@ func WithEncoderLevel(l EncoderLevel) EOption { // WithZeroFrames will encode 0 length input as full frames. // This can be needed for compatibility with zstandard usage, // but is not needed for this package. +// Can be changed with ResetWithOptions. func WithZeroFrames(b bool) EOption { return func(o *encoderOptions) error { o.fullZero = b @@ -259,6 +280,7 @@ func WithZeroFrames(b bool) EOption { // Disabling this will skip incompressible data faster, but in cases with no matches but // skewed character distribution compression is lost. // Default value depends on the compression level selected. +// Can be changed with ResetWithOptions. func WithAllLitEntropyCompression(b bool) EOption { return func(o *encoderOptions) error { o.customALEntropy = true @@ -270,6 +292,7 @@ func WithAllLitEntropyCompression(b bool) EOption { // WithNoEntropyCompression will always skip entropy compression of literals. // This can be useful if content has matches, but unlikely to benefit from entropy // compression. Usually the slight speed improvement is not worth enabling this. +// Can be changed with ResetWithOptions. func WithNoEntropyCompression(b bool) EOption { return func(o *encoderOptions) error { o.noEntropy = b @@ -287,6 +310,7 @@ func WithNoEntropyCompression(b bool) EOption { // This is only a recommendation, each decoder is free to support higher or lower limits, depending on local limitations. // If this is not specified, block encodes will automatically choose this based on the input size and the window size. // This setting has no effect on streamed encodes. +// Can be changed with ResetWithOptions. func WithSingleSegment(b bool) EOption { return func(o *encoderOptions) error { o.single = &b @@ -298,8 +322,12 @@ func WithSingleSegment(b bool) EOption { // slower encoding speed. // This will not change the window size which is the primary function for reducing // memory usage. See WithWindowSize. +// Cannot be changed with ResetWithOptions. func WithLowerEncoderMem(b bool) EOption { return func(o *encoderOptions) error { + if o.resetOpt && b != o.lowMem { + return errors.New("WithLowerEncoderMem cannot be changed on Reset") + } o.lowMem = b return nil } @@ -311,6 +339,7 @@ func WithLowerEncoderMem(b bool) EOption { // "zstd --train" from the Zstandard reference implementation. // // The encoder *may* choose to use no dictionary instead for certain payloads. +// Can be changed with ResetWithOptions. // // [dictionary format]: https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#dictionary-format func WithEncoderDict(dict []byte) EOption { @@ -328,6 +357,7 @@ func WithEncoderDict(dict []byte) EOption { // // The slice content may contain arbitrary data. It will be used as an initial // history. +// Can be changed with ResetWithOptions. func WithEncoderDictRaw(id uint32, content []byte) EOption { return func(o *encoderOptions) error { if bits.UintSize > 32 && uint(len(content)) > dictMaxLength { @@ -337,3 +367,12 @@ func WithEncoderDictRaw(id uint32, content []byte) EOption { return nil } } + +// WithEncoderDictDelete clears the dictionary, so no dictionary will be used. +// Should be used with ResetWithOptions. +func WithEncoderDictDelete() EOption { + return func(o *encoderOptions) error { + o.dict = nil + return nil + } +} diff --git a/vendor/github.com/klauspost/compress/zstd/framedec.go b/vendor/github.com/klauspost/compress/zstd/framedec.go index e47af66e7c..d88f067e5c 100644 --- a/vendor/github.com/klauspost/compress/zstd/framedec.go +++ b/vendor/github.com/klauspost/compress/zstd/framedec.go @@ -238,10 +238,7 @@ func (d *frameDec) reset(br byteBuffer) error { if d.WindowSize == 0 && d.SingleSegment { // We may not need window in this case. - d.WindowSize = d.FrameContentSize - if d.WindowSize < MinWindowSize { - d.WindowSize = MinWindowSize - } + d.WindowSize = max(d.FrameContentSize, MinWindowSize) if d.WindowSize > d.o.maxDecodedSize { if debugDecoder { printf("window size %d > max %d\n", d.WindowSize, d.o.maxWindowSize) diff --git a/vendor/github.com/klauspost/compress/zstd/fse_decoder_amd64.go b/vendor/github.com/klauspost/compress/zstd/fse_decoder_amd64.go index d04a829b0a..b8c8607b5d 100644 --- a/vendor/github.com/klauspost/compress/zstd/fse_decoder_amd64.go +++ b/vendor/github.com/klauspost/compress/zstd/fse_decoder_amd64.go @@ -1,5 +1,4 @@ //go:build amd64 && !appengine && !noasm && gc -// +build amd64,!appengine,!noasm,gc package zstd diff --git a/vendor/github.com/klauspost/compress/zstd/fse_decoder_generic.go b/vendor/github.com/klauspost/compress/zstd/fse_decoder_generic.go index 8adfebb029..2138f8091a 100644 --- a/vendor/github.com/klauspost/compress/zstd/fse_decoder_generic.go +++ b/vendor/github.com/klauspost/compress/zstd/fse_decoder_generic.go @@ -1,5 +1,4 @@ //go:build !amd64 || appengine || !gc || noasm -// +build !amd64 appengine !gc noasm package zstd diff --git a/vendor/github.com/klauspost/compress/zstd/fse_encoder.go b/vendor/github.com/klauspost/compress/zstd/fse_encoder.go index ab26326a8f..3a0f4e7fbe 100644 --- a/vendor/github.com/klauspost/compress/zstd/fse_encoder.go +++ b/vendor/github.com/klauspost/compress/zstd/fse_encoder.go @@ -149,7 +149,7 @@ func (s *fseEncoder) buildCTable() error { if v > largeLimit { s.zeroBits = true } - for nbOccurrences := int16(0); nbOccurrences < v; nbOccurrences++ { + for range v { tableSymbol[position] = symbol position = (position + step) & tableMask for position > highThreshold { diff --git a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_other.go b/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_other.go index 0be16cefc7..9576426e68 100644 --- a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_other.go +++ b/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_other.go @@ -1,5 +1,4 @@ //go:build (!amd64 && !arm64) || appengine || !gc || purego || noasm -// +build !amd64,!arm64 appengine !gc purego noasm package xxhash diff --git a/vendor/github.com/klauspost/compress/zstd/matchlen_amd64.go b/vendor/github.com/klauspost/compress/zstd/matchlen_amd64.go index f41932b7a4..1ed18927f9 100644 --- a/vendor/github.com/klauspost/compress/zstd/matchlen_amd64.go +++ b/vendor/github.com/klauspost/compress/zstd/matchlen_amd64.go @@ -1,5 +1,4 @@ //go:build amd64 && !appengine && !noasm && gc -// +build amd64,!appengine,!noasm,gc // Copyright 2019+ Klaus Post. All rights reserved. // License information can be found in the LICENSE file. diff --git a/vendor/github.com/klauspost/compress/zstd/matchlen_generic.go b/vendor/github.com/klauspost/compress/zstd/matchlen_generic.go index bea1779e97..379746c96c 100644 --- a/vendor/github.com/klauspost/compress/zstd/matchlen_generic.go +++ b/vendor/github.com/klauspost/compress/zstd/matchlen_generic.go @@ -1,5 +1,4 @@ //go:build !amd64 || appengine || !gc || noasm -// +build !amd64 appengine !gc noasm // Copyright 2019+ Klaus Post. All rights reserved. // License information can be found in the LICENSE file. diff --git a/vendor/github.com/klauspost/compress/zstd/seqdec.go b/vendor/github.com/klauspost/compress/zstd/seqdec.go index 9a7de82f9e..0bfb0e43c7 100644 --- a/vendor/github.com/klauspost/compress/zstd/seqdec.go +++ b/vendor/github.com/klauspost/compress/zstd/seqdec.go @@ -231,10 +231,7 @@ func (s *sequenceDecs) decodeSync(hist []byte) error { llTable, mlTable, ofTable := s.litLengths.fse.dt[:maxTablesize], s.matchLengths.fse.dt[:maxTablesize], s.offsets.fse.dt[:maxTablesize] llState, mlState, ofState := s.litLengths.state.state, s.matchLengths.state.state, s.offsets.state.state out := s.out - maxBlockSize := maxCompressedBlockSize - if s.windowSize < maxBlockSize { - maxBlockSize = s.windowSize - } + maxBlockSize := min(s.windowSize, maxCompressedBlockSize) if debugDecoder { println("decodeSync: decoding", seqs, "sequences", br.remain(), "bits remain on stream") diff --git a/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.go b/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.go index c59f17e07a..18c3703ddc 100644 --- a/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.go +++ b/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.go @@ -1,5 +1,4 @@ //go:build amd64 && !appengine && !noasm && gc -// +build amd64,!appengine,!noasm,gc package zstd @@ -79,10 +78,7 @@ func (s *sequenceDecs) decodeSyncSimple(hist []byte) (bool, error) { br := s.br - maxBlockSize := maxCompressedBlockSize - if s.windowSize < maxBlockSize { - maxBlockSize = s.windowSize - } + maxBlockSize := min(s.windowSize, maxCompressedBlockSize) ctx := decodeSyncAsmContext{ llTable: s.litLengths.fse.dt[:maxTablesize], @@ -237,10 +233,7 @@ func sequenceDecs_decode_56_bmi2(s *sequenceDecs, br *bitReader, ctx *decodeAsmC func (s *sequenceDecs) decode(seqs []seqVals) error { br := s.br - maxBlockSize := maxCompressedBlockSize - if s.windowSize < maxBlockSize { - maxBlockSize = s.windowSize - } + maxBlockSize := min(s.windowSize, maxCompressedBlockSize) ctx := decodeAsmContext{ llTable: s.litLengths.fse.dt[:maxTablesize], diff --git a/vendor/github.com/klauspost/compress/zstd/seqdec_generic.go b/vendor/github.com/klauspost/compress/zstd/seqdec_generic.go index 7cec2197cd..516cd9b070 100644 --- a/vendor/github.com/klauspost/compress/zstd/seqdec_generic.go +++ b/vendor/github.com/klauspost/compress/zstd/seqdec_generic.go @@ -1,5 +1,4 @@ //go:build !amd64 || appengine || !gc || noasm -// +build !amd64 appengine !gc noasm package zstd diff --git a/vendor/github.com/klauspost/compress/zstd/simple_go124.go b/vendor/github.com/klauspost/compress/zstd/simple_go124.go new file mode 100644 index 0000000000..2efc0497bf --- /dev/null +++ b/vendor/github.com/klauspost/compress/zstd/simple_go124.go @@ -0,0 +1,56 @@ +// Copyright 2025+ Klaus Post. All rights reserved. +// License information can be found in the LICENSE file. + +//go:build go1.24 + +package zstd + +import ( + "errors" + "runtime" + "sync" + "weak" +) + +var weakMu sync.Mutex +var simpleEnc weak.Pointer[Encoder] +var simpleDec weak.Pointer[Decoder] + +// EncodeTo appends the encoded data from src to dst. +func EncodeTo(dst []byte, src []byte) []byte { + weakMu.Lock() + enc := simpleEnc.Value() + if enc == nil { + var err error + enc, err = NewWriter(nil, WithEncoderConcurrency(runtime.NumCPU()), WithWindowSize(1<<20), WithLowerEncoderMem(true), WithZeroFrames(true)) + if err != nil { + panic("failed to create simple encoder: " + err.Error()) + } + simpleEnc = weak.Make(enc) + } + weakMu.Unlock() + + return enc.EncodeAll(src, dst) +} + +// DecodeTo appends the decoded data from src to dst. +// The maximum decoded size is 1GiB, +// not including what may already be in dst. +func DecodeTo(dst []byte, src []byte) ([]byte, error) { + weakMu.Lock() + dec := simpleDec.Value() + if dec == nil { + var err error + dec, err = NewReader(nil, WithDecoderConcurrency(runtime.NumCPU()), WithDecoderLowmem(true), WithDecoderMaxMemory(1<<30)) + if err != nil { + weakMu.Unlock() + return nil, errors.New("failed to create simple decoder: " + err.Error()) + } + runtime.SetFinalizer(dec, func(d *Decoder) { + d.Close() + }) + simpleDec = weak.Make(dec) + } + weakMu.Unlock() + return dec.DecodeAll(src, dst) +} diff --git a/vendor/github.com/klauspost/compress/zstd/snappy.go b/vendor/github.com/klauspost/compress/zstd/snappy.go index a17381b8f8..336c288930 100644 --- a/vendor/github.com/klauspost/compress/zstd/snappy.go +++ b/vendor/github.com/klauspost/compress/zstd/snappy.go @@ -257,7 +257,7 @@ func (r *SnappyConverter) Convert(in io.Reader, w io.Writer) (int64, error) { if !r.readFull(r.buf[:len(snappyMagicBody)], false) { return written, r.err } - for i := 0; i < len(snappyMagicBody); i++ { + for i := range len(snappyMagicBody) { if r.buf[i] != snappyMagicBody[i] { println("r.buf[i] != snappyMagicBody[i]", r.buf[i], snappyMagicBody[i], i) r.err = ErrSnappyCorrupt diff --git a/vendor/github.com/klauspost/compress/zstd/zip.go b/vendor/github.com/klauspost/compress/zstd/zip.go index 29c15c8c4e..3198d71892 100644 --- a/vendor/github.com/klauspost/compress/zstd/zip.go +++ b/vendor/github.com/klauspost/compress/zstd/zip.go @@ -19,7 +19,7 @@ const ZipMethodWinZip = 93 const ZipMethodPKWare = 20 // zipReaderPool is the default reader pool. -var zipReaderPool = sync.Pool{New: func() interface{} { +var zipReaderPool = sync.Pool{New: func() any { z, err := NewReader(nil, WithDecoderLowmem(true), WithDecoderMaxWindow(128<<20), WithDecoderConcurrency(1)) if err != nil { panic(err) diff --git a/vendor/github.com/klauspost/compress/zstd/zstd.go b/vendor/github.com/klauspost/compress/zstd/zstd.go index 6252b46ae6..1a869710d2 100644 --- a/vendor/github.com/klauspost/compress/zstd/zstd.go +++ b/vendor/github.com/klauspost/compress/zstd/zstd.go @@ -98,13 +98,13 @@ var ( ErrDecoderNilInput = errors.New("nil input provided as reader") ) -func println(a ...interface{}) { +func println(a ...any) { if debug || debugDecoder || debugEncoder { log.Println(a...) } } -func printf(format string, a ...interface{}) { +func printf(format string, a ...any) { if debug || debugDecoder || debugEncoder { log.Printf(format, a...) } diff --git a/vendor/modules.txt b/vendor/modules.txt index 1cf1949c3a..7b7f7a41b1 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -505,8 +505,8 @@ github.com/kballard/go-shellquote # github.com/kevinburke/ssh_config v1.2.0 ## explicit github.com/kevinburke/ssh_config -# github.com/klauspost/compress v1.18.0 -## explicit; go 1.22 +# github.com/klauspost/compress v1.18.7 +## explicit; go 1.24 github.com/klauspost/compress github.com/klauspost/compress/flate github.com/klauspost/compress/fse From 743a2eeb5541760552741af75824a9b4cef48ce4 Mon Sep 17 00:00:00 2001 From: icanhasmath Date: Fri, 23 Oct 2026 11:21:05 -0500 Subject: [PATCH 3/6] check-format: don't fail when a PR changes no files. With an empty diff, grep -v exits 1 and set -e aborts the script with no output, so a PR that changes nothing (e.g. merging beta into a version branch) fails Check Format on every OS. Co-Authored-By: Claude Opus 5.5 --- activestate.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/activestate.yaml b/activestate.yaml index 9523cd6a0f..1a15e7d772 100644 --- a/activestate.yaml +++ b/activestate.yaml @@ -434,7 +434,7 @@ scripts: git fetch --quiet origin $TARGET_BRANCH:refs/remotes/origin/$TARGET_BRANCH - CHANGED=$(git diff --name-only origin/$TARGET_BRANCH | grep -v testdata | grep -v vendor) + CHANGED=$(git diff --name-only origin/$TARGET_BRANCH | grep -v testdata | grep -v vendor || true) NO_NEWLINE=0 for FILE in $CHANGED; do if file "$FILE" | grep -q -E 'text|ASCII'; then From 785725eaeb6954357cadf6746a89b474d62e0af6 Mon Sep 17 00:00:00 2001 From: icanhasmath Date: Fri, 23 Oct 2026 12:21:02 -0500 Subject: [PATCH 4/6] Update to Go 1.26.10 and golang.org/x/net v0.60.0 to remediate CVEs. The CI Trivy scan fails on 13 Go stdlib CVEs (fixed in 1.26.9/1.26.10) and 11 golang.org/x/net findings (fixed in v0.60.0) published since the 0.48.1 scan. Re-vendored. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/build.yml | 2 +- changelog.md | 1 + go.mod | 4 +- go.sum | 4 +- vendor/golang.org/x/net/http2/ascii.go | 14 +- vendor/golang.org/x/net/http2/databuffer.go | 5 +- vendor/golang.org/x/net/http2/flow.go | 103 +++++++-- vendor/golang.org/x/net/http2/frame.go | 34 ++- vendor/golang.org/x/net/http2/h2c/h2c.go | 5 + vendor/golang.org/x/net/http2/server.go | 110 +++++---- .../golang.org/x/net/http2/server_common.go | 47 ++++ vendor/golang.org/x/net/http2/transport.go | 144 ++++++++---- .../x/net/http2/transport_common.go | 79 ++++++- .../golang.org/x/net/http2/transport_wrap.go | 216 ++++++++++++++++-- vendor/golang.org/x/net/http2/writesched.go | 7 +- .../x/net/internal/httpcommon/README.md | 7 + .../x/net/internal/httpcommon/gzip.go | 132 +++++++++++ .../x/net/internal/httpcommon/request.go | 34 ++- vendor/modules.txt | 4 +- 19 files changed, 784 insertions(+), 168 deletions(-) create mode 100644 vendor/golang.org/x/net/internal/httpcommon/README.md create mode 100644 vendor/golang.org/x/net/internal/httpcommon/gzip.go diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 72c19bf0ba..e540a6a6f9 100755 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -36,7 +36,7 @@ jobs: strategy: matrix: go-version: - - 1.26.6 + - 1.26.10 sys: - { os: ubuntu-latest } - { os: macos-15-intel, shell: zsh } diff --git a/changelog.md b/changelog.md index 3a1258d236..0908eecfae 100644 --- a/changelog.md +++ b/changelog.md @@ -34,6 +34,7 @@ and this project adheres to ### Security - Updated `github.com/klauspost/compress` to v1.18.7 to remediate CVE-2026-63209 (GHSA-259r-337f-4rfw). +- Updated to Go 1.26.10 and `golang.org/x/net` v0.60.0 to remediate 14 CVEs in the Go standard library and HTTP/2 handling. ## 0.48.1 diff --git a/go.mod b/go.mod index a8b2517f1c..d2b63f4abb 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/ActiveState/cli -go 1.26.6 +go 1.26.10 require ( github.com/99designs/gqlgen v0.17.54 @@ -52,7 +52,7 @@ require ( github.com/vbauerster/mpb/v7 v7.1.5 github.com/vektah/gqlparser/v2 v2.5.16 golang.org/x/crypto v0.57.0 // indirect - golang.org/x/net v0.58.0 + golang.org/x/net v0.60.0 golang.org/x/sys v0.48.0 golang.org/x/term v0.46.0 golang.org/x/text v0.42.0 diff --git a/go.sum b/go.sum index 2c6b0ddb99..7dab0fdf09 100644 --- a/go.sum +++ b/go.sum @@ -850,8 +850,8 @@ golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= -golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= +golang.org/x/net v0.60.0 h1:79p50tfZlm0J9YfoDsSi639qSXNGVwEzOPLCxM2FsYU= +golang.org/x/net v0.60.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= diff --git a/vendor/golang.org/x/net/http2/ascii.go b/vendor/golang.org/x/net/http2/ascii.go index 17caa20586..3a7aed12c5 100644 --- a/vendor/golang.org/x/net/http2/ascii.go +++ b/vendor/golang.org/x/net/http2/ascii.go @@ -4,7 +4,9 @@ package http2 -import "strings" +import ( + "strings" +) // The HTTP protocols are defined in terms of ASCII, not Unicode. This file // contains helper functions which may use Unicode-aware functions which would @@ -43,6 +45,16 @@ func isASCIIPrint(s string) bool { return true } +// isASCII returns whether s is ASCII. +func isASCII(s string) bool { + for i := 0; i < len(s); i++ { + if s[i] > 0x7f { + return false + } + } + return true +} + // asciiToLower returns the lowercase version of s if s is ASCII and printable, // and whether or not it was. func asciiToLower(s string) (lower string, ok bool) { diff --git a/vendor/golang.org/x/net/http2/databuffer.go b/vendor/golang.org/x/net/http2/databuffer.go index e6f55cbd16..9763ab134c 100644 --- a/vendor/golang.org/x/net/http2/databuffer.go +++ b/vendor/golang.org/x/net/http2/databuffer.go @@ -121,10 +121,7 @@ func (b *dataBuffer) Write(p []byte) (int, error) { // If the last chunk is empty, allocate a new chunk. Try to allocate // enough to fully copy p plus any additional bytes we expect to // receive. However, this may allocate less than len(p). - want := int64(len(p)) - if b.expected > want { - want = b.expected - } + want := max(int64(len(p)), b.expected) chunk := b.lastChunkOrAlloc(want) n := copy(chunk[b.w:], p) p = p[n:] diff --git a/vendor/golang.org/x/net/http2/flow.go b/vendor/golang.org/x/net/http2/flow.go index b7dbd18695..5066a39912 100644 --- a/vendor/golang.org/x/net/http2/flow.go +++ b/vendor/golang.org/x/net/http2/flow.go @@ -10,6 +10,9 @@ package http2 // flow control window update. const inflowMinRefresh = 4 << 10 +// maxFlowWindow is the maximum size of a flow control window. +const maxFlowWindow = (1 << 31) - 1 + // inflow accounts for an inbound flow control window. // It tracks both the latest window sent to the peer (used for enforcement) // and the accumulated unsent window. @@ -74,47 +77,99 @@ func takeInflows(f1, f2 *inflow, n uint32) bool { return true } -// outflow is the outbound flow control window's size. +// connOutflow is connection-level outbound flow control. +type connOutflow struct { + initial int32 // SETTINGS_INITIAL_WINDOW_SIZE, changes with settings updates + n int32 // connection-level flow control window + flowErr bool // set when a flow control error is encountered +} + +func (f *connOutflow) init() { + f.initial = initialWindowSize // initial stream window size + f.n = initialWindowSize // current connection window size +} + +func (f *connOutflow) changeInitialWindowSize(size int64) bool { + if size > maxFlowWindow { + f.flowErr = true + return false + } + f.initial = int32(size) + return true +} + +func (f *connOutflow) add(n int32) bool { + sum := int64(f.n) + int64(n) + if sum > maxFlowWindow { + f.flowErr = true + return false + } + f.n += n + return true +} + +// outflow is the stream-level outbound flow control window's size. type outflow struct { _ incomparable - // n is the number of DATA bytes we're allowed to send. - // An outflow is kept both on a conn and a per-stream. - n int32 + // delta is the difference between the stream's flow control window and + // the connection's initial window size (conn.initial). + // + // Another view is that delta is the number of flow control bytes provided to this + // stream in WINDOW_UPDATE frames, less the number of bytes sent on the stream. + delta int32 // conn points to the shared connection-level outflow that is - // shared by all streams on that conn. It is nil for the outflow - // that's on the conn directly. - conn *outflow + // shared by all streams on that conn. + conn *connOutflow } -func (f *outflow) setConnFlow(cf *outflow) { f.conn = cf } - -func (f *outflow) available() int32 { - n := f.n - if f.conn != nil && f.conn.n < n { - n = f.conn.n +func (f *outflow) available() (int32, bool) { + if f.conn == nil { + return maxFlowWindow, true // only happens in tests } - return n + if f.conn.flowErr { + // Block all sending once any stream observes a flow control error. + return 0, false + } + n := int64(f.conn.initial) + int64(f.delta) + if n > maxFlowWindow { + f.conn.flowErr = true + return 0, false + } + return min(int32(n), f.conn.n), true } func (f *outflow) take(n int32) { - if n > f.available() { - panic("internal error: took too much") + if f.conn == nil { + return // only happens in tests } - f.n -= n - if f.conn != nil { - f.conn.n -= n + avail, _ := f.available() + if n > avail { + panic("internal error: took too much") } + f.delta -= n + f.conn.n -= n } // add adds n bytes (positive or negative) to the flow control window. // It returns false if the sum would exceed 2^31-1. func (f *outflow) add(n int32) bool { - sum := f.n + n - if (sum > n) == (f.n > 0) { - f.n = sum - return true + if f.conn == nil { + return true // only happens in tests + } + avail := int64(f.conn.initial) + int64(f.delta) + if avail > maxFlowWindow { + // An earlier change to the initial window pushed this stream over the limit. + // This is a connection-level flow control error. + f.conn.flowErr = true + return false } - return false + if avail+int64(n) > maxFlowWindow { + // This update would push the stream over the limit. + // This is a stream-level flow control error. + return false + } + f.delta += n + return true } diff --git a/vendor/golang.org/x/net/http2/frame.go b/vendor/golang.org/x/net/http2/frame.go index be75badcc0..3363bbafc8 100644 --- a/vendor/golang.org/x/net/http2/frame.go +++ b/vendor/golang.org/x/net/http2/frame.go @@ -1711,7 +1711,8 @@ func (fr *Framer) readMetaFrame(hf *HeadersFrame) (Frame, error) { mh := &MetaHeadersFrame{ HeadersFrame: hf, } - var remainSize = fr.maxHeaderListSize() + var headersRemainSize = fr.maxHeaderListSize() + var trailersRemainSize = fr.maxHeaderListSize() var sawRegular bool var invalid error // pseudo header field errors @@ -1743,14 +1744,29 @@ func (fr *Framer) readMetaFrame(hf *HeadersFrame) (Frame, error) { return } - size := hf.Size() - if size > remainSize { + var remainSize *uint32 + var size uint32 + if hf.Name == "trailer" { + remainSize = &trailersRemainSize + fieldCount := strings.Count(hf.Value, ",") + 1 + // Rather than actually constructing hpack.HeaderField for each + // trailer field and cumulatively adding its Size, just do the math + // manually to avoid unnecessary work. This does make it so + // whitespaces after comma are counted against the budget, but that + // should be innocuous. + size = uint32(len(hf.Value)-fieldCount+1) + uint32(fieldCount)*hpack.HeaderField{}.Size() + } else { + remainSize = &headersRemainSize + size = hf.Size() + } + if size > *remainSize { hdec.SetEmitEnabled(false) mh.Truncated = true - remainSize = 0 + headersRemainSize = 0 + trailersRemainSize = 0 return } - remainSize -= size + *remainSize -= size mh.Fields = append(mh.Fields, hf) }) @@ -1766,10 +1782,10 @@ func (fr *Framer) readMetaFrame(hf *HeadersFrame) (Frame, error) { // skip parsing the fragment and close the connection. // // "Too much" is either any CONTINUATION frame after we've already - // exceeded the max header list size (in which case remainSize is 0), - // or a frame whose encoded size is more than twice the remaining - // header list bytes we're willing to accept. - if int64(len(frag)) > int64(2*remainSize) { + // exceeded the max header list size (if so, both budgets are 0), or a + // frame whose encoded size is more than twice the remaining header + // list bytes we're willing to accept. + if int64(len(frag)) > 2*int64(headersRemainSize+trailersRemainSize) { if VerboseLogs { log.Printf("http2: header list too large") } diff --git a/vendor/golang.org/x/net/http2/h2c/h2c.go b/vendor/golang.org/x/net/http2/h2c/h2c.go index b5807d557f..23470b4960 100644 --- a/vendor/golang.org/x/net/http2/h2c/h2c.go +++ b/vendor/golang.org/x/net/http2/h2c/h2c.go @@ -24,6 +24,7 @@ // client := &http.Client{Transport: tr} // // Deprecated: This package is deprecated. +// It is not maintained and should not be used. package h2c import ( @@ -84,6 +85,10 @@ type h2cHandler struct { // NewHandler is deprecated. // +// The Handler returned by NewHandler will read the first request on a connection entirely +// into memory. To limit the memory consumed by this request, wrap the result of NewHandler +// in an http.MaxBytesHandler. +// // Deprecated: Set the [http.Server] Protocols field to use unencrypted HTTP/2 instead. func NewHandler(h http.Handler, s *http2.Server) http.Handler { return &h2cHandler{ diff --git a/vendor/golang.org/x/net/http2/server.go b/vendor/golang.org/x/net/http2/server.go index a7d2053b6c..a6cd66226d 100644 --- a/vendor/golang.org/x/net/http2/server.go +++ b/vendor/golang.org/x/net/http2/server.go @@ -280,7 +280,6 @@ func (s *Server) serveConn(c net.Conn, opts *ServeConnOpts, newf func(*serverCon doneServing: make(chan struct{}), clientMaxStreams: math.MaxUint32, // Section 6.5.2: "Initially, there is no limit to this value" advMaxStreams: conf.MaxConcurrentStreams, - initialStreamSendWindowSize: initialWindowSize, initialStreamRecvWindowSize: conf.MaxUploadBufferPerStream, maxFrameSize: initialMaxFrameSize, pingTimeout: conf.PingTimeout, @@ -320,7 +319,7 @@ func (s *Server) serveConn(c net.Conn, opts *ServeConnOpts, newf func(*serverCon // These start at the RFC-specified defaults. If there is a higher // configured value for inflow, that will be updated when we send a // WINDOW_UPDATE shortly after sending SETTINGS. - sc.flow.add(initialWindowSize) + sc.flow.init() sc.inflow.init(initialWindowSize) sc.hpackEncoder = hpack.NewEncoder(&sc.headerWriteBuf) sc.hpackEncoder.SetMaxDynamicTableSizeLimit(conf.MaxEncoderHeaderTableSize) @@ -427,7 +426,7 @@ type serverConn struct { wroteFrameCh chan frameWriteResult // from writeFrameAsync -> serve, tickles more frame writes bodyReadCh chan bodyReadMsg // from handlers -> serve serveMsgCh chan interface{} // misc messages & code to send to / run on the serve loop - flow outflow // conn-wide (not stream-specific) outbound flow control + flow connOutflow // conn-wide (not stream-specific) outbound flow control inflow inflow // conn-wide inbound flow control tlsState *tls.ConnectionState // shared by all handlers, like net/http remoteAddrStr string @@ -441,6 +440,9 @@ type serverConn struct { sawFirstSettings bool // got the initial SETTINGS frame after the preface needToSendSettingsAck bool unackedSettings int // how many SETTINGS have we sent without ACKs? + pendingEncoderTableSize bool // peer changed SETTINGS_HEADER_TABLE_SIZE; apply to hpackEncoder before the next frame write + encoderTableSizeMin uint32 // smallest SETTINGS_HEADER_TABLE_SIZE since the last apply + encoderTableSize uint32 // latest SETTINGS_HEADER_TABLE_SIZE queuedControlFrames int // control frames in the writeSched queue clientMaxStreams uint32 // SETTINGS_MAX_CONCURRENT_STREAMS from client (our PUSH_PROMISE limit) advMaxStreams uint32 // our SETTINGS_MAX_CONCURRENT_STREAMS advertised the client @@ -451,7 +453,6 @@ type serverConn struct { maxPushPromiseID uint32 // ID of the last push promise (even), or 0 if there have been no pushes streams map[uint32]*stream unstartedHandlers []unstartedHandler - initialStreamSendWindowSize int32 initialStreamRecvWindowSize int32 maxFrameSize int32 peerMaxHeaderListSize uint32 // zero means unknown (default) @@ -521,7 +522,8 @@ type stream struct { // immutable: sc *serverConn id uint32 - body *pipe // non-nil if expecting DATA frames + body *pipe // non-nil if expecting DATA frames + reqBody *requestBody cw closeWaiter // closed wait stream transitions to closed state ctx context.Context cancelCtx func() @@ -1172,6 +1174,16 @@ func (sc *serverConn) startFrameWrite(wr FrameWriteRequest) { sc.writingFrame = true sc.needsFrameFlush = true + if sc.pendingEncoderTableSize { + // hpackEncoder may be in use by writeFrameAsync, so SETTINGS + // changes to it are deferred until no frame is being written. + // Replaying the smallest size before the latest one keeps the + // encoder's view identical to having applied every change + // (RFC 7541, Section 4.2). + sc.pendingEncoderTableSize = false + sc.hpackEncoder.SetMaxDynamicTableSize(sc.encoderTableSizeMin) + sc.hpackEncoder.SetMaxDynamicTableSize(sc.encoderTableSize) + } if wr.write.staysWithinBuffer(sc.bw.Available()) { sc.writingFrameAsync = false err := wr.write.writeFrame(sc) @@ -1271,6 +1283,11 @@ func (sc *serverConn) scheduleFrameWrite() { } sc.inFrameScheduleLoop = true for !sc.writingFrameAsync { + if sc.flow.flowErr && (!sc.inGoAway || sc.goAwayCode == ErrCodeNo) { + sc.inGoAway = true + sc.needToSendGoAway = true + sc.goAwayCode = ErrCodeFlowControl + } if sc.needToSendGoAway { sc.needToSendGoAway = false sc.startFrameWrite(FrameWriteRequest{ @@ -1294,6 +1311,9 @@ func (sc *serverConn) scheduleFrameWrite() { sc.startFrameWrite(wr) continue } + if sc.flow.flowErr { + continue + } } if sc.needsFrameFlush { sc.startFrameWrite(FrameWriteRequest{write: flushFrameWriter{}}) @@ -1526,6 +1546,10 @@ func (sc *serverConn) processWindowUpdate(f *WindowUpdateFrame) error { return nil } if !st.flow.add(int32(f.Increment)) { + if st.flow.conn.flowErr { + // This is a lazily-detected connection-level flow control error. + return sc.countError("bad_flow", ConnectionError(ErrCodeFlowControl)) + } return sc.countError("bad_flow", streamError(f.StreamID, ErrCodeFlowControl)) } default: // connection-level flow control @@ -1584,10 +1608,6 @@ func (sc *serverConn) closeStream(st *stream, err error) { } } if p := st.body; p != nil { - // Return any buffered unread bytes worth of conn-level flow control. - // See golang.org/issue/16481 - sc.sendWindowUpdate(nil, p.Len()) - p.CloseWithError(err) } if e, ok := err.(StreamError); ok { @@ -1641,7 +1661,12 @@ func (sc *serverConn) processSetting(s Setting) error { } switch s.ID { case SettingHeaderTableSize: - sc.hpackEncoder.SetMaxDynamicTableSize(s.Val) + // Applied by startFrameWrite; see comment there. + if !sc.pendingEncoderTableSize || s.Val < sc.encoderTableSizeMin { + sc.encoderTableSizeMin = s.Val + } + sc.encoderTableSize = s.Val + sc.pendingEncoderTableSize = true case SettingEnablePush: sc.pushEnabled = s.Val != 0 case SettingMaxConcurrentStreams: @@ -1672,28 +1697,14 @@ func (sc *serverConn) processSetting(s Setting) error { func (sc *serverConn) processSettingInitialWindowSize(val uint32) error { sc.serveG.check() - // Note: val already validated to be within range by - // processSetting's Valid call. - - // "A SETTINGS frame can alter the initial flow control window - // size for all current streams. When the value of - // SETTINGS_INITIAL_WINDOW_SIZE changes, a receiver MUST - // adjust the size of all stream flow control windows that it - // maintains by the difference between the new value and the - // old value." - old := sc.initialStreamSendWindowSize - sc.initialStreamSendWindowSize = int32(val) - growth := int32(val) - old // may be negative - for _, st := range sc.streams { - if !st.flow.add(growth) { - // 6.9.2 Initial Flow Control Window Size - // "An endpoint MUST treat a change to - // SETTINGS_INITIAL_WINDOW_SIZE that causes any flow - // control window to exceed the maximum size as a - // connection error (Section 5.4.1) of type - // FLOW_CONTROL_ERROR." - return sc.countError("setting_win_size", ConnectionError(ErrCodeFlowControl)) - } + if !sc.flow.changeInitialWindowSize(int64(val)) { + // 6.9.2 Initial Flow Control Window Size + // "An endpoint MUST treat a change to + // SETTINGS_INITIAL_WINDOW_SIZE that causes any flow + // control window to exceed the maximum size as a + // connection error (Section 5.4.1) of type + // FLOW_CONTROL_ERROR." + return sc.countError("setting_win_size", ConnectionError(ErrCodeFlowControl)) } return nil } @@ -1966,7 +1977,7 @@ func (sc *serverConn) processHeaders(f *MetaHeadersFrame) error { if st.reqTrailer != nil { st.trailer = make(http.Header) } - st.body = req.Body.(*requestBody).pipe // may be nil + st.body = st.reqBody.pipe // may be nil st.declBodyBytes = req.ContentLength handler := sc.handler.ServeHTTP @@ -1989,7 +2000,7 @@ func (sc *serverConn) processHeaders(f *MetaHeadersFrame) error { st.readDeadline = time.AfterFunc(sc.hs.ReadTimeout, st.onReadTimeout) } - return sc.scheduleHandler(id, rw, req, handler) + return sc.scheduleHandler(st, rw, req, handler) } func (sc *serverConn) upgradeRequest(req *http.Request) { @@ -2102,7 +2113,6 @@ func (sc *serverConn) newStream(id, pusherID uint32, state streamState, priority } st.cw.Init() st.flow.conn = &sc.flow // link to conn-level counter - st.flow.add(sc.initialStreamSendWindowSize) st.inflow.init(sc.initialStreamRecvWindowSize) if sc.hs.WriteTimeout > 0 { st.writeDeadline = time.AfterFunc(sc.hs.WriteTimeout, st.onWriteTimeout) @@ -2184,7 +2194,7 @@ func (sc *serverConn) newWriterAndRequest(st *stream, f *MetaHeadersFrame) (*res } else { req.ContentLength = -1 } - req.Body.(*requestBody).pipe = &pipe{ + st.reqBody.pipe = &pipe{ b: &dataBuffer{expected: req.ContentLength}, } } @@ -2204,7 +2214,7 @@ func (sc *serverConn) newWriterAndRequestNoBody(st *stream, rp httpcommon.Server return nil, nil, sc.countError(res.InvalidReason, streamError(st.id, ErrCodeProtocol)) } - body := &requestBody{ + st.reqBody = &requestBody{ conn: sc, stream: st, needsContinue: res.NeedsContinue, @@ -2220,7 +2230,7 @@ func (sc *serverConn) newWriterAndRequestNoBody(st *stream, rp httpcommon.Server ProtoMinor: 0, TLS: tlsState, Host: rp.Authority, - Body: body, + Body: st.reqBody, Trailer: res.Trailer, }).WithContext(st.ctx) rw := sc.newResponseWriter(st, req) @@ -2244,11 +2254,12 @@ type unstartedHandler struct { rw *responseWriter req *http.Request handler func(http.ResponseWriter, *http.Request) + body *pipe } // scheduleHandler starts a handler goroutine, // or schedules one to start as soon as an existing handler finishes. -func (sc *serverConn) scheduleHandler(streamID uint32, rw *responseWriter, req *http.Request, handler func(http.ResponseWriter, *http.Request)) error { +func (sc *serverConn) scheduleHandler(st *stream, rw *responseWriter, req *http.Request, handler func(http.ResponseWriter, *http.Request)) error { sc.serveG.check() maxHandlers := sc.advMaxStreams if sc.curHandlers < maxHandlers { @@ -2260,10 +2271,11 @@ func (sc *serverConn) scheduleHandler(streamID uint32, rw *responseWriter, req * return sc.countError("too_many_early_resets", ConnectionError(ErrCodeEnhanceYourCalm)) } sc.unstartedHandlers = append(sc.unstartedHandlers, unstartedHandler{ - streamID: streamID, + streamID: st.id, rw: rw, req: req, handler: handler, + body: st.body, }) return nil } @@ -2277,6 +2289,10 @@ func (sc *serverConn) handlerDone() { u := sc.unstartedHandlers[i] if sc.streams[u.streamID] == nil { // This stream was reset before its goroutine had a chance to start. + if u.body != nil { + u.body.BreakWithError(errClosedBody) + sc.sendWindowUpdate(nil, u.body.Len()) + } continue } if sc.curHandlers >= maxHandlers { @@ -2298,6 +2314,12 @@ func (sc *serverConn) runHandler(rw *responseWriter, req *http.Request, handler didPanic := true defer func() { rw.rws.stream.cancelCtx() + if b := rw.rws.stream.reqBody; b != nil { + // Closing the body refunds flow control credit for any unconsumed data. + // (reqBody is nil for Upgrade: h2c requests, but those do not use flow + // control for the request body.) + b.Close() + } if req.MultipartForm != nil { req.MultipartForm.RemoveAll() } @@ -2396,7 +2418,7 @@ func (sc *serverConn) noteBodyReadFromHandler(st *stream, n int, err error) { func (sc *serverConn) noteBodyRead(st *stream, n int) { sc.serveG.check() sc.sendWindowUpdate(nil, n) // conn-level - if st.state != stateHalfClosedRemote && st.state != stateClosed { + if st != nil && st.state != stateHalfClosedRemote && st.state != stateClosed { // Don't send this WINDOW_UPDATE if the stream is closed // remotely. sc.sendWindowUpdate(st, n) @@ -2444,6 +2466,9 @@ func (b *requestBody) Close() error { b.closeOnce.Do(func() { if b.pipe != nil { b.pipe.BreakWithError(errClosedBody) + if unread := b.pipe.Len(); unread > 0 { + b.conn.noteBodyReadFromHandler(nil, unread, errClosedBody) + } } }) return nil @@ -2461,9 +2486,6 @@ func (b *requestBody) Read(p []byte) (n int, err error) { if err == io.EOF { b.sawEOF = true } - if b.conn == nil { - return - } b.conn.noteBodyReadFromHandler(b.stream, n, err) return } diff --git a/vendor/golang.org/x/net/http2/server_common.go b/vendor/golang.org/x/net/http2/server_common.go index 449538c861..fec369092a 100644 --- a/vendor/golang.org/x/net/http2/server_common.go +++ b/vendor/golang.org/x/net/http2/server_common.go @@ -25,6 +25,8 @@ import ( // // https://golang.org/pkg/net/http/#ResponseWriter // https://golang.org/pkg/net/http/#example_ResponseWriter_trailers +// +// Deprecated: Use [http.TrailerPrefix] instead. const TrailerPrefix = "Trailer:" // Push errors. @@ -38,16 +40,22 @@ var ( // The configuration conf may be nil. // // ConfigureServer must be called before s begins serving. +// +// Deprecated: Set [http.Server.Protocols] instead. func ConfigureServer(s *http.Server, conf *Server) error { return configureServer(s, conf) } // Server is an HTTP/2 server. +// +// Deprecated: Use [http.Server] instead. type Server struct { // MaxHandlers limits the number of http.Handler ServeHTTP goroutines // which may run at a time over all connections. // Negative or zero no limit. // TODO: implement + // + // Deprecated: This field has never had any effect. MaxHandlers int // MaxConcurrentStreams optionally specifies the number of @@ -56,6 +64,9 @@ type Server struct { // which may be active globally, which is MaxHandlers. // If zero, MaxConcurrentStreams defaults to at least 100, per // the HTTP/2 spec's recommendations. + // + // Deprecated: Use [http.Server.HTTP2] and + // [http.HTTP2Config.MaxConcurrentStreams] instead. MaxConcurrentStreams uint32 // MaxDecoderHeaderTableSize optionally specifies the http2 @@ -63,44 +74,67 @@ type Server struct { // informs the remote endpoint of the maximum size of the header compression // table used to decode header blocks, in octets. If zero, the default value // of 4096 is used. + // + // Deprecated: Use [http.Server.HTTP2] and + // [http.HTTP2Config.MaxDecoderHeaderTableSize] instead. MaxDecoderHeaderTableSize uint32 // MaxEncoderHeaderTableSize optionally specifies an upper limit for the // header compression table used for encoding request headers. Received // SETTINGS_HEADER_TABLE_SIZE settings are capped at this limit. If zero, // the default value of 4096 is used. + // + // Deprecated: Use [http.Server.HTTP2] and + // [http.HTTP2Config.MaxEncoderHeaderTableSize] instead. MaxEncoderHeaderTableSize uint32 // MaxReadFrameSize optionally specifies the largest frame // this server is willing to read. A valid value is between // 16k and 16M, inclusive. If zero or otherwise invalid, a // default value is used. + // + // Deprecated: Use [http.Server.HTTP2] and + // [http.HTTP2Config.MaxReadFrameSize] instead. MaxReadFrameSize uint32 // PermitProhibitedCipherSuites, if true, permits the use of // cipher suites prohibited by the HTTP/2 spec. + // + // Deprecated: Use [http.Server.HTTP2] and + // [http.HTTP2Config.PermitProhibitedCipherSuites] instead. PermitProhibitedCipherSuites bool // IdleTimeout specifies how long until idle clients should be // closed with a GOAWAY frame. PING frames are not considered // activity for the purposes of IdleTimeout. // If zero or negative, there is no timeout. + // + // Deprecated: Use [http.Server.IdleTimeout] instead. IdleTimeout time.Duration // ReadIdleTimeout is the timeout after which a health check using a ping // frame will be carried out if no frame is received on the connection. // If zero, no health check is performed. + // + // Deprecated: Use [http.Server.HTTP2] and + // [http.HTTP2Config.SendPingTimeout] instead. ReadIdleTimeout time.Duration // PingTimeout is the timeout after which the connection will be closed // if a response to a ping is not received. // If zero, a default of 15 seconds is used. + // + // Deprecated: Use [http.Server.HTTP2] and + // [http.HTTP2Config.PingTimeout] instead. PingTimeout time.Duration // WriteByteTimeout is the timeout after which a connection will be // closed if no data can be written to it. The timeout begins when data is // available to write, and is extended whenever any bytes are written. // If zero or negative, there is no timeout. + // + // Deprecated: Use [http.Server.HTTP2] and + // [http.HTTP2Config.WriteByteTimeout] instead. WriteByteTimeout time.Duration // MaxUploadBufferPerConnection is the size of the initial flow @@ -108,12 +142,18 @@ type Server struct { // allow this to be smaller than 65535 or larger than 2^32-1. // If the value is outside this range, a default value will be // used instead. + // + // Deprecated: Use [http.Server.HTTP2] and + // [http.HTTP2Config.MaxReceiveBufferPerConnection] instead. MaxUploadBufferPerConnection int32 // MaxUploadBufferPerStream is the size of the initial flow control // window for each stream. The HTTP/2 spec does not allow this to // be larger than 2^32-1. If the value is zero or larger than the // maximum, a default value will be used instead. + // + // Deprecated: Use [http.Server.HTTP2] and + // [http.HTTP2Config.MaxReceiveBufferPerStream] instead. MaxUploadBufferPerStream int32 // NewWriteScheduler constructs a write scheduler for a connection. @@ -126,6 +166,9 @@ type Server struct { // It's intended to increment a metric for monitoring, such // as an expvar or Prometheus metric. // The errType consists of only ASCII word characters. + // + // Deprecated: Use [http.Server.HTTP2] and + // [http.HTTP2Config.CountError] instead. CountError func(errType string) // Internal state. This is a pointer (rather than embedded directly) @@ -135,6 +178,8 @@ type Server struct { } // ServeConnOpts are options for the Server.ServeConn method. +// +// Deprecated: ServeConnOpts is deprecated. type ServeConnOpts struct { // Context is the base context to use. // If nil, context.Background is used. @@ -178,6 +223,8 @@ type ServeConnOpts struct { // implemented in terms of providing a suitably-behaving net.Conn. // // The opts parameter is optional. If nil, default values are used. +// +// Deprecated: Use [http.Server.Serve] or [http.Server.ServeTLS] instead. func (s *Server) ServeConn(c net.Conn, opts *ServeConnOpts) { if opts == nil { opts = &ServeConnOpts{} diff --git a/vendor/golang.org/x/net/http2/transport.go b/vendor/golang.org/x/net/http2/transport.go index 08ac409b0e..d894c6019c 100644 --- a/vendor/golang.org/x/net/http2/transport.go +++ b/vendor/golang.org/x/net/http2/transport.go @@ -27,6 +27,7 @@ import ( "net/http" "net/http/httptrace" "net/textproto" + "slices" "strconv" "sync" "sync/atomic" @@ -184,6 +185,8 @@ func (t *Transport) initConnPool() { // ClientConn is the state of a single HTTP/2 client connection to an // HTTP/2 server. +// +// Deprecated: Use [http.ClientConn] instead. type ClientConn struct { t *Transport tconn net.Conn // usually *tls.Conn, except specialized impls @@ -201,7 +204,7 @@ type ClientConn struct { mu sync.Mutex // guards following cond *sync.Cond // hold mu; broadcast on flow/closed changes - flow outflow // our conn-level flow control quota (cs.outflow is per stream) + flow connOutflow // our conn-level flow control quota (cs.outflow is per stream) inflow inflow // peer's conn-level flow control doNotReuse bool // whether conn is marked to not be reused for any future requests closing bool @@ -225,7 +228,6 @@ type ClientConn struct { maxConcurrentStreams uint32 peerMaxHeaderListSize uint64 peerMaxHeaderTableSize uint32 - initialWindowSize uint32 initialStreamRecvWindowSize int32 readIdleTimeout time.Duration pingTimeout time.Duration @@ -470,7 +472,6 @@ func (t *Transport) newClientConn(c net.Conn, singleUse bool, internalStateHook readerDone: make(chan struct{}), nextStreamID: 1, maxFrameSize: 16 << 10, // spec default - initialWindowSize: 65535, // spec default initialStreamRecvWindowSize: conf.MaxUploadBufferPerStream, maxConcurrentStreams: initialMaxConcurrentStreams, // "infinite", per spec. Use a smaller value until we have received server settings. strictMaxConcurrentStreams: conf.StrictMaxConcurrentRequests, @@ -495,7 +496,7 @@ func (t *Transport) newClientConn(c net.Conn, singleUse bool, internalStateHook } cc.cond = sync.NewCond(&cc.mu) - cc.flow.add(int32(initialWindowSize)) + cc.flow.init() // TODO: adjust this writer size to account for frame size + // MTU + crypto/tls record padding. @@ -874,6 +875,33 @@ func (cc *ClientConn) sendGoAway() error { return nil } +func (cc *ClientConn) goAwayAndClose(code ErrCode) { + cc.mu.Lock() + closed := cc.closed + cc.closing = true + cc.closed = true + cc.mu.Unlock() + if closed { + return + } + if f := cc.fr.countError; f != nil { + f(fmt.Sprintf("conn_close_error_%s", code.stringToken())) + } + done := make(chan struct{}) + go func() { + defer close(done) + cc.wmu.Lock() + cc.fr.WriteGoAway(0, code, nil) + cc.bw.Flush() + cc.wmu.Unlock() + }() + select { + case <-done: + case <-time.After(250 * time.Millisecond): + } + cc.closeForError(fmt.Errorf("http2: closing connection with %v", code)) +} + // closes the client connection immediately. In-flight requests are interrupted. // err is sent to streams. func (cc *ClientConn) closeForError(err error) { @@ -1317,7 +1345,9 @@ func (cs *clientStream) cleanupWriteRequest(err error) { } if err != nil { cs.abortStream(err) // possibly redundant, but harmless - if cs.sentHeaders { + if ce, ok := err.(ConnectionError); ok { + cc.goAwayAndClose(ErrCode(ce)) + } else if cs.sentHeaders { if se, ok := err.(StreamError); ok { if se.Cause != errFromPeer { cc.writeStreamReset(cs.ID, se.Code, false, err) @@ -1646,8 +1676,12 @@ func (cs *clientStream) awaitFlowControl(maxBytes int) (taken int32, err error) return 0, errRequestCanceled default: } - if a := cs.flow.available(); a > 0 { - take := a + avail, ok := cs.flow.available() + if !ok { + return 0, ConnectionError(ErrCodeFlowControl) + } + if avail > 0 { + take := avail if int(take) > maxBytes { take = int32(maxBytes) // can't truncate int; take is int32 @@ -1708,8 +1742,7 @@ type resAndError struct { // requires cc.mu be held. func (cc *ClientConn) addStreamLocked(cs *clientStream) { - cs.flow.add(int32(cc.initialWindowSize)) - cs.flow.setConnFlow(&cc.flow) + cs.flow.conn = &cc.flow cs.inflow.init(cc.initialStreamRecvWindowSize) cs.ID = cc.nextStreamID cc.nextStreamID += 2 @@ -2093,18 +2126,42 @@ func (rl *clientConnReadLoop) handleResponse(cs *clientStream, f *MetaHeadersFra return nil, nil } + // Delete various headers that might mess up framing for HTTP/1. This is + // not a problem for HTTP/2, but someone might use HTTP/2 transport as a + // reverse proxy which forwards the response to an HTTP/1 client. Our + // HTTP/1 client transport will properly reject improper headers such as + // multiple conflicting Content-Length headers, but other implementations + // might not. + // TODO: just reject such responses? We deleted them for compatibility + // since this was done in a security fix (go.dev/issue/81115). However, + // rejecting them seems entirely reasonable and relatively safe. + + // Connection-specific header fields must not appear in an HTTP/2 message, + // and any message containing them is malformed. RFC 9113, Section 8.2.2. + for _, k := range connHeaders { + delete(res.Header, k) + } res.ContentLength = -1 - if clens := res.Header["Content-Length"]; len(clens) == 1 { - if cl, err := strconv.ParseUint(clens[0], 10, 63); err == nil { - res.ContentLength = int64(cl) + if clens, ok := res.Header["Content-Length"]; ok { + // Repeated Content-Length values may be collapsed into one only if + // they are identical per RFC 9110 Section 8.6. + // No need to trim whitespace, HTTP/2 header values must not have + // extraneous whitespace per RFC 9113 Section 8.2.1. + // Non-canonical headers are already rejected by our framer at this + // point. + conflicting := slices.ContainsFunc(clens[1:], func(clen string) bool { + return clen != clens[0] + }) + cl, err := strconv.ParseUint(clens[0], 10, 63) + if conflicting || err != nil { + delete(res.Header, "Content-Length") } else { - // TODO: care? unlike http/1, it won't mess up our framing, so it's - // more safe smuggling-wise to ignore. + res.Header["Content-Length"] = clens[:1] + res.ContentLength = int64(cl) } - } else if len(clens) > 1 { - // TODO: care? unlike http/1, it won't mess up our framing, so it's - // more safe smuggling-wise to ignore. - } else if f.StreamEnded() && !cs.isHead { + } + + if res.ContentLength < 0 && f.StreamEnded() && !cs.isHead { res.ContentLength = 0 } @@ -2415,6 +2472,10 @@ const ( func (rl *clientConnReadLoop) streamByID(id uint32, headerOrData bool) *clientStream { rl.cc.mu.Lock() defer rl.cc.mu.Unlock() + return rl.streamByIDLocked(id, headerOrData) +} + +func (rl *clientConnReadLoop) streamByIDLocked(id uint32, headerOrData bool) *clientStream { if headerOrData { // Work around an unfortunate gRPC behavior. // See comment on ClientConn.rstStreamPingsBlocked for details. @@ -2497,24 +2558,10 @@ func (rl *clientConnReadLoop) processSettingsNoWrite(f *SettingsFrame) error { case SettingMaxHeaderListSize: cc.peerMaxHeaderListSize = uint64(s.Val) case SettingInitialWindowSize: - // Values above the maximum flow-control - // window size of 2^31-1 MUST be treated as a - // connection error (Section 5.4.1) of type - // FLOW_CONTROL_ERROR. - if s.Val > math.MaxInt32 { + if !cc.flow.changeInitialWindowSize(int64(s.Val)) { return ConnectionError(ErrCodeFlowControl) } - - // Adjust flow control of currently-open - // frames by the difference of the old initial - // window size and this one. - delta := int32(s.Val) - int32(cc.initialWindowSize) - for _, cs := range cc.streams { - cs.flow.add(delta) - } cc.cond.Broadcast() - - cc.initialWindowSize = s.Val case SettingHeaderTableSize: cc.henc.SetMaxDynamicTableSize(s.Val) cc.peerMaxHeaderTableSize = s.Val @@ -2556,29 +2603,30 @@ func (rl *clientConnReadLoop) processSettingsNoWrite(f *SettingsFrame) error { func (rl *clientConnReadLoop) processWindowUpdate(f *WindowUpdateFrame) error { cc := rl.cc - cs := rl.streamByID(f.StreamID, notHeaderOrDataFrame) - if f.StreamID != 0 && cs == nil { - return nil - } - cc.mu.Lock() defer cc.mu.Unlock() - - fl := &cc.flow - if cs != nil { - fl = &cs.flow - } - if !fl.add(int32(f.Increment)) { - // For stream, the sender sends RST_STREAM with an error code of FLOW_CONTROL_ERROR - if cs != nil { + if f.StreamID == 0 { + if !cc.flow.add(int32(f.Increment)) { + return ConnectionError(ErrCodeFlowControl) + } + } else { + cs := rl.streamByIDLocked(f.StreamID, notHeaderOrDataFrame) + if cs == nil { + return nil + } + if !cs.flow.add(int32(f.Increment)) { + if cs.flow.conn.flowErr { + // This is a lazily-detected connection-level flow control error. + return ConnectionError(ErrCodeFlowControl) + } + // For stream, the sender sends RST_STREAM with + // an error code of FLOW_CONTROL_ERROR. rl.endStreamErrorLocked(cs, StreamError{ StreamID: f.StreamID, Code: ErrCodeFlowControl, }) return nil } - - return ConnectionError(ErrCodeFlowControl) } cc.cond.Broadcast() return nil diff --git a/vendor/golang.org/x/net/http2/transport_common.go b/vendor/golang.org/x/net/http2/transport_common.go index b9f52932e9..4f0333a703 100644 --- a/vendor/golang.org/x/net/http2/transport_common.go +++ b/vendor/golang.org/x/net/http2/transport_common.go @@ -24,6 +24,8 @@ import ( // It returns an error if t1 has already been HTTP/2-enabled. // // Use ConfigureTransports instead to configure the HTTP/2 Transport. +// +// Deprecated: Set [http.Transport.Protocols] instead. func ConfigureTransport(t1 *http.Transport) error { return configureTransport(t1) } @@ -31,6 +33,8 @@ func ConfigureTransport(t1 *http.Transport) error { // ConfigureTransports configures a net/http HTTP/1 Transport to use HTTP/2. // It returns a new HTTP/2 Transport for further configuration. // It returns an error if t1 has already been HTTP/2-enabled. +// +// Deprecated: Set [http.Transport.Protocols] instead. func ConfigureTransports(t1 *http.Transport) (*Transport, error) { return configureTransports(t1) } @@ -39,6 +43,8 @@ func ConfigureTransports(t1 *http.Transport) (*Transport, error) { // // A Transport internally caches connections to servers. It is safe // for concurrent use by multiple goroutines. +// +// Deprecated: Use [http.Transport] instead. type Transport struct { // DialTLSContext specifies an optional dial function with context for // creating TLS connections for requests. @@ -47,6 +53,8 @@ type Transport struct { // // If the returned net.Conn has a ConnectionState method like tls.Conn, // it will be used to set http.Response.TLS. + // + // Deprecated: Use [http.Transport.DialTLSContext] instead. DialTLSContext func(ctx context.Context, network, addr string, cfg *tls.Config) (net.Conn, error) // DialTLS specifies an optional dial function for creating @@ -54,17 +62,21 @@ type Transport struct { // // If DialTLSContext and DialTLS is nil, tls.Dial is used. // - // Deprecated: Use DialTLSContext instead, which allows the transport - // to cancel dials as soon as they are no longer needed. - // If both are set, DialTLSContext takes priority. + // Deprecated: Use [http.Transport.DialTLSContext] instead. DialTLS func(network, addr string, cfg *tls.Config) (net.Conn, error) // TLSClientConfig specifies the TLS configuration to use with // tls.Client. If nil, the default configuration is used. + // + // Deprecated: Use [http.Transport.TLSClientConfig] instead. TLSClientConfig *tls.Config // ConnPool optionally specifies an alternate connection pool to use. // If nil, the default is used. + // + // Deprecated: To create a custom connection pool, implement + // [http.RoundTripper]. Use [http.Transport.NewClientConn] + // to create connections for the pool. ConnPool ClientConnPool // DisableCompression, if true, prevents the Transport from @@ -75,10 +87,14 @@ type Transport struct { // decoded in the Response.Body. However, if the user // explicitly requested gzip it is not automatically // uncompressed. + // + // Deprecated: Use [http.Transport.DisableCompression] instead. DisableCompression bool // AllowHTTP, if true, permits HTTP/2 requests using the insecure, // plain-text "http" scheme. Note that this does not enable h2c support. + // + // Deprecated: Use [http.Transport.Protocols] instead. AllowHTTP bool // MaxHeaderListSize is the http2 SETTINGS_MAX_HEADER_LIST_SIZE to @@ -88,6 +104,8 @@ type Transport struct { // want to advertise an unlimited value to the peer, Transport // interprets the highest possible value here (0xffffffff or 1<<32-1) // to mean no limit. + // + // Deprecated: Use [http.Transport.MaxResponseHeaderBytes] instead. MaxHeaderListSize uint32 // MaxReadFrameSize is the http2 SETTINGS_MAX_FRAME_SIZE to send in the @@ -97,6 +115,9 @@ type Transport struct { // according to the spec: // https://datatracker.ietf.org/doc/html/rfc7540#section-6.5.2. // Values are bounded in the range 16k to 16M. + // + // Deprecated: Use [http.Transport.HTTP2] and + // [http.HTTP2Config.MaxReadFrameSize] instead. MaxReadFrameSize uint32 // MaxDecoderHeaderTableSize optionally specifies the http2 @@ -104,12 +125,18 @@ type Transport struct { // informs the remote endpoint of the maximum size of the header compression // table used to decode header blocks, in octets. If zero, the default value // of 4096 is used. + // + // Deprecated: Use [http.Transport.HTTP2] and + // [http.HTTP2Config.MaxDecoderHeaderTableSize] instead. MaxDecoderHeaderTableSize uint32 // MaxEncoderHeaderTableSize optionally specifies an upper limit for the // header compression table used for encoding request headers. Received // SETTINGS_HEADER_TABLE_SIZE settings are capped at this limit. If zero, // the default value of 4096 is used. + // + // Deprecated: Use [http.Transport.HTTP2] and + // [http.HTTP2Config.MaxEncoderHeaderTableSize] instead. MaxEncoderHeaderTableSize uint32 // StrictMaxConcurrentStreams controls whether the server's @@ -120,12 +147,17 @@ type Transport struct { // server's SETTINGS_MAX_CONCURRENT_STREAMS is interpreted as // a global limit and callers of RoundTrip block when needed, // waiting for their turn. + // + // Deprecated: Use [http.Transport.HTTP2] and + // [http.HTTP2Config.StrictMaxConcurrentRequests] instead. StrictMaxConcurrentStreams bool // IdleConnTimeout is the maximum amount of time an idle // (keep-alive) connection will remain idle before closing // itself. // Zero means no limit. + // + // Deprecated: Use [http.Transport.IdleConnTimeout] instead. IdleConnTimeout time.Duration // ReadIdleTimeout is the timeout after which a health check using ping @@ -134,22 +166,34 @@ type Transport struct { // there is no other traffic on the connection, the health check will // be performed every ReadIdleTimeout interval. // If zero, no health check is performed. + // + // Deprecated: Use [http.Transport.HTTP2] and + // [http.HTTP2Config.SendPingTimeout] instead. ReadIdleTimeout time.Duration // PingTimeout is the timeout after which the connection will be closed // if a response to Ping is not received. // Defaults to 15s. + // + // Deprecated: Use [http.Transport.HTTP2] and + // [http.HTTP2Config.PingTimeout] instead. PingTimeout time.Duration // WriteByteTimeout is the timeout after which the connection will be // closed no data can be written to it. The timeout begins when data is // available to write, and is extended whenever any bytes are written. + // + // Deprecated: Use [http.Transport.HTTP2] and + // [http.HTTP2Config.WriteByteTimeout] instead. WriteByteTimeout time.Duration // CountError, if non-nil, is called on HTTP/2 transport errors. // It's intended to increment a metric for monitoring, such // as an expvar or Prometheus metric. // The errType consists of only ASCII word characters. + // + // Deprecated: Use [http.Transport.HTTP2] and + // [http.HTTP2Config.CountError] instead. CountError func(errType string) // Internal state, differs between wrapped and non-wrapped implementations. @@ -165,6 +209,10 @@ var ( ) // ClientConnPool manages a pool of HTTP/2 client connections. +// +// Deprecated: To create a custom connection pool, implement +// [http.RoundTripper]. Use [http.Transport.NewClientConn] +// to create connections for the pool. type ClientConnPool interface { // GetClientConn returns a specific HTTP/2 connection (usually // a TLS-TCP connection) to an HTTP/2 server. On success, the @@ -177,6 +225,8 @@ type ClientConnPool interface { } // ClientConnState describes the state of a ClientConn. +// +// Deprecated: Use [http.ClientConn] instead. type ClientConnState struct { // Closed is whether the connection is closed. Closed bool @@ -210,6 +260,8 @@ type ClientConnState struct { } // RoundTripOpt are options for the Transport.RoundTripOpt method. +// +// Deprecated: There are no options to set. type RoundTripOpt struct { // OnlyCachedConn controls whether RoundTripOpt may // create a new TCP connection. If set true and @@ -222,11 +274,14 @@ type RoundTripOpt struct { allowHTTP bool // allow http:// URLs } +// Deprecated: Use [http.Transport.RoundTrip] instead. func (t *Transport) RoundTrip(req *http.Request) (*http.Response, error) { return t.RoundTripOpt(req, RoundTripOpt{}) } // RoundTripOpt is like RoundTrip, but takes options. +// +// Deprecated: Use [http.Transport.RoundTrip] instead. func (t *Transport) RoundTripOpt(req *http.Request, opt RoundTripOpt) (*http.Response, error) { return t.roundTripOpt(req, opt) } @@ -234,10 +289,15 @@ func (t *Transport) RoundTripOpt(req *http.Request, opt RoundTripOpt) (*http.Res // CloseIdleConnections closes any connections which were previously // connected from previous requests but are now sitting idle. // It does not interrupt any connections currently in use. +// +// Deprecated: Use [http.Transport.CloseIdleConnections] instead. func (t *Transport) CloseIdleConnections() { t.closeIdleConnections() } +// NewClientConn is deprecated. +// +// Deprecated: Use [http.Transport.NewClientConn] instead. func (t *Transport) NewClientConn(c net.Conn) (*ClientConn, error) { return t.newUserClientConn(c) } @@ -256,8 +316,15 @@ func authorityAddr(scheme string, authority string) (addr string) { port = "80" } } - if a, err := idna.ToASCII(host); err == nil { - host = a + // Skip IDNA processing on hosts which are already ASCII. + // This is consistent with net/http and the WHATWG URL Specification. + // (We currently don't follow all of WHATWG, but we're aligned on + // permitting all-ASCII hostnames which fail IDNA validation. + // There are existing, valid domain names which TR #46 processing rejects.) + if !isASCII(host) { + if a, err := idna.Lookup.ToASCII(host); err == nil && a != "" { + host = a + } } // IPv6 address literal, without a port: if strings.HasPrefix(host, "[") && strings.HasSuffix(host, "]") { @@ -414,6 +481,8 @@ func (t *Transport) dialTLSWithContext(ctx context.Context, network, addr string // GoAwayError is returned by the Transport when the server closes the // TCP connection after sending a GOAWAY frame. +// +// Deprecated: GoAwayError is deprecated. type GoAwayError struct { LastStreamID uint32 ErrCode ErrCode diff --git a/vendor/golang.org/x/net/http2/transport_wrap.go b/vendor/golang.org/x/net/http2/transport_wrap.go index 741fb97062..eeaa95f03c 100644 --- a/vendor/golang.org/x/net/http2/transport_wrap.go +++ b/vendor/golang.org/x/net/http2/transport_wrap.go @@ -119,33 +119,27 @@ type http2TransportContextKey struct{} // DialFromContext dials a new connection using the http2.Transport's DialTLS/DialTLSContext. func (t transportConfig) DialFromContext(ctx context.Context, network, address string) (net.Conn, error) { - if ctx.Value(http2TransportContextKey{}) == nil { + dial, _ := ctx.Value(http2TransportContextKey{}).(*transportRoundTripState) + if dial == nil { // We're being called from a RoundTrip that did not start with an http2.Transport. // Use the http.Transport's dialer. return nil, errors.ErrUnsupported } - - tlsConf := t.t.TLSClientConfig - if tlsConf == nil { - tlsConf = &tls.Config{} - } else { - tlsConf = tlsConf.Clone() - } - if !slices.Contains(tlsConf.NextProtos, "h2") { - tlsConf.NextProtos = append([]string{"h2"}, tlsConf.NextProtos...) - } - if tlsConf.ServerName == "" { - host, _, err := net.SplitHostPort(address) - if err == nil { - tlsConf.ServerName = host - } - } - return t.t.dialTLS(ctx, network, address, tlsConf) + return t.t.dialForRoundTrip(ctx, dial, network, address) } type transportInternal struct { initOnce sync.Once lazyt1 *http.Transport + + dialMu sync.Mutex + dials map[string]*transportDialState +} + +type transportDialState struct { + dialc chan struct{} // close when dial returns + dialErr error // dial result, set before dialc is closed + rtdonec chan struct{} // closed when RoundTrip initiating the dial returns } func (t *Transport) init() *http.Transport { @@ -167,6 +161,52 @@ func (t *Transport) configure(t1 *http.Transport) { } } +// transportRoundTripState is the state of the dial for an http2.Transport.RoundTrip. +type transportRoundTripState struct { + mu sync.Mutex + rtdone bool // set when RoundTrip returns + rtdonec chan struct{} // closed when RoundTrip returns + gotconnc chan struct{} // closed when GotConn hook is called +} + +// startDial is called when a dial starts. +// +// It returns done=true if the RoundTrip has already returned, +// in which case we should skip dialing. +func (dial *transportRoundTripState) startDial() (gotconnc chan struct{}, done bool) { + dial.mu.Lock() + defer dial.mu.Unlock() + if dial.rtdone { + return nil, true + } + if dial.rtdonec == nil { + dial.rtdonec = make(chan struct{}) + } + dial.gotconnc = make(chan struct{}) + return dial.gotconnc, false +} + +// gotConn is called when RoundTrip gets a connection. +// This may happen multiple times per RoundTrip, if a request is retried. +func (dial *transportRoundTripState) gotConn() { + dial.mu.Lock() + defer dial.mu.Unlock() + if dial.gotconnc != nil { + close(dial.gotconnc) + dial.gotconnc = nil + } +} + +// roundTripDone is called when RoundTrip returns. +func (dial *transportRoundTripState) roundTripDone() { + dial.mu.Lock() + defer dial.mu.Unlock() + dial.rtdone = true + if dial.rtdonec != nil { + close(dial.rtdonec) + } +} + func (t *Transport) roundTripOpt(req *http.Request, opt RoundTripOpt) (*http.Response, error) { t1 := t.init() @@ -186,12 +226,146 @@ func (t *Transport) roundTripOpt(req *http.Request, opt RoundTripOpt) (*http.Res // Both http.Transport and http2.Transport allow the user to provide a custom // dial function, and historically you only get the dial function from the // Transport you're calling RoundTrip on. - ctx := context.WithValue(req.Context(), http2TransportContextKey{}, t) + // + // In addition, http2.Transport coalesces dials, which http.Transport historically + // has not. + dial := &transportRoundTripState{} + defer dial.roundTripDone() + ctx := context.WithValue(req.Context(), http2TransportContextKey{}, dial) + ctx = httptrace.WithClientTrace(ctx, &httptrace.ClientTrace{ + GotConn: func(httptrace.GotConnInfo) { + // Tell dialForRoundTrip that we have received a connection. + dial.gotConn() + }, + }) req = req.WithContext(ctx) - return t1.RoundTrip(req) } +var errCoalescedDialAbandoned = errors.New("http2: abandoned coalesced dial") + +const coalescedDialRetryTimeout = 50 * time.Millisecond + +// dialForRoundTrip is called (indirectly) by net/http when dialing a new connection +// for a request call which originated as an http2.Transport.RoundTrip call. +// +// It uses the http2.Transport's Dial hooks and coalesces dials. +func (t *Transport) dialForRoundTrip(ctx context.Context, dial *transportRoundTripState, network, address string) (net.Conn, error) { + gotconnc, rtdone := dial.startDial() + if rtdone { + // RoundTrip returned, no need for this dial to proceed. + return nil, errCoalescedDialAbandoned + } + + var state *transportDialState + for { + // The first dial to an address registers itself with t.dials. + // When the dial finishes, it records the outcome and removes itself from t.dials + // so future dials will start a new round of coalescing. + // + // Subsequent dials observe that a dial is in progress, and skip dialing. + t.dialMu.Lock() + if t.dials == nil { + t.dials = make(map[string]*transportDialState) + } + leader := t.dials[address] + if leader == nil { + // No entry in t.dials to coalesce with. Add ourselves as leader. + state = &transportDialState{ + dialc: make(chan struct{}), + rtdonec: dial.rtdonec, + } + t.dials[address] = state + } + t.dialMu.Unlock() + if leader == nil { + // We are the leader, so we should dial for real. + break + } + + // Coalesce with a previous dial. + var ( + done = false + leaderdialc = leader.dialc + leaderrtdonec = leader.rtdonec + timerc <-chan time.Time + ) + for !done { + if leaderdialc == nil && leaderrtdonec == nil && timerc == nil { + // The leader finished dialing, and its RoundTrip finished. + // + // There are several possibilities, which reduce to: + // - The RoundTrip we are dialing for is about to receive + // a connection (possibly the one the leader just dialed), + // but we haven't observed it yet. + // - There's something wrong with the leader's connection, + // such as a TLS handshake failure. + // + // We have no good way to distinguish between these cases. + // + // Wait a short time for the leader's connection (or some + // other usable connection) to be delivered to our RoundTrip. + // If it is not, start a dial of our own. + timerc = time.After(coalescedDialRetryTimeout) + } + select { + case <-leaderdialc: + // The leader's dial completed. + if leader.dialErr != nil { + return nil, leader.dialErr + } + leaderdialc = nil + case <-leaderrtdonec: + // The leader's RoundTrip completed. + leaderrtdonec = nil + case <-timerc: + // The leader's dial and RoundTrip completed, and some time + // has passed. Assume we're not getting a connection and + // restart the dial process. + done = true + case <-dial.rtdonec: + // Our RoundTrip returned. + return nil, errCoalescedDialAbandoned + case <-gotconnc: + // Our RoundTrip got a connection. + return nil, errCoalescedDialAbandoned + case <-ctx.Done(): + // net/http abandoned the dial. + return nil, ctx.Err() + } + } + } + + // Dial for real. + tlsConf := t.TLSClientConfig + if tlsConf == nil { + tlsConf = &tls.Config{} + } else { + tlsConf = tlsConf.Clone() + } + if !slices.Contains(tlsConf.NextProtos, "h2") { + tlsConf.NextProtos = append([]string{"h2"}, tlsConf.NextProtos...) + } + if tlsConf.ServerName == "" { + host, _, err := net.SplitHostPort(address) + if err == nil { + tlsConf.ServerName = host + } + } + nc, err := t.dialTLS(ctx, network, address, tlsConf) + + // Remove this dial from t.dials, so future dial attempts will not coalesce with it. + t.dialMu.Lock() + delete(t.dials, address) + t.dialMu.Unlock() + + // Notify any dial coalesced with this one that we are done. + state.dialErr = err + close(state.dialc) + + return nc, err +} + func (t *Transport) closeIdleConnections() { t1 := t.init() t1.CloseIdleConnections() @@ -216,6 +390,8 @@ func (t *Transport) newUserClientConn(c net.Conn) (*ClientConn, error) { // ClientConn is the state of a single HTTP/2 client connection to an // HTTP/2 server. +// +// Deprecated: Use [http.ClientConn] instead. type ClientConn struct { cc *http.ClientConn tconn net.Conn diff --git a/vendor/golang.org/x/net/http2/writesched.go b/vendor/golang.org/x/net/http2/writesched.go index 36ad5e32b4..f1903a8c66 100644 --- a/vendor/golang.org/x/net/http2/writesched.go +++ b/vendor/golang.org/x/net/http2/writesched.go @@ -79,10 +79,11 @@ func (wr FrameWriteRequest) Consume(n int32) (FrameWriteRequest, FrameWriteReque } // Might need to split after applying limits. - allowed := wr.stream.flow.available() - if n < allowed { - allowed = n + avail, ok := wr.stream.flow.available() + if !ok { + return empty, empty, 0 } + allowed := min(n, avail) if wr.stream.sc.maxFrameSize < allowed { allowed = wr.stream.sc.maxFrameSize } diff --git a/vendor/golang.org/x/net/internal/httpcommon/README.md b/vendor/golang.org/x/net/internal/httpcommon/README.md new file mode 100644 index 0000000000..951c346c2d --- /dev/null +++ b/vendor/golang.org/x/net/internal/httpcommon/README.md @@ -0,0 +1,7 @@ +These two packages are identical: + +- net/http/internal/httpcommon +- golang.org/x/net/internal/httpcommon + +The two are kept in sync manually. +Any change to one should be made in the other. diff --git a/vendor/golang.org/x/net/internal/httpcommon/gzip.go b/vendor/golang.org/x/net/internal/httpcommon/gzip.go new file mode 100644 index 0000000000..819dd650cb --- /dev/null +++ b/vendor/golang.org/x/net/internal/httpcommon/gzip.go @@ -0,0 +1,132 @@ +// Copyright 2026 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package httpcommon + +import ( + "compress/flate" + "compress/gzip" + "errors" + "io" + "io/fs" + "sync" +) + +var errConcurrentRead = errors.New("http: concurrent read on response body") + +// incomparable is a zero-width, non-comparable type. Adding it to a struct +// makes that struct also non-comparable, and generally doesn't add +// any size (as long as it's first). +type incomparable [0]func() + +// GzipReader wraps a response body so it can lazily +// get gzip.Reader from the pool on the first call to Read. +// After Close is called it puts gzip.Reader to the pool immediately +// if there is no Read in progress or later when Read completes. +type GzipReader struct { + _ incomparable + Body io.ReadCloser // underlying Response.Body + mu sync.Mutex // guards zr and zerr + zr *gzip.Reader // stores gzip reader from the pool between reads + zerr error // sticky gzip reader init error or sentinel value to detect concurrent read and read after close +} + +type eofReader struct{} + +func (eofReader) Read([]byte) (int, error) { return 0, io.EOF } +func (eofReader) ReadByte() (byte, error) { return 0, io.EOF } + +var gzipPool = sync.Pool{New: func() any { return new(gzip.Reader) }} + +// gzipPoolGet gets a gzip.Reader from the pool and resets it to read from r. +func gzipPoolGet(r io.Reader) (*gzip.Reader, error) { + zr := gzipPool.Get().(*gzip.Reader) + if err := zr.Reset(r); err != nil { + gzipPoolPut(zr) + return nil, err + } + return zr, nil +} + +// gzipPoolPut puts a gzip.Reader back into the pool. +func gzipPoolPut(zr *gzip.Reader) { + // Reset will allocate bufio.Reader if we pass it anything + // other than a flate.Reader, so ensure that it's getting one. + var r flate.Reader = eofReader{} + zr.Reset(r) + gzipPool.Put(zr) +} + +// acquire returns a gzip.Reader for reading response body. +// The reader must be released after use. +func (gz *GzipReader) acquire() (*gzip.Reader, error) { + gz.mu.Lock() + defer gz.mu.Unlock() + if gz.zerr != nil { + return nil, gz.zerr + } + if gz.zr == nil { + // gzipPoolGet might block indefinitely since it reads the gzip header. + // Therefore, drop mu temporarily when using gzipPoolGet. + // We set zerr to errConcurrentRead to prevent concurrent read + // even when mu is temporarily dropped. + gz.zerr = errConcurrentRead + gz.mu.Unlock() + zr, err := gzipPoolGet(gz.Body) + gz.mu.Lock() + // Guard against Close being called while gzipPoolGet is running. + if gz.zerr != errConcurrentRead { + if zr != nil { + gzipPoolPut(zr) + } + return nil, gz.zerr + } + gz.zr, gz.zerr = zr, err + if gz.zerr != nil { + return nil, gz.zerr + } + } + ret := gz.zr + gz.zr, gz.zerr = nil, errConcurrentRead + return ret, nil +} + +// release returns the gzip.Reader to the pool if Close was called during Read. +func (gz *GzipReader) release(zr *gzip.Reader) { + gz.mu.Lock() + defer gz.mu.Unlock() + if gz.zerr == errConcurrentRead { + gz.zr, gz.zerr = zr, nil + } else { // fs.ErrClosed + gzipPoolPut(zr) + } +} + +// close returns the gzip.Reader to the pool immediately or +// signals release to do so after Read completes. +func (gz *GzipReader) close() { + gz.mu.Lock() + defer gz.mu.Unlock() + if gz.zerr == nil && gz.zr != nil { + gzipPoolPut(gz.zr) + gz.zr = nil + } + gz.zerr = fs.ErrClosed +} + +func (gz *GzipReader) Read(p []byte) (n int, err error) { + zr, err := gz.acquire() + if err != nil { + return 0, err + } + defer gz.release(zr) + + return zr.Read(p) +} + +func (gz *GzipReader) Close() error { + gz.close() + + return gz.Body.Close() +} diff --git a/vendor/golang.org/x/net/internal/httpcommon/request.go b/vendor/golang.org/x/net/internal/httpcommon/request.go index d94c87429a..5aaca5989b 100644 --- a/vendor/golang.org/x/net/internal/httpcommon/request.go +++ b/vendor/golang.org/x/net/internal/httpcommon/request.go @@ -391,6 +391,7 @@ type ServerRequestParam struct { // ServerRequestResult is the result of NewServerRequest. type ServerRequestResult struct { // Various http.Request fields. + Host string URL *url.URL RequestURI string Trailer map[string][]string @@ -433,20 +434,46 @@ func NewServerRequest(rp ServerRequestParam) ServerRequestResult { } delete(rp.Header, "Trailer") + authority := rp.Authority + if host := rp.Header["Host"]; len(host) == 1 { + // HTTP/2 and HTTP/3 permit the Host header to be present, + // but it must match the :authority pseudo-header. + if authority == "" { + authority = host[0] + } else if host[0] != authority { + return ServerRequestResult{ + InvalidReason: "authority_host_mismatch", + } + } + delete(rp.Header, "Host") + } else if len(host) > 1 { + // HTTP/1.1 rejects any request containing more than one Host header. + // HTTP/2 and HTTP/3 don't use the Host header, but reject multiple anyway. + return ServerRequestResult{ + InvalidReason: "multiple_host_headers", + } + } + // "':authority' MUST NOT include the deprecated userinfo subcomponent // for "http" or "https" schemed URIs." // https://www.rfc-editor.org/rfc/rfc9113.html#section-8.3.1-2.3.8 - if strings.IndexByte(rp.Authority, '@') != -1 && (rp.Scheme == "http" || rp.Scheme == "https") { + if strings.IndexByte(authority, '@') != -1 && (rp.Scheme == "http" || rp.Scheme == "https") { return ServerRequestResult{ InvalidReason: "userinfo_in_authority", } } + if authority != "" && !httpguts.ValidHostHeader(authority) { + return ServerRequestResult{ + InvalidReason: "invalid_authority", + } + } + var url_ *url.URL var requestURI string if rp.Method == "CONNECT" && rp.Protocol == "" { - url_ = &url.URL{Host: rp.Authority} - requestURI = rp.Authority // mimic HTTP/1 server behavior + url_ = &url.URL{Host: authority} + requestURI = authority // mimic HTTP/1 server behavior } else { // "[The :path] pseudo-header field MUST NOT be empty [...]" // https://www.rfc-editor.org/rfc/rfc9113.html#section-8.3.1-2.4.2 @@ -467,6 +494,7 @@ func NewServerRequest(rp ServerRequestParam) ServerRequestResult { } return ServerRequestResult{ + Host: authority, URL: url_, NeedsContinue: needsContinue, RequestURI: requestURI, diff --git a/vendor/modules.txt b/vendor/modules.txt index 7b7f7a41b1..7b70fa60f1 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -803,8 +803,8 @@ golang.org/x/crypto/ssh/knownhosts # golang.org/x/mod v0.41.0 ## explicit; go 1.26.0 golang.org/x/mod/semver -# golang.org/x/net v0.58.0 -## explicit; go 1.25.0 +# golang.org/x/net v0.60.0 +## explicit; go 1.26.0 golang.org/x/net/context golang.org/x/net/http/httpguts golang.org/x/net/http2 From caff2a3dfaa6eba5f192a3e7a7993d8fa15581eb Mon Sep 17 00:00:00 2001 From: icanhasmath Date: Fri, 23 Oct 2026 12:34:55 -0500 Subject: [PATCH 5/6] Compare RSA keys by value in keypair tests. The tests compared a generated keypair with the same key parsed back from PEM using deep struct equality, which also compares rsa.PrivateKey's precomputed internals. Under Go 1.26.10 those can differ (e.g. a leading zero byte in dP) even though the keys are identical, so the tests failed intermittently. Compare with rsa.PrivateKey.Equal instead. Co-Authored-By: Claude Opus 5.5 --- internal/keypairs/equal_test.go | 22 ++++++++++++++++++++++ internal/keypairs/generate_test.go | 8 ++++---- internal/keypairs/local_save_test.go | 4 ++-- internal/keypairs/rsa_keypair_test.go | 6 +++--- 4 files changed, 31 insertions(+), 9 deletions(-) create mode 100644 internal/keypairs/equal_test.go diff --git a/internal/keypairs/equal_test.go b/internal/keypairs/equal_test.go new file mode 100644 index 0000000000..22501e808c --- /dev/null +++ b/internal/keypairs/equal_test.go @@ -0,0 +1,22 @@ +package keypairs_test + +import ( + "testing" + + "github.com/ActiveState/cli/internal/keypairs" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// requireSameRSAKey asserts that two keypairs hold the same RSA private key. +// Comparing the structs directly also compares rsa.PrivateKey's precomputed +// internals, whose representation can differ between a generated key and the +// same key parsed back from PEM. +func requireSameRSAKey(t *testing.T, expected, actual keypairs.Keypair) { + t.Helper() + expectedRSA, ok := expected.(*keypairs.RSAKeypair) + require.True(t, ok, "expected keypair is not an RSA keypair") + actualRSA, ok := actual.(*keypairs.RSAKeypair) + require.True(t, ok, "actual keypair is not an RSA keypair") + assert.True(t, expectedRSA.PrivateKey.Equal(actualRSA.PrivateKey), "RSA private keys differ") +} diff --git a/internal/keypairs/generate_test.go b/internal/keypairs/generate_test.go index 837d7b2d89..184f7d02b3 100644 --- a/internal/keypairs/generate_test.go +++ b/internal/keypairs/generate_test.go @@ -58,7 +58,7 @@ func (suite *KeypairGenerateTestSuite) TestGenerate_NoPassphrase() { validationKeypair, err := keypairs.ParseRSA(encKeypair.EncodedPrivateKey) suite.Require().Nil(err) suite.Require().NotNil(validationKeypair) - suite.Equal(encKeypair.Keypair, validationKeypair) + requireSameRSAKey(suite.T(), encKeypair.Keypair, validationKeypair) // verify encoded public key matches generated keypair's public key validationPublicKey, err := keypairs.ParseRSAPublicKey(encKeypair.EncodedPublicKey) @@ -79,7 +79,7 @@ func (suite *KeypairGenerateTestSuite) TestGenerate_WithPassphrase() { validationKeypair, err := keypairs.ParseEncryptedRSA(encKeypair.EncodedPrivateKey, "tuxedomoon") suite.Require().Nil(err) suite.Require().NotNil(validationKeypair) - suite.Equal(encKeypair.Keypair, validationKeypair) + requireSameRSAKey(suite.T(), encKeypair.Keypair, validationKeypair) // verify encoded public key matches generated keypair's public key validationPublicKey, err := keypairs.ParseRSAPublicKey(encKeypair.EncodedPublicKey) @@ -123,7 +123,7 @@ func (suite *KeypairGenerateTestSuite) TestGenerateAndSave_Success_NoPassphrase( validationKeypair, err := keypairs.ParseRSA(encKeypair.EncodedPrivateKey) suite.Require().Nil(err) suite.Require().NotNil(validationKeypair) - suite.Equal(encKeypair.Keypair, validationKeypair) + requireSameRSAKey(suite.T(), encKeypair.Keypair, validationKeypair) // verify encoded public key matches generated keypair's public key validationPublicKey, err := keypairs.ParseRSAPublicKey(encKeypair.EncodedPublicKey) @@ -149,7 +149,7 @@ func (suite *KeypairGenerateTestSuite) TestGenerateAndSave_Success_WithPassphras validationKeypair, err := keypairs.ParseEncryptedRSA(encKeypair.EncodedPrivateKey, "bauhaus") suite.Require().Nil(err) suite.Require().NotNil(validationKeypair) - suite.Equal(encKeypair.Keypair, validationKeypair) + requireSameRSAKey(suite.T(), encKeypair.Keypair, validationKeypair) // verify encoded public key matches generated keypair's public key validationPublicKey, err := keypairs.ParseRSAPublicKey(encKeypair.EncodedPublicKey) diff --git a/internal/keypairs/local_save_test.go b/internal/keypairs/local_save_test.go index d5fd397e83..eda6825af2 100644 --- a/internal/keypairs/local_save_test.go +++ b/internal/keypairs/local_save_test.go @@ -37,7 +37,7 @@ func (suite *KeypairLocalSaveTestSuite) TestSave_Success() { kp2, err := keypairs.Load(suite.cfg, "save-testing") suite.Require().Nil(err) - suite.Equal(kp, kp2) + requireSameRSAKey(suite.T(), kp, kp2) fileInfo := suite.statConfigDirFile("save-testing.key") if runtime.GOOS != "windows" { @@ -54,7 +54,7 @@ func (suite *KeypairLocalSaveTestSuite) TestSaveWithDefaults_Success() { kp2, err := keypairs.Load(suite.cfg, constants.KeypairLocalFileName) suite.Require().Nil(err) - suite.Equal(kp, kp2) + requireSameRSAKey(suite.T(), kp, kp2) fileInfo := suite.statConfigDirFile(constants.KeypairLocalFileName + ".key") if runtime.GOOS != "windows" { diff --git a/internal/keypairs/rsa_keypair_test.go b/internal/keypairs/rsa_keypair_test.go index a65a5f9e89..554a7ad0b4 100644 --- a/internal/keypairs/rsa_keypair_test.go +++ b/internal/keypairs/rsa_keypair_test.go @@ -85,7 +85,7 @@ func (suite *RSAKeypairTestSuite) TestParseRSA_ParsesKeypair() { kp2, err := keypairs.ParseRSA(encPrivKey) suite.Require().Nil(err) suite.Implements((*keypairs.Keypair)(nil), kp2) - suite.Equal(kp, kp2) + requireSameRSAKey(suite.T(), kp, kp2) } func (suite *RSAKeypairTestSuite) TestParseRSA_EncodingNotOfPrivateKey() { @@ -137,7 +137,7 @@ func (suite *RSAKeypairTestSuite) TestParseRSA_EmptyPassphraseButStillEncrypted( kp2, err := keypairs.ParseRSA(encrPrivKey) suite.Require().Nil(err) suite.Implements((*keypairs.Keypair)(nil), kp2) - suite.Equal(kp, kp2) + requireSameRSAKey(suite.T(), kp, kp2) } func (suite *RSAKeypairTestSuite) TestParseRSA_KeypairNotPEMEncoded() { @@ -156,7 +156,7 @@ func (suite *RSAKeypairTestSuite) TestParseEncryptedRSA_ParsesKeypair() { kp2, err := keypairs.ParseEncryptedRSA(keyPEM, "abc123") suite.Require().Nil(err) suite.Implements((*keypairs.Keypair)(nil), kp2) - suite.Equal(kp, kp2) + requireSameRSAKey(suite.T(), kp, kp2) } func (suite *RSAKeypairTestSuite) TestParseEncryptedRSA_KeypairNotPEMEncoded() { From 6ef8df3bb11bc4a3ca1699f76f0c0d9a98eaa8a2 Mon Sep 17 00:00:00 2001 From: Marc Gutman <43051639+icanhasmath@users.noreply.github.com> Date: Fri, 23 Oct 2026 13:25:28 -0500 Subject: [PATCH 6/6] Allow for slow sw_vers in the macOS OSVersion performance test. (#3866) TestPerformance/OSVersion averages ten calls against a 10ms budget, so the single uncached call bounds it at ~100ms. On macOS that call shells out to sw_vers, which took 115-716ms on macos-latest runners and failed 3 of 4 runs. Cached calls take nanoseconds. Keep 10ms elsewhere; allow 100ms on darwin. Co-authored-by: Claude Opus 5.5 --- pkg/sysinfo/sysinfo_test.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pkg/sysinfo/sysinfo_test.go b/pkg/sysinfo/sysinfo_test.go index 9fb43e1c57..2d79d9bb4f 100644 --- a/pkg/sysinfo/sysinfo_test.go +++ b/pkg/sysinfo/sysinfo_test.go @@ -1,6 +1,7 @@ package sysinfo import ( + "runtime" "testing" "time" @@ -24,6 +25,11 @@ func TestPerformance(t *testing.T) { t.Run("OSVersion", func(t *testing.T) { maxDuration := 10 * time.Millisecond + if runtime.GOOS == "darwin" { + // The first, uncached call shells out to sw_vers, which takes 100ms-1s on macOS CI + // runners and dominates the average. Cached calls take nanoseconds. + maxDuration = 100 * time.Millisecond + } err := performance.TimeIt(t, func() { _, err := OSVersion() assert.NoError(t, err)