java: move CycloneDX deduplication to the coalescer

java/bom/package.go

@@ -1,6 +1,7 @@
 package bom
 
 import (
+	"encoding/hex"
 	"net/url"
 	"path"
 	"unique"
@@ -41,6 +42,17 @@ func PopulatePackage(pkg *claircore.Package, bp Package, sbomFile string) error
 	for _, w := range bp.CPEs {
 		hint.Add("cpe", w.String())
 	}
+	// path is the layer path a JAR scan should not also report. The coalescer
+	// reads it back with the component digests.
+	if bp.Location != "" {
+		hint.Set("path", n)
+	}
+	if sum := bp.Hashes[HashSHA1]; len(sum) != 0 {
+		hint.Set("sha1", hex.EncodeToString(sum))
+	}
+	if sum := bp.Hashes[HashSHA256]; len(sum) != 0 {
+		hint.Set("sha256", hex.EncodeToString(sum))
+	}
 	if len(bp.CPEs) == 1 {
 		pkg.CPE = bp.CPEs[0]
 	}

java/coalescer.go

@@ -19,6 +19,8 @@ func (*coalescer) Coalesce(ctx context.Context, ls []*indexer.LayerArtifacts) (*
 		Repositories: map[string]*claircore.Repository{},
 	}
 
+	claims := newSbomClaims(ctx, ls)
+
 	for _, l := range ls {
 		// If we didn't find at least one repository in this layer
 		// no point in searching for packages.
@@ -32,6 +34,10 @@ func (*coalescer) Coalesce(ctx context.Context, ls []*indexer.LayerArtifacts) (*
 			}
 		}
 		for _, pkg := range l.Pkgs {
+			if claims.covers(pkg) {
+				slog.DebugContext(ctx, "skipping jar covered by sbom", "package", pkg.Name, "path", pkg.Filepath)
+				continue
+			}
 			ir.Packages[pkg.ID] = pkg
 			rs := mavenIDs
 			if strings.HasPrefix(pkg.PackageDB, "sbom:") {
@@ -54,6 +60,75 @@ func (*coalescer) Coalesce(ctx context.Context, ls []*indexer.LayerArtifacts) (*
 	return ir, nil
 }
 
+// sbomClaims is the set of JARs named by SBOM packages across every layer.
+// An SBOM in one layer can name a JAR in another, and an SBOM inside a JAR
+// can name a JAR outside that archive. Both claims are on the hint.
+type sbomClaims struct {
+	path   map[string]struct{}
+	sha1   map[string]struct{}
+	sha256 map[string]struct{}
+}
+
+func newSbomClaims(ctx context.Context, ls []*indexer.LayerArtifacts) sbomClaims {
+	c := sbomClaims{
+		path:   map[string]struct{}{},
+		sha1:   map[string]struct{}{},
+		sha256: map[string]struct{}{},
+	}
+	for _, l := range ls {
+		for _, pkg := range l.Pkgs {
+			if !strings.HasPrefix(pkg.PackageDB, "sbom:") {
+				continue
+			}
+			q, err := url.ParseQuery(pkg.RepositoryHint)
+			if err != nil {
+				slog.DebugContext(ctx, "sbom repository hint", "reason", err)
+				continue
+			}
+			if p := q.Get("path"); p != "" {
+				c.path[p] = struct{}{}
+			}
+			for _, sum := range q["sha1"] {
+				c.sha1[strings.ToLower(sum)] = struct{}{}
+			}
+			for _, sum := range q["sha256"] {
+				c.sha256[strings.ToLower(sum)] = struct{}{}
+			}
+		}
+	}
+	return c
+}
+
+// covers reports whether pkg is a JAR scan that an SBOM already named, either
+// by layer path or by digest.
+func (c sbomClaims) covers(pkg *claircore.Package) bool {
+	switch {
+	case strings.HasPrefix(pkg.PackageDB, "file:"),
+		strings.HasPrefix(pkg.PackageDB, "jar:"),
+		strings.HasPrefix(pkg.PackageDB, "maven:"):
+	default:
+		return false
+	}
+	if _, ok := c.path[pkg.Filepath]; ok {
+		return true
+	}
+	q, err := url.ParseQuery(pkg.RepositoryHint)
+	if err != nil {
+		return false
+	}
+	if sum := strings.ToLower(q.Get("sha1")); sum != "" {
+		if _, ok := c.sha1[sum]; ok {
+			return true
+		}
+	}
+	if sum := strings.ToLower(q.Get("sha256")); sum != "" {
+		if _, ok := c.sha256[sum]; ok {
+			return true
+		}
+	}
+	return false
+}
+
 func sbomRepositoryIDs(ctx context.Context, pkg *claircore.Package, repos []*claircore.Repository) []string {
 	q, err := url.ParseQuery(pkg.RepositoryHint)
 	if err != nil {

java/coalescer_test.go

@@ -1,6 +1,7 @@
 package java
 
 import (
+	"context"
 	"net/url"
 	"testing"
 
@@ -92,3 +93,126 @@ func TestCoalescerMultipleProductCPEs(t *testing.T) {
 		t.Fatal("sbom package recorded Maven Central")
 	}
 }
+
+func TestCoalescerDropsCoveredJars(t *testing.T) {
+	t.Parallel()
+	w := cpe.MustUnbind("cpe:/a:redhat:jboss_enterprise_application_platform:8.1")
+	cpeRepo := &claircore.Repository{
+		ID:   "1",
+		Name: w.String(),
+		Key:  RedHatCPERepositoryKey,
+		CPE:  w,
+	}
+	maven := Repository
+	maven.ID = "2"
+
+	sbomPkg := func(id, name, db, filepath, covered, sum string) *claircore.Package {
+		return &claircore.Package{
+			ID:        id,
+			Name:      name,
+			PackageDB: db,
+			Filepath:  filepath,
+			RepositoryHint: url.Values{
+				"cpe":    {w.String()},
+				"path":   {covered},
+				"sha256": {sum},
+			}.Encode(),
+		}
+	}
+	jarPkg := func(id, name, db, filepath, sum string) *claircore.Package {
+		return &claircore.Package{
+			ID:             id,
+			Name:           name,
+			PackageDB:      db,
+			Filepath:       filepath,
+			RepositoryHint: url.Values{
+				"sha1":   {"0123456789abcdef0123456789abcdef01234567"},
+				"sha256": {sum},
+			}.Encode(),
+		}
+	}
+
+	t.Run("path", func(t *testing.T) {
+		t.Parallel()
+		ctx := test.Logging(t)
+		doc := sbomPkg("10", "org.example:covered", "sbom:sbom.cdx.json:lib/covered-1.0.jar", "lib/covered-1.0.jar", "lib/covered-1.0.jar", "aa")
+		covered := jarPkg("11", "covered", "file:lib/covered-1.0.jar", "lib/covered-1.0.jar", "bb")
+		other := jarPkg("12", "other", "file:lib/other-1.0.jar", "lib/other-1.0.jar", "cc")
+		ir := coalesce(t, ctx, []*indexer.LayerArtifacts{{
+			Hash:  test.RandomSHA256Digest(t),
+			Pkgs:  []*claircore.Package{doc, covered, other},
+			Repos: []*claircore.Repository{cpeRepo, &maven},
+		}})
+		if _, ok := ir.Packages[covered.ID]; ok {
+			t.Fatal("jar named by path was kept")
+		}
+		if _, ok := ir.Packages[doc.ID]; !ok {
+			t.Fatal("sbom package was dropped")
+		}
+		if _, ok := ir.Packages[other.ID]; !ok {
+			t.Fatal("unnamed jar was dropped")
+		}
+		if got := ir.Environments[doc.ID][0].RepositoryIDs; len(got) != 1 || got[0] != cpeRepo.ID {
+			t.Fatalf("sbom repositories: %v", got)
+		}
+	})
+
+	t.Run("hash", func(t *testing.T) {
+		t.Parallel()
+		ctx := test.Logging(t)
+		doc := sbomPkg("10", "org.example:hashed", "sbom:meta/app.cdx.json:elsewhere/hashed-1.0.jar", "meta/elsewhere/hashed-1.0.jar", "meta/elsewhere/hashed-1.0.jar", "deadbeef")
+		hashed := jarPkg("11", "hashed", "maven:lib/hashed-1.0.jar", "lib/hashed-1.0.jar", "deadbeef")
+		ir := coalesce(t, ctx, []*indexer.LayerArtifacts{
+			{
+				Hash:  test.RandomSHA256Digest(t),
+				Pkgs:  []*claircore.Package{doc},
+				Repos: []*claircore.Repository{cpeRepo},
+			},
+			{
+				Hash:  test.RandomSHA256Digest(t),
+				Pkgs:  []*claircore.Package{hashed},
+				Repos: []*claircore.Repository{&maven},
+			},
+		})
+		if _, ok := ir.Packages[hashed.ID]; ok {
+			t.Fatal("jar named by digest was kept")
+		}
+		if _, ok := ir.Packages[doc.ID]; !ok {
+			t.Fatal("sbom package was dropped")
+		}
+	})
+
+	t.Run("embedded", func(t *testing.T) {
+		t.Parallel()
+		ctx := test.Logging(t)
+		doc := sbomPkg("10", "org.example:embedded", "sbom:aaa-app-1.0.jar:META-INF/sbom/dependency.cdx.json:lib/sibling-1.0.jar", "aaa-app-1.0.jar", "lib/sibling-1.0.jar", "abc123")
+		sibling := jarPkg("11", "sibling", "jar:lib/sibling-1.0.jar", "lib/sibling-1.0.jar", "ffffff")
+		byHash := jarPkg("12", "outside", "file:opt/outside-1.0.jar", "opt/outside-1.0.jar", "ABC123")
+		ir := coalesce(t, ctx, []*indexer.LayerArtifacts{{
+			Hash:  test.RandomSHA256Digest(t),
+			Pkgs:  []*claircore.Package{doc, sibling, byHash},
+			Repos: []*claircore.Repository{cpeRepo, &maven},
+		}})
+		if _, ok := ir.Packages[sibling.ID]; ok {
+			t.Fatal("jar named by embedded path was kept")
+		}
+		if _, ok := ir.Packages[byHash.ID]; ok {
+			t.Fatal("jar named by embedded digest was kept")
+		}
+		if _, ok := ir.Packages[doc.ID]; !ok {
+			t.Fatal("embedded sbom package was dropped")
+		}
+		if got := ir.Environments[doc.ID][0].RepositoryIDs; len(got) != 1 || got[0] != cpeRepo.ID {
+			t.Fatalf("sbom repositories: %v", got)
+		}
+	})
+}
+
+func coalesce(t *testing.T, ctx context.Context, ls []*indexer.LayerArtifacts) *claircore.IndexReport {
+	t.Helper()
+	ir, err := (*coalescer)(nil).Coalesce(ctx, ls)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return ir
+}

java/packagescanner.go

@@ -8,6 +8,7 @@ import (
 	"context"
 	"crypto/sha1"
 	"crypto/sha256"
+	"encoding/hex"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -22,7 +23,6 @@ import (
 	"strconv"
 	"strings"
 	"time"
-	"unique"
 
 	"github.com/quay/claircore"
 	"github.com/quay/claircore/indexer"
@@ -82,7 +82,7 @@ type Scanner struct {
 func (*Scanner) Name() string { return "java" }
 
 // Version implements scanner.VersionedScanner.
-func (*Scanner) Version() string { return "9" }
+func (*Scanner) Version() string { return "10" }
 
 // Kind implements scanner.VersionedScanner.
 func (*Scanner) Kind() string { return "package" }
@@ -141,6 +141,7 @@ func (s *Scanner) Scan(ctx context.Context, layer *claircore.Layer) ([]*claircor
 	h1, h256 := sha1.New(), sha256.New()
 	hs := io.MultiWriter(h1, h256)
 	ck1 := make([]byte, 0, sha1.Size)
+	ck256 := make([]byte, 0, sha256.Size)
 	doSearch := s.root != nil
 	defer putBuf(buf)
 	set, err := rpm.NewPathSet(ctx, layer)
@@ -149,11 +150,6 @@ func (s *Scanner) Scan(ctx context.Context, layer *claircore.Layer) ([]*claircor
 	}
 	seed := maphash.MakeSeed()
 	sbomDup := make(map[uint64]struct{})
-	sbomPath := make(map[uint64]struct{})
-	sbomHash := map[unique.Handle[string]]map[uint64]struct{}{
-		bom.HashSHA1:   make(map[uint64]struct{}),
-		bom.HashSHA256: make(map[uint64]struct{}),
-	}
 
 	seq, walkErr := walkInteresting(ctx, sys)
 	var sboms, jars []string
@@ -223,15 +219,6 @@ func (s *Scanner) Scan(ctx context.Context, layer *claircore.Layer) ([]*claircor
 			if err != nil {
 				return nil, err
 			}
-			n := path.Join(dir, bp.Location)
-			sbomPath[maphash.String(seed, n)] = struct{}{}
-			for k, b := range bp.Hashes {
-				h, ok := sbomHash[k]
-				if !ok {
-					continue
-				}
-				h[maphash.Bytes(seed, b)] = struct{}{}
-			}
 			var pkg claircore.Package
 			if err := bom.PopulatePackage(&pkg, bp, p); err != nil {
 				return nil, err
@@ -241,18 +228,10 @@ func (s *Scanner) Scan(ctx context.Context, layer *claircore.Layer) ([]*claircor
 	}
 	for _, p := range jars {
 		log := slog.With("path", p)
-		if coveredSBOMPath(seed, p, sbomPath) {
-			log.DebugContext(ctx, "skipping jar covered by sbom path")
-			continue
-		}
 		sz, err := read(p)
 		if err != nil {
 			return nil, err
 		}
-		if coveredSBOMHash(seed, h1.Sum(nil), h256.Sum(nil), sbomHash) {
-			log.DebugContext(ctx, "skipping jar covered by sbom hash")
-			continue
-		}
 		// Let the zip reader determine if this is actually a valid zip file.
 		// We cannot just check the header, as it's possible the jar file
 		// starts off with a script. This scenario is explicitly mentioned in
@@ -278,6 +257,7 @@ func (s *Scanner) Scan(ctx context.Context, layer *claircore.Layer) ([]*claircor
 			return nil, err
 		}
 		ck1 := h1.Sum(ck1)
+		ck256 := h256.Sum(ck256)
 		ps := make([]*claircore.Package, len(infos))
 		for j := range infos {
 			i := &infos[j]
@@ -306,13 +286,31 @@ func (s *Scanner) Scan(ctx context.Context, layer *claircore.Layer) ([]*claircor
 				for _, w := range i.CPEs {
 					hint.Add("cpe", w.String())
 				}
+				// An embedded document names files outside this archive by the
+				// occurrence location, not by a path joined onto the member.
+				if i.Location != "" {
+					hint.Set("path", i.Location)
+				}
+				if len(i.SHA1) != 0 {
+					hint.Set("sha1", hex.EncodeToString(i.SHA1))
+				}
+				if len(i.SHA256) != 0 {
+					hint.Set("sha256", hex.EncodeToString(i.SHA256))
+				}
 				pkg.RepositoryHint = hint.Encode()
 			} else {
 				b := ck1
 				if len(i.SHA1) != 0 {
 					b = i.SHA1
 				}
-				pkg.RepositoryHint = fmt.Sprintf(`sha1:%40x`, b)
+				sum := ck256
+				if len(i.SHA256) != 0 {
+					sum = i.SHA256
+				}
+				hint := url.Values{}
+				hint.Set("sha1", hex.EncodeToString(b))
+				hint.Set("sha256", hex.EncodeToString(sum))
+				pkg.RepositoryHint = hint.Encode()
 			}
 			if len(i.CPEs) == 1 {
 				pkg.CPE = i.CPEs[0]
@@ -352,19 +350,6 @@ func (s *Scanner) Scan(ctx context.Context, layer *claircore.Layer) ([]*claircor
 	return ret, nil
 }
 
-func coveredSBOMPath(seed maphash.Seed, p string, paths map[uint64]struct{}) bool {
-	_, ok := paths[maphash.String(seed, p)]
-	return ok
-}
-
-func coveredSBOMHash(seed maphash.Seed, sha1Sum, sha256Sum []byte, hashes map[unique.Handle[string]]map[uint64]struct{}) bool {
-	if _, ok := hashes[bom.HashSHA1][maphash.Bytes(seed, sha1Sum)]; ok {
-		return true
-	}
-	_, ok := hashes[bom.HashSHA256][maphash.Bytes(seed, sha256Sum)]
-	return ok
-}
-
 // DefaultRepository implements [indexer.DefaultRepoScanner].
 func (Scanner) DefaultRepository(ctx context.Context) *claircore.Repository {
 	return &Repository