diff --git a/docs/modgraph.md b/docs/modgraph.md index a31d5910..e0256b22 100644 --- a/docs/modgraph.md +++ b/docs/modgraph.md @@ -74,6 +74,7 @@ module of the build. ```go sources, err := modgraph.Discover(modgraph.DiscoverOptions{ Dir: projectDir, // the directory of the main go.mod + Work: workFile, // the go.work that uses it, or "" outside a workspace ModuleCache: moduleCache, // GOMODCACHE; the caller finds it LocalRoot: scanRoot, // a local replace directive is read only inside it }) @@ -109,6 +110,44 @@ a replaced module, and `Source.Origin` is the module that the source holds. `replace` directives. A directive that `golang.org/x/mod` does not know fails the parse, and the caller keeps no graph. +## Workspaces + +With `DiscoverOptions.Work`, `Discover` gives the build list of the +workspace for the module in `Dir`, as the go command builds each module of +a workspace. The caller finds the `go.work` file, as the go command does from +`GOWORK` and the parents of the module directory. + +- The build list is the result of minimal version selection with module + graph pruning, as `readModGraph` of the go command builds it. The graph + holds the requirements of each module of the workspace and of the + `go.mod` of each such requirement. A `go.mod` below go 1.17, a module of + the workspace too, adds the requirements of each `go.mod` that it + requires, and so on down the chain. Then, while a module that the build + needs requires a module below its selected version, the `go.mod` of the + selected version joins the graph with each `go.mod` that it requires. + Each path takes its highest version in the graph. +- A requirement on a module of the workspace at a version reads the + `go.mod` of that version. The build uses the code of the workspace. +- Each other module of the workspace is a local source at no version. Its + code is read inside `LocalRoot`, and `Source.Origin` holds its directory. +- A `replace` directive of `go.work` wins. Else the `replace` directive of a + module of the workspace applies. Two modules that replace one module with + other code make `Discover` fail, as they make the go command fail. +- The go command reads the workspace from the `vendor` directory next to + `go.work` when the `go` directive of `go.work` is go 1.14 or later and + the first line of `vendor/modules.txt` holds the `## workspace` + annotation that `go work vendor` writes. Each module that `modules.txt` + lists with a package then comes from `vendor`, at the version of that + list. Outside a workspace, `Discover` reads no `vendor` directory that + holds this annotation. +- `go.work.sum` and the `go.sum` of each module give the hashes. + +A wrong version can hide an edge. So `Discover` fails when it cannot know +the build list: a `go.mod` that the graph needs is not on the disk, a module +of the workspace is outside `LocalRoot`, or a `go.mod` has an `exclude` +directive. The caller then keeps no graph. `go mod download` in the +workspace fetches each `go.mod` that the graph needs. + `examples/modgraph` prints the graph of a module. Run `go mod download` first: a module with no source on the disk is open, and one open node that the build reaches makes each module reached. @@ -129,12 +168,18 @@ full. repository for four platforms and checks that the graph holds each module edge, or that the edge leaves an open node. `MODGRAPH_ORACLE_DIRS` adds more projects, and a project with `vendor/modules.txt` reads its dependencies -from `vendor`. The test needs the `go` command and a module cache, and skips +from `vendor`. `TestBuildHoldsEachEdgeOfGoListInAWorkspace` puts this +repository in a workspace with a probe module. For a workspace, and for a +project of `MODGRAPH_ORACLE_DIRS` with a `go.work`, the test checks each +module that the workspace uses. It also checks that `Discover` gives each +version of the build list that `go list -m all` gives. +`TestDiscoverGivesTheBuildListOfTheGoCommand` serves synthetic modules +from a proxy on the disk and compares `Discover` with `go list -m all` for +workspaces where the versions of two modules differ. The test needs the `go` command and a module cache, and skips with neither. It sets `GOPROXY=off`, so it downloads nothing. ## Not covered -- `go.work` workspaces. A caller keeps no graph for a workspace. - A file larger than 32 MiB whose first 64 KiB does not parse makes its module open. - A file budget, 200,000 files by default, caps a run. Past it the module diff --git a/examples/modgraph/main.go b/examples/modgraph/main.go index 1c55820e..69556732 100644 --- a/examples/modgraph/main.go +++ b/examples/modgraph/main.go @@ -3,9 +3,12 @@ // them. It finds the source of each module with modgraph.Discover in the // vendor directory and in the module cache, which -modcache names, or // $GOMODCACHE, or $GOPATH/pkg/mod. It runs no go command and downloads -// nothing. Run go mod download first for a graph with no open node. +// nothing. Run go mod download first for a graph with no open node. With +// -work, the build list is that of the workspace, and the modules of the +// workspace are read inside the directory of go.work. // // go run ./examples/modgraph -dir ../vet +// go run ./examples/modgraph -dir ../vet -work ../vet/go.work // go run ./examples/modgraph -dir ../vet -json package main @@ -25,16 +28,21 @@ import ( func main() { dir := flag.String("dir", ".", "the directory of the main go.mod") modcache := flag.String("modcache", "", "the module cache, else $GOMODCACHE, else $GOPATH/pkg/mod, else $HOME/go/pkg/mod") + work := flag.String("work", "", "the go.work file of a workspace that uses the module") asJSON := flag.Bool("json", false, "print the graph as JSON") flag.Parse() - if err := run(os.Stdout, *dir, *modcache, *asJSON); err != nil { + if err := run(os.Stdout, *dir, *work, *modcache, *asJSON); err != nil { fmt.Fprintln(os.Stderr, "modgraph:", err) os.Exit(1) } } -func run(w io.Writer, dir, modcache string, asJSON bool) error { - sources, err := modgraph.Discover(modgraph.DiscoverOptions{Dir: dir, ModuleCache: moduleCache(modcache), LocalRoot: dir}) +func run(w io.Writer, dir, work, modcache string, asJSON bool) error { + root := dir + if work != "" { + root = filepath.Dir(work) + } + sources, err := modgraph.Discover(modgraph.DiscoverOptions{Dir: dir, Work: work, ModuleCache: moduleCache(modcache), LocalRoot: root}) if err != nil { return err } diff --git a/examples/modgraph/main_test.go b/examples/modgraph/main_test.go index 2dfcf468..0915902b 100644 --- a/examples/modgraph/main_test.go +++ b/examples/modgraph/main_test.go @@ -26,13 +26,13 @@ func TestRunPrintsTheGraph(t *testing.T) { } var out strings.Builder - require.NoError(t, run(&out, dir, cache, false)) + require.NoError(t, run(&out, dir, "", cache, false)) assert.Contains(t, out.String(), "reached imports example.com/dep@v1.0.0") assert.Contains(t, out.String(), "example.com/app -> example.com/dep (imports, main.go:3)") assert.Contains(t, out.String(), "example.com/dep -> std (imports, d.go:3)") out.Reset() - require.NoError(t, run(&out, dir, cache, true)) + require.NoError(t, run(&out, dir, "", cache, true)) var g modgraph.Graph require.NoError(t, json.Unmarshal([]byte(out.String()), &g)) dep, ok := g.Node("example.com/dep") diff --git a/modgraph/buildlist_test.go b/modgraph/buildlist_test.go new file mode 100644 index 00000000..0b4e56e5 --- /dev/null +++ b/modgraph/buildlist_test.go @@ -0,0 +1,133 @@ +package modgraph + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + gomodule "golang.org/x/mod/module" + modzip "golang.org/x/mod/zip" +) + +// proxyModule is one module version of a test proxy, with its go.mod. +type proxyModule struct { + path, version, gomod string +} + +// writeProxy writes a module proxy that a file:// GOPROXY serves: a list, +// an info file, a go.mod and a zip for each module version. +func writeProxy(t *testing.T, dir string, mods []proxyModule) { + t.Helper() + versions := map[string][]string{} + for _, m := range mods { + esc, err := gomodule.EscapePath(m.path) + require.NoError(t, err) + at := filepath.Join(dir, filepath.FromSlash(esc), "@v") + src := t.TempDir() + name := m.path[strings.LastIndex(m.path, "/")+1:] + writeFiles(t, src, map[string]string{"go.mod": m.gomod, name + ".go": "package " + name + "\n"}) + writeFiles(t, at, map[string]string{ + m.version + ".info": `{"Version":"` + m.version + `","Time":"2026-01-01T00:00:00Z"}`, + m.version + ".mod": m.gomod, + }) + f, err := os.Create(filepath.Join(at, m.version+".zip")) + require.NoError(t, err) + require.NoError(t, modzip.CreateFromDir(f, gomodule.Version{Path: m.path, Version: m.version}, src)) + require.NoError(t, f.Close()) + versions[at] = append(versions[at], m.version) + } + for at, vs := range versions { + writeFiles(t, at, map[string]string{"list": strings.Join(vs, "\n") + "\n"}) + } +} + +func mod(path, version string, requires ...string) proxyModule { + gomod := "module " + path + "\n\ngo 1.22\n" + for _, r := range requires { + gomod += "\nrequire " + r + "\n" + } + return proxyModule{path: path, version: version, gomod: gomod} +} + +// The go command decides the build list of a workspace. Each case runs it +// on a proxy on the disk, and compares Discover with go list -m all. +func TestDiscoverGivesTheBuildListOfTheGoCommand(t *testing.T) { + if testing.Short() { + t.Skip("the test runs the go command") + } + if _, err := exec.LookPath("go"); err != nil { + t.Skip("no go command") + } + cases := []struct { + name string + files map[string]string + modules []proxyModule + }{ + { + name: "a module of the workspace requires a lower version than another", + files: map[string]string{ + "go.work": "go 1.22\n\nuse ./a\nuse ./b\n", + "a/go.mod": "module example.com/a\n\ngo 1.22\n\nrequire (\n\texample.com/x v1.0.0\n\texample.com/z v1.0.0\n)\n", + "b/go.mod": "module example.com/b\n\ngo 1.22\n\nrequire example.com/x v1.2.0\n", + }, + modules: []proxyModule{ + mod("example.com/x", "v1.0.0"), + mod("example.com/x", "v1.2.0", "example.com/y v1.0.0"), + mod("example.com/y", "v1.0.0", "example.com/z v1.5.0"), + mod("example.com/z", "v1.0.0"), + mod("example.com/z", "v1.5.0"), + }, + }, + { + name: "a requirement on a module of the workspace at a version", + files: map[string]string{ + "go.work": "go 1.22\n\nuse ./a\nuse ./b\n", + "a/go.mod": "module example.com/a\n\ngo 1.22\n\nrequire (\n\texample.com/b v0.1.0\n\texample.com/z v1.0.0\n)\n", + "b/go.mod": "module example.com/b\n\ngo 1.22\n\nrequire example.com/z v1.0.0\n", + }, + modules: []proxyModule{ + mod("example.com/b", "v0.1.0", "example.com/z v1.5.0"), + mod("example.com/z", "v1.0.0"), + mod("example.com/z", "v1.5.0"), + }, + }, + { + name: "a module of the workspace below go 1.17", + files: map[string]string{ + "go.work": "go 1.22\n\nuse ./a\nuse ./c\n", + "a/go.mod": "module example.com/a\n\ngo 1.22\n\nrequire (\n\texample.com/x v1.0.0\n\texample.com/z v1.0.0\n)\n", + "c/go.mod": "module example.com/c\n\ngo 1.16\n\nrequire example.com/x v1.0.0\n", + }, + modules: []proxyModule{ + mod("example.com/x", "v1.0.0", "example.com/y v1.0.0"), + mod("example.com/y", "v1.0.0", "example.com/z v1.5.0"), + mod("example.com/z", "v1.0.0"), + mod("example.com/z", "v1.5.0"), + }, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + root, proxy, cache := t.TempDir(), t.TempDir(), t.TempDir() + writeFiles(t, root, tc.files) + writeProxy(t, proxy, tc.modules) + work := filepath.Join(root, "go.work") + env := []string{ + "GOWORK=" + work, "GOMODCACHE=" + cache, "GOPROXY=file://" + filepath.ToSlash(proxy), + "GOSUMDB=off", "GONOSUMDB=*", "GOFLAGS=-modcacherw", + } + dir := filepath.Join(root, "a") + _, err := goCommand(dir, env, "mod", "download", "all") + require.NoError(t, err) + want := goListBuildList(t, dir, env) + sources, err := Discover(DiscoverOptions{Dir: dir, Work: work, ModuleCache: cache, LocalRoot: root}) + require.NoError(t, err) + got := buildListOf("example.com/a", sources) + assert.Equal(t, want, got) + }) + } +} diff --git a/modgraph/discover.go b/modgraph/discover.go index 48ecdf2b..14e26128 100644 --- a/modgraph/discover.go +++ b/modgraph/discover.go @@ -8,22 +8,27 @@ import ( "io/fs" "os" "path/filepath" - "slices" "strings" "golang.org/x/mod/modfile" gomodule "golang.org/x/mod/module" + "golang.org/x/mod/semver" ) // DiscoverOptions says where Discover looks for the source of each module. type DiscoverOptions struct { // Dir is the directory of the main go.mod. Dir string + // Work is the go.work file of a workspace that uses Dir, or "" for a + // module that the go command builds alone. The caller finds it, as the + // go command does from GOWORK and the parents of Dir. + Work string // ModuleCache is the module cache of the go command, as GOMODCACHE, or "" // for none. The caller finds it, so Discover reads no environment. ModuleCache string - // LocalRoot bounds a replace directive with a local path: Discover reads - // a replacement only inside it, as the root of the scan. "" reads none. + // LocalRoot bounds a replace directive with a local path, and each module + // of a workspace: Discover reads such a directory only inside it, as the + // root of the scan. "" reads none. LocalRoot string } @@ -33,77 +38,148 @@ type DiscoverOptions struct { // module at its version, then the module cache. A module with no source // gets the go.mod of the module cache, when the cache has it. Discover never // downloads, and a module that it cannot find has no source and no go.mod, -// so Build makes it open. +// so Build makes it open. With Work, the build list is that of the +// workspace, and each other module of the workspace is a local source. func Discover(opts DiscoverOptions) ([]Source, error) { - data, err := os.ReadFile(filepath.Join(opts.Dir, "go.mod")) - if err != nil { - return nil, fmt.Errorf("modgraph: read go.mod: %w", err) + // A relative path would meet the resolved, absolute paths of the + // workspace and of each local source. + for _, p := range []*string{&opts.Dir, &opts.Work, &opts.LocalRoot} { + if *p == "" { + continue + } + abs, err := filepath.Abs(*p) + if err != nil { + return nil, fmt.Errorf("modgraph: %w", err) + } + *p = abs + } + if opts.Work != "" { + return discoverWorkspace(opts) } - // A lax parse drops the replace directives, and a replaced module read - // from the module cache is other code. So go.mod must parse in full. - f, err := modfile.Parse("go.mod", data, nil) + f, err := parseGoMod(filepath.Join(opts.Dir, "go.mod")) if err != nil { - return nil, fmt.Errorf("modgraph: parse go.mod: %w", err) + return nil, err + } + d := finder{ + opts: opts, + vendorDir: filepath.Join(opts.Dir, "vendor"), + sums: readSums(filepath.Join(opts.Dir, "go.sum")), + } + // The go command reads no vendor directory of a workspace outside + // workspace mode. + if modules := filepath.Join(opts.Dir, "vendor", "modules.txt"); !vendorForWorkspace(modules) { + d.vendored = readVendorList(modules) } - vendored := readVendorList(filepath.Join(opts.Dir, "vendor", "modules.txt")) - sums := readSums(filepath.Join(opts.Dir, "go.sum")) var out []Source for _, r := range f.Require { - s := Source{Module: Module{Path: r.Mod.Path, Version: r.Mod.Version}} - src := replacement(f, r.Mod) - if src != r.Mod { - s.Origin = Module{Path: src.Path, Version: src.Version} - } - // In vendor mode the go command reads each module from vendor, a - // local replacement too. - switch { - case vendored[r.Mod.Path] == src: - s.FS = os.DirFS(filepath.Join(opts.Dir, "vendor", filepath.FromSlash(r.Mod.Path))) - case src.Version == "": - s.FS, s.GoMod = localSource(opts, src.Path) - default: - s.FS, s.GoMod = cacheSource(opts.ModuleCache, src) - s.Immutable = s.FS != nil - s.Sum = sums[src] + t := target{base: opts.Dir, mod: r.Mod} + if rep := findReplace(f.Replace, r.Mod); rep != nil { + t.mod = rep.New } - out = append(out, s) + out = append(out, d.source(r.Mod, t)) } return out, nil } -// replacement returns the module that the go command builds for a module of -// the build list: the target of the replace directive of that version, else -// of a replace directive of every version, else the module itself. -func replacement(f *modfile.File, m gomodule.Version) gomodule.Version { +// parseGoMod reads a go.mod in full. A lax parse drops the replace +// directives, and a replaced module read from the module cache is other +// code. +func parseGoMod(name string) (*modfile.File, error) { + data, err := os.ReadFile(name) + if err != nil { + return nil, fmt.Errorf("modgraph: read go.mod: %w", err) + } + f, err := modfile.Parse(name, data, nil) + if err != nil { + return nil, fmt.Errorf("modgraph: parse go.mod: %w", err) + } + return f, nil +} + +// target is the module that the go command builds for a module of the +// build list, as a replace directive writes it. A local path is relative to +// base. +type target struct { + base string + mod gomodule.Version +} + +func (t target) local() bool { return t.mod.Version == "" } + +// dir is the directory of a local target. +func (t target) dir() string { + dir := filepath.FromSlash(t.mod.Path) + if !filepath.IsAbs(dir) { + dir = filepath.Join(t.base, dir) + } + return filepath.Clean(dir) +} + +// same reports whether two targets name the same code. +func (t target) same(o target) bool { + if t.local() || o.local() { + return t.local() && o.local() && t.dir() == o.dir() + } + return t.mod == o.mod +} + +// findReplace returns the replace directive of a module version, else the +// one of each version of its path, else nil. +func findReplace(rs []*modfile.Replace, m gomodule.Version) *modfile.Replace { var all *modfile.Replace - for _, r := range f.Replace { + for _, r := range rs { switch { case r.Old.Path != m.Path: case r.Old.Version == m.Version: - return r.New + return r case r.Old.Version == "": all = r } } - if all != nil { - return all.New + return all +} + +// finder finds the source of each module of one build list. +type finder struct { + opts DiscoverOptions + vendorDir string + vendored map[string]vendored + // vendorMode reads each module that vendor lists from vendor, as the + // go command does for a workspace. Else vendor serves a module only + // when it holds the version and the replacement of go.mod. + vendorMode bool + sums map[gomodule.Version]string +} + +// source finds the source of the module m, which the go command builds +// from t. In vendor mode the go command reads each module from vendor, a +// local replacement too. +func (d finder) source(m gomodule.Version, t target) Source { + s := Source{Module: Module{Path: m.Path, Version: m.Version}} + if t.mod != m { + s.Origin = Module{Path: t.mod.Path, Version: t.mod.Version} + } + switch v, ok := d.vendored[m.Path]; { + case ok && (d.vendorMode || v.holds(m.Path) == t.mod): + s.FS = os.DirFS(filepath.Join(d.vendorDir, filepath.FromSlash(m.Path))) + case t.local(): + s.FS, s.GoMod = localSource(d.opts.LocalRoot, t.dir()) + default: + s.FS, s.GoMod = cacheSource(d.opts.ModuleCache, t.mod) + s.Immutable = s.FS != nil + s.Sum = d.sums[t.mod] } - return m + return s } -// localSource returns the source of a replacement with a local path, inside -// LocalRoot only. Both paths are resolved through links first, and each read -// of the source resolves its links again, so a link cannot lead the read out -// of LocalRoot. -func localSource(opts DiscoverOptions, p string) (fs.FS, []byte) { - if opts.LocalRoot == "" { +// localSource returns the source of a local directory, inside root only. +// Both paths are resolved through links first, and each read of the source +// resolves its links again, so a link cannot lead the read out of root. +func localSource(root, dir string) (fs.FS, []byte) { + if root == "" { return nil, nil } - dir := filepath.FromSlash(p) - if !filepath.IsAbs(dir) { - dir = filepath.Join(opts.Dir, dir) - } - root, err := filepath.EvalSymlinks(opts.LocalRoot) + root, err := filepath.EvalSymlinks(root) if err != nil { return nil, nil } @@ -176,39 +252,99 @@ func readOptional(name string) []byte { return data } -// readVendorList reads the modules that vendor/modules.txt lists, each with -// the module that the vendor directory holds for it: the module itself, or -// the target of its replace. A missing file lists none. -func readVendorList(name string) map[string]gomodule.Version { +// vendored is a module that vendor/modules.txt lists with a package: the +// version that the build list holds, and the target of its replace +// directive, or the zero module. +type vendored struct { + version string + replacement gomodule.Version +} + +// holds returns the module that the vendor directory holds for v. +func (v vendored) holds(path string) gomodule.Version { + if v.replacement.Path != "" { + return v.replacement + } + return gomodule.Version{Path: path, Version: v.version} +} + +// readVendorList reads vendor/modules.txt as the go command does. A module +// enters the list only from a "# path version" line that package lines +// follow. A line with "=> target" records the replacement of that version, +// or of each version when the line has no version. A missing file lists +// none. +func readVendorList(name string) map[string]vendored { data, err := os.ReadFile(name) if err != nil { return nil } - out := map[string]gomodule.Version{} + out := map[string]vendored{} + replaced := map[gomodule.Version]gomodule.Version{} + var mod gomodule.Version sc := bufio.NewScanner(bytes.NewReader(data)) for sc.Scan() { - // A module line is "# path version", or - // "# path [version] => target [version]". - fields := strings.Fields(sc.Text()) - if len(fields) < 3 || fields[0] != "#" { + line := sc.Text() + if strings.HasPrefix(line, "# ") { + f := strings.Fields(line) + mod = gomodule.Version{} + switch { + case len(f) < 3: + continue + case semver.IsValid(f[2]): + mod, f = gomodule.Version{Path: f[1], Version: f[2]}, f[3:] + case f[2] == "=>": + mod, f = gomodule.Version{Path: f[1]}, f[2:] + default: + continue + } + switch { + case len(f) == 2 && f[0] == "=>": + replaced[mod] = gomodule.Version{Path: f[1]} + case len(f) == 3 && f[0] == "=>" && semver.IsValid(f[2]): + replaced[mod] = gomodule.Version{Path: f[1], Version: f[2]} + } continue } - path, rest := fields[1], fields[2:] - if i := slices.Index(rest, "=>"); i >= 0 { - target := rest[i+1:] - switch len(target) { - case 1: - out[path] = gomodule.Version{Path: target[0]} - case 2: - out[path] = gomodule.Version{Path: target[0], Version: target[1]} - } + f := strings.Fields(line) + if mod.Version == "" || len(f) != 1 || strings.HasPrefix(line, "## ") || gomodule.CheckImportPath(f[0]) != nil { continue } - out[path] = gomodule.Version{Path: path, Version: rest[0]} + if v, ok := out[mod.Path]; !ok || semver.Compare(v.version, mod.Version) < 0 { + out[mod.Path] = vendored{version: mod.Version} + } + } + for path, v := range out { + if r, ok := replaced[gomodule.Version{Path: path, Version: v.version}]; ok { + v.replacement = r + } else if r, ok := replaced[gomodule.Version{Path: path}]; ok { + v.replacement = r + } + out[path] = v } return out } +// vendorForWorkspace reports whether the first line of vendor/modules.txt +// marks it as the vendor directory of a workspace, as go work vendor +// writes it. +func vendorForWorkspace(name string) bool { + data, err := os.ReadFile(name) + if err != nil { + return false + } + line, _, _ := strings.Cut(string(data), "\n") + annotations, ok := strings.CutPrefix(line, "## ") + if !ok { + return false + } + for entry := range strings.SplitSeq(annotations, ";") { + if strings.TrimSpace(entry) == "workspace" { + return true + } + } + return false +} + // readSums reads the h1 hash of each module version in go.sum. A go.mod // hash line is not the hash of the source, so it does not count. func readSums(name string) map[gomodule.Version]string { diff --git a/modgraph/oracle_test.go b/modgraph/oracle_test.go index 42aad3de..74cf2c81 100644 --- a/modgraph/oracle_test.go +++ b/modgraph/oracle_test.go @@ -9,6 +9,7 @@ import ( "os" "os/exec" "path/filepath" + "slices" "strings" "testing" "time" @@ -23,11 +24,13 @@ import ( // edge that go list gives must be in the graph, unless the node that it // leaves is open, and the node that it reaches must be reached. go list // runs for each platform pair below, with the test packages of the main -// module. The test runs on this repository. MODGRAPH_ORACLE_DIRS adds more -// projects, as a list of directories that filepath.SplitList reads. A -// project with vendor/modules.txt reads its dependencies from vendor. The -// test needs the go command and a module cache that holds the build list, -// and skips with neither. It never downloads: GOPROXY is off. +// module. The test runs on this repository, alone and in a workspace with +// a probe module. MODGRAPH_ORACLE_DIRS adds more projects, as a list of +// directories that filepath.SplitList reads. A project with +// vendor/modules.txt reads its dependencies from vendor. A project with a +// go.work is checked from each module that the workspace uses. The test +// needs the go command and a module cache that holds the build list, and +// skips with neither. It never downloads: GOPROXY is off. var oraclePlatforms = [][2]string{ {"linux", "amd64"}, @@ -53,64 +56,24 @@ func TestBuildHoldsEachEdgeOfGoList(t *testing.T) { } for _, dir := range dirs { t.Run(filepath.Base(dir), func(t *testing.T) { - sources, ok := oracleSources(t, dir) + if work := filepath.Join(dir, "go.work"); dir != root && fileExists(work) { + checkWorkspace(t, work) + return + } + sources, ok := oracleSources(t, dir, "") if !ok { t.Skip("the module cache does not hold the build list") } - st := memstore.New() - start := time.Now() - g, err := Build(context.Background(), Input{MainFS: os.DirFS(dir), Modules: sources, Store: st}) - require.NoError(t, err) - cold := time.Since(start) - start = time.Now() - warm, err := Build(context.Background(), Input{MainFS: os.DirFS(dir), Modules: sources, Store: st}) - require.NoError(t, err) - assert.Equal(t, g.Edges, warm.Edges, "the cache gives the same graph") - assert.Equal(t, g.Nodes, warm.Nodes, "the cache gives the same nodes") - t.Logf("Build: %d files, %d packages, %v; with the cache: %d files, %d cached packages, %v", - g.Stats.Files, g.Stats.Packages, cold.Round(time.Millisecond), warm.Stats.Files, warm.Stats.CachedPackages, time.Since(start).Round(time.Millisecond)) - for _, n := range g.Nodes { - if n.Open() && n.Reached { - t.Logf("open: %s: %s", n.Module.Path, n.Reason) - } - } - edges := map[[2]string]bool{} - for _, e := range g.Edges { - edges[[2]string{e.From.Path, e.To.Path}] = true - } - // An open node passes each edge, so an edge of go list must leave a - // module that has source, or the test compares nothing. - sourced := map[string]bool{g.Main.Path: true} - for _, s := range sources { - sourced[s.Module.Path] = s.FS != nil - } - for _, p := range oraclePlatforms { - listed, dropped := goListEdges(t, dir, p[0], p[1]) - t.Logf("%s/%s: go list has %d module edges, the graph %d, with %d nodes", p[0], p[1], len(listed), len(g.Edges), len(g.Nodes)) - if dir == root { - assert.Empty(t, dropped, "%s/%s: go list resolves each import of this repository", p[0], p[1]) - } else if len(dropped) > 0 { - t.Logf("%s/%s: go list resolves no module for %d imports, such as %s", p[0], p[1], len(dropped), dropped[0]) - } - for e := range listed { - from, ok := g.Node(e[0]) - require.True(t, ok, "the graph has no node %s", e[0]) - require.True(t, sourced[e[0]], "go list reads %s, and Discover finds no source of it", e[0]) - assert.True(t, from.Reached, "%s/%s: go list reaches %s, the graph does not", p[0], p[1], e[0]) - to, ok := g.Node(e[1]) - assert.True(t, ok && to.Reached, "%s/%s: go list reaches %s, the graph does not", p[0], p[1], e[1]) - assert.True(t, edges[e] || from.Open(), "%s/%s: go list has the edge %s -> %s", p[0], p[1], e[0], e[1]) - } - } + checkAgainstGoList(t, dir, sources, nil, dir == root) // With no source, each node holds only its go.mod, and each module // that go list reaches must still be reached. for i := range sources { sources[i].FS = nil } - g, err = Build(context.Background(), Input{MainFS: os.DirFS(dir), Modules: sources}) + g, err := Build(context.Background(), Input{MainFS: os.DirFS(dir), Modules: sources}) require.NoError(t, err) - listed, _ := goListEdges(t, dir, "linux", "amd64") + listed, _ := goListEdges(t, dir, nil, "linux", "amd64") for e := range listed { to, ok := g.Node(e[1]) assert.True(t, ok && to.Reached, "go.mod only: go list reaches %s, the graph does not", e[1]) @@ -119,16 +82,158 @@ func TestBuildHoldsEachEdgeOfGoList(t *testing.T) { } } +func TestBuildHoldsEachEdgeOfGoListInAWorkspace(t *testing.T) { + if testing.Short() { + t.Skip("the oracle test runs the go command") + } + if _, err := exec.LookPath("go"); err != nil { + t.Skip("no go command") + } + root, err := filepath.Abs("..") + require.NoError(t, err) + self, err := parseGoMod(filepath.Join(root, "go.mod")) + require.NoError(t, err) + probe, ws := t.TempDir(), t.TempDir() + writeFiles(t, probe, map[string]string{ + "go.mod": "module example.com/probe\n\ngo " + self.Go.Version + "\n", + "probe.go": "package probe\n\nimport _ \"github.com/safedep/code/modgraph\"\n", + }) + work := filepath.Join(ws, "go.work") + writeFiles(t, ws, map[string]string{ + "go.work": "go " + self.Go.Version + "\n\nuse (\n\t" + filepath.ToSlash(root) + "\n\t" + filepath.ToSlash(probe) + "\n)\n", + }) + checkWorkspace(t, work) +} + +// checkWorkspace checks the graph of each module that a workspace uses. +func checkWorkspace(t *testing.T, work string) { + t.Helper() + w, err := readWorkspace(work, string(filepath.Separator)) + require.NoError(t, err) + for _, m := range w.mains { + t.Run(m.path, func(t *testing.T) { + sources, ok := oracleSources(t, m.dir, work) + if !ok { + t.Skip("the module cache does not hold the build list") + } + env := []string{"GOWORK=" + work} + // go list -m all refuses vendor mode. + if !w.vendorMode() { + assert.Equal(t, goListBuildList(t, m.dir, env), buildListOf(m.path, sources), "the build list of the workspace") + } + checkAgainstGoList(t, m.dir, sources, env, false) + }) + } +} + +// goListBuildList returns the version of each module of the build list that +// go list -m all gives, with no version for a module of the workspace. +func goListBuildList(t *testing.T, dir string, env []string) map[string]string { + t.Helper() + out, err := goCommand(dir, env, "list", "-m", "-e", "-json", "all") + require.NoError(t, err, "go list -m") + list := map[string]string{} + dec := json.NewDecoder(bytes.NewReader(out)) + for { + var m struct { + Path string + Version string + Main bool + } + if err := dec.Decode(&m); errors.Is(err, io.EOF) { + break + } else { + require.NoError(t, err) + } + list[m.Path] = m.Version + } + return list +} + +func buildListOf(main string, sources []Source) map[string]string { + list := map[string]string{main: ""} + for _, s := range sources { + list[s.Module.Path] = s.Module.Version + } + return list +} + +func fileExists(name string) bool { + _, err := os.Stat(name) + return err == nil +} + +// checkAgainstGoList builds the graph of the module in dir and checks it +// against go list. strict also asks go list to resolve each import. +func checkAgainstGoList(t *testing.T, dir string, sources []Source, env []string, strict bool) { + t.Helper() + st := memstore.New() + start := time.Now() + g, err := Build(context.Background(), Input{MainFS: os.DirFS(dir), Modules: sources, Store: st}) + require.NoError(t, err) + cold := time.Since(start) + start = time.Now() + warm, err := Build(context.Background(), Input{MainFS: os.DirFS(dir), Modules: sources, Store: st}) + require.NoError(t, err) + assert.Equal(t, g.Edges, warm.Edges, "the cache gives the same graph") + assert.Equal(t, g.Nodes, warm.Nodes, "the cache gives the same nodes") + t.Logf("Build: %d files, %d packages, %v; with the cache: %d files, %d cached packages, %v", + g.Stats.Files, g.Stats.Packages, cold.Round(time.Millisecond), warm.Stats.Files, warm.Stats.CachedPackages, time.Since(start).Round(time.Millisecond)) + for _, n := range g.Nodes { + if n.Open() && n.Reached { + t.Logf("open: %s: %s", n.Module.Path, n.Reason) + } + } + edges := map[[2]string]bool{} + for _, e := range g.Edges { + edges[[2]string{e.From.Path, e.To.Path}] = true + } + // An open node passes each edge, so an edge of go list must leave a + // module that has source, or the test compares nothing. + sourced := map[string]bool{g.Main.Path: true} + for _, s := range sources { + sourced[s.Module.Path] = s.FS != nil + } + for _, p := range oraclePlatforms { + listed, dropped := goListEdges(t, dir, env, p[0], p[1]) + t.Logf("%s/%s: go list has %d module edges, the graph %d, with %d nodes", p[0], p[1], len(listed), len(g.Edges), len(g.Nodes)) + if strict { + assert.Empty(t, dropped, "%s/%s: go list resolves each import of this repository", p[0], p[1]) + } else if len(dropped) > 0 { + t.Logf("%s/%s: go list resolves no module for %d imports, such as %s", p[0], p[1], len(dropped), dropped[0]) + } + for e := range listed { + from, ok := g.Node(e[0]) + require.True(t, ok, "the graph has no node %s", e[0]) + require.True(t, sourced[e[0]], "go list reads %s, and Discover finds no source of it", e[0]) + assert.True(t, from.Reached, "%s/%s: go list reaches %s, the graph does not", p[0], p[1], e[0]) + to, ok := g.Node(e[1]) + assert.True(t, ok && to.Reached, "%s/%s: go list reaches %s, the graph does not", p[0], p[1], e[1]) + assert.True(t, edges[e] || from.Open(), "%s/%s: go list has the edge %s -> %s", p[0], p[1], e[0], e[1]) + } + } +} + // oracleSources finds the sources with Discover and the module cache of // the go command. -func oracleSources(t *testing.T, dir string) ([]Source, bool) { +func oracleSources(t *testing.T, dir, work string) ([]Source, bool) { t.Helper() out, err := goCommand(dir, nil, "env", "GOMODCACHE") if err != nil { t.Logf("go env: %v", err) return nil, false } - sources, err := Discover(DiscoverOptions{Dir: dir, ModuleCache: strings.TrimSpace(string(out)), LocalRoot: dir}) + root := dir + if work != "" { + root = string(filepath.Separator) + } + sources, err := Discover(DiscoverOptions{Dir: dir, Work: work, ModuleCache: strings.TrimSpace(string(out)), LocalRoot: root}) + // A module cache that lacks a go.mod of the build list skips the test. + // Each other error is a defect. + if work != "" && err != nil && strings.Contains(err.Error(), "is on the disk") { + t.Logf("Discover: %v", err) + return nil, false + } require.NoError(t, err) return sources, true } @@ -144,9 +249,9 @@ type listPackage struct { // -test gives for a platform, and each import that go list resolves to no // module. go list -e gives such an import with no package of a module, so // it leaves no edge to compare. -func goListEdges(t *testing.T, dir, goos, goarch string) (map[[2]string]bool, []string) { +func goListEdges(t *testing.T, dir string, env []string, goos, goarch string) (map[[2]string]bool, []string) { t.Helper() - out, err := goCommand(dir, []string{"GOOS=" + goos, "GOARCH=" + goarch, "CGO_ENABLED=1"}, "list", "-e", "-deps", "-test", "-json", "./...") + out, err := goCommand(dir, append(slices.Clone(env), "GOOS="+goos, "GOARCH="+goarch, "CGO_ENABLED=1"), "list", "-e", "-deps", "-test", "-json", "./...") require.NoError(t, err, "go list -deps") var pkgs []listPackage dec := json.NewDecoder(bytes.NewReader(out)) diff --git a/modgraph/workspace.go b/modgraph/workspace.go new file mode 100644 index 00000000..cfa51204 --- /dev/null +++ b/modgraph/workspace.go @@ -0,0 +1,390 @@ +package modgraph + +import ( + "fmt" + "go/version" + "maps" + "os" + "path/filepath" + "slices" + "strings" + + "golang.org/x/mod/modfile" + gomodule "golang.org/x/mod/module" + "golang.org/x/mod/semver" +) + +// workspace is a go.work file and the modules that it uses. +type workspace struct { + dir string + file *modfile.WorkFile + mains []mainModule +} + +// mainModule is a module that a workspace uses. +type mainModule struct { + path string + dir string + file *modfile.File +} + +// discoverWorkspace finds the sources of the build list of a workspace for +// the main module in opts.Dir. +func discoverWorkspace(opts DiscoverOptions) ([]Source, error) { + w, err := readWorkspace(opts.Work, opts.LocalRoot) + if err != nil { + return nil, err + } + main := w.mainAt(opts.Dir) + if main == nil { + return nil, fmt.Errorf("modgraph: %s does not use the module in %s", opts.Work, opts.Dir) + } + sums := map[gomodule.Version]string{} + maps.Copy(sums, readSums(filepath.Join(w.dir, "go.work.sum"))) + for _, m := range w.mains { + maps.Copy(sums, readSums(filepath.Join(m.dir, "go.sum"))) + } + d := finder{opts: opts, vendorDir: filepath.Join(w.dir, "vendor"), sums: sums} + + var list []gomodule.Version + if w.vendorMode() { + d.vendorMode = true + d.vendored = readVendorList(filepath.Join(w.dir, "vendor", "modules.txt")) + for path, v := range d.vendored { + list = append(list, gomodule.Version{Path: path, Version: v.version}) + } + } else if list, err = w.buildList(opts); err != nil { + return nil, err + } + + var out []Source + for _, m := range list { + if w.main(m.Path) != nil { + continue + } + t, err := w.replacement(m) + if err != nil { + return nil, err + } + out = append(out, d.source(m, t)) + } + for _, m := range w.mains { + if m.dir == main.dir { + continue + } + s := Source{Module: Module{Path: m.path}, Origin: Module{Path: filepath.ToSlash(m.dir)}} + s.FS, s.GoMod = localSource(opts.LocalRoot, m.dir) + out = append(out, s) + } + slices.SortFunc(out, func(a, b Source) int { return strings.Compare(a.Module.Path, b.Module.Path) }) + return out, nil +} + +// readWorkspace reads a go.work file and the go.mod of each module that it +// uses, inside root only. A module of the workspace whose go.mod does not +// read leaves the build list unknown, so it is an error. +func readWorkspace(name, root string) (*workspace, error) { + data, err := os.ReadFile(name) + if err != nil { + return nil, fmt.Errorf("modgraph: read go.work: %w", err) + } + f, err := modfile.ParseWork(name, data, nil) + if err != nil { + return nil, fmt.Errorf("modgraph: parse go.work: %w", err) + } + w := &workspace{dir: filepath.Dir(name), file: f} + seen := map[string]string{} + for _, u := range f.Use { + dir := filepath.FromSlash(u.Path) + if !filepath.IsAbs(dir) { + dir = filepath.Join(w.dir, dir) + } + dir = resolved(dir) + _, data := localSource(root, dir) + if data == nil { + return nil, fmt.Errorf("modgraph: read the go.mod in %s: no go.mod inside the local root", dir) + } + mf, err := modfile.Parse(filepath.Join(dir, "go.mod"), data, nil) + if err != nil { + return nil, fmt.Errorf("modgraph: parse go.mod: %w", err) + } + if mf.Module == nil || mf.Module.Mod.Path == "" { + return nil, fmt.Errorf("modgraph: the go.mod in %s has no module directive", dir) + } + // An exclude directive can move the build list off the highest + // version, and the build list below does not model it. + if len(mf.Exclude) > 0 { + return nil, fmt.Errorf("modgraph: the go.mod in %s has an exclude directive, which a workspace build list here does not model", dir) + } + path := mf.Module.Mod.Path + if prev, ok := seen[path]; ok { + return nil, fmt.Errorf("modgraph: %s uses %s twice, in %s and %s", name, path, prev, dir) + } + seen[path] = dir + w.mains = append(w.mains, mainModule{path: path, dir: dir, file: mf}) + } + return w, nil +} + +// resolved returns a path with its links resolved, or the clean path when +// it does not resolve. +func resolved(p string) string { + if r, err := filepath.EvalSymlinks(p); err == nil { + return r + } + return filepath.Clean(p) +} + +func (w *workspace) main(path string) *mainModule { + for i := range w.mains { + if w.mains[i].path == path { + return &w.mains[i] + } + } + return nil +} + +func (w *workspace) mainAt(dir string) *mainModule { + dir = resolved(dir) + for i := range w.mains { + if w.mains[i].dir == dir { + return &w.mains[i] + } + } + return nil +} + +// vendorMode reports whether the go command reads the workspace from its +// vendor directory: a vendor directory next to go.work, a go directive of +// go 1.14 or later in go.work, and a vendor/modules.txt that go work vendor +// wrote for a workspace. +func (w *workspace) vendorMode() bool { + if w.file.Go == nil || version.Compare("go"+w.file.Go.Version, "go1.14") < 0 { + return false + } + if info, err := os.Stat(filepath.Join(w.dir, "vendor")); err != nil || !info.IsDir() { + return false + } + return vendorForWorkspace(filepath.Join(w.dir, "vendor", "modules.txt")) +} + +// replacement returns the module that the go command builds for m: the +// replace directive of go.work, else the one of a module of the workspace. +// Two modules that replace m with other code make the go command fail. +func (w *workspace) replacement(m gomodule.Version) (target, error) { + if r := findReplace(w.file.Replace, m); r != nil { + return target{base: w.dir, mod: r.New}, nil + } + var found *target + for _, mm := range w.mains { + r := findReplace(mm.file.Replace, m) + if r == nil { + continue + } + t := target{base: mm.dir, mod: r.New} + if found != nil && !found.same(t) { + return target{}, fmt.Errorf("modgraph: the modules of the workspace replace %s with other code", m) + } + found = &t + } + if found != nil { + return *found, nil + } + return target{mod: m}, nil +} + +// buildList returns the build list of the workspace as readModGraph of the +// go command gives it in workspace mode: the highest version of each path +// in a graph of go.mod requirements. The graph holds the requirements of +// each module of the workspace and of the go.mod of each such requirement. +// A go.mod below go 1.17 adds the go.mod of each module that it requires, +// and so on down the chain. Then, while a module that the build needs +// requires a module below its selected version, the go.mod of the selected +// version joins the graph with each go.mod that it requires. A go.mod that +// the graph needs and that the disk lacks leaves the build list unknown, so +// it is an error. A wrong version can hide an edge. +func (w *workspace) buildList(opts DiscoverOptions) ([]gomodule.Version, error) { + g := &modGraph{w: w, opts: opts, required: map[gomodule.Version][]gomodule.Version{}, loading: map[loadKey]bool{}, selected: map[string]string{}} + for _, mm := range w.mains { + g.enqueue(gomodule.Version{Path: mm.path}, loadWorkspace) + } + if err := g.drain(); err != nil { + return nil, err + } + + // The reload pass of readModGraph. + needed := map[string]bool{} + for _, mm := range w.mains { + needed[mm.path] = true + } + seen := map[gomodule.Version]bool{} + for { + next := map[gomodule.Version]bool{} + for p := range needed { + m := gomodule.Version{Path: p, Version: g.selectedOf(p)} + if !seen[m] { + next[m] = true + continue + } + for _, r := range g.required[m] { + s := gomodule.Version{Path: r.Path, Version: g.selectedOf(r.Path)} + if laterVersion(s.Version, r.Version) && !seen[s] { + next[s] = true + } + } + } + if len(next) == 0 { + break + } + for m := range next { + g.enqueue(m, loadWorkspace) + seen[m] = true + needed[m.Path] = true + } + if err := g.drain(); err != nil { + return nil, err + } + } + + out := make([]gomodule.Version, 0, len(g.selected)) + for path, v := range g.selected { + out = append(out, gomodule.Version{Path: path, Version: v}) + } + return out, nil +} + +// laterVersion compares two versions as the go command does, where "" is +// the version of a module of the workspace, after each other version. +func laterVersion(a, b string) bool { + switch { + case a == b: + return false + case a == "": + return true + case b == "": + return false + } + return semver.Compare(a, b) > 0 +} + +// loadMode is the pruning that a go.mod is loaded at, as in readModGraph. +type loadMode int + +const ( + // loadPruned loads the requirements of a go.mod and does not follow + // them, unless the go.mod is below go 1.17. + loadPruned loadMode = iota + // loadUnpruned follows each requirement down the chain. + loadUnpruned + // loadWorkspace loads a go.mod and each go.mod that it requires. + loadWorkspace +) + +type loadKey struct { + mod gomodule.Version + mode loadMode +} + +// modGraph is the graph of go.mod requirements of a workspace. +type modGraph struct { + w *workspace + opts DiscoverOptions + required map[gomodule.Version][]gomodule.Version + unpruned map[gomodule.Version]bool + loading map[loadKey]bool + queue []loadKey + selected map[string]string +} + +// selectedOf returns the selected version of a path, "" for a module of +// the workspace. +func (g *modGraph) selectedOf(path string) string { + if g.w.main(path) != nil { + return "" + } + return g.selected[path] +} + +func (g *modGraph) enqueue(m gomodule.Version, mode loadMode) { + if m.Version == "none" || g.loading[loadKey{m, mode}] { + return + } + g.loading[loadKey{m, mode}] = true + g.queue = append(g.queue, loadKey{m, mode}) +} + +func (g *modGraph) drain() error { + for len(g.queue) > 0 { + k := g.queue[0] + g.queue = g.queue[1:] + reqs, unpruned, err := g.load(k.mod) + if err != nil { + return err + } + next := loadPruned + if unpruned || k.mode == loadUnpruned { + next = loadUnpruned + } + for _, r := range reqs { + if k.mode != loadPruned || unpruned { + g.enqueue(r, next) + } + } + } + return nil +} + +// load reads the requirements of a module version once, adds them to the +// graph, and reports whether its go.mod is below go 1.17. A module of the +// workspace at no version is its own go.mod. A versioned requirement on a +// module of the workspace reads that version, as the go command does. +func (g *modGraph) load(m gomodule.Version) ([]gomodule.Version, bool, error) { + if reqs, ok := g.required[m]; ok { + return reqs, g.unpruned[m], nil + } + var f *modfile.File + if mm := g.w.main(m.Path); mm != nil && m.Version == "" { + f = mm.file + } else { + t, err := g.w.replacement(m) + if err != nil { + return nil, false, err + } + if f, err = g.w.goModOf(g.opts, t); err != nil { + return nil, false, err + } + } + reqs := make([]gomodule.Version, 0, len(f.Require)) + for _, r := range f.Require { + reqs = append(reqs, r.Mod) + if v, ok := g.selected[r.Mod.Path]; g.w.main(r.Mod.Path) == nil && r.Mod.Version != "none" && (!ok || laterVersion(r.Mod.Version, v)) { + g.selected[r.Mod.Path] = r.Mod.Version + } + } + unpruned := f.Go == nil || version.Compare("go"+f.Go.Version, "go1.17") < 0 + g.required[m] = reqs + if g.unpruned == nil { + g.unpruned = map[gomodule.Version]bool{} + } + g.unpruned[m] = unpruned + return reqs, unpruned, nil +} + +// goModOf reads the go.mod of the code of a target: a local directory +// inside LocalRoot, or the .mod file of the module cache. +func (w *workspace) goModOf(opts DiscoverOptions, t target) (*modfile.File, error) { + var data []byte + if t.local() { + _, data = localSource(opts.LocalRoot, t.dir()) + } else { + _, data = cacheSource(opts.ModuleCache, t.mod) + } + if data == nil { + return nil, fmt.Errorf("modgraph: no go.mod of %s is on the disk, so the build list of the workspace is not known", t.mod) + } + f, err := modfile.ParseLax("go.mod", data, nil) + if err != nil { + return nil, fmt.Errorf("modgraph: parse the go.mod of %s: %w", t.mod, err) + } + return f, nil +} diff --git a/modgraph/workspace_test.go b/modgraph/workspace_test.go new file mode 100644 index 00000000..5bbba748 --- /dev/null +++ b/modgraph/workspace_test.go @@ -0,0 +1,276 @@ +package modgraph + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func cachedModule(path, version, gomod string, files map[string]string) map[string]string { + out := map[string]string{"cache/download/" + path + "/@v/" + version + ".mod": gomod} + for name, data := range files { + out[path+"@"+version+"/"+name] = data + } + return out +} + +func mergeFiles(sets ...map[string]string) map[string]string { + out := map[string]string{} + for _, s := range sets { + for k, v := range s { + out[k] = v + } + } + return out +} + +func TestDiscoverGivesTheBuildListOfTheWorkspace(t *testing.T) { + root, cache := t.TempDir(), t.TempDir() + writeFiles(t, root, map[string]string{ + "go.work": "go 1.22\n\nuse (\n\t./a\n\t./b\n)\n", + "a/go.mod": "module example.com/a\n\ngo 1.22\n\nrequire (\n\texample.com/x v1.0.0\n\texample.com/b v0.1.0\n)\n", + "a/a.go": "package a\n\nimport _ \"example.com/b\"\n", + "b/go.mod": "module example.com/b\n\ngo 1.22\n\nrequire (\n\texample.com/x v1.2.0\n\texample.com/y v1.0.0\n\texample.com/old v1.0.0\n)\n", + "b/b.go": "package b\n\nimport _ \"example.com/x\"\n", + "b/go.sum": "example.com/x v1.2.0 h1:x=\n", + "go.work.sum": "example.com/y v1.0.0 h1:y=\n", + }) + writeFiles(t, cache, mergeFiles( + cachedModule("example.com/x", "v1.0.0", "module example.com/x\n\ngo 1.22\n", map[string]string{"x.go": "package x\n"}), + cachedModule("example.com/x", "v1.2.0", "module example.com/x\n\ngo 1.22\n", map[string]string{"x.go": "package x\n\nimport _ \"example.com/z\"\n"}), + cachedModule("example.com/y", "v1.0.0", "module example.com/y\n\ngo 1.22\n\nrequire example.com/z v1.1.0\n", nil), + cachedModule("example.com/old", "v1.0.0", "module example.com/old\n\ngo 1.16\n\nrequire example.com/w v1.0.0\n", nil), + cachedModule("example.com/w", "v1.0.0", "module example.com/w\n\ngo 1.22\n\nrequire example.com/q v1.3.0\n", nil), + cachedModule("example.com/b", "v0.1.0", "module example.com/b\n\ngo 1.22\n", nil), + cachedModule("example.com/q", "v1.3.0", "module example.com/q\n\ngo 1.22\n", nil), + cachedModule("example.com/z", "v1.1.0", "module example.com/z\n\ngo 1.22\n", map[string]string{"z.go": "package z\n"}), + )) + + sources, err := Discover(DiscoverOptions{Dir: filepath.Join(root, "a"), Work: filepath.Join(root, "go.work"), ModuleCache: cache, LocalRoot: root}) + require.NoError(t, err) + got := map[string]string{} + for _, s := range sources { + got[s.Module.Path] = s.Module.Version + } + assert.Equal(t, map[string]string{ + "example.com/b": "", + "example.com/x": "v1.2.0", + "example.com/y": "v1.0.0", + "example.com/z": "v1.1.0", + "example.com/old": "v1.0.0", + "example.com/w": "v1.0.0", + "example.com/q": "v1.3.0", + }, got, "the highest version in the graph, the go.mod of a requirement, and the go.mod chain below go 1.17") + + b := sourceOf(t, sources, "example.com/b") + assert.True(t, hasFile(t, b.FS, "b.go"), "another module of the workspace is local code") + assert.Equal(t, Module{Path: filepath.ToSlash(resolved(filepath.Join(root, "b")))}, b.Origin) + assert.Equal(t, "h1:x=", sourceOf(t, sources, "example.com/x").Sum, "each go.sum of the workspace gives a hash") + assert.Equal(t, "h1:y=", sourceOf(t, sources, "example.com/y").Sum, "go.work.sum gives a hash") + + g, err := Build(context.Background(), Input{MainFS: os.DirFS(filepath.Join(root, "a")), Modules: sources}) + require.NoError(t, err) + assert.Equal(t, []string{"example.com/a", "example.com/b", "example.com/x", "example.com/z"}, g.Path([]string{"example.com/a"}, "example.com/z"), + "the code of x at the version of the workspace imports z") +} + +func TestDiscoverAppliesTheReplacementsOfTheWorkspace(t *testing.T) { + cases := []struct { + name string + work string + modA string + modB string + want Module + wantErr string + }{ + { + name: "go.work wins over a module", + work: "replace example.com/x => example.com/workfork v1.0.0\n", + modA: "replace example.com/x v1.0.0 => example.com/modfork v1.0.0\n", + want: Module{Path: "example.com/workfork", Version: "v1.0.0"}, + }, + { + name: "a local replace of go.work is relative to its directory", + work: "replace example.com/x => ./fork\n", + want: Module{Path: "./fork"}, + }, + { + name: "a replace of one module", + modB: "replace example.com/x => example.com/modfork v1.0.0\n", + want: Module{Path: "example.com/modfork", Version: "v1.0.0"}, + }, + { + name: "two modules replace it with other code", + modA: "replace example.com/x => example.com/modfork v1.0.0\n", + modB: "replace example.com/x => example.com/workfork v1.0.0\n", + wantErr: "replace example.com/x@v1.0.0 with other code", + }, + { + name: "two modules replace it with one directory", + modA: "replace example.com/x => ../fork\n", + modB: "replace example.com/x => ./../fork\n", + want: Module{Path: "./../fork"}, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + root, cache := t.TempDir(), t.TempDir() + writeFiles(t, root, map[string]string{ + "go.work": "go 1.22\n\nuse ./a\nuse ./b\n" + tc.work, + "a/go.mod": "module example.com/a\n\ngo 1.22\n\nrequire example.com/x v1.0.0\n" + tc.modA, + "b/go.mod": "module example.com/b\n\ngo 1.22\n" + tc.modB, + "fork/go.mod": "module example.com/x\n\ngo 1.22\n", + "fork/fork.go": "package x\n", + }) + writeFiles(t, cache, mergeFiles( + cachedModule("example.com/workfork", "v1.0.0", "module example.com/x\n\ngo 1.22\n", map[string]string{"x.go": "package x\n"}), + cachedModule("example.com/modfork", "v1.0.0", "module example.com/x\n\ngo 1.22\n", map[string]string{"x.go": "package x\n"}), + )) + sources, err := Discover(DiscoverOptions{Dir: filepath.Join(root, "a"), Work: filepath.Join(root, "go.work"), ModuleCache: cache, LocalRoot: root}) + if tc.wantErr != "" { + assert.ErrorContains(t, err, tc.wantErr) + return + } + require.NoError(t, err) + x := sourceOf(t, sources, "example.com/x") + assert.Equal(t, tc.want, x.Origin) + assert.NotNil(t, x.FS) + }) + } +} + +func TestDiscoverReadsTheVendorDirectoryOfTheWorkspace(t *testing.T) { + cases := []struct { + name string + work string + modules string + vendored bool + }{ + {"go work vendor wrote it", "go 1.21", "## workspace\n# example.com/v v1.0.0\n## explicit; go 1.21\nexample.com/v\n", true}, + {"a vendor directory of one module", "go 1.22", "# example.com/v v1.0.0\n## explicit; go 1.21\nexample.com/v\n", false}, + {"a go.work before go 1.14", "go 1.13", "## workspace\n# example.com/v v1.0.0\nexample.com/v\n", false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + root, cache := t.TempDir(), t.TempDir() + writeFiles(t, root, map[string]string{ + "go.work": tc.work + "\n\nuse ./a\n", + "a/go.mod": "module example.com/a\n\ngo 1.21\n\nrequire example.com/v v1.0.0\n", + "vendor/modules.txt": tc.modules, + "vendor/example.com/v/v.go": "package v\n", + }) + writeFiles(t, cache, cachedModule("example.com/v", "v1.0.0", "module example.com/v\n\ngo 1.21\n", map[string]string{"c.go": "package v\n"})) + sources, err := Discover(DiscoverOptions{Dir: filepath.Join(root, "a"), Work: filepath.Join(root, "go.work"), ModuleCache: cache, LocalRoot: root}) + require.NoError(t, err) + v := sourceOf(t, sources, "example.com/v") + assert.Equal(t, "v1.0.0", v.Module.Version) + assert.Equal(t, tc.vendored, hasFile(t, v.FS, "v.go")) + assert.Equal(t, !tc.vendored, hasFile(t, v.FS, "c.go")) + }) + } +} + +func TestDiscoverReadsAReplacedModuleFromTheVendorDirectoryOfTheWorkspace(t *testing.T) { + root := t.TempDir() + writeFiles(t, root, map[string]string{ + "go.work": "go 1.22\n\nuse ./a\n", + "a/go.mod": "module example.com/a\n\ngo 1.22\n\nrequire example.com/x v1.0.0\n\nreplace example.com/x => ../fork\n", + // go work vendor writes the local path relative to go.work, and adds + // a line for the replace of each version. + "vendor/modules.txt": "## workspace\n# example.com/x v1.0.0 => ./fork\n## explicit; go 1.22\nexample.com/x\n# example.com/x => ./fork\n", + "vendor/example.com/x/x.go": "package x\n", + "fork/go.mod": "module example.com/x\n\ngo 1.22\n", + "fork/f.go": "package x\n", + }) + sources, err := Discover(DiscoverOptions{Dir: filepath.Join(root, "a"), Work: filepath.Join(root, "go.work"), LocalRoot: root}) + require.NoError(t, err) + x := sourceOf(t, sources, "example.com/x") + assert.Equal(t, "v1.0.0", x.Module.Version, "the wildcard line keeps the version") + assert.True(t, hasFile(t, x.FS, "x.go"), "the go command builds the vendor copy") +} + +func TestDiscoverSkipsTheVendorDirectoryOfAWorkspaceOutsideWorkspaceMode(t *testing.T) { + root, cache := t.TempDir(), t.TempDir() + writeFiles(t, root, map[string]string{ + "go.mod": "module example.com/a\n\ngo 1.22\n\nrequire example.com/v v1.0.0\n", + "vendor/modules.txt": "## workspace\n# example.com/v v1.0.0\nexample.com/v\n", + "vendor/example.com/v/v.go": "package v\n", + }) + writeFiles(t, cache, cachedModule("example.com/v", "v1.0.0", "module example.com/v\n", map[string]string{"c.go": "package v\n"})) + sources, err := Discover(DiscoverOptions{Dir: root, ModuleCache: cache}) + require.NoError(t, err) + assert.True(t, hasFile(t, sources[0].FS, "c.go")) +} + +func TestDiscoverRefusesAWorkspaceWhoseBuildListItCannotKnow(t *testing.T) { + cases := []struct { + name string + files map[string]string + dir string + wantErr string + }{ + { + name: "a go.mod that the graph needs is missing", + files: map[string]string{"go.work": "go 1.22\n\nuse ./a\n", "a/go.mod": "module example.com/a\n\ngo 1.22\n\nrequire example.com/gone v1.0.0\n"}, + wantErr: "no go.mod of example.com/gone@v1.0.0 is on the disk", + }, + { + name: "an exclude directive", + files: map[string]string{"go.work": "go 1.22\n\nuse ./a\n", "a/go.mod": "module example.com/a\n\ngo 1.22\n\nexclude example.com/x v1.0.0\n"}, + wantErr: "exclude directive", + }, + { + name: "a workspace that does not use the module", + files: map[string]string{"go.work": "go 1.22\n\nuse ./b\n", "a/go.mod": "module example.com/a\n\ngo 1.22\n", "b/go.mod": "module example.com/b\n\ngo 1.22\n"}, + wantErr: "does not use the module", + }, + { + name: "a module of the workspace with no go.mod", + files: map[string]string{"go.work": "go 1.22\n\nuse ./a\nuse ./b\n", "a/go.mod": "module example.com/a\n\ngo 1.22\n"}, + wantErr: "no go.mod inside the local root", + }, + { + name: "a go.work that does not parse", + files: map[string]string{"go.work": "go 1.22\n\nuse (\n", "a/go.mod": "module example.com/a\n\ngo 1.22\n"}, + wantErr: "parse go.work", + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + root := t.TempDir() + writeFiles(t, root, tc.files) + _, err := Discover(DiscoverOptions{Dir: filepath.Join(root, "a"), Work: filepath.Join(root, "go.work"), ModuleCache: t.TempDir(), LocalRoot: root}) + assert.ErrorContains(t, err, tc.wantErr) + }) + } +} + +func TestDiscoverReadsNoWorkspaceModuleOutsideTheLocalRoot(t *testing.T) { + root, outside := t.TempDir(), t.TempDir() + writeFiles(t, outside, map[string]string{"go.mod": "module example.com/b\n\ngo 1.22\n", "b.go": "package b\n"}) + writeFiles(t, root, map[string]string{ + "go.work": "go 1.22\n\nuse ./a\nuse " + filepath.ToSlash(outside) + "\n", + "a/go.mod": "module example.com/a\n\ngo 1.22\n", + }) + _, err := Discover(DiscoverOptions{Dir: filepath.Join(root, "a"), Work: filepath.Join(root, "go.work"), LocalRoot: root}) + assert.ErrorContains(t, err, "no go.mod inside the local root", "its requirements are part of the build list") +} + +func TestDiscoverTakesRelativePaths(t *testing.T) { + root := t.TempDir() + writeFiles(t, root, map[string]string{ + "ws/go.work": "go 1.22\n\nuse ./a\nuse ./b\n", + "ws/a/go.mod": "module example.com/a\n\ngo 1.22\n", + "ws/b/go.mod": "module example.com/b\n\ngo 1.22\n", + "ws/b/b.go": "package b\n", + "other/x.txt": "", + }) + t.Chdir(filepath.Join(root, "other")) + sources, err := Discover(DiscoverOptions{Dir: "../ws/a", Work: "../ws/go.work", LocalRoot: "../ws"}) + require.NoError(t, err) + assert.True(t, hasFile(t, sourceOf(t, sources, "example.com/b").FS, "b.go")) +}