From c00726c6325c86faa11ad9a972ccd7b4c33b70be Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 17:50:55 +0000 Subject: [PATCH 1/3] perf(indexer): give a cached plugin the call graph of the index The run builds the call graph of each file in the workers, for the index. A cached plugin that reads the call graph, as the file plugin of vet, built it again in Analyze. The extra plugins run on the goroutine of the run, so that second build is serial work. On gitea it took 8.9 s of the 21 s that the plugin of vet spent on that goroutine, and the goroutine set the wall time of the run. The worker now keeps the call graph of the index in its outcome. A plugin that implements the new GraphPlugin gets it through AnalyzeGraph. A file with no call graph, as one whose index failed, still goes to Analyze. The index is complete before the plugin runs, so the plugin cannot change the index. The callgraph plugin gets AnalyzeGraph too, so a caller with a graph runs its callback and its partial check without a second build. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PFQbqPqd9xn1fT6hBsyoUm Signed-off-by: Claude --- docs/projectgraph.md | 6 +++++ plugin/callgraph/context_test.go | 26 ++++++++++++++++++++ plugin/callgraph/plugin.go | 6 +++++ plugin/indexer/cache.go | 18 ++++++++++++++ plugin/indexer/cache_test.go | 42 +++++++++++++++++++++++++++++++- plugin/indexer/gounit.go | 2 +- plugin/indexer/indexer.go | 11 ++++++++- plugin/indexer/indexer_pool.go | 9 ++++--- plugin/indexer/plugin.go | 7 +++++- plugin/indexer/project.go | 15 ++++++------ 10 files changed, 128 insertions(+), 14 deletions(-) diff --git a/docs/projectgraph.md b/docs/projectgraph.md index 078744d5..9f13e94c 100644 --- a/docs/projectgraph.md +++ b/docs/projectgraph.md @@ -106,6 +106,12 @@ version. On the next run the store serves the artifact and the plugin gets it through `Restore` with no parse. A new plugin version misses every older artifact. `Prune` removes artifacts as it removes indexes. +A cached plugin that reads the call graph of a file implements +`indexer.GraphPlugin`. The run builds the call graph once for the index, and +gives it to `AnalyzeGraph` with the tree. The plugin does not build the graph +again. The extra plugins run on one goroutine, so a second build would slow +the whole run. + ## Link ```go diff --git a/plugin/callgraph/context_test.go b/plugin/callgraph/context_test.go index de732189..a7e7c7d2 100644 --- a/plugin/callgraph/context_test.go +++ b/plugin/callgraph/context_test.go @@ -119,3 +119,29 @@ func TestDoneContextStopsBuildCallGraphAtTheImports(t *testing.T) { _, err = Build(ctx, tree) assert.ErrorIs(t, err, context.Canceled) } + +func TestAnalyzeGraphFailsOnAPartialGraphAsAnalyzeTreeDoes(t *testing.T) { + python, err := lang.NewPythonLanguage() + require.NoError(t, err) + tree, err := parser.ParseBytes(t.Context(), python, []byte(deadlineSource), parser.WithFileName("calls.py")) + require.NoError(t, err) + defer tree.Close() + imports, err := python.Resolvers().ResolveImports(t.Context(), tree) + require.NoError(t, err) + treeData, err := tree.Data() + require.NoError(t, err) + partial := newCallGraph("calls.py", python.Meta().Code) + partial.build(&nodeDeadline{Context: t.Context(), graph: partial, nodes: 2 * ctxCheckInterval}, tree.Tree().RootNode(), imports, *treeData) + full, err := Build(t.Context(), tree) + require.NoError(t, err) + + var got []*CallGraph + pl := NewCallGraphPlugin(func(_ context.Context, cg *CallGraph) error { + got = append(got, cg) + return nil + }) + require.NoError(t, pl.AnalyzeGraph(t.Context(), full)) + err = pl.AnalyzeGraph(t.Context(), partial) + assert.ErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, []*CallGraph{full, partial}, got, "the callback gets the graph of the caller, the partial one too") +} diff --git a/plugin/callgraph/plugin.go b/plugin/callgraph/plugin.go index 8c8fb167..a97a6f83 100644 --- a/plugin/callgraph/plugin.go +++ b/plugin/callgraph/plugin.go @@ -37,7 +37,13 @@ func (p *callgraphPlugin) AnalyzeTree(ctx context.Context, tree core.ParseTree) if err != nil { return fmt.Errorf("failed to build call graph: %w", err) } + return p.AnalyzeGraph(ctx, cg) +} +// AnalyzeGraph runs the callback on a call graph that the caller built, as +// the indexer builds one for the index of each file. It fails on a partial +// graph, as AnalyzeTree does. +func (p *callgraphPlugin) AnalyzeGraph(ctx context.Context, cg *CallGraph) error { if err := p.callgraphCallback(ctx, cg); err != nil { return err } diff --git a/plugin/indexer/cache.go b/plugin/indexer/cache.go index a31a9fbd..ec6d29a9 100644 --- a/plugin/indexer/cache.go +++ b/plugin/indexer/cache.go @@ -6,6 +6,7 @@ import ( "github.com/safedep/code/core" "github.com/safedep/code/fileindex/store" + "github.com/safedep/code/plugin/callgraph" "github.com/safedep/dry/log" ) @@ -33,6 +34,23 @@ type CachedPlugin interface { Restore(ctx context.Context, file core.File, data []byte) error } +// GraphPlugin is a CachedPlugin that reads the call graph of a file. The +// run builds the call graph of each file once, for the index, and gives it +// to AnalyzeGraph, so the plugin does not build it again. A file with no +// call graph, as one whose index failed, goes to Analyze. +// +// The index of the file is complete before AnalyzeGraph runs, so the +// plugin cannot change the index through cg. The graph of a file that the +// deadline cut is partial, as the graph that Analyze would build is. +type GraphPlugin interface { + CachedPlugin + + // AnalyzeGraph runs the plugin on the tree and the call graph of a + // file, and returns the artifact to store. It must return the artifact + // that Analyze returns for the tree. + AnalyzeGraph(ctx context.Context, file core.File, tree core.ParseTree, cg *callgraph.CallGraph) ([]byte, error) +} + // artifactKey returns the key of the artifact of a cached plugin for a // file. func artifactKey(p pending, pl CachedPlugin) store.ArtifactKey { diff --git a/plugin/indexer/cache_test.go b/plugin/indexer/cache_test.go index 9273e4db..456f45a4 100644 --- a/plugin/indexer/cache_test.go +++ b/plugin/indexer/cache_test.go @@ -10,6 +10,7 @@ import ( "github.com/safedep/code/core" "github.com/safedep/code/fileindex" "github.com/safedep/code/fileindex/store/memstore" + "github.com/safedep/code/plugin/callgraph" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -34,11 +35,15 @@ func (p *countingPlugin) SupportedLanguages() []core.LanguageCode { } func (p *countingPlugin) Analyze(_ context.Context, file core.File, tree core.ParseTree) ([]byte, error) { + p.analyzed = append(p.analyzed, file.Name()) + return lineCount(tree) +} + +func lineCount(tree core.ParseTree) ([]byte, error) { data, err := tree.Data() if err != nil { return nil, err } - p.analyzed = append(p.analyzed, file.Name()) return []byte(strconv.Itoa(strings.Count(string(*data), "\n"))), nil } @@ -131,3 +136,38 @@ func TestIndexerServesACachedPluginFromTheStore(t *testing.T) { assert.Equal(t, []string{"main.js"}, broken.analyzed) }) } + +// graphCountingPlugin is a countingPlugin that reads the call graph of the +// index. It records the file of each graph that it got. +type graphCountingPlugin struct { + *countingPlugin + graphs []string +} + +func (p *graphCountingPlugin) AnalyzeGraph(_ context.Context, file core.File, tree core.ParseTree, cg *callgraph.CallGraph) ([]byte, error) { + if cg.FileName != file.Name() || len(cg.Nodes) == 0 { + return nil, errors.New("not the call graph of the file") + } + p.graphs = append(p.graphs, file.Name()) + return lineCount(tree) +} + +func TestIndexerGivesAGraphPluginTheCallGraphOfTheIndex(t *testing.T) { + dir := t.TempDir() + writeFiles(t, dir, projectFiles) + fsys := appFileSystem(t, dir) + s := memstore.New() + + pl := &graphCountingPlugin{countingPlugin: newCountingPlugin("1")} + result, err := runIndexer(t, context.Background(), fsys, Options{Languages: projectLanguages(t), Store: s, Plugins: []core.Plugin{pl}}) + require.NoError(t, err) + assert.Empty(t, result.Report.Failed) + assert.Equal(t, ecmascriptNames, pl.graphs) + assert.Empty(t, pl.analyzed, "the run builds no second call graph through Analyze") + + warm := &graphCountingPlugin{countingPlugin: newCountingPlugin("1")} + _, err = runIndexer(t, context.Background(), fsys, Options{Languages: projectLanguages(t), Store: s, Plugins: []core.Plugin{warm}}) + require.NoError(t, err) + assert.Equal(t, ecmascriptNames, warm.restored, "the artifact of AnalyzeGraph serves the next run") + assert.Equal(t, map[string]string{"app/util.ts": "1", "lib/helper.js": "1", "main.js": "2"}, warm.lines) +} diff --git a/plugin/indexer/gounit.go b/plugin/indexer/gounit.go index 421c8150..562b80a6 100644 --- a/plugin/indexer/gounit.go +++ b/plugin/indexer/gounit.go @@ -169,7 +169,7 @@ func skeletonIndex(ctx context.Context, p contentParser, file pending, tree core return nil, err } defer skeleton.Close() - out, err := project(ctx, skeleton) + out, _, err := project(ctx, skeleton) if err != nil { return nil, err } diff --git a/plugin/indexer/indexer.go b/plugin/indexer/indexer.go index 2c8e4a7e..038428fe 100644 --- a/plugin/indexer/indexer.go +++ b/plugin/indexer/indexer.go @@ -433,7 +433,7 @@ func (r *run) collect(ctx context.Context, p pending, o outcome) (*fileindex.Ind if p.restored[pl.Name()] { continue } - data, err := cached.Analyze(ctx, p.file, o.tree) + data, err := analyzeCached(ctx, cached, p.file, o) if err != nil { report.Failed = append(report.Failed, failure(p.file, pl, err)) } else { @@ -454,6 +454,15 @@ func (r *run) collect(ctx context.Context, p pending, o outcome) (*fileindex.Ind return o.index, artifacts } +// analyzeCached runs a cached plugin on the tree of a file, and gives a +// GraphPlugin the call graph of the index. +func analyzeCached(ctx context.Context, pl CachedPlugin, file core.File, o outcome) ([]byte, error) { + if gp, ok := pl.(GraphPlugin); ok && o.graph != nil { + return gp.AnalyzeGraph(ctx, file, o.tree, o.graph) + } + return pl.Analyze(ctx, file, o.tree) +} + // failure is the report entry of a plugin that failed on a file. func failure(file core.File, pl core.Plugin, err error) plugin.FileError { return plugin.FileError{File: file.Name(), Stage: core.Stage(pl.Name()), Err: err} diff --git a/plugin/indexer/indexer_pool.go b/plugin/indexer/indexer_pool.go index 40797ffb..33d0b70d 100644 --- a/plugin/indexer/indexer_pool.go +++ b/plugin/indexer/indexer_pool.go @@ -9,6 +9,7 @@ import ( "github.com/safedep/code/fileindex" "github.com/safedep/code/parser" "github.com/safedep/code/plugin" + "github.com/safedep/code/plugin/callgraph" ) // contentParser parses the bytes of a file that the run already read. @@ -37,6 +38,8 @@ type job struct { type outcome struct { tree core.ParseTree index *fileindex.Index + // graph is the call graph that the index plugin built, or nil. + graph *callgraph.CallGraph // indexErr is the failure of the index plugin, at the stage stage. indexErr error @@ -114,10 +117,10 @@ func analyze(ctx context.Context, p contentParser, file pending) outcome { } var index *fileindex.Index - indexPlugin := NewFileIndexPlugin(func(_ context.Context, ix *fileindex.Index) error { + indexPlugin := &fileIndexPlugin{callback: func(_ context.Context, ix *fileindex.Index) error { index = ix return nil - }) + }} if !slices.Contains(indexPlugin.SupportedLanguages(), file.language.Meta().Code) { return outcome{tree: tree} } @@ -125,5 +128,5 @@ func analyze(ctx context.Context, p contentParser, file pending) outcome { if err == nil { index, err = skeletonIndex(ctx, p, file, tree, index) } - return outcome{tree: tree, index: index, indexErr: err, stage: core.Stage(indexPlugin.Name())} + return outcome{tree: tree, index: index, graph: indexPlugin.graph, indexErr: err, stage: core.Stage(indexPlugin.Name())} } diff --git a/plugin/indexer/plugin.go b/plugin/indexer/plugin.go index 54406580..ea202eaf 100644 --- a/plugin/indexer/plugin.go +++ b/plugin/indexer/plugin.go @@ -17,6 +17,10 @@ import ( type fileIndexPlugin struct { callback core.PluginCallback[*fileindex.Index] + + // graph is the call graph of the last tree, which the run gives to a + // GraphPlugin. + graph *callgraph.CallGraph } var _ core.TreePlugin = (*fileIndexPlugin)(nil) @@ -37,10 +41,11 @@ func (p *fileIndexPlugin) SupportedLanguages() []core.LanguageCode { } func (p *fileIndexPlugin) AnalyzeTree(ctx context.Context, tree core.ParseTree) error { - ix, err := project(ctx, tree) + ix, cg, err := project(ctx, tree) if err != nil { return fmt.Errorf("failed to index file: %w", err) } + p.graph = cg if err := p.callback(ctx, ix); err != nil { return err diff --git a/plugin/indexer/project.go b/plugin/indexer/project.go index 790406a5..874db990 100644 --- a/plugin/indexer/project.go +++ b/plugin/indexer/project.go @@ -53,19 +53,20 @@ type projection struct { exportStatements []*sitter.Node } -// project builds the call graph of tree and projects it into an index. -func project(ctx context.Context, tree core.ParseTree) (*fileindex.Index, error) { +// project builds the call graph of tree and projects it into an index. It +// returns the call graph too, so the run can give it to a GraphPlugin. +func project(ctx context.Context, tree core.ParseTree) (*fileindex.Index, *callgraph.CallGraph, error) { language, err := tree.Language() if err != nil { - return nil, fmt.Errorf("failed to get language: %w", err) + return nil, nil, fmt.Errorf("failed to get language: %w", err) } data, err := tree.Data() if err != nil { - return nil, fmt.Errorf("failed to get tree data: %w", err) + return nil, nil, fmt.Errorf("failed to get tree data: %w", err) } cg, err := callgraph.Build(ctx, tree) if err != nil { - return nil, err + return nil, nil, err } p := &projection{ @@ -106,7 +107,7 @@ func project(ctx context.Context, tree core.ParseTree) (*fileindex.Index, error) p.addUnresolved() p.run(rules.prune) if err := p.addClasses(ctx); err != nil { - return nil, err + return nil, nil, err } for _, def := range p.defs { def.Exported = p.exported(def) @@ -114,7 +115,7 @@ func project(ctx context.Context, tree core.ParseTree) (*fileindex.Index, error) } p.ix.Sort() - return p.ix, nil + return p.ix, cg, nil } // run runs a pass of the rules of the language, when the language has it. From f314b3f840a0f5d0232f0f9ce976625b44d1a14a Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 18:07:30 +0000 Subject: [PATCH 2/3] perf(indexer): keep the call graph only for a GraphPlugin A review found three gaps in the shared call graph. Each outcome kept its call graph, also in a run with no GraphPlugin. The pool keeps two outcomes per worker in flight, so a run with no plugin held graphs that it never read. The pool now keeps a graph only when the run has a GraphPlugin. The GraphPlugin doc said that a file whose index failed goes to Analyze. A file that the deadline cut has a failed index and a partial graph, and it goes to AnalyzeGraph. The doc now says that a file whose call graph did not build goes to Analyze. It also says that a file that the workers finished before the deadline gets its full graph. The run test checked only the file name of the graph. It now compares the graph with the one that callgraph.Build gives for the tree. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PFQbqPqd9xn1fT6hBsyoUm Signed-off-by: Claude --- docs/projectgraph.md | 3 ++- plugin/indexer/cache.go | 8 ++++--- plugin/indexer/cache_test.go | 43 ++++++++++++++++++++++++++++++++-- plugin/indexer/indexer.go | 6 ++++- plugin/indexer/indexer_pool.go | 15 ++++++++---- plugin/indexer/plugin.go | 2 +- 6 files changed, 65 insertions(+), 12 deletions(-) diff --git a/docs/projectgraph.md b/docs/projectgraph.md index 9f13e94c..3c37e618 100644 --- a/docs/projectgraph.md +++ b/docs/projectgraph.md @@ -110,7 +110,8 @@ A cached plugin that reads the call graph of a file implements `indexer.GraphPlugin`. The run builds the call graph once for the index, and gives it to `AnalyzeGraph` with the tree. The plugin does not build the graph again. The extra plugins run on one goroutine, so a second build would slow -the whole run. +the whole run. A file that the workers finished before the deadline passed +gets its full graph. With a done context, `Analyze` would build none. ## Link diff --git a/plugin/indexer/cache.go b/plugin/indexer/cache.go index ec6d29a9..569c6708 100644 --- a/plugin/indexer/cache.go +++ b/plugin/indexer/cache.go @@ -36,12 +36,14 @@ type CachedPlugin interface { // GraphPlugin is a CachedPlugin that reads the call graph of a file. The // run builds the call graph of each file once, for the index, and gives it -// to AnalyzeGraph, so the plugin does not build it again. A file with no -// call graph, as one whose index failed, goes to Analyze. +// to AnalyzeGraph, so the plugin does not build it again. A file whose call +// graph did not build goes to Analyze. // // The index of the file is complete before AnalyzeGraph runs, so the // plugin cannot change the index through cg. The graph of a file that the -// deadline cut is partial, as the graph that Analyze would build is. +// deadline cut is partial. A file that the workers finished before the +// deadline passed gets its full graph, where Analyze would have built +// none with a done ctx. type GraphPlugin interface { CachedPlugin diff --git a/plugin/indexer/cache_test.go b/plugin/indexer/cache_test.go index 456f45a4..b62004f3 100644 --- a/plugin/indexer/cache_test.go +++ b/plugin/indexer/cache_test.go @@ -1,6 +1,7 @@ package indexer import ( + "bytes" "context" "errors" "strconv" @@ -10,6 +11,7 @@ import ( "github.com/safedep/code/core" "github.com/safedep/code/fileindex" "github.com/safedep/code/fileindex/store/memstore" + "github.com/safedep/code/parser" "github.com/safedep/code/plugin/callgraph" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -144,8 +146,19 @@ type graphCountingPlugin struct { graphs []string } -func (p *graphCountingPlugin) AnalyzeGraph(_ context.Context, file core.File, tree core.ParseTree, cg *callgraph.CallGraph) ([]byte, error) { - if cg.FileName != file.Name() || len(cg.Nodes) == 0 { +func (p *graphCountingPlugin) AnalyzeGraph(ctx context.Context, file core.File, tree core.ParseTree, cg *callgraph.CallGraph) ([]byte, error) { + built, err := callgraph.Build(ctx, tree) + if err != nil { + return nil, err + } + var got, want bytes.Buffer + if err := cg.Dump(&got); err != nil { + return nil, err + } + if err := built.Dump(&want); err != nil { + return nil, err + } + if cg.FileName != file.Name() || len(cg.Nodes) == 0 || got.String() != want.String() { return nil, errors.New("not the call graph of the file") } p.graphs = append(p.graphs, file.Name()) @@ -171,3 +184,29 @@ func TestIndexerGivesAGraphPluginTheCallGraphOfTheIndex(t *testing.T) { assert.Equal(t, ecmascriptNames, warm.restored, "the artifact of AnalyzeGraph serves the next run") assert.Equal(t, map[string]string{"app/util.ts": "1", "lib/helper.js": "1", "main.js": "2"}, warm.lines) } + +func TestAnalyzeKeepsTheCallGraphOnlyForAGraphPlugin(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + writeFiles(t, dir, map[string]string{"main.js": projectFiles["main.js"]}) + langs := projectLanguages(t) + files, _, err := enumerate(ctx, appFileSystem(t, dir), Options{Languages: langs}) + require.NoError(t, err) + require.Len(t, files, 1) + byExtension, err := parser.LanguagesByExtension(langs) + require.NoError(t, err) + data, err := read(files[0], parser.DefaultMaxBytes) + require.NoError(t, err) + p, err := parser.NewParser(langs) + require.NoError(t, err) + defer p.Close() + + file := pending{file: files[0], language: byExtension[".js"], data: data} + for _, keep := range []bool{false, true} { + o := analyze(ctx, p, file, keep) + require.NoError(t, o.err) + require.NotNil(t, o.index) + o.tree.Close() + assert.Equal(t, keep, o.graph != nil, "a run with no GraphPlugin drops the graph after the index") + } +} diff --git a/plugin/indexer/indexer.go b/plugin/indexer/indexer.go index 038428fe..8a9acad4 100644 --- a/plugin/indexer/indexer.go +++ b/plugin/indexer/indexer.go @@ -207,7 +207,11 @@ func (ix *Indexer) Run(ctx context.Context, fsys core.ImportAwareFileSystem) (*R }, strs: interner{}, } - if r.pool, err = newPool(ctx, ix.opts.Languages, ix.opts.MaxFileBytes, ix.opts.Workers); err != nil { + keepGraphs := slices.ContainsFunc(ix.opts.Plugins, func(pl core.Plugin) bool { + _, ok := pl.(GraphPlugin) + return ok + }) + if r.pool, err = newPool(ctx, ix.opts.Languages, ix.opts.MaxFileBytes, ix.opts.Workers, keepGraphs); err != nil { return nil, err } defer r.pool.close() diff --git a/plugin/indexer/indexer_pool.go b/plugin/indexer/indexer_pool.go index 33d0b70d..86756c50 100644 --- a/plugin/indexer/indexer_pool.go +++ b/plugin/indexer/indexer_pool.go @@ -48,7 +48,10 @@ type outcome struct { err error } -func newPool(ctx context.Context, languages []core.Language, maxBytes, size int) (*pool, error) { +// newPool starts the workers. keepGraphs keeps the call graph of each file +// in its outcome for a GraphPlugin. A run with none drops each graph after +// the index, so the files in flight do not hold it. +func newPool(ctx context.Context, languages []core.Language, maxBytes, size int, keepGraphs bool) (*pool, error) { parsers := make([]contentParser, 0, max(size, 1)) for range cap(parsers) { p, err := parser.NewParser(languages, parser.WithMaxBytes(maxBytes)) @@ -68,7 +71,7 @@ func newPool(ctx context.Context, languages []core.Language, maxBytes, size int) defer pl.wg.Done() defer p.Close() for j := range pl.jobs { - j.out <- analyze(ctx, p, j.file) + j.out <- analyze(ctx, p, j.file, keepGraphs) } }() } @@ -107,7 +110,7 @@ func (pl *pool) each(ctx context.Context, files []pending, fn func(pending, outc } // analyze parses one file and runs the index plugin on its tree. -func analyze(ctx context.Context, p contentParser, file pending) outcome { +func analyze(ctx context.Context, p contentParser, file pending, keepGraph bool) outcome { if ctx.Err() != nil { return outcome{} } @@ -128,5 +131,9 @@ func analyze(ctx context.Context, p contentParser, file pending) outcome { if err == nil { index, err = skeletonIndex(ctx, p, file, tree, index) } - return outcome{tree: tree, index: index, graph: indexPlugin.graph, indexErr: err, stage: core.Stage(indexPlugin.Name())} + o := outcome{tree: tree, index: index, indexErr: err, stage: core.Stage(indexPlugin.Name())} + if keepGraph { + o.graph = indexPlugin.graph + } + return o } diff --git a/plugin/indexer/plugin.go b/plugin/indexer/plugin.go index ea202eaf..50792ae8 100644 --- a/plugin/indexer/plugin.go +++ b/plugin/indexer/plugin.go @@ -18,7 +18,7 @@ import ( type fileIndexPlugin struct { callback core.PluginCallback[*fileindex.Index] - // graph is the call graph of the last tree, which the run gives to a + // graph is the call graph of the tree, which the run gives to a // GraphPlugin. graph *callgraph.CallGraph } From 2e58e5d97bed1d513f5bd30c1d17d12d5078e2f6 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 18:34:37 +0000 Subject: [PATCH 3/3] perf(callgraph): log no debug line for each call or definition The call graph build wrote a debug line for each call, each definition and each resolved call or constructor. A debug index run of gitea wrote 185,860 lines, and 160,836 of them were "Go call". The log took 2.3 s of a 15.8 s run. The lines named what the graph holds, so they did not help to find a fault. dry/log has no trace level, so the build no longer writes them. It keeps the line for each file and the lines that report a problem, as a node with no name or a limit that the build hit. The debug run of gitea writes 5,566 lines and takes 13.8 s. A new test builds a file with one call and with 50 calls in Go, JavaScript, Python and Java, and expects the same number of log lines. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PFQbqPqd9xn1fT6hBsyoUm Signed-off-by: Claude --- plugin/callgraph/context_test.go | 44 ++++++++++++++++++++++++++++++++ plugin/callgraph/golang.go | 5 +--- plugin/callgraph/java.go | 4 --- plugin/callgraph/javascript.go | 7 +---- plugin/callgraph/processors.go | 3 --- plugin/callgraph/python.go | 2 -- 6 files changed, 46 insertions(+), 19 deletions(-) diff --git a/plugin/callgraph/context_test.go b/plugin/callgraph/context_test.go index a7e7c7d2..373ffd45 100644 --- a/plugin/callgraph/context_test.go +++ b/plugin/callgraph/context_test.go @@ -12,6 +12,7 @@ import ( "github.com/safedep/code/parser" "github.com/safedep/code/pkg/test" "github.com/safedep/code/plugin" + "github.com/safedep/dry/log" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -145,3 +146,46 @@ func TestAnalyzeGraphFailsOnAPartialGraphAsAnalyzeTreeDoes(t *testing.T) { assert.ErrorIs(t, err, context.DeadlineExceeded) assert.Equal(t, []*CallGraph{full, partial}, got, "the callback gets the graph of the caller, the partial one too") } + +// A debug line for each call made the debug log of a large project run to +// hundreds of thousands of lines. The log of a build must not grow with +// the number of calls. +func TestBuildLogsNoLineForEachCall(t *testing.T) { + for _, tc := range []struct { + name string + language func() (core.Language, error) + file string + source func(calls int) string + }{ + {"go", func() (core.Language, error) { return lang.NewGoLanguage() }, "main.go", func(n int) string { + return "package main\n\nfunc f() {}\n\nfunc g() {}\n\nfunc main() {\n" + strings.Repeat("\tf()\n", n) + "}\n" + }}, + {"javascript", func() (core.Language, error) { return lang.NewJavascriptLanguage() }, "main.js", func(n int) string { + return "function f() {}\nconst g = () => {};\n" + strings.Repeat("f();\nnew Map();\n", n) + }}, + {"python", func() (core.Language, error) { return lang.NewPythonLanguage() }, "main.py", func(n int) string { + return "class C:\n pass\n\ndef f():\n pass\n\n" + strings.Repeat("f()\nC()\n", n) + }}, + {"java", func() (core.Language, error) { return lang.NewJavaLanguage() }, "Main.java", func(n int) string { + return "class Main {\n void f() {}\n void m() {\n" + strings.Repeat(" f();\n new Main();\n this.f();\n", n) + " }\n}\n" + }}, + } { + t.Run(tc.name, func(t *testing.T) { + language, err := tc.language() + require.NoError(t, err) + lines := func(calls int) int { + tree, err := parser.ParseBytes(t.Context(), language, []byte(tc.source(calls)), parser.WithFileName(tc.file)) + require.NoError(t, err) + defer tree.Close() + var buf strings.Builder + restore := log.SwapGlobalForTest(&buf) + defer restore() + cg, err := Build(t.Context(), tree) + require.NoError(t, err) + require.NotEmpty(t, cg.Nodes) + return strings.Count(buf.String(), "\n") + } + assert.Equal(t, lines(1), lines(50)) + }) + } +} diff --git a/plugin/callgraph/golang.go b/plugin/callgraph/golang.go index 70cc5e30..4d04b122 100644 --- a/plugin/callgraph/golang.go +++ b/plugin/callgraph/golang.go @@ -106,7 +106,6 @@ func goCallExpressionProcessor(callNode *sitter.Node, treeData []byte, currentNa ref := callGraph.reference(callee, CallCall, functionNode, callArguments) ref.Unbound = unbound callGraph.addCall(currentNamespace, nil, ref) - log.Debugf("Go call: %s -> %s", currentNamespace, callee) } return result @@ -279,9 +278,7 @@ func goMethodDeclarationProcessor(methodDefNode *sitter.Node, treeData []byte, c methodNamespace = currentNamespace + namespaceSeparator + receiverType + namespaceSeparator + methodName } - if callGraph.addDefinition(methodNamespace, methodDefNode) { - log.Debugf("Register Go method definition for %s on %s - %s", methodName, receiverType, methodNamespace) - } + callGraph.addDefinition(methodNamespace, methodDefNode) return processFunctionBody(methodDefNode, treeData, methodNamespace, callGraph, metadata) } diff --git a/plugin/callgraph/java.go b/plugin/callgraph/java.go index 53c832a1..e55ec0d6 100644 --- a/plugin/callgraph/java.go +++ b/plugin/callgraph/java.go @@ -222,8 +222,6 @@ func objectCreationExpressionProcessor(objectCreationNode *sitter.Node, treeData classNode, classResolvedBySearch := searchSymbolInScopeChain(constructedClassName, currentNamespace, callGraph) if classResolvedBySearch { - log.Debugf("Constructor %s searched (direct) & resolved to %s\n", constructedClassName, classNode.Namespace) - callGraph.addEdge(currentNamespace, CallNew, constructedClassNode, classNode.Namespace, nil, argsResult) markClassAssignment(classNode) } else if constructedClassNode.Type() == "scoped_type_identifier" { @@ -289,7 +287,6 @@ func methodInvocationProcessor(methodInvocationNode *sitter.Node, treeData []byt functionAssignmentNode, functionResolvedBySearch := searchSymbolInScopeChain(methodName, currentNamespace, callGraph) if functionResolvedBySearch { - log.Debugf("Method invocation %s searched (direct) & resolved to %s\n", methodName, functionAssignmentNode.Namespace) callGraph.addEdge( currentNamespace, CallCall, methodInvocationNode, functionAssignmentNode.Namespace, nil, @@ -341,7 +338,6 @@ func resolveQualifiedInvocation(invocation, object *sitter.Node, methodName stri if !chained { callee = receiver + namespaceSeparator + methodName } - log.Debugf("Method invocation %s searched (object qualified) & resolved to - %s\n", methodName, callee) callGraph.addEdge(currentNamespace, CallCall, invocation, callee, nil, args) } } diff --git a/plugin/callgraph/javascript.go b/plugin/callgraph/javascript.go index fc5db9b2..5e634aed 100644 --- a/plugin/callgraph/javascript.go +++ b/plugin/callgraph/javascript.go @@ -209,7 +209,6 @@ func jsCallExpressionProcessor(callNode *sitter.Node, treeData []byte, currentNa ref := callGraph.reference(callee, kind, functionNode, callArguments) ref.Unbound = unbound callGraph.addCall(currentNamespace, nil, ref) - log.Debugf("JS call: %s -> %s", currentNamespace, callee) // The file cannot tell a factory of a dependency from a function of // it. The value of the call is the callee, as for new X(), so // app.get() in const app = express() keeps the import, as Python @@ -463,9 +462,7 @@ func arrowFunctionProcessor(arrowNode *sitter.Node, treeData []byte, currentName if functionName != "" { arrowFunctionNamespace = currentNamespace + namespaceSeparator + functionName - if callGraph.addDefinition(arrowFunctionNamespace, arrowNode) { - log.Debugf("Register arrow function definition for %s - %s", functionName, arrowFunctionNamespace) - } + callGraph.addDefinition(arrowFunctionNamespace, arrowNode) } results.addResults(processFunctionBody(arrowNode, treeData, arrowFunctionNamespace, callGraph, metadata)) @@ -652,8 +649,6 @@ func jsNewExpressionProcessor(newNode *sitter.Node, treeData []byte, currentName } for _, constructor := range constructors { - log.Debugf("JS constructor resolved to %s", constructor) - ref := callGraph.reference(constructor, CallNew, constructorNode, callArguments) ref.Unbound = unbound callGraph.addCall(currentNamespace, nil, ref) diff --git a/plugin/callgraph/processors.go b/plugin/callgraph/processors.go index 1c8eb701..90955db3 100644 --- a/plugin/callgraph/processors.go +++ b/plugin/callgraph/processors.go @@ -217,7 +217,6 @@ func defineClass(classDefNode *sitter.Node, treeData []byte, classNamespace stri metadata.insideClass = true processChildren(classBody, treeData, classNamespace, callGraph, metadata) - log.Debugf("Register class definition %s", classNamespace) } // functionDefinitionProcessor processes a function or a method in its own @@ -245,8 +244,6 @@ func functionDefinitionProcessor(functionDefNode *sitter.Node, treeData []byte, } if callGraph.addDefinition(functionNamespace, functionDefNode) { - log.Debugf("Register function definition for %s - %s", funcName, functionNamespace) - if metadata.insideClass { if keyword := profile.instanceKeyword; keyword != "" { instanceNamespace := currentNamespace + namespaceSeparator + keyword + namespaceSeparator + funcName diff --git a/plugin/callgraph/python.go b/plugin/callgraph/python.go index 1a29f8a2..c857981c 100644 --- a/plugin/callgraph/python.go +++ b/plugin/callgraph/python.go @@ -167,7 +167,6 @@ func functionCallProcessor(functionCallNode *sitter.Node, argumentsNode *sitter. case classAssignmentNode == nil: case callGraph.classConstructors[classAssignmentNode.Namespace]: result.ImmediateAssignments = append(result.ImmediateAssignments, classAssignmentNode) - log.Debugf("Class constructed - %s in fncall for %s\n", classAssignmentNode.Namespace, functionName) case callGraph.imported(classAssignmentNode.Namespace) && !callGraph.isPythonDynamicImport(classAssignmentNode.Namespace): // The file cannot tell an imported class from an imported // function. The value of the call is the callee, as for new X() @@ -196,7 +195,6 @@ func functionCallProcessor(functionCallNode *sitter.Node, argumentsNode *sitter. switch function.Type() { case "identifier": if assignment, found := searchSymbolInScopeChain(functionName, currentNamespace, callGraph); found { - log.Debugf("Call %s searched (direct) & resolved to %s\n", functionName, assignment.Namespace) addEdge(assignment.Namespace, nil) markClassAssignment(assignment) return result