diff --git a/CLAUDE.md b/CLAUDE.md index 24a3b7d..bb3a48c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -14,7 +14,7 @@ A spec flows through three stages, one per `internal/` package: 1. **parse** (`internal/parser`): load and validate the spec with libopenapi, resolve `$ref`s. 2. **analyze** (`internal/analyzer`): lower the spec into the intermediate representation (`internal/ir`): operations, params, types, pagination. This is where naming, disambiguation, and pagination detection happen. -3. **generate** (`internal/generator`): render the IR through `text/template` files (`internal/templates/*.tmpl`, embedded via `embed.go`) and post-process each file with `goimports` (`WriteFiles`). +3. **generate** (`internal/generator`): render the IR through `text/template` files (`internal/templates/*.tmpl`, embedded via `embed.go`) then `WriteFiles` fixes each file's imports against a fixed stdlib table and gofmts it (large files print in parallel chunks). Supporting packages: `internal/naming` (OpenAPI identifier → Go identifier), `cmd/` (the Cobra CLI). diff --git a/go.mod b/go.mod index 8800a4e..959fc6f 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/giraffesyo/openapi-go-naming v0.1.1 github.com/pb33f/libopenapi v0.40.1 github.com/spf13/cobra v1.10.2 - golang.org/x/tools v0.50.0 + go.yaml.in/yaml/v4 v4.0.0-rc.6 ) require ( @@ -16,7 +16,5 @@ require ( github.com/pb33f/jsonpath v0.8.3 // indirect github.com/pb33f/ordered-map/v2 v2.3.1 // indirect github.com/spf13/pflag v1.0.9 // indirect - go.yaml.in/yaml/v4 v4.0.0-rc.6 // indirect - golang.org/x/mod v0.41.0 // indirect golang.org/x/sync v0.23.0 // indirect ) diff --git a/go.sum b/go.sum index e2a7c46..459b784 100644 --- a/go.sum +++ b/go.sum @@ -7,8 +7,6 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/giraffesyo/openapi-go-naming v0.1.1 h1:5/kK3FAIPtpEpmwvPk/moYVrdlznzY9s7015nBYMsZY= github.com/giraffesyo/openapi-go-naming v0.1.1/go.mod h1:20Q7TQ9sJpWQ5aUvmMdNlp70IRpl3JlLp6rH7dBD2ns= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/pb33f/jsonpath v0.8.3 h1:gdOUYn31vic8iRSSR4I2SEv7UQf2WxLeDcUnDPpGZNE= @@ -31,12 +29,8 @@ github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= go.yaml.in/yaml/v4 v4.0.0-rc.6 h1:1h7H1ohdUh93/FyE4YaDa1Zh64K6VVbjF4K6WUxMtH4= go.yaml.in/yaml/v4 v4.0.0-rc.6/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0= -golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c= -golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o= golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk= golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= -golang.org/x/tools v0.50.0 h1:c2ifzfcuY7L90lZ2aKd8S4K2NpASF08SZx9ZuJkHmSU= -golang.org/x/tools v0.50.0/go.mod h1:7ulVMw3831Mwi5EZD6RomGyffr4VFjuNYXf2BbCEAV0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/generator/e2e_combinations_test.go b/internal/generator/e2e_combinations_test.go index 3ad7555..099c4fe 100644 --- a/internal/generator/e2e_combinations_test.go +++ b/internal/generator/e2e_combinations_test.go @@ -96,7 +96,7 @@ func TestE2E_Combinations(t *testing.T) { // containsCollapsed reports whether haystack holds needle once the runs of // whitespace in both are flattened. The generated files reach this test before -// goimports aligns them, so an assertion written the way the output looks would +// gofmt aligns them, so an assertion written the way the output looks would // depend on alignment that has not happened yet. func containsCollapsed(haystack, needle string) bool { return strings.Contains(strings.Join(strings.Fields(haystack), " "), strings.Join(strings.Fields(needle), " ")) diff --git a/internal/generator/generator.go b/internal/generator/generator.go index f0e7c4d..513b391 100644 --- a/internal/generator/generator.go +++ b/internal/generator/generator.go @@ -2,7 +2,9 @@ package generator import ( "bytes" + "errors" "fmt" + "sync" "text/template" "github.com/parallelworks/openapi-client-generator/internal/ir" @@ -56,13 +58,22 @@ func (g *Generator) Generate() ([]GeneratedFile, error) { {"streaming.go.tmpl", "streaming.go"}, } - var files []GeneratedFile - for _, spec := range specs { - var buf bytes.Buffer - if err := g.templates.ExecuteTemplate(&buf, spec.tmplName, g.pkg); err != nil { - return nil, fmt.Errorf("executing template %s: %w", spec.tmplName, err) - } - files = append(files, GeneratedFile{Name: spec.fileName, Content: buf.Bytes()}) + files := make([]GeneratedFile, len(specs)) + errs := make([]error, len(specs)) + var wg sync.WaitGroup + for i, spec := range specs { + wg.Go(func() { + var buf bytes.Buffer + if err := g.templates.ExecuteTemplate(&buf, spec.tmplName, g.pkg); err != nil { + errs[i] = fmt.Errorf("executing template %s: %w", spec.tmplName, err) + return + } + files[i] = GeneratedFile{Name: spec.fileName, Content: buf.Bytes()} + }) + } + wg.Wait() + if err := errors.Join(errs...); err != nil { + return nil, err } return files, nil } diff --git a/internal/generator/generator_test.go b/internal/generator/generator_test.go index 27c8cfe..2980a6d 100644 --- a/internal/generator/generator_test.go +++ b/internal/generator/generator_test.go @@ -1,11 +1,12 @@ package generator import ( + "go/parser" + "go/token" "strings" "testing" "github.com/parallelworks/openapi-client-generator/internal/ir" - "golang.org/x/tools/imports" ) func TestGenerate_StructAndEnum(t *testing.T) { @@ -127,7 +128,7 @@ func TestGenerate_StructAndEnum(t *testing.T) { } } -func TestGenerate_GoimportsSucceeds(t *testing.T) { +func TestGenerate_OutputParses(t *testing.T) { pkg := &ir.Package{ Name: "testpkg", Types: []*ir.TypeDef{ @@ -160,14 +161,8 @@ func TestGenerate_GoimportsSucceeds(t *testing.T) { t.Fatal("expected at least 1 generated file") } - // Verify goimports can process the output without error - _, err = imports.Process(files[0].Name, files[0].Content, &imports.Options{ - Comments: true, - TabIndent: true, - TabWidth: 8, - }) - if err != nil { - t.Errorf("goimports failed on generated output: %v\n\nRaw output:\n%s", err, string(files[0].Content)) + if _, err := parser.ParseFile(token.NewFileSet(), files[0].Name, files[0].Content, parser.ParseComments); err != nil { + t.Errorf("generated output does not parse: %v\n\nRaw output:\n%s", err, string(files[0].Content)) } } diff --git a/internal/generator/writer.go b/internal/generator/writer.go index 626f32c..158e052 100644 --- a/internal/generator/writer.go +++ b/internal/generator/writer.go @@ -1,35 +1,268 @@ package generator import ( + "bytes" + "errors" "fmt" + "go/ast" + "go/format" + "go/parser" + "go/token" "log" "os" + "path" "path/filepath" - - "golang.org/x/tools/imports" + "slices" + "strconv" + "sync" ) -// WriteFiles formats each generated file with goimports and writes them to outputDir. +// stdImports resolves a package name a template references without importing it. +var stdImports = map[string]string{ + "base64": "encoding/base64", + "bufio": "bufio", + "bytes": "bytes", + "context": "context", + "encoding": "encoding", + "errors": "errors", + "fmt": "fmt", + "http": "net/http", + "io": "io", + "json": "encoding/json", + "math": "math", + "mime": "mime", + "multipart": "mime/multipart", + "rand": "math/rand", + "reflect": "reflect", + "sort": "sort", + "strconv": "strconv", + "strings": "strings", + "textproto": "net/textproto", + "time": "time", + "tls": "crypto/tls", + "url": "net/url", +} + +// WriteFiles fixes imports in and gofmts each generated file, then writes them to outputDir. func WriteFiles(outputDir string, files []GeneratedFile) error { if err := os.MkdirAll(outputDir, 0o755); err != nil { return fmt.Errorf("creating output directory: %w", err) } - for _, f := range files { - formatted, err := imports.Process(f.Name, f.Content, &imports.Options{ - Comments: true, - TabIndent: true, - TabWidth: 8, + fset := token.NewFileSet() + parsed := make([]*ast.File, len(files)) + parseErrs := make([]error, len(files)) + var wg sync.WaitGroup + for i, f := range files { + wg.Go(func() { + parsed[i], parseErrs[i] = parser.ParseFile(fset, f.Name, f.Content, parser.ParseComments) + }) + } + wg.Wait() + + // A name declared in a sibling file is not a package reference, even where + // it shares a name with one. + declared := map[string]bool{} + for _, af := range parsed { + if af != nil { + for name := range af.Scope.Objects { + declared[name] = true + } + } + } + + writeErrs := make([]error, len(files)) + for i, f := range files { + wg.Go(func() { + out := f.Content + if parseErrs[i] != nil { + log.Printf("WARNING: formatting failed for %s: %v (writing unformatted)", f.Name, parseErrs[i]) + } else { + fixImports(parsed[i], declared) + formatted, err := formatFile(fset, parsed[i], f.Content) + if err != nil { + log.Printf("WARNING: formatting failed for %s: %v (writing unformatted)", f.Name, err) + } else { + out = formatted + } + } + p := filepath.Join(outputDir, f.Name) + if err := os.WriteFile(p, out, 0o644); err != nil { + writeErrs[i] = fmt.Errorf("writing %s: %w", p, err) + } }) + } + wg.Wait() + return errors.Join(writeErrs...) +} + +// fixImports drops imports the file never references and adds the standard +// library ones it references without importing. +func fixImports(f *ast.File, declared map[string]bool) { + used := map[string]bool{} + ast.Inspect(f, func(n ast.Node) bool { + if sel, ok := n.(*ast.SelectorExpr); ok { + if id, ok := sel.X.(*ast.Ident); ok && id.Obj == nil && !declared[id.Name] { + used[id.Name] = true + } + } + return true + }) + + var have, want []string + for _, spec := range f.Imports { + p, err := strconv.Unquote(spec.Path.Value) if err != nil { - log.Printf("WARNING: goimports failed for %s: %v (writing unformatted)", f.Name, err) - formatted = f.Content + return + } + have = append(have, p) + if used[path.Base(p)] { + want = append(want, p) + delete(used, path.Base(p)) + } + } + for name := range used { + if p, ok := stdImports[name]; ok { + want = append(want, p) + } + } + slices.Sort(have) + slices.Sort(want) + if slices.Equal(have, want) { + return + } + + var decl *ast.GenDecl + decls := f.Decls[:0] + for _, d := range f.Decls { + if gd, ok := d.(*ast.GenDecl); ok && gd.Tok == token.IMPORT { + if decl == nil && len(want) > 0 { + decl = gd + decls = append(decls, d) + } + continue } + decls = append(decls, d) + } + f.Decls = decls + f.Imports = nil + if len(want) == 0 { + return + } + if decl == nil { + decl = &ast.GenDecl{Tok: token.IMPORT, TokPos: f.Name.End()} + f.Decls = slices.Insert(f.Decls, 0, ast.Decl(decl)) + } + decl.Specs = decl.Specs[:0] + for _, p := range want { + spec := &ast.ImportSpec{Path: &ast.BasicLit{Kind: token.STRING, Value: strconv.Quote(p)}} + decl.Specs = append(decl.Specs, spec) + f.Imports = append(f.Imports, spec) + } + switch { + case len(want) == 1: + decl.Lparen, decl.Rparen = token.NoPos, token.NoPos + case !decl.Lparen.IsValid(): + decl.Lparen, decl.Rparen = decl.TokPos, decl.TokPos + } +} + +const formatChunkSize = 64 << 10 - path := filepath.Join(outputDir, f.Name) - if err := os.WriteFile(path, formatted, 0o644); err != nil { - return fmt.Errorf("writing %s: %w", path, err) +// formatFile gofmts a file, printing a large one in parallel chunks. A chunk +// boundary is a blank line between top-level declarations: gofmt prints that +// gap as exactly one blank line, and no alignment or comment spans it. +func formatFile(fset *token.FileSet, f *ast.File, src []byte) ([]byte, error) { + tf := fset.File(f.Pos()) + offset := func(p token.Pos) int { return tf.Offset(p) } + var cuts []int + last := 0 + for i := 1; i < len(f.Decls); i++ { + start := f.Decls[i].Pos() + if doc := declDoc(f.Decls[i]); doc != nil { + start = doc.Pos() + } + // fixImports may have rebuilt the import decl without positions. + if gd, ok := f.Decls[i-1].(*ast.GenDecl); ok && gd.Tok == token.IMPORT { + continue + } + from, to := offset(f.Decls[i-1].End()), offset(start) + if from-last < formatChunkSize || !isBlankGap(src[from:to]) { + continue } + cuts = append(cuts, i) + last = from + } + if len(cuts) == 0 { + var buf bytes.Buffer + err := format.Node(&buf, fset, f) + return buf.Bytes(), err + } + + bounds := append(append([]int{0}, cuts...), len(f.Decls)) + chunks := make([][]byte, len(bounds)-1) + errs := make([]error, len(chunks)) + var wg sync.WaitGroup + for c := range chunks { + wg.Go(func() { + decls := f.Decls[bounds[c]:bounds[c+1]] + lo, hi := token.NoPos, decls[len(decls)-1].End() + if c > 0 { + lo = decls[0].Pos() + if doc := declDoc(decls[0]); doc != nil { + lo = doc.Pos() + } + } + var comments []*ast.CommentGroup + for _, cg := range f.Comments { + if cg.Pos() >= lo && (c == len(chunks)-1 || cg.End() <= hi) { + comments = append(comments, cg) + } + } + // The printer only takes comments alongside a whole file, so every + // chunk prints as one and the later ones drop the package clause. + part := *f + part.Decls, part.Comments = decls, comments + if c > 0 { + part.Doc = nil + } + // token.File locks on every position lookup, so chunks sharing one + // serialize; a private copy at the same base keeps positions valid. + own := token.NewFileSet() + own.AddFile(tf.Name(), tf.Base(), tf.Size()).SetLines(tf.Lines()) + var buf bytes.Buffer + if errs[c] = format.Node(&buf, own, &part); errs[c] != nil { + return + } + out := buf.Bytes() + if c > 0 { + clause := "package " + f.Name.Name + "\n\n" + if !bytes.HasPrefix(out, []byte(clause)) { + errs[c] = fmt.Errorf("chunk %d does not start with %q", c, clause) + return + } + out = out[len(clause):] + } + chunks[c] = bytes.TrimRight(out, "\n") + }) + } + wg.Wait() + if err := errors.Join(errs...); err != nil { + return nil, err + } + return append(bytes.Join(chunks, []byte("\n\n")), '\n'), nil +} + +func declDoc(d ast.Decl) *ast.CommentGroup { + switch d := d.(type) { + case *ast.FuncDecl: + return d.Doc + case *ast.GenDecl: + return d.Doc } return nil } + +func isBlankGap(gap []byte) bool { + return len(bytes.TrimSpace(gap)) == 0 && bytes.Count(gap, []byte("\n")) >= 2 +} diff --git a/internal/generator/writer_test.go b/internal/generator/writer_test.go new file mode 100644 index 0000000..7131240 --- /dev/null +++ b/internal/generator/writer_test.go @@ -0,0 +1,151 @@ +package generator + +import ( + "os" + "path/filepath" + "testing" +) + +func TestWriteFiles_FixesImports(t *testing.T) { + tests := []struct { + name string + in string + want string + }{ + { + name: "adds missing and drops unused", + in: `package p + +import ( + "fmt" + "net/http" +) + +func f(r *http.Request) time.Time { return time.Now() } +`, + want: `package p + +import ( + "net/http" + "time" +) + +func f(r *http.Request) time.Time { return time.Now() } +`, + }, + { + name: "collapses a block left with one import", + in: `package p + +import ( + "fmt" + "net/http" +) + +var c http.Client +`, + want: `package p + +import "net/http" + +var c http.Client +`, + }, + { + name: "removes an import block nothing uses", + in: `package p + +import "fmt" + +var x = 1 +`, + want: `package p + +var x = 1 +`, + }, + { + name: "adds a block where there was none", + in: `package p + +func f() error { return errors.New(strings.ToUpper("x")) } +`, + want: `package p + +import ( + "errors" + "strings" +) + +func f() error { return errors.New(strings.ToUpper("x")) } +`, + }, + { + name: "ignores locals and literal keys named like packages", + in: `package p + +import "net/url" + +type T struct{ url string } + +func f(url *T) T { return T{url: url.url} } +`, + want: `package p + +type T struct{ url string } + +func f(url *T) T { return T{url: url.url} } +`, + }, + { + name: "ignores names declared in a sibling file", + in: `package p + +func f() string { return rand.Name } +`, + want: `package p + +func f() string { return rand.Name } +`, + }, + { + name: "leaves a correct file as it was", + in: `package p + +import ( + "context" + + "net/http" +) + +func f(ctx context.Context, c *http.Client) {} +`, + want: `package p + +import ( + "context" + + "net/http" +) + +func f(ctx context.Context, c *http.Client) {} +`, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + dir := t.TempDir() + sibling := GeneratedFile{Name: "sibling.go", Content: []byte("package p\n\nvar rand struct{ Name string }\n")} + if err := WriteFiles(dir, []GeneratedFile{{Name: "f.go", Content: []byte(tt.in)}, sibling}); err != nil { + t.Fatalf("WriteFiles: %v", err) + } + got, err := os.ReadFile(filepath.Join(dir, "f.go")) + if err != nil { + t.Fatal(err) + } + if string(got) != tt.want { + t.Errorf("got:\n%s\nwant:\n%s", got, tt.want) + } + }) + } +} diff --git a/internal/parser/jsonnode.go b/internal/parser/jsonnode.go new file mode 100644 index 0000000..c3bf0b3 --- /dev/null +++ b/internal/parser/jsonnode.go @@ -0,0 +1,325 @@ +package parser + +import ( + "errors" + "strconv" + "unicode/utf16" + "unicode/utf8" + + "go.yaml.in/yaml/v4" +) + +// errNotFastJSON sends a document the JSON fast path cannot reproduce +// exactly back to the YAML parser. +var errNotFastJSON = errors.New("not handled by the JSON fast path") + +// parseJSONNode builds the node tree the YAML parser produces for a JSON +// document (same kinds, tags, styles, values, lines and columns), several +// times faster than tokenizing the JSON as YAML flow syntax. +func parseJSONNode(data []byte) (*yaml.Node, error) { + p := &jsonNodeParser{data: data, line: 1, col: 1, strs: map[string]string{}} + p.skipSpace() + doc := p.node() + doc.Kind, doc.Line, doc.Column = yaml.DocumentNode, p.line, p.col + root, err := p.value() + if err != nil { + return nil, err + } + p.skipSpace() + if p.pos != len(data) { + return nil, errNotFastJSON + } + doc.Content = []*yaml.Node{root} + return doc, nil +} + +type jsonNodeParser struct { + data []byte + pos int + line, col int + slab []yaml.Node + stack []*yaml.Node + strs map[string]string + buf []byte +} + +func (p *jsonNodeParser) node() *yaml.Node { + if len(p.slab) == 0 { + p.slab = make([]yaml.Node, 1024) + } + n := &p.slab[0] + p.slab = p.slab[1:] + return n +} + +func (p *jsonNodeParser) skipSpace() { + for p.pos < len(p.data) { + switch p.data[p.pos] { + case ' ', '\t': + p.col++ + case '\n': + p.line++ + p.col = 1 + case '\r': + if p.pos+1 < len(p.data) && p.data[p.pos+1] == '\n' { + p.pos++ + } + p.line++ + p.col = 1 + default: + return + } + p.pos++ + } +} + +func (p *jsonNodeParser) value() (*yaml.Node, error) { + if p.pos >= len(p.data) { + return nil, errNotFastJSON + } + n := p.node() + n.Line, n.Column = p.line, p.col + switch c := p.data[p.pos]; { + case c == '{': + return n, p.container(n, yaml.MappingNode, "!!map", '}') + case c == '[': + return n, p.container(n, yaml.SequenceNode, "!!seq", ']') + case c == '"': + s, err := p.str() + if err != nil { + return nil, err + } + n.Kind, n.Tag, n.Style, n.Value = yaml.ScalarNode, "!!str", yaml.DoubleQuotedStyle, s + return n, nil + default: + return n, p.literal(n) + } +} + +func (p *jsonNodeParser) container(n *yaml.Node, kind yaml.Kind, tag string, end byte) error { + n.Kind, n.Tag, n.Style = kind, tag, yaml.FlowStyle + p.pos++ + p.col++ + base := len(p.stack) + p.skipSpace() + if p.pos < len(p.data) && p.data[p.pos] == end { + p.pos++ + p.col++ + return nil + } + for { + p.skipSpace() + if kind == yaml.MappingNode { + if p.pos >= len(p.data) || p.data[p.pos] != '"' { + return errNotFastJSON + } + key, err := p.value() + if err != nil { + return err + } + p.stack = append(p.stack, key) + p.skipSpace() + if p.pos >= len(p.data) || p.data[p.pos] != ':' { + return errNotFastJSON + } + p.pos++ + p.col++ + p.skipSpace() + } + v, err := p.value() + if err != nil { + return err + } + p.stack = append(p.stack, v) + p.skipSpace() + if p.pos >= len(p.data) { + return errNotFastJSON + } + c := p.data[p.pos] + p.pos++ + p.col++ + if c == end { + break + } + if c != ',' { + return errNotFastJSON + } + } + n.Content = append([]*yaml.Node(nil), p.stack[base:]...) + clear(p.stack[base:]) + p.stack = p.stack[:base] + return nil +} + +func (p *jsonNodeParser) str() (string, error) { + p.pos++ + p.col++ + start := p.pos + escaped := false + for { + if p.pos >= len(p.data) { + return "", errNotFastJSON + } + c := p.data[p.pos] + if c == '"' { + break + } + if c < 0x20 { + return "", errNotFastJSON + } + if c == '\\' { + escaped = true + p.pos++ + p.col++ + if p.pos >= len(p.data) { + return "", errNotFastJSON + } + c = p.data[p.pos] + } + if c < utf8.RuneSelf || c >= 0xC0 { + p.col++ + } + p.pos++ + } + raw := p.data[start:p.pos] + p.pos++ + p.col++ + if !escaped { + return p.intern(raw), nil + } + return p.unescape(raw) +} + +func (p *jsonNodeParser) intern(b []byte) string { + if len(b) > 64 { + return string(b) + } + if s, ok := p.strs[string(b)]; ok { + return s + } + s := string(b) + p.strs[s] = s + return s +} + +func (p *jsonNodeParser) unescape(raw []byte) (string, error) { + out := p.buf[:0] + for i := 0; i < len(raw); i++ { + c := raw[i] + if c != '\\' { + out = append(out, c) + continue + } + i++ + switch raw[i] { + case '"', '\\': + out = append(out, raw[i]) + case 'b': + out = append(out, '\b') + case 'f': + out = append(out, '\f') + case 'n': + out = append(out, '\n') + case 'r': + out = append(out, '\r') + case 't': + out = append(out, '\t') + case 'u': + if i+5 > len(raw) { + return "", errNotFastJSON + } + r, err := strconv.ParseUint(string(raw[i+1:i+5]), 16, 16) + if err != nil || utf16.IsSurrogate(rune(r)) { + return "", errNotFastJSON + } + out = utf8.AppendRune(out, rune(r)) + i += 4 + default: + // Includes \/, which the YAML parser only accepts after + // libopenapi rewrites it, shifting the columns that follow. + return "", errNotFastJSON + } + } + p.buf = out + return p.intern(out), nil +} + +func (p *jsonNodeParser) literal(n *yaml.Node) error { + start := p.pos + for p.pos < len(p.data) { + c := p.data[p.pos] + if c == ',' || c == '}' || c == ']' || c == ' ' || c == '\t' || c == '\n' || c == '\r' { + break + } + p.pos++ + } + raw := p.data[start:p.pos] + p.col += len(raw) + n.Kind = yaml.ScalarNode + switch string(raw) { + case "true", "false": + n.Tag, n.Value = "!!bool", string(raw) + return nil + case "null": + n.Tag, n.Value = "!!null", "null" + return nil + } + tag, ok := numberTag(raw) + if !ok { + return errNotFastJSON + } + n.Tag, n.Value = tag, p.intern(raw) + return nil +} + +// numberTag resolves a JSON number the way the YAML core schema does. +func numberTag(raw []byte) (string, bool) { + i := 0 + if i < len(raw) && raw[i] == '-' { + i++ + } + digits := func() int { + n := 0 + for i < len(raw) && raw[i] >= '0' && raw[i] <= '9' { + i++ + n++ + } + return n + } + intDigits := digits() + if intDigits == 0 || (intDigits > 1 && raw[i-intDigits] == '0') { + return "", false + } + isFloat := false + if i < len(raw) && raw[i] == '.' { + i++ + if digits() == 0 { + return "", false + } + isFloat = true + } + if i < len(raw) && (raw[i] == 'e' || raw[i] == 'E') { + i++ + if i < len(raw) && (raw[i] == '+' || raw[i] == '-') { + i++ + } + if digits() == 0 { + return "", false + } + isFloat = true + } + if i != len(raw) { + return "", false + } + if !isFloat { + s := string(raw) + if _, err := strconv.ParseInt(s, 10, 64); err == nil { + return "!!int", true + } + if _, err := strconv.ParseUint(s, 10, 64); err == nil { + return "!!int", true + } + return "", false + } + return "!!float", true +} diff --git a/internal/parser/jsonnode_test.go b/internal/parser/jsonnode_test.go new file mode 100644 index 0000000..b405bba --- /dev/null +++ b/internal/parser/jsonnode_test.go @@ -0,0 +1,135 @@ +package parser + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/pb33f/libopenapi/datamodel" + "go.yaml.in/yaml/v4" +) + +// diffNodes reports the first place two node trees differ, or "". +func diffNodes(path string, got, want *yaml.Node) string { + if got.Kind != want.Kind || got.Style != want.Style || got.Tag != want.Tag || got.Value != want.Value || + got.Line != want.Line || got.Column != want.Column || len(got.Content) != len(want.Content) { + return fmt.Sprintf("%s:\n got kind=%v style=%v tag=%q value=%q at %d:%d with %d children\n want kind=%v style=%v tag=%q value=%q at %d:%d with %d children", + path, got.Kind, got.Style, got.Tag, got.Value, got.Line, got.Column, len(got.Content), + want.Kind, want.Style, want.Tag, want.Value, want.Line, want.Column, len(want.Content)) + } + for i := range got.Content { + if d := diffNodes(fmt.Sprintf("%s/%d", path, i), got.Content[i], want.Content[i]); d != "" { + return d + } + } + return "" +} + +func assertSpecInfoMatches(t *testing.T, spec []byte) { + t.Helper() + got, err := extractSpecInfo(spec, &datamodel.DocumentConfiguration{}) + if err != nil { + t.Fatalf("extractSpecInfo: %v", err) + } + want, err := datamodel.ExtractSpecInfoWithConfig(spec, &datamodel.DocumentConfiguration{}) + if err != nil { + t.Fatalf("ExtractSpecInfoWithConfig: %v", err) + } + type fields struct { + SpecType, Version, SpecFormat, SpecFileType, APISchema, Self string + VersionNumeric float32 + NumLines, OriginalIndentation int + } + project := func(i *datamodel.SpecInfo) fields { + return fields{i.SpecType, i.Version, i.SpecFormat, i.SpecFileType, i.APISchema, i.Self, i.VersionNumeric, i.NumLines, i.OriginalIndentation} + } + if g, w := project(got), project(want); g != w { + t.Errorf("SpecInfo differs:\n got %+v\n want %+v", g, w) + } + if d := diffNodes("", got.RootNode, want.RootNode); d != "" { + t.Error(d) + } +} + +func assertMatchesYAML(t *testing.T, data []byte) { + t.Helper() + got, err := parseJSONNode(data) + if err != nil { + t.Fatalf("parseJSONNode: %v", err) + } + var want yaml.Node + if err := yaml.Unmarshal(data, &want); err != nil { + t.Fatalf("yaml.Unmarshal: %v", err) + } + if d := diffNodes("", got, &want); d != "" { + t.Error(d) + } +} + +func TestParseJSONNode_MatchesYAMLParser(t *testing.T) { + cases := map[string]string{ + "scalars": `{"s": "x", "e": "", "t": true, "f": false, "n": null, "i": 42, "neg": -7, "zero": 0, + "fl": 1.5, "exp": 1e5, "Exp": -2.5E-3, "big": 18446744073709551615, "nested": {"a": [1, [2, {}], []]}}`, + "escapes": `{"q": "a\"b\\c\nd\te\r\b\f", "u": "caf\u00e9 \u4e2d", "after": "x"}`, + "unicode": "{\"k\u00e9\": \"\u4e2d\u6587\", \"emoji\": \"\U0001F600\", \"after\": [1, 2]}", + "crlf": "{\r\n \"a\": 1,\r\n \"b\": [\r\n true\r\n ]\r\n}\r\n", + "compact": `{"a":{"b":[1,2,{"c":"d"}]},"e":[]}`, + "leadingWS": "\n\n {\"a\": 1}\n", + } + for name, in := range cases { + t.Run(name, func(t *testing.T) { assertMatchesYAML(t, []byte(in)) }) + } + + specs, err := filepath.Glob(filepath.Join(projectRoot(), "testdata", "*.yaml")) + if err != nil || len(specs) == 0 { + t.Fatalf("no testdata specs: %v", err) + } + for _, spec := range specs { + t.Run(filepath.Base(spec), func(t *testing.T) { + src, err := os.ReadFile(spec) + if err != nil { + t.Fatal(err) + } + var v any + if err := yaml.Unmarshal(src, &v); err != nil { + t.Fatal(err) + } + indented, err := json.MarshalIndent(v, "", " ") + if err != nil { + t.Fatal(err) + } + assertMatchesYAML(t, indented) + assertSpecInfoMatches(t, indented) + compact, err := json.Marshal(v) + if err != nil { + t.Fatal(err) + } + assertMatchesYAML(t, compact) + }) + } +} + +func TestParseJSONNode_DefersToYAMLParser(t *testing.T) { + for name, in := range map[string]string{ + "escaped slash": `{"a": "x\/y"}`, + "surrogate pair": `{"a": "\ud83d\ude00"}`, + "lone surrogate": `{"a": "\ud83d"}`, + "bom": "\ufeff{\"a\": 1}", + "trailing comma": `{"a": 1,}`, + "leading zero": `{"a": 01}`, + "int overflow": `{"a": 99999999999999999999}`, + "bare word": `{"a": yes}`, + "trailing data": `{"a": 1} x`, + "unterminated": `{"a": "b`, + "control in text": "{\"a\": \"b\nc\"}", + } { + t.Run(name, func(t *testing.T) { + if _, err := parseJSONNode([]byte(in)); !errors.Is(err, errNotFastJSON) { + t.Errorf("got %v, want errNotFastJSON", err) + } + }) + } +} diff --git a/internal/parser/parser.go b/internal/parser/parser.go index 5fc2a2b..fb6a75f 100644 --- a/internal/parser/parser.go +++ b/internal/parser/parser.go @@ -1,20 +1,26 @@ package parser import ( + "bytes" + "encoding/json" + "errors" "fmt" "os" "path/filepath" + "unicode/utf8" - "github.com/pb33f/libopenapi" "github.com/pb33f/libopenapi/datamodel" v3high "github.com/pb33f/libopenapi/datamodel/high/v3" + lowbase "github.com/pb33f/libopenapi/datamodel/low/base" + v3low "github.com/pb33f/libopenapi/datamodel/low/v3" + "github.com/pb33f/libopenapi/index" + "github.com/pb33f/libopenapi/utils" ) -// ParseResult holds the parsed OpenAPI document and its high-level model. +// ParseResult holds the parsed OpenAPI document's high-level model. type ParseResult struct { - Document libopenapi.Document - Model *v3high.Document - Version string + Model *v3high.Document + Version string } // Config controls parser behavior. @@ -40,24 +46,80 @@ func Parse(specPath string, cfg Config) (*ParseResult, error) { BasePath: basePath, AllowFileReferences: true, AllowRemoteReferences: cfg.AllowRemoteRefs, + // The analyzer reads only the model, never the index's diagnostic + // metadata, and collecting it is a large share of parse allocations. + SkipMetadataCollection: true, } - doc, err := libopenapi.NewDocumentWithConfiguration(specBytes, docConfig) + // The root libopenapi package would do these steps for us, but importing it + // compiles its diffing and Arazzo models too, which dominate a cold build. + info, err := extractSpecInfo(specBytes, docConfig) if err != nil { return nil, fmt.Errorf("parsing spec: %w", err) } + switch info.SpecFormat { + case datamodel.OAS3, datamodel.OAS31, datamodel.OAS32: + default: + return nil, fmt.Errorf("parsing spec: unsupported spec version %q, only OpenAPI 3.x is supported", info.Version) + } + + lowDoc, docErr := v3low.CreateDocumentFromConfig(info, docConfig) + if lowDoc == nil { + return nil, fmt.Errorf("failed to build OpenAPI v3 model: %w", docErr) + } + for _, err := range utils.UnwrapErrors(docErr) { + var refErr *index.ResolvingError + if errors.As(err, &refErr) && refErr.CircularReference == nil { + return nil, fmt.Errorf("failed to build OpenAPI v3 model: %w", docErr) + } + } + if docErr != nil { + fmt.Fprintf(os.Stderr, "warning: %v\n", docErr) + } + + model := v3high.NewDocument(lowDoc) + model.Rolodex = lowDoc.Index.GetRolodex() + lowbase.SchemaQuickHashMap.Clear() + + return &ParseResult{Model: model, Version: info.Version}, nil +} - docModel, err := doc.BuildV3Model() +// extractSpecInfo is datamodel.ExtractSpecInfoWithConfig with a faster parse +// for JSON documents, which are the common machine-generated case. +func extractSpecInfo(spec []byte, cfg *datamodel.DocumentConfiguration) (*datamodel.SpecInfo, error) { + trimmed := bytes.TrimSpace(spec) + if len(trimmed) == 0 || trimmed[0] != '{' || trimmed[len(trimmed)-1] != '}' || !utf8.Valid(spec) { + return datamodel.ExtractSpecInfoWithConfig(spec, cfg) + } + root, err := parseJSONNode(spec) if err != nil { - fmt.Fprintf(os.Stderr, "warning: %v\n", err) + return datamodel.ExtractSpecInfoWithConfig(spec, cfg) } - if docModel == nil { - return nil, fmt.Errorf("failed to build OpenAPI v3 model") + _, openAPI3 := utils.FindKeyNode(utils.OpenApi3, root.Content) + _, openAPI2 := utils.FindKeyNode(utils.OpenApi2, root.Content) + _, asyncAPI := utils.FindKeyNode(utils.AsyncApi, root.Content) + if openAPI3 == nil || openAPI2 != nil || asyncAPI != nil { + return datamodel.ExtractSpecInfoWithConfig(spec, cfg) } - return &ParseResult{ - Document: doc, - Model: &docModel.Model, - Version: doc.GetVersion(), - }, nil + // Every other SpecInfo field follows from the version alone, so a one-key + // document reuses libopenapi's version handling without its slow parse. + stub, err := json.Marshal(map[string]string{utils.OpenApi3: openAPI3.Value}) + if err != nil { + return nil, err + } + info, err := datamodel.ExtractSpecInfoWithConfig(stub, cfg) + if err != nil { + return datamodel.ExtractSpecInfoWithConfig(spec, cfg) + } + info.SpecBytes = &spec + info.RootNode = root + info.NumLines = bytes.Count(spec, []byte{'\n'}) + 1 + info.OriginalIndentation = utils.DetermineWhitespaceLengthBytes(spec) + if info.SpecFormat == datamodel.OAS31 || info.SpecFormat == datamodel.OAS32 { + if _, self := utils.FindKeyNode("$self", root.Content); self != nil && self.Value != "" { + info.Self = self.Value + } + } + return info, nil }