From d6f516e0229c00d062087c20337c91a602f5ffaa Mon Sep 17 00:00:00 2001 From: Leo Cavalcante Date: Tue, 29 Sep 2026 18:08:03 -0300 Subject: [PATCH] test: lock TestGen coverage to non-field operations Record which suites TestGen should generate and fail when that case list drifts from the condition table. --- Makefile | 2 +- docs/internals.md | 4 +- testgen/README.md | 53 +++++++----- testgen/coverage_test.go | 183 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 218 insertions(+), 24 deletions(-) create mode 100644 testgen/coverage_test.go diff --git a/Makefile b/Makefile index 4acde81..50e8c8d 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,7 @@ clean: unittests: @echo "Running unit tests" go clean -testcache - go test -v ./internal/... ./types/... + go test -v ./internal/... ./types/... ./testgen/ benchtests: build @echo "Running bench tests" diff --git a/docs/internals.md b/docs/internals.md index b5c896e..9760c35 100644 --- a/docs/internals.md +++ b/docs/internals.md @@ -165,6 +165,6 @@ TestGen is `package main` in `testgen/`. `make testgen` runs it and moves the fi - `generated_function_code_no_pointer_test.go` and `generated_function_code_pointer_test.go` move to `internal/codegenerator/`. - `generated_cmp_perf_no_pointer_test.go` and `generated_cmp_perf_pointer_test.go` move to `tests/cmpbenchtests/`. -The case list is `typesValidation` in `testgen/validations.go`. [testgen/README.md](../testgen/README.md) records which suites that list drives and which suites are still written by hand. +The case list is `typesValidation` in `testgen/validations.go`. [testgen/README.md](../testgen/README.md) records which suites that list drives and which suites stay hand-written. -Hand-written tests cover the parser, the analyzer, operation checks, and condition-table cases that TestGen does not emit. `make unittests` runs `go test` on `./internal/...` and `./types/...`. `make endtoendtests` builds `bin/validgen`, deletes existing `validator__.go` files under `tests/endtoend/`, runs the generator there, and executes `go run .` in that directory. It then does the same for `tests/jsonunmarshal/`, passing `-unmarshal-json`. +Hand-written tests cover the parser, the analyzer operation checks, and condition-table cases. `make unittests` runs `go test` on `./internal/...`, `./types/...`, and `./testgen/`. `make endtoendtests` builds `bin/validgen`, deletes existing `validator__.go` files under `tests/endtoend/`, runs the generator there, and executes `go run .` in that directory. It then does the same for `tests/jsonunmarshal/`, passing `-unmarshal-json`. diff --git a/testgen/README.md b/testgen/README.md index 72033c4..dad1bf9 100644 --- a/testgen/README.md +++ b/testgen/README.md @@ -72,27 +72,38 @@ Previously, ValidGen had two separate test generators: However, these generators lacked a common configuration, didn't implement all tests for all cases, and keeping the separate configuration files in sync was difficult. -## What TestGen does - -TestGen generates the following tests (without field operations): -- [x] Benchmark tests between ValidGen and GoValidator -- [x] End-to-end tests with all possible use cases (all validations vs all types vs valid and invalid inputs) -- [x] Unit tests to validate the "buildValidationCode" function - -High priority generators: -- [ ] Unit tests to validate the "condition table" (get_test_elements_*_test.go) -- [ ] Benchmark tests between ValidGen and GoValidator with field operations -- [ ] End-to-end tests with all possible use cases (all validations vs all types vs valid and invalid inputs) with field operations -- [ ] Unit tests to validate the "buildValidationCode" function with field operations - -Low priority generators (already exist, but could be automated): -- [ ] Unit tests to validate operations (func TestOperationsIsValid) -- [ ] Unit tests to validate operation vs type (func TestOperationsIsValidByType) -- [ ] Unit tests to validate field operations (func TestOperationsIsFieldOperation) -- [ ] Unit tests to validate argument count by operation (func TestOperationsArgsCount) -- [ ] Examples (in _examples/) could be generated - -Where applicable, TestGen generates both valid and invalid test scenarios. +## What TestGen generates + +`typesValidation` in `validations.go` is the case list. `make testgen` walks that list and writes four suites. Each suite is a pointer file and a non-pointer file. The list contains non-field operations only. + +- Benchmark tests between ValidGen and GoValidator, in `tests/cmpbenchtests/generated_cmp_perf_*`. +- End-to-end tests for each validation, type class, and valid or invalid input, in `tests/endtoend/generated_endtoend_*`. +- Unit tests for `BuildValidationCode`, in `internal/codegenerator/generated_validation_code_*`. +- Unit tests for the generated validator function, in `internal/codegenerator/generated_function_code_*`. + +A case with both inputs emits the valid input and the invalid input. Array `required` cases set `excludeIf` to `noPointer` because a non-pointer Go array cannot be empty. + +`go test ./testgen` checks that list. `TestTypesValidationListsNonFieldOperations` requires one entry for each operation below, with `isFieldValidation` false, the same argument count as `operations.New()`, and a case for every type `IsValidByType` accepts. `TestTypesValidationCasesBuildValidationCode` calls `BuildValidationCode` for each concrete type, including the pointer form, and checks that the generated error text contains the catalog message. + +| Operation | Generated | +| - | - | +| email, required, eq, neq | yes | +| gt, gte, lt, lte | yes | +| min, max, len | yes | +| eq_ignore_case, neq_ignore_case | yes | +| in, nin | yes | +| eqfield, neqfield, gtfield, gtefield, ltfield, ltefield | hand-written | + +## Suites that stay hand-written + +These groups already have tests beside the code they check. A generator that reads the same list would only compare that list with itself. + +- Operation checks in `internal/analyzer/operations/operations_test.go`. The functions are `TestOperationsIsValid`, `TestOperationsIsValidByType`, `TestOperationsIsFieldOperation`, and `TestOperationsArgsCount`. +- Condition-table checks in `internal/codegenerator/get_test_elements_*_test.go`. Each case stores the condition string and the error string passed to `DefineTestElements`. +- Parser checks in `internal/parser/parser_test.go`. They compare parsed structs with source text. +- Examples under `_examples/`. + +Field-operation rows in the four generated suites wait on integer field operations in issue #78. Complex ordering and `dive` in issue #7 are separate work. ## Usage diff --git a/testgen/coverage_test.go b/testgen/coverage_test.go new file mode 100644 index 0000000..164b07c --- /dev/null +++ b/testgen/coverage_test.go @@ -0,0 +1,183 @@ +package main + +import ( + "strings" + "testing" + + "github.com/opencodeco/validgen/internal/analyzer" + "github.com/opencodeco/validgen/internal/analyzer/operations" + "github.com/opencodeco/validgen/internal/codegenerator" + "github.com/opencodeco/validgen/internal/common" +) + +// generatedOperations is the non-field set make testgen emits. +var generatedOperations = []string{ + "email", + "required", + "eq", + "neq", + "gt", + "gte", + "lt", + "lte", + "min", + "max", + "eq_ignore_case", + "neq_ignore_case", + "len", + "in", + "nin", +} + +// fieldOperations stay in hand-written tests. Integer field operations are issue #78. +var fieldOperations = []string{ + "eqfield", + "neqfield", + "gtfield", + "gtefield", + "ltfield", + "ltefield", +} + +// normalizedTypeClasses is every class HelperFromNormalizedToFieldTypes accepts. +var normalizedTypeClasses = []string{ + "", "", "", "", "", + "[]", "[]", "[]", "[]", "[]", + "[N]", "[N]", "[N]", "[N]", "[N]", + "map[]", "map[]", "map[]", "map[]", "map[]", +} + +func TestTypesValidationListsNonFieldOperations(t *testing.T) { + ops := operations.New() + byTag := map[string]int{} + for _, entry := range typesValidation { + byTag[entry.tag]++ + if entry.isFieldValidation { + t.Errorf("%s is marked as a field validation", entry.tag) + } + if !ops.IsValid(entry.tag) { + t.Errorf("%s is missing from the operations list", entry.tag) + continue + } + if ops.IsFieldOperation(entry.tag) { + t.Errorf("%s is a field operation", entry.tag) + } + if got := ops.ArgsCount(entry.tag); got != entry.argsCount { + t.Errorf("%s argument count is %v, catalog has %v", entry.tag, got, entry.argsCount) + } + + classes := map[string]struct{}{} + for _, tc := range entry.testCases { + classes[tc.typeClass] = struct{}{} + if tc.validCase == "" || tc.errorMessage == "" { + t.Errorf("%s %s is missing a valid case or error message", entry.tag, tc.typeClass) + } + if tc.invalidCase == "" { + t.Errorf("%s %s is missing an invalid case", entry.tag, tc.typeClass) + } + if tc.invalidCase == "--" && tc.excludeIf&noPointer == 0 { + t.Errorf("%s %s uses -- without noPointer", entry.tag, tc.typeClass) + } + } + for _, class := range normalizedTypeClasses { + if !ops.IsValidByType(entry.tag, class) { + continue + } + if _, ok := classes[class]; !ok { + t.Errorf("%s accepts %s and typesValidation has no case", entry.tag, class) + } + } + } + + inCatalog := map[string]struct{}{} + for tag := range byTag { + inCatalog[tag] = struct{}{} + } + + for _, op := range generatedOperations { + if byTag[op] != 1 { + t.Errorf("%s appears %d times in typesValidation, want 1", op, byTag[op]) + } + delete(byTag, op) + } + for tag := range byTag { + t.Errorf("%s is in typesValidation and is outside the generated set", tag) + } + + for _, op := range fieldOperations { + if !ops.IsValid(op) || !ops.IsFieldOperation(op) { + t.Errorf("%s should stay a field operation in the operations list", op) + } + if _, ok := inCatalog[op]; ok { + t.Errorf("%s is a field operation and is in typesValidation", op) + } + } +} + +func TestTypesValidationCasesBuildValidationCode(t *testing.T) { + gv := codegenerator.GenValidations{} + + for _, entry := range typesValidation { + for _, tc := range entry.testCases { + for _, pointer := range []bool{false, true} { + if tc.excludeIf&noPointer != 0 && !pointer { + continue + } + + normalizedType := tc.typeClass + if pointer { + normalizedType = "*" + normalizedType + } + + fieldTypes, err := common.HelperFromNormalizedToFieldTypes(normalizedType) + if err != nil { + t.Errorf("%s %s pointer %v: %v", entry.tag, tc.typeClass, pointer, err) + continue + } + + validation := entry.tag + if entry.argsCount != common.ZeroValue { + validation += "=" + tc.validation + } + parsed, err := analyzer.ParserValidation(validation) + if err != nil { + t.Errorf("parse %s: %v", validation, err) + continue + } + + for _, fieldType := range fieldTypes { + const fieldName = "Field" + got, err := gv.BuildValidationCode(fieldName, fieldType, []*analyzer.Validation{parsed}) + if err != nil { + t.Errorf("%s %s %s pointer %v: %v", entry.tag, tc.typeClass, fieldType.ToType(), pointer, err) + continue + } + if !errorTextMatches(got, fieldName, tc.errorMessage) { + t.Errorf("%s %s %s pointer %v: generated error text missing %q\n%s", + entry.tag, tc.typeClass, fieldType.ToType(), pointer, tc.errorMessage, got) + } + } + } + } + } +} + +func errorTextMatches(generated, fieldName, catalogMessage string) bool { + message := strings.ReplaceAll(catalogMessage, "{{.FieldName}}", fieldName) + message = strings.ReplaceAll(message, "{{.Target}}", "\x00") + message = strings.ReplaceAll(message, "{{.Targets}}", "\x00") + + from := 0 + for _, part := range strings.Split(message, "\x00") { + part = strings.TrimSpace(part) + if part == "" { + continue + } + idx := strings.Index(generated[from:], part) + if idx < 0 { + return false + } + from += idx + len(part) + } + return from > 0 +}