test: lock TestGen coverage to non-field operations

Makefile

@@ -16,7 +16,7 @@ clean:
 unittests:
 	@echo "Running unit tests"
 	go clean -testcache
-	go test -v ./internal/... ./types/...
+	go test -v ./internal/... ./types/... ./testgen/
 
 benchtests: build
 	@echo "Running bench tests"

docs/internals.md

@@ -165,6 +165,6 @@ TestGen is `package main` in `testgen/`. `make testgen` runs it and moves the fi
 - `generated_function_code_no_pointer_test.go` and `generated_function_code_pointer_test.go` move to `internal/codegenerator/`.
 - `generated_cmp_perf_no_pointer_test.go` and `generated_cmp_perf_pointer_test.go` move to `tests/cmpbenchtests/`.
 
-The case list is `typesValidation` in `testgen/validations.go`. [testgen/README.md](../testgen/README.md) records which suites that list drives and which suites are still written by hand.
+The case list is `typesValidation` in `testgen/validations.go`. [testgen/README.md](../testgen/README.md) records which suites that list drives and which suites stay hand-written.
 
-Hand-written tests cover the parser, the analyzer, operation checks, and condition-table cases that TestGen does not emit. `make unittests` runs `go test` on `./internal/...` and `./types/...`. `make endtoendtests` builds `bin/validgen`, deletes existing `validator__.go` files under `tests/endtoend/`, runs the generator there, and executes `go run .` in that directory. It then does the same for `tests/jsonunmarshal/`, passing `-unmarshal-json`.
+Hand-written tests cover the parser, the analyzer operation checks, and condition-table cases. `make unittests` runs `go test` on `./internal/...`, `./types/...`, and `./testgen/`. `make endtoendtests` builds `bin/validgen`, deletes existing `validator__.go` files under `tests/endtoend/`, runs the generator there, and executes `go run .` in that directory. It then does the same for `tests/jsonunmarshal/`, passing `-unmarshal-json`.

testgen/README.md

@@ -72,27 +72,38 @@ Previously, ValidGen had two separate test generators:
 
 However, these generators lacked a common configuration, didn't implement all tests for all cases, and keeping the separate configuration files in sync was difficult.
 
-## What TestGen does
-
-TestGen generates the following tests (without field operations):
-- [x] Benchmark tests between ValidGen and GoValidator
-- [x] End-to-end tests with all possible use cases (all validations vs all types vs valid and invalid inputs)
-- [x] Unit tests to validate the "buildValidationCode" function
-
-High priority generators:
-- [ ] Unit tests to validate the "condition table" (get_test_elements_*_test.go)
-- [ ] Benchmark tests between ValidGen and GoValidator with field operations
-- [ ] End-to-end tests with all possible use cases (all validations vs all types vs valid and invalid inputs) with field operations
-- [ ] Unit tests to validate the "buildValidationCode" function with field operations
-
-Low priority generators (already exist, but could be automated):
-- [ ] Unit tests to validate operations (func TestOperationsIsValid)
-- [ ] Unit tests to validate operation vs type (func TestOperationsIsValidByType)
-- [ ] Unit tests to validate field operations (func TestOperationsIsFieldOperation)
-- [ ] Unit tests to validate argument count by operation (func TestOperationsArgsCount)
-- [ ] Examples (in _examples/) could be generated
-
-Where applicable, TestGen generates both valid and invalid test scenarios.
+## What TestGen generates
+
+`typesValidation` in `validations.go` is the case list. `make testgen` walks that list and writes four suites. Each suite is a pointer file and a non-pointer file. The list contains non-field operations only.
+
+- Benchmark tests between ValidGen and GoValidator, in `tests/cmpbenchtests/generated_cmp_perf_*`.
+- End-to-end tests for each validation, type class, and valid or invalid input, in `tests/endtoend/generated_endtoend_*`.
+- Unit tests for `BuildValidationCode`, in `internal/codegenerator/generated_validation_code_*`.
+- Unit tests for the generated validator function, in `internal/codegenerator/generated_function_code_*`.
+
+A case with both inputs emits the valid input and the invalid input. Array `required` cases set `excludeIf` to `noPointer` because a non-pointer Go array cannot be empty.
+
+`go test ./testgen` checks that list. `TestTypesValidationListsNonFieldOperations` requires one entry for each operation below, with `isFieldValidation` false, the same argument count as `operations.New()`, and a case for every type `IsValidByType` accepts. `TestTypesValidationCasesBuildValidationCode` calls `BuildValidationCode` for each concrete type, including the pointer form, and checks that the generated error text contains the catalog message.
+
+| Operation | Generated |
+| - | - |
+| email, required, eq, neq | yes |
+| gt, gte, lt, lte | yes |
+| min, max, len | yes |
+| eq_ignore_case, neq_ignore_case | yes |
+| in, nin | yes |
+| eqfield, neqfield, gtfield, gtefield, ltfield, ltefield | hand-written |
+
+## Suites that stay hand-written
+
+These groups already have tests beside the code they check. A generator that reads the same list would only compare that list with itself.
+
+- Operation checks in `internal/analyzer/operations/operations_test.go`. The functions are `TestOperationsIsValid`, `TestOperationsIsValidByType`, `TestOperationsIsFieldOperation`, and `TestOperationsArgsCount`.
+- Condition-table checks in `internal/codegenerator/get_test_elements_*_test.go`. Each case stores the condition string and the error string passed to `DefineTestElements`.
+- Parser checks in `internal/parser/parser_test.go`. They compare parsed structs with source text.
+- Examples under `_examples/`.
+
+Field-operation rows in the four generated suites wait on integer field operations in issue #78. Complex ordering and `dive` in issue #7 are separate work.
 
 ## Usage
 

testgen/coverage_test.go

@@ -0,0 +1,183 @@
+package main
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/opencodeco/validgen/internal/analyzer"
+	"github.com/opencodeco/validgen/internal/analyzer/operations"
+	"github.com/opencodeco/validgen/internal/codegenerator"
+	"github.com/opencodeco/validgen/internal/common"
+)
+
+// generatedOperations is the non-field set make testgen emits.
+var generatedOperations = []string{
+	"email",
+	"required",
+	"eq",
+	"neq",
+	"gt",
+	"gte",
+	"lt",
+	"lte",
+	"min",
+	"max",
+	"eq_ignore_case",
+	"neq_ignore_case",
+	"len",
+	"in",
+	"nin",
+}
+
+// fieldOperations stay in hand-written tests. Integer field operations are issue #78.
+var fieldOperations = []string{
+	"eqfield",
+	"neqfield",
+	"gtfield",
+	"gtefield",
+	"ltfield",
+	"ltefield",
+}
+
+// normalizedTypeClasses is every class HelperFromNormalizedToFieldTypes accepts.
+var normalizedTypeClasses = []string{
+	"<STRING>", "<INT>", "<FLOAT>", "<BOOL>", "<COMPLEX>",
+	"[]<STRING>", "[]<INT>", "[]<FLOAT>", "[]<BOOL>", "[]<COMPLEX>",
+	"[N]<STRING>", "[N]<INT>", "[N]<FLOAT>", "[N]<BOOL>", "[N]<COMPLEX>",
+	"map[<STRING>]", "map[<INT>]", "map[<FLOAT>]", "map[<BOOL>]", "map[<COMPLEX>]",
+}
+
+func TestTypesValidationListsNonFieldOperations(t *testing.T) {
+	ops := operations.New()
+	byTag := map[string]int{}
+	for _, entry := range typesValidation {
+		byTag[entry.tag]++
+		if entry.isFieldValidation {
+			t.Errorf("%s is marked as a field validation", entry.tag)
+		}
+		if !ops.IsValid(entry.tag) {
+			t.Errorf("%s is missing from the operations list", entry.tag)
+			continue
+		}
+		if ops.IsFieldOperation(entry.tag) {
+			t.Errorf("%s is a field operation", entry.tag)
+		}
+		if got := ops.ArgsCount(entry.tag); got != entry.argsCount {
+			t.Errorf("%s argument count is %v, catalog has %v", entry.tag, got, entry.argsCount)
+		}
+
+		classes := map[string]struct{}{}
+		for _, tc := range entry.testCases {
+			classes[tc.typeClass] = struct{}{}
+			if tc.validCase == "" || tc.errorMessage == "" {
+				t.Errorf("%s %s is missing a valid case or error message", entry.tag, tc.typeClass)
+			}
+			if tc.invalidCase == "" {
+				t.Errorf("%s %s is missing an invalid case", entry.tag, tc.typeClass)
+			}
+			if tc.invalidCase == "--" && tc.excludeIf&noPointer == 0 {
+				t.Errorf("%s %s uses -- without noPointer", entry.tag, tc.typeClass)
+			}
+		}
+		for _, class := range normalizedTypeClasses {
+			if !ops.IsValidByType(entry.tag, class) {
+				continue
+			}
+			if _, ok := classes[class]; !ok {
+				t.Errorf("%s accepts %s and typesValidation has no case", entry.tag, class)
+			}
+		}
+	}
+
+	inCatalog := map[string]struct{}{}
+	for tag := range byTag {
+		inCatalog[tag] = struct{}{}
+	}
+
+	for _, op := range generatedOperations {
+		if byTag[op] != 1 {
+			t.Errorf("%s appears %d times in typesValidation, want 1", op, byTag[op])
+		}
+		delete(byTag, op)
+	}
+	for tag := range byTag {
+		t.Errorf("%s is in typesValidation and is outside the generated set", tag)
+	}
+
+	for _, op := range fieldOperations {
+		if !ops.IsValid(op) || !ops.IsFieldOperation(op) {
+			t.Errorf("%s should stay a field operation in the operations list", op)
+		}
+		if _, ok := inCatalog[op]; ok {
+			t.Errorf("%s is a field operation and is in typesValidation", op)
+		}
+	}
+}
+
+func TestTypesValidationCasesBuildValidationCode(t *testing.T) {
+	gv := codegenerator.GenValidations{}
+
+	for _, entry := range typesValidation {
+		for _, tc := range entry.testCases {
+			for _, pointer := range []bool{false, true} {
+				if tc.excludeIf&noPointer != 0 && !pointer {
+					continue
+				}
+
+				normalizedType := tc.typeClass
+				if pointer {
+					normalizedType = "*" + normalizedType
+				}
+
+				fieldTypes, err := common.HelperFromNormalizedToFieldTypes(normalizedType)
+				if err != nil {
+					t.Errorf("%s %s pointer %v: %v", entry.tag, tc.typeClass, pointer, err)
+					continue
+				}
+
+				validation := entry.tag
+				if entry.argsCount != common.ZeroValue {
+					validation += "=" + tc.validation
+				}
+				parsed, err := analyzer.ParserValidation(validation)
+				if err != nil {
+					t.Errorf("parse %s: %v", validation, err)
+					continue
+				}
+
+				for _, fieldType := range fieldTypes {
+					const fieldName = "Field"
+					got, err := gv.BuildValidationCode(fieldName, fieldType, []*analyzer.Validation{parsed})
+					if err != nil {
+						t.Errorf("%s %s %s pointer %v: %v", entry.tag, tc.typeClass, fieldType.ToType(), pointer, err)
+						continue
+					}
+					if !errorTextMatches(got, fieldName, tc.errorMessage) {
+						t.Errorf("%s %s %s pointer %v: generated error text missing %q\n%s",
+							entry.tag, tc.typeClass, fieldType.ToType(), pointer, tc.errorMessage, got)
+					}
+				}
+			}
+		}
+	}
+}
+
+func errorTextMatches(generated, fieldName, catalogMessage string) bool {
+	message := strings.ReplaceAll(catalogMessage, "{{.FieldName}}", fieldName)
+	message = strings.ReplaceAll(message, "{{.Target}}", "\x00")
+	message = strings.ReplaceAll(message, "{{.Targets}}", "\x00")
+
+	from := 0
+	for _, part := range strings.Split(message, "\x00") {
+		part = strings.TrimSpace(part)
+		if part == "" {
+			continue
+		}
+		idx := strings.Index(generated[from:], part)
+		if idx < 0 {
+			return false
+		}
+		from += idx + len(part)
+	}
+	return from > 0
+}