diff --git a/README.md b/README.md index 328c2b5..4eef4c0 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,20 @@ In order to remove the duplicate tests, run the following command: sudo -E keploy-enterprise dedup --rm ``` +### Notes + +- **No Go toolchain needed in the app image.** Keploy's agent places a + `keploy-covdata-` binary next to the sockets, and the SDK uses it to + read the coverage data. If it is not there or it cannot start, + `go tool covdata` is used when `go` is on `PATH`. +- **`KEPLOY_COVERAGE_DIR`** is the folder that holds the two sockets + (`coverage_control.sock`, `coverage_data.sock`) and the coverage temp files. + It defaults to `/tmp`. In Kubernetes, Keploy sets it for you (the app sees + the shared folder at `/keploy-sockets`); you don't need to set it yourself. +- **Not measured.** When a test's coverage can't be taken (for example, the app + was not built with `-covermode=atomic`), the SDK tells Keploy why instead + of sending nothing. + # keploytest: Go tests under keploy's Universal Mocker Package `github.com/keploy/go-sdk/v3/keploytest` joins a `go test` suite to diff --git a/keploy/keploy.go b/keploy/keploy.go index 1bca4dd..0347e06 100644 --- a/keploy/keploy.go +++ b/keploy/keploy.go @@ -10,33 +10,126 @@ package keploy import ( "bufio" "bytes" + "context" "encoding/json" + "errors" "fmt" "log" "net" "os" "os/exec" "path/filepath" + "runtime" "runtime/coverage" + "runtime/debug" "sort" "strings" "sync" + "time" "golang.org/x/tools/cover" ) const ( + // coverageDirEnv names the folder Keploy shares with the app for the two + // sockets and the coverage temp files. In k8s the app sees the shared folder + // at /keploy-sockets, so Keploy sets it there. Unset keeps the old behaviour: + // sockets in /tmp, temp files in the OS default temp folder. + coverageDirEnv = "KEPLOY_COVERAGE_DIR" + // defaultSocketDir holds the sockets when coverageDirEnv is unset. + defaultSocketDir = "/tmp" + // covdataTimeout bounds one covdata run. Keploy waits on END with no + // deadline, so a hung covdata would stall the whole replay. + covdataTimeout = 30 * time.Second +) + +// errCovdataMissing means neither Keploy's covdata nor the go tool is there. +var errCovdataMissing = errors.New("no covdata tool: Keploy's keploy-covdata was not found and 'go' is not on PATH") + +var ( + // coverageDir is the value of coverageDirEnv, or "" when it is unset. + coverageDir = strings.TrimSpace(os.Getenv(coverageDirEnv)) // controlSocketPath is used by Keploy to send commands (START/END) to the app. - controlSocketPath = "/tmp/coverage_control.sock" + controlSocketPath = filepath.Join(socketDir(coverageDir), "coverage_control.sock") // dataSocketPath is used by the app to send coverage data back to Keploy. - dataSocketPath = "/tmp/coverage_data.sock" + dataSocketPath = filepath.Join(socketDir(coverageDir), "coverage_data.sock") ) +// socketDir returns the folder that holds the sockets: dir, or /tmp when dir is +// empty. +func socketDir(dir string) string { + if dir == "" { + return defaultSocketDir + } + return dir +} + +// coverageRecord is what the app sends on the data socket for one test. +type coverageRecord struct { + ID string `json:"id"` + ExecutedLinesByFile map[string][]int `json:"executedLinesByFile"` + // ExecutedBlocksByFile is file -> the blocks that ran, with columns. It is + // sent next to ExecutedLinesByFile, which older agents still need. + ExecutedBlocksByFile map[string][]codeBlock `json:"executedBlocksByFile,omitempty"` + // Totals are Go's own statement counts for the main module. + Totals *statementTotals `json:"totals,omitempty"` + // MetaHash names the build's coverage meta-data (the covmeta. file). + MetaHash string `json:"metaHash,omitempty"` + // BuildBlocks is file -> every block of the build, run or not. It goes with + // the first record of each test set only, since it is the same for every + // test of one build. + BuildBlocks map[string][]codeBlock `json:"buildBlocks,omitempty"` + // Reason says why the test has no coverage; nil when coverage was taken. + Reason *noCoverageReason `json:"reason,omitempty"` +} + +// codeBlock is one Go coverage block as +// [startLine, startCol, endLine, endCol, numStmt]. +type codeBlock [5]int + +// statementTotals are statement counts over the main module's blocks. +type statementTotals struct { + Statements int `json:"statements"` + CoveredStatements int `json:"coveredStatements"` +} + +// noCoverageReason explains a test with no coverage, so Keploy can show +// "not measured" with the cause instead of a silent gap. +type noCoverageReason struct { + Code string `json:"code"` + Message string `json:"message"` +} + +// Reason codes for noCoverageReason.Code. +const ( + reasonNoStart = "no_start" // END came with no START before it + reasonEndMismatch = "end_mismatch" // END named another test than the last START + reasonDumpFailed = "dump_failed" // writing the counters or meta-data failed + reasonCovdataFailed = "covdata_failed" // converting or reading the coverage data failed + reasonCovdataMissing = "covdata_missing" // no covdata tool to convert the data with +) + +// reportError carries the reason code of an error from reportCoverage. +type reportError struct { + code string + err error +} + +func (e *reportError) Error() string { return e.err.Error() } +func (e *reportError) Unwrap() error { return e.err } + var ( // controlMu protects access to the currentTestID, ensuring command handling is atomic. controlMu sync.Mutex // currentTestID stores the ID of the test case currently being recorded. currentTestID string + // buildTableSent and buildTableSentForSet record which test set's first + // record already carried the build's block table. + buildTableSent bool + buildTableSentForSet string + // useGoCovdata is set once Keploy's covdata could not start, so later + // tests go straight to 'go tool covdata'. + useGoCovdata bool ) // init starts the background control server that listens for commands from the Keploy test runner. @@ -111,19 +204,26 @@ func handleControlRequest(conn net.Conn) { case "END": if currentTestID != id { log.Printf("[Agent] Warning: Mismatched END command. Expected '%s', got '%s'. Skipping coverage report to avoid inconsistent state.", currentTestID, id) + code := reasonEndMismatch + if currentTestID == "" { + code = reasonNoStart + } + sendReason(id, code, fmt.Sprintf("END for %q, but the last START was for %q", id, currentTestID)) + ack(conn) return } err := reportCoverage(id) if err != nil { log.Printf("[Agent] 🚨 Error reporting coverage for test %s: %v", id, err) + var re *reportError + if errors.As(err, &re) { + sendReason(id, re.code, re.Error()) + } } // Reset the currentTestID to an empty string to indicate that no test is currently being recorded. currentTestID = "" - _, err = conn.Write([]byte("ACK\n")) - if err != nil { - log.Printf("[Agent] Error sending ACK to controller: %v", err) - } + ack(conn) default: log.Printf("[Agent] Unrecognized command: %s", action) } @@ -131,15 +231,10 @@ func handleControlRequest(conn net.Conn) { // reportCoverage dumps, processes, and sends the coverage data. func reportCoverage(testID string) error { - // Only take the part before the first slash, - // e.g. "test-2" from "test-set-0/test-2" - parts := strings.SplitN(testID, "/", 2) - baseID := parts[1] - // Create a temporary directory to store the coverage data. - tempDir, err := os.MkdirTemp("", fmt.Sprintf("keploy-coverage-%s-", baseID)) + tempDir, err := os.MkdirTemp(coverageDir, tempDirPattern(testID)) if err != nil { - return fmt.Errorf("failed to create temp dir: %w", err) + return &reportError{reasonDumpFailed, fmt.Errorf("failed to create temp dir: %w", err)} } defer func() { if err := os.RemoveAll(tempDir); err != nil { @@ -149,34 +244,128 @@ func reportCoverage(testID string) error { err = coverage.WriteCountersDir(tempDir) if err != nil { - return fmt.Errorf("failed to write coverage counters. Ensure the application was built with '-cover -covermode=atomic'. Original error: %w", err) + return &reportError{reasonDumpFailed, fmt.Errorf("failed to write coverage counters. Ensure the application was built with '-cover -covermode=atomic'. Original error: %w", err)} } err = coverage.WriteMetaDir(tempDir) if err != nil { - return fmt.Errorf("failed to write meta dir: %w", err) + return &reportError{reasonDumpFailed, fmt.Errorf("failed to write meta dir: %w", err)} } - processedData, err := processCoverageProfilesUsingCovdata(tempDir) + processed, err := processCoverageProfilesUsingCovdata(tempDir) if err != nil { - return fmt.Errorf("failed to process coverage profiles: %w", err) + code := reasonCovdataFailed + if errors.Is(err, errCovdataMissing) { + code = reasonCovdataMissing + } + return &reportError{code, fmt.Errorf("failed to process coverage profiles: %w", err)} } - if len(processedData) == 0 { + if len(processed.executedLines) == 0 { log.Printf("[Agent-Warning] No covered lines were found for test %s. The report will be empty.", testID) } - payload := map[string]interface{}{ - "id": testID, - "executedLinesByFile": processedData, - } + set := testSetOf(testID) + withTable := needsBuildTable(set) + record := buildRecord(testID, processed, metaHash(tempDir), withTable) - jsonData, err := json.Marshal(payload) + jsonData, err := json.Marshal(record) if err != nil { return fmt.Errorf("failed to marshal coverage data to JSON: %w", err) } - return sendToSocket(jsonData) + if err := sendToSocket(jsonData); err != nil { + return err + } + if withTable { + buildTableSent, buildTableSentForSet = true, set + } + return nil +} + +// buildRecord assembles the record for one test: the blocks that ran, and +// every block of the build when withTable is set. +func buildRecord(testID string, processed processedCoverage, hash string, withTable bool) coverageRecord { + totals := processed.totals() + record := coverageRecord{ + ID: testID, + ExecutedLinesByFile: processed.executedLines, + ExecutedBlocksByFile: processed.codeBlocks(true), + Totals: &totals, + MetaHash: hash, + } + if withTable { + record.BuildBlocks = processed.codeBlocks(false) + } + return record +} + +// needsBuildTable reports whether the record for a test of set must carry the +// build's block table: no record carried it yet, or the last one that did was +// for another set. The caller holds controlMu. +func needsBuildTable(set string) bool { + return !buildTableSent || set != buildTableSentForSet +} + +// testSetOf returns the test set part of a test ID ("test-set-0" from +// "test-set-0/test-2"), or "" when there is no slash. +func testSetOf(testID string) string { + if set, _, ok := strings.Cut(testID, "/"); ok { + return set + } + return "" +} + +// metaHash returns the hash from the covmeta. file WriteMetaDir wrote to +// dir, or "" when there is none. +func metaHash(dir string) string { + entries, err := os.ReadDir(dir) + if err != nil { + return "" + } + for _, e := range entries { + if hash, ok := strings.CutPrefix(e.Name(), "covmeta."); ok { + return hash + } + } + return "" +} + +// tempDirPattern builds the os.MkdirTemp pattern for a test ID. It takes the +// part after the first slash, e.g. "test-2" from "test-set-0/test-2", or the +// whole ID when there is no slash. Path separators left in the name are +// replaced, because MkdirTemp rejects a pattern that contains one. +func tempDirPattern(testID string) string { + _, name, found := strings.Cut(testID, "/") + if !found { + name = testID + } + name = strings.NewReplacer("/", "_", `\`, "_").Replace(name) + return fmt.Sprintf("keploy-coverage-%s-", name) +} + +// sendReason tells Keploy why a test has no coverage. A failure is only +// logged; the test is then missing, as it was before reasons existed. +func sendReason(testID, code, message string) { + data, err := json.Marshal(coverageRecord{ + ID: testID, + ExecutedLinesByFile: map[string][]int{}, + Reason: &noCoverageReason{Code: code, Message: message}, + }) + if err != nil { + log.Printf("[Agent] Error encoding the no-coverage reason for test %s: %v", testID, err) + return + } + if err := sendToSocket(data); err != nil { + log.Printf("[Agent] Error sending the no-coverage reason for test %s: %v", testID, err) + } +} + +// ack tells Keploy that the END command is done. +func ack(conn net.Conn) { + if _, err := conn.Write([]byte("ACK\n")); err != nil { + log.Printf("[Agent] Error sending ACK to controller: %v", err) + } } // sendToSocket connects to the Keploy data socket and writes the JSON payload. @@ -195,13 +384,70 @@ func sendToSocket(data []byte) error { return err } +// covdataPath is where Keploy's agent drops a covdata binary for this arch. +func covdataPath() string { + return filepath.Join(socketDir(coverageDir), "keploy-covdata-"+runtime.GOARCH) +} + +// covdataCommand returns the command that converts the binary coverage data in +// dir to text in out. It prefers the covdata Keploy delivers, so the app image +// needs no Go toolchain, and falls back to 'go tool covdata' when it is missing +// or could not start before. +func covdataCommand(ctx context.Context, dir, out string) (*exec.Cmd, error) { + args := []string{"textfmt", "-i=" + dir, "-o=" + out} + if info, err := os.Stat(covdataPath()); err == nil && info.Mode().IsRegular() && !useGoCovdata { + return exec.CommandContext(ctx, covdataPath(), args...), nil + } + if _, err := exec.LookPath("go"); err != nil { + return nil, errCovdataMissing + } + return exec.CommandContext(ctx, "go", append([]string{"tool", "covdata"}, args...)...), nil +} + +// processedCoverage is one test's coverage, read from a covdata text profile. +// Only files of the main module are kept (see fileKey). +type processedCoverage struct { + // executedLines is file -> sorted lines of the blocks that ran. + executedLines map[string][]int + // blocks is file -> every block of the file, run or not, with its count. + blocks map[string][]cover.ProfileBlock +} + +// codeBlocks returns file -> the blocks that ran (onlyRun) or every block. +func (p processedCoverage) codeBlocks(onlyRun bool) map[string][]codeBlock { + out := make(map[string][]codeBlock) + for file, blocks := range p.blocks { + for _, b := range blocks { + if onlyRun && b.Count <= 0 { + continue + } + out[file] = append(out[file], codeBlock{b.StartLine, b.StartCol, b.EndLine, b.EndCol, b.NumStmt}) + } + } + return out +} + +// totals sums the statements of every block, and of the blocks that ran. +func (p processedCoverage) totals() statementTotals { + var t statementTotals + for _, blocks := range p.blocks { + for _, b := range blocks { + t.Statements += b.NumStmt + if b.Count > 0 { + t.CoveredStatements += b.NumStmt + } + } + } + return t +} + // processCoverageProfilesUsingCovdata uses the covdata tool to convert binary coverage data to text format // and then processes it using the standard cover package. -func processCoverageProfilesUsingCovdata(dir string) (map[string][]int, error) { +func processCoverageProfilesUsingCovdata(dir string) (processedCoverage, error) { // Create a temporary file for the text format output - textFile, err := os.CreateTemp("", "coverage-*.txt") + textFile, err := os.CreateTemp(coverageDir, "coverage-*.txt") if err != nil { - return nil, fmt.Errorf("failed to create temp file for text coverage: %w", err) + return processedCoverage{}, fmt.Errorf("failed to create temp file for text coverage: %w", err) } defer func() { if err := os.Remove(textFile.Name()); err != nil { @@ -216,53 +462,58 @@ func processCoverageProfilesUsingCovdata(dir string) (map[string][]int, error) { }() // Use covdata to convert binary format to text format - cmd := exec.Command("go", "tool", "covdata", "textfmt", "-i="+dir, "-o="+textFile.Name()) + ctx, cancel := context.WithTimeout(context.Background(), covdataTimeout) + defer cancel() + cmd, err := covdataCommand(ctx, dir, textFile.Name()) + if err != nil { + return processedCoverage{}, err + } var stderr bytes.Buffer cmd.Stderr = &stderr err = cmd.Run() - if err != nil { - return nil, fmt.Errorf("failed to convert coverage data to text format: %w\nStderr: %s", err, stderr.String()) + var exitErr *exec.ExitError + if err != nil && cmd.Path == covdataPath() && !errors.As(err, &exitErr) { + // Keploy's covdata is there but could not start, e.g. its folder is + // mounted noexec. Use 'go tool covdata' instead, now and from then on. + // A covdata that ran and exited non-zero is not retried. + if _, lookErr := exec.LookPath("go"); lookErr == nil { + log.Printf("[Agent-Warning] Keploy's covdata could not start (%v). Using 'go tool covdata' from now on.", err) + useGoCovdata = true + cmd, err = covdataCommand(ctx, dir, textFile.Name()) + if err != nil { + return processedCoverage{}, err + } + cmd.Stderr = &stderr + err = cmd.Run() + } } - - // Get the module path (e.g., "your/module/path") to resolve file paths correctly. - modulePathCmd := exec.Command("go", "list", "-m") - var stderrModPath bytes.Buffer - modulePathCmd.Stderr = &stderrModPath - modulePathBytes, err := modulePathCmd.Output() if err != nil { - return nil, fmt.Errorf("failed to get module path with 'go list -m': %w\nStderr: %s", err, stderrModPath.String()) + return processedCoverage{}, fmt.Errorf("failed to convert coverage data to text format: %w\nStderr: %s", err, stderr.String()) } - modulePath := strings.TrimSpace(string(modulePathBytes)) - // Get the module's root directory on the filesystem. - moduleDirCmd := exec.Command("go", "list", "-m", "-f", "{{.Dir}}") - var stderrModDir bytes.Buffer - moduleDirCmd.Stderr = &stderrModDir - moduleDirBytes, err := moduleDirCmd.Output() - if err != nil { - return nil, fmt.Errorf("failed to get module directory with 'go list -m -f {{.Dir}}': %w\nStderr: %s", err, stderrModDir.String()) - } - moduleDir := strings.TrimSpace(string(moduleDirBytes)) + // The main module path (e.g., "your/module/path") from the binary itself, so + // no Go toolchain or source tree is needed in the app's image. + return parseProfiles(textFile.Name(), mainModulePath()) +} +// parseProfiles reads a covdata text profile into a processedCoverage. +func parseProfiles(path, modulePath string) (processedCoverage, error) { // Parse the text format using the standard cover package. - profiles, err := cover.ParseProfiles(textFile.Name()) + profiles, err := cover.ParseProfiles(path) if err != nil { - return nil, fmt.Errorf("failed to parse text coverage profile: %w", err) + return processedCoverage{}, fmt.Errorf("failed to parse text coverage profile: %w", err) } executedLinesByFile := make(map[string][]int) + blocksByFile := make(map[string][]cover.ProfileBlock) for _, profile := range profiles { - var absolutePath string - if strings.HasPrefix(profile.FileName, modulePath) { - relativePath := strings.TrimPrefix(profile.FileName, modulePath) - absolutePath = filepath.Join(moduleDir, relativePath) - } else if !filepath.IsAbs(profile.FileName) { + key, ok := fileKey(profile.FileName, modulePath) + if !ok { continue - } else { - absolutePath = profile.FileName } + blocksByFile[key] = profile.Blocks lineSet := make(map[int]bool) @@ -283,9 +534,34 @@ func processCoverageProfilesUsingCovdata(dir string) (map[string][]int, error) { lines = append(lines, line) } sort.Ints(lines) - executedLinesByFile[absolutePath] = lines + executedLinesByFile[key] = lines } } - return executedLinesByFile, nil + return processedCoverage{executedLines: executedLinesByFile, blocks: blocksByFile}, nil +} + +// mainModulePath returns the app's main module path from its build info, e.g. +// "example.com/app", or "" when the binary carries no build info. +func mainModulePath() string { + info, ok := debug.ReadBuildInfo() + if !ok { + return "" + } + return info.Main.Path +} + +// fileKey turns a file name from the coverage profile into the key Keploy +// stores. A main-module file becomes module-relative ("handlers/user.go"), the +// same on every machine. An absolute name, which Go uses for files built +// outside a module (go build -cover main.go), is kept as it is. A file of any +// other module is skipped (ok is false). +func fileKey(fileName, modulePath string) (key string, ok bool) { + if modulePath != "" && strings.HasPrefix(fileName, modulePath+"/") { + return strings.TrimPrefix(fileName, modulePath+"/"), true + } + if filepath.IsAbs(fileName) { + return fileName, true + } + return "", false } diff --git a/keploy/keploy_test.go b/keploy/keploy_test.go new file mode 100644 index 0000000..7e2d086 --- /dev/null +++ b/keploy/keploy_test.go @@ -0,0 +1,520 @@ +package keploy + +import ( + "context" + "encoding/json" + "errors" + "io" + "net" + "os" + "os/exec" + "path/filepath" + "reflect" + "runtime" + "runtime/coverage" + "strings" + "testing" + "time" +) + +func TestTempDirPattern(t *testing.T) { + tests := []struct { + name string + testID string + want string + }{ + {"set and case", "test-set-0/test-2", "keploy-coverage-test-2-"}, + {"no slash", "test-2", "keploy-coverage-test-2-"}, + {"slash in case name", "test-set-0/a/b", "keploy-coverage-a_b-"}, + {"backslash in case name", `test-set-0/a\b`, "keploy-coverage-a_b-"}, + {"empty", "", "keploy-coverage--"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tempDirPattern(tt.testID); got != tt.want { + t.Errorf("tempDirPattern(%q) = %q, want %q", tt.testID, got, tt.want) + } + }) + } +} + +func TestSocketDir(t *testing.T) { + if got := socketDir(""); got != "/tmp" { + t.Errorf(`socketDir("") = %q, want "/tmp"`, got) + } + if got := socketDir("/keploy-sockets"); got != "/keploy-sockets" { + t.Errorf(`socketDir("/keploy-sockets") = %q, want "/keploy-sockets"`, got) + } +} + +// Every pattern must be accepted by os.MkdirTemp; a raw "a/b" case name made +// it fail with "pattern contains path separator". +func TestTempDirPatternWorksWithMkdirTemp(t *testing.T) { + for _, testID := range []string{"test-set-0/test-2", "test-2", "test-set-0/a/b", `test-set-0/a\b`} { + dir, err := os.MkdirTemp(t.TempDir(), tempDirPattern(testID)) + if err != nil { + t.Fatalf("MkdirTemp with pattern for %q: %v", testID, err) + } + if _, err := os.Stat(dir); err != nil { + t.Fatalf("temp dir for %q not created: %v", testID, err) + } + } +} + +// A normal record must look exactly as before: no "reason" key. +func TestCoverageRecordWithoutReason(t *testing.T) { + data, err := json.Marshal(coverageRecord{ID: "set-0/test-1", ExecutedLinesByFile: map[string][]int{"a.go": {3}}}) + if err != nil { + t.Fatal(err) + } + got := string(data) + if strings.Contains(got, "reason") || !strings.Contains(got, `"executedLinesByFile":{"a.go":[3]}`) { + t.Fatalf("record JSON = %s", got) + } +} + +func TestEndWithoutStartSendsReason(t *testing.T) { + records := listenData(t) + setCurrentTestID(t, "") + + if reply := sendCommand(t, "END set-0/test-1"); reply != "ACK\n" { + t.Fatalf("reply = %q, want ACK", reply) + } + rec := receive(t, records) + if rec.ID != "set-0/test-1" || rec.Reason == nil || rec.Reason.Code != reasonNoStart { + t.Fatalf("record = %+v, want a %s reason for set-0/test-1", rec, reasonNoStart) + } +} + +func TestMismatchedEndSendsReason(t *testing.T) { + records := listenData(t) + setCurrentTestID(t, "set-0/test-0") + + if reply := sendCommand(t, "END set-0/test-1"); reply != "ACK\n" { + t.Fatalf("reply = %q, want ACK", reply) + } + rec := receive(t, records) + if rec.ID != "set-0/test-1" || rec.Reason == nil || rec.Reason.Code != reasonEndMismatch { + t.Fatalf("record = %+v, want an %s reason for set-0/test-1", rec, reasonEndMismatch) + } + if got := getCurrentTestID(); got != "set-0/test-0" { + t.Fatalf("currentTestID = %q, want it unchanged", got) + } +} + +func TestFailedDumpSendsReason(t *testing.T) { + if coverage.ClearCounters() == nil { + t.Skip("test binary built with -covermode=atomic, so the dump would succeed") + } + records := listenData(t) + setCurrentTestID(t, "set-0/test-2") + + if reply := sendCommand(t, "END set-0/test-2"); reply != "ACK\n" { + t.Fatalf("reply = %q, want ACK", reply) + } + rec := receive(t, records) + if rec.ID != "set-0/test-2" || rec.Reason == nil || rec.Reason.Code != reasonDumpFailed { + t.Fatalf("record = %+v, want a %s reason for set-0/test-2", rec, reasonDumpFailed) + } + if got := getCurrentTestID(); got != "" { + t.Fatalf("currentTestID = %q, want it reset", got) + } +} + +func TestCovdataCommandPrefersKeployCovdata(t *testing.T) { + dir := t.TempDir() + setCoverageDir(t, dir) + tool := filepath.Join(dir, "keploy-covdata-"+runtime.GOARCH) + if err := os.WriteFile(tool, []byte("#!/bin/sh\n"), 0o755); err != nil { + t.Fatal(err) + } + + cmd, err := covdataCommand(context.Background(), "in", "out") + if err != nil { + t.Fatal(err) + } + if cmd.Path != tool || strings.Join(cmd.Args[1:], " ") != "textfmt -i=in -o=out" { + t.Fatalf("command = %s %v, want %s textfmt -i=in -o=out", cmd.Path, cmd.Args, tool) + } +} + +func TestCovdataCommandFallsBackToGo(t *testing.T) { + setCoverageDir(t, t.TempDir()) + bin := t.TempDir() + goTool := filepath.Join(bin, "go") + if err := os.WriteFile(goTool, []byte("#!/bin/sh\n"), 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", bin) + + cmd, err := covdataCommand(context.Background(), "in", "out") + if err != nil { + t.Fatal(err) + } + if cmd.Path != goTool || strings.Join(cmd.Args[1:], " ") != "tool covdata textfmt -i=in -o=out" { + t.Fatalf("command = %s %v, want go tool covdata textfmt", cmd.Path, cmd.Args) + } +} + +func TestCovdataCommandMissing(t *testing.T) { + setCoverageDir(t, t.TempDir()) + t.Setenv("PATH", t.TempDir()) + + if _, err := covdataCommand(context.Background(), "in", "out"); !errors.Is(err, errCovdataMissing) { + t.Fatalf("err = %v, want errCovdataMissing", err) + } +} + +// Keploy's covdata is there but cannot start (e.g. /tmp is mounted noexec): +// 'go tool covdata' converts the data instead, and is used from then on. +func TestCovdataFallsBackToGoWhenKeployCovdataCannotStart(t *testing.T) { + covDir := writeCovdata(t) + dir := t.TempDir() + setCoverageDir(t, dir) + setUseGoCovdata(t, false) + tool := filepath.Join(dir, "keploy-covdata-"+runtime.GOARCH) + if err := os.WriteFile(tool, []byte("#!/bin/sh\n"), 0o644); err != nil { + t.Fatal(err) + } + + got, err := processCoverageProfilesUsingCovdata(covDir) + if err != nil { + t.Fatalf("err = %v, want 'go tool covdata' to convert the data", err) + } + if len(got.executedLines) == 0 { + t.Fatal("no executed lines: the data was not converted") + } + + // The next test must not try Keploy's covdata again. Make it one that + // starts, leaves a mark and fails. + mark := filepath.Join(t.TempDir(), "ran") + if err := os.WriteFile(tool, []byte("#!/bin/sh\ntouch "+mark+"\nexit 1\n"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.Chmod(tool, 0o755); err != nil { + t.Fatal(err) + } + if _, err := processCoverageProfilesUsingCovdata(covDir); err != nil { + t.Fatalf("second test: err = %v, want 'go tool covdata' to convert the data", err) + } + if _, err := os.Stat(mark); err == nil { + t.Fatal("Keploy's covdata was tried again after it could not start") + } +} + +// Keploy's covdata started and exited non-zero: its error is returned, and +// 'go tool covdata' is not tried. +func TestCovdataNoFallbackWhenKeployCovdataFails(t *testing.T) { + covDir := writeCovdata(t) + dir := t.TempDir() + setCoverageDir(t, dir) + setUseGoCovdata(t, false) + tool := filepath.Join(dir, "keploy-covdata-"+runtime.GOARCH) + if err := os.WriteFile(tool, []byte("#!/bin/sh\nexit 1\n"), 0o755); err != nil { + t.Fatal(err) + } + + _, err := processCoverageProfilesUsingCovdata(covDir) + var exitErr *exec.ExitError + if !errors.As(err, &exitErr) { + t.Fatalf("err = %v, want the exit error of Keploy's covdata", err) + } + if useGoCovdata { + t.Fatal("useGoCovdata was set, want Keploy's covdata kept") + } +} + +func TestFileKey(t *testing.T) { + const mod = "example.com/app" + tests := []struct { + name, fileName, modulePath, want string + ok bool + }{ + {"main module file", "example.com/app/handlers/user.go", mod, "handlers/user.go", true}, + {"main module root file", "example.com/app/main.go", mod, "main.go", true}, + {"module with the same prefix", "example.com/apptools/x.go", mod, "", false}, + {"other module", "github.com/lib/pq/conn.go", mod, "", false}, + {"absolute name, no module", "/src/app/main.go", "command-line-arguments", "/src/app/main.go", true}, + {"no build info", "example.com/app/main.go", "", "", false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, ok := fileKey(tt.fileName, tt.modulePath) + if got != tt.want || ok != tt.ok { + t.Errorf("fileKey(%q, %q) = %q, %v; want %q, %v", tt.fileName, tt.modulePath, got, ok, tt.want, tt.ok) + } + }) + } +} + +// sampleProfile is a covdata text profile as `go tool covdata textfmt` writes +// it: every block of the build, count 0 included. +const sampleProfile = `mode: atomic +example.com/app/handlers/hello.go:6.2,6.40 1 1 +example.com/app/handlers/hello.go:6.40,9.3 2 0 +example.com/app/handlers/hello.go:10.2,10.48 1 1 +example.com/app/main.go:10.13,13.2 2 1 +github.com/lib/x/x.go:1.1,2.2 1 1 +` + +func TestParseProfiles(t *testing.T) { + path := filepath.Join(t.TempDir(), "profile.txt") + if err := os.WriteFile(path, []byte(sampleProfile), 0o644); err != nil { + t.Fatal(err) + } + + got, err := parseProfiles(path, "example.com/app") + if err != nil { + t.Fatal(err) + } + wantLines := map[string][]int{"handlers/hello.go": {6, 10}, "main.go": {10, 11, 12, 13}} + if !reflect.DeepEqual(got.executedLines, wantLines) { + t.Errorf("executedLines = %v, want %v", got.executedLines, wantLines) + } + if n := len(got.blocks["handlers/hello.go"]); n != 3 { + t.Errorf("hello.go has %d blocks, want 3 (the block that did not run is kept)", n) + } + if _, ok := got.blocks["github.com/lib/x/x.go"]; ok { + t.Error("a file of another module was kept") + } + + wantBlocks := map[string][]codeBlock{ + "handlers/hello.go": {{6, 2, 6, 40, 1}, {10, 2, 10, 48, 1}}, + "main.go": {{10, 13, 13, 2, 2}}, + } + if blocks := got.codeBlocks(true); !reflect.DeepEqual(blocks, wantBlocks) { + t.Errorf("codeBlocks(true) = %v, want %v", blocks, wantBlocks) + } + + wantAll := map[string][]codeBlock{ + "handlers/hello.go": {{6, 2, 6, 40, 1}, {6, 40, 9, 3, 2}, {10, 2, 10, 48, 1}}, + "main.go": {{10, 13, 13, 2, 2}}, + } + if blocks := got.codeBlocks(false); !reflect.DeepEqual(blocks, wantAll) { + t.Errorf("codeBlocks(false) = %v, want %v", blocks, wantAll) + } + + if totals := got.totals(); totals != (statementTotals{Statements: 6, CoveredStatements: 4}) { + t.Errorf("totals() = %+v, want 6 statements, 4 covered", totals) + } +} + +func TestTestSetOf(t *testing.T) { + for id, want := range map[string]string{"test-set-0/test-2": "test-set-0", "a/b/c": "a", "test-2": ""} { + if got := testSetOf(id); got != want { + t.Errorf("testSetOf(%q) = %q, want %q", id, got, want) + } + } +} + +// A record carries only the blocks that ran, and the build's block table only +// with the first test of each test set. +func TestBuildRecord(t *testing.T) { + path := filepath.Join(t.TempDir(), "profile.txt") + if err := os.WriteFile(path, []byte(sampleProfile), 0o644); err != nil { + t.Fatal(err) + } + processed, err := parseProfiles(path, "example.com/app") + if err != nil { + t.Fatal(err) + } + setBuildTableSent(t, false, "") + + for _, step := range []struct { + testID string + wantTable bool + }{ + {"test-set-0/test-1", true}, + {"test-set-0/test-2", false}, + {"test-set-1/test-1", true}, + } { + set := testSetOf(step.testID) + withTable := needsBuildTable(set) + rec := buildRecord(step.testID, processed, "5f3e9a", withTable) + if rec.ID != step.testID || rec.MetaHash != "5f3e9a" || rec.Totals == nil || *rec.Totals != processed.totals() { + t.Errorf("%s: record = %+v, want its id, meta hash and totals", step.testID, rec) + } + if !reflect.DeepEqual(rec.ExecutedBlocksByFile, processed.codeBlocks(true)) { + t.Errorf("%s: ExecutedBlocksByFile = %v, want only the blocks that ran", step.testID, rec.ExecutedBlocksByFile) + } + if step.wantTable && !reflect.DeepEqual(rec.BuildBlocks, processed.codeBlocks(false)) { + t.Errorf("%s: BuildBlocks = %v, want every block of the build", step.testID, rec.BuildBlocks) + } + if !step.wantTable && rec.BuildBlocks != nil { + t.Errorf("%s: BuildBlocks = %v, want none after the set's first test", step.testID, rec.BuildBlocks) + } + if withTable { + // What reportCoverage does once the record is sent. + buildTableSent, buildTableSentForSet = true, set + } + } +} + +func TestMetaHash(t *testing.T) { + dir := t.TempDir() + for _, name := range []string{"covcounters.abc.1.2", "covmeta.5f3e9a"} { + if err := os.WriteFile(filepath.Join(dir, name), nil, 0o644); err != nil { + t.Fatal(err) + } + } + if got := metaHash(dir); got != "5f3e9a" { + t.Errorf("metaHash = %q, want 5f3e9a", got) + } + if got := metaHash(t.TempDir()); got != "" { + t.Errorf("metaHash of an empty dir = %q, want empty", got) + } +} + +// Blocks go out as plain arrays, next to the old lines field. +func TestCoverageRecordWithBlocks(t *testing.T) { + data, err := json.Marshal(coverageRecord{ + ID: "set-0/test-1", + ExecutedLinesByFile: map[string][]int{"a.go": {3}}, + ExecutedBlocksByFile: map[string][]codeBlock{"a.go": {{3, 2, 3, 20, 1}}}, + }) + if err != nil { + t.Fatal(err) + } + want := `{"id":"set-0/test-1","executedLinesByFile":{"a.go":[3]},"executedBlocksByFile":{"a.go":[[3,2,3,20,1]]}}` + if string(data) != want { + t.Fatalf("record JSON = %s, want %s", data, want) + } +} + +func TestMainModulePath(t *testing.T) { + if got := mainModulePath(); got != "github.com/keploy/go-sdk/v3" { + t.Errorf("mainModulePath() = %q, want the test binary's module", got) + } +} + +func setCoverageDir(t *testing.T, dir string) { + t.Helper() + old := coverageDir + coverageDir = dir + t.Cleanup(func() { coverageDir = old }) +} + +func setUseGoCovdata(t *testing.T, v bool) { + t.Helper() + old := useGoCovdata + useGoCovdata = v + t.Cleanup(func() { useGoCovdata = old }) +} + +func setBuildTableSent(t *testing.T, sent bool, set string) { + t.Helper() + oldSent, oldSet := buildTableSent, buildTableSentForSet + buildTableSent, buildTableSentForSet = sent, set + t.Cleanup(func() { buildTableSent, buildTableSentForSet = oldSent, oldSet }) +} + +// writeCovdata builds a small program with -cover, runs it, and returns the +// folder with its binary coverage data. +func writeCovdata(t *testing.T) string { + t.Helper() + goTool, err := exec.LookPath("go") + if err != nil { + t.Skip("'go' is not on PATH") + } + src, covDir := t.TempDir(), t.TempDir() + app := filepath.Join(t.TempDir(), "app") + if err := os.WriteFile(filepath.Join(src, "main.go"), []byte("package main\n\nfunc main() {\n\tprintln()\n}\n"), 0o644); err != nil { + t.Fatal(err) + } + build := exec.Command(goTool, "build", "-cover", "-o", app, "main.go") + build.Dir = src + if out, err := build.CombinedOutput(); err != nil { + t.Fatalf("go build -cover: %v\n%s", err, out) + } + run := exec.Command(app) + run.Env = append(os.Environ(), "GOCOVERDIR="+covDir) + if out, err := run.CombinedOutput(); err != nil { + t.Fatalf("running the covered program: %v\n%s", err, out) + } + return covDir +} + +// listenData points dataSocketPath at a fresh socket and returns the records +// sent to it. +func listenData(t *testing.T) <-chan coverageRecord { + t.Helper() + // A short folder: a unix socket path must stay under ~107 bytes. + dir, err := os.MkdirTemp("", "gsdk") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.RemoveAll(dir) }) + path := filepath.Join(dir, "d.sock") + ln, err := net.Listen("unix", path) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = ln.Close() }) + + old := dataSocketPath + dataSocketPath = path + t.Cleanup(func() { dataSocketPath = old }) + + records := make(chan coverageRecord, 4) + go func() { + for { + conn, err := ln.Accept() + if err != nil { + return + } + var rec coverageRecord + if err := json.NewDecoder(conn).Decode(&rec); err == nil { + records <- rec + } + _ = conn.Close() + } + }() + return records +} + +// sendCommand runs one control command through handleControlRequest and +// returns the app's reply. +func sendCommand(t *testing.T, command string) string { + t.Helper() + client, server := net.Pipe() + done := make(chan struct{}) + go func() { + handleControlRequest(server) + close(done) + }() + if _, err := client.Write([]byte(command + "\n")); err != nil { + t.Fatal(err) + } + reply, _ := io.ReadAll(client) + <-done + return string(reply) +} + +func receive(t *testing.T, records <-chan coverageRecord) coverageRecord { + t.Helper() + select { + case rec := <-records: + return rec + case <-time.After(2 * time.Second): + t.Fatal("no record arrived on the data socket") + return coverageRecord{} + } +} + +func setCurrentTestID(t *testing.T, id string) { + t.Helper() + controlMu.Lock() + currentTestID = id + controlMu.Unlock() + t.Cleanup(func() { + controlMu.Lock() + currentTestID = "" + controlMu.Unlock() + }) +} + +func getCurrentTestID() string { + controlMu.Lock() + defer controlMu.Unlock() + return currentTestID +}