diff --git a/.github/workflows/build-develop.yml b/.github/workflows/build-develop.yml index e6ec6d0ad..345daa1a4 100644 --- a/.github/workflows/build-develop.yml +++ b/.github/workflows/build-develop.yml @@ -140,7 +140,7 @@ jobs: - name: Configure CMake shell: bash working-directory: ${{github.workspace}}\build - run: PKG_CONFIG_PATH=${{github.workspace}}\local\lib\pkgconfig cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DCMAKE_PROJECT_TOP_LEVEL_INCLUDES=./cmake/conan_provider.cmake + run: PKG_CONFIG_PATH=${{github.workspace}}\local\lib\pkgconfig cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DCMAKE_PROJECT_TOP_LEVEL_INCLUDES=./cmake/conan_provider.cmake -DCONAN_INSTALL_BUILD_CONFIGURATIONS=$BUILD_TYPE - name: Get VERSION shell: pwsh diff --git a/.github/workflows/build-release.yml b/.github/workflows/build-release.yml index 19c036c5e..f8fcc57d5 100644 --- a/.github/workflows/build-release.yml +++ b/.github/workflows/build-release.yml @@ -184,7 +184,7 @@ jobs: - name: Configure CMake shell: bash working-directory: ${{github.workspace}}\build - run: PKG_CONFIG_PATH=${{github.workspace}}\local\lib\pkgconfig cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DCMAKE_PROJECT_TOP_LEVEL_INCLUDES=./cmake/conan_provider.cmake + run: PKG_CONFIG_PATH=${{github.workspace}}\local\lib\pkgconfig cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DCMAKE_PROJECT_TOP_LEVEL_INCLUDES=./cmake/conan_provider.cmake -DCONAN_INSTALL_BUILD_CONFIGURATIONS=$BUILD_TYPE - name: Get VERSION shell: pwsh diff --git a/CMakeLists.txt b/CMakeLists.txt index 7e673df1c..62d945918 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -108,12 +108,11 @@ enable_testing() message(STATUS "Building pktvisor version ${CMAKE_PROJECT_VERSION_MAJOR}.${CMAKE_PROJECT_VERSION_MINOR}.${CMAKE_PROJECT_VERSION_PATCH}${VISOR_PRERELEASE}") -# opentelemetry-cpp 1.26.0 generates proto headers with dllexport_decl=OPENTELEMETRY_PROTO_API; the Conan recipe doesn't propagate the macro. On Windows the proto package is a DLL (consumers must import); elsewhere it expands to nothing. -if(WIN32) - add_compile_definitions("OPENTELEMETRY_PROTO_API=__declspec(dllimport)") -else() - add_compile_definitions(OPENTELEMETRY_PROTO_API=) -endif() +# opentelemetry-cpp generates its proto headers with dllexport_decl=OPENTELEMETRY_PROTO_API and the +# Conan recipe does not propagate the macro to consumers. The recipe builds opentelemetry_proto as a +# static library on every platform (the shared option is removed on Windows), so the macro must +# expand to nothing; declaring dllimport against a static .lib leaves every proto symbol unresolved. +add_compile_definitions(OPENTELEMETRY_PROTO_API=) add_subdirectory(3rd) add_subdirectory(libs) diff --git a/README.md b/README.md index c6a27f5b9..efabd5375 100644 --- a/README.md +++ b/README.md @@ -251,7 +251,9 @@ or -H HOSTSPEC Specify subnets (comma separated) to consider HOST, in CIDR form. In live capture this /may/ be detected automatically from capture device but /must/ be specified for pcaps. Example: "10.0.1.0/24,10.0.2.1/32,2001:db8::/64" - Specifying this for live capture will append to any automatic detection. + For live capture, specifying this defines the host set explicitly and + disables automatic detection from the capture device; list every address + family (IPv4 and IPv6) you need. ``` @@ -396,7 +398,9 @@ docker run --rm netboxlabs/pktvisor pktvisor-reader --help --geo-asn FILE GeoLite2 ASN database to use for IP to ASN mapping (if enabled) -H HOSTSPEC Specify subnets (comma separated) to consider HOST, in CIDR form. In live capture this /may/ be detected automatically from capture device but /must/ be specified for pcaps. Example: "10.0.1.0/24,10.0.2.1/32,2001:db8::/64" - Specifying this for live capture will append to any automatic detection. + For live capture, specifying this defines the host set explicitly and + disables automatic detection from the capture device; list every address + family (IPv4 and IPv6) you need. ``` @@ -527,6 +531,9 @@ using [remote write](https://prometheus.io/docs/operating/integrations/#remote-e cloud providers, there is a [docker image available](https://hub.docker.com/r/netboxlabs/pktvisor-prom-write) to make this easy. See [centralized_collection/prometheus](centralized_collection/prometheus) for more. +To run pktvisor as a sidecar in Kubernetes and scrape it with Prometheus, see +[centralized_collection/k8s](centralized_collection/k8s). + Also see [getorb.io](https://getorb.io) for information on connecting pktvisor agents to the Orb observability platform. ### REST API diff --git a/centralized_collection/README.md b/centralized_collection/README.md index 694c5e4aa..863fb8c46 100644 --- a/centralized_collection/README.md +++ b/centralized_collection/README.md @@ -9,3 +9,4 @@ See the individual READMEs for more information: * [Prometheus](prometheus/README.md) * [Elasticsearch](elastic/README.md) +* [Kubernetes sidecar (scraped by Prometheus)](k8s/README.md) diff --git a/k8s/README.md b/centralized_collection/k8s/README.md similarity index 97% rename from k8s/README.md rename to centralized_collection/k8s/README.md index 4305689a6..bd6b8efd7 100644 --- a/k8s/README.md +++ b/centralized_collection/k8s/README.md @@ -43,8 +43,8 @@ the same `eth0` as your application container. ## Deploy ```shell -# from the repo root (or use the bare filename from inside k8s/) -kubectl apply -f k8s/pktvisor-sidecar.yaml +# from the repo root (or use the bare filename from inside centralized_collection/k8s/) +kubectl apply -f centralized_collection/k8s/pktvisor-sidecar.yaml ``` Creates a `pktvisor-demo` Deployment with three containers: your app (`nginx` @@ -140,7 +140,7 @@ explicit selection, do it inside a pktvisor-only scrape job. ## Grafana dashboard Import the community dashboard **ID 14221**, or the JSON at -`../centralized_collection/prometheus/grafana-dashboard-prometheus.json`. +`../prometheus/grafana-dashboard-prometheus.json`. ## Use with your own workload diff --git a/k8s/pktvisor-sidecar.yaml b/centralized_collection/k8s/pktvisor-sidecar.yaml similarity index 100% rename from k8s/pktvisor-sidecar.yaml rename to centralized_collection/k8s/pktvisor-sidecar.yaml diff --git a/conanfile.py b/conanfile.py index 783f94b69..87560f2e8 100644 --- a/conanfile.py +++ b/conanfile.py @@ -7,14 +7,14 @@ class Pktvisor(ConanFile): generators = "CMakeToolchain", "CMakeDeps" def requirements(self): - self.requires("catch2/3.15.1") - self.requires("cpp-httplib/0.27.0") + self.requires("catch2/3.16.0") + self.requires("cpp-httplib/0.47.0") self.requires("docopt.cpp/0.6.3") self.requires("fast-cpp-csv-parser/cci.20240102") self.requires("json-schema-validator/2.4.0") self.requires("libmaxminddb/1.12.2") self.requires("nlohmann_json/3.12.0", force=True) - self.requires("openssl/3.6.3") + self.requires("openssl/3.6.4") if self.settings.os != "Windows": self.requires("libpcap/1.10.6", force=True) else: @@ -27,7 +27,7 @@ def requirements(self): self.requires("uvw/3.4.0") self.requires("yaml-cpp/0.9.0") self.requires("robin-hood-hashing/3.11.5") - self.requires("libcurl/8.21.0") + self.requires("libcurl/8.22.0") self.requires("libnghttp2/1.68.1") if ( "libc" not in self.settings.compiler.fields @@ -37,6 +37,7 @@ def requirements(self): def configure(self): self.options["libcurl"].with_nghttp2 = True + self.options["cpp-httplib"].use_non_blocking_getaddrinfo = False def build_requirements(self): self.tool_requires("protobuf/6.33.5") diff --git a/src/handlers/netprobe/README.md b/src/handlers/netprobe/README.md index 0de224719..078e372ea 100644 --- a/src/handlers/netprobe/README.md +++ b/src/handlers/netprobe/README.md @@ -162,7 +162,7 @@ All metrics are per-target (keyed by the name given in the `targets` config map) | `dns_lookup_failures` | DNS resolution failures | | `packets_timeout` | Probes that timed out | | `http_status_failures` | HTTP/DoH responses whose HTTP status failed the configured status checks (default: any status outside 2xx/3xx). See [Success semantics](#success-semantics) above for the full `failure_status`/`expected_status` precedence — this counter fires whenever that evaluation lands on "fail," whether by the default 2xx/3xx rule, an `expected_status` miss, or a `failure_status` hit. | -| `content_failures` | HTTP responses whose status passed the status check but the configured `expected_body`/`expected_body_regex` check(s) did not match. Never incremented together with `successes` or `http_status_failures` for the same response — HTTP-only (not applicable to `doh`, which has no body-check config). | +| `content_failures` | HTTP responses whose status passed the status check but at least one configured response assertion failed — body (`expected_body`, `expected_body_regex`, `not_contains`, `body_not_matches_regex`), JSON (`json_path`/`json_equals`), size (`min_response_size_bytes`/`max_response_size_bytes`), header (`fail_if_header_matches`/`fail_if_header_not_matches`, `max_last_modified_diff_secs`) or protocol version (`valid_http_versions`). Never incremented together with `successes` or `http_status_failures` for the same response — HTTP-only (not applicable to `doh`, which has no response-assertion config). | | `top_status_codes` | Top HTTP status codes observed (e.g. `"200"`, `"404"`, `"503"`) | | `dns_response_failures` | DoH responses with HTTP 2xx/3xx but a non-NOERROR or unparseable DNS rcode | | `top_rcodes` | Top DNS response codes observed in DoH probes (e.g. `"NOERROR"`, `"NXDOMAIN"`, `"SRVFAIL"`, `"PARSE_ERROR"`) | diff --git a/src/inputs/netprobe/test_netprobe.cpp b/src/inputs/netprobe/test_netprobe.cpp index 513b7c256..266e31a13 100644 --- a/src/inputs/netprobe/test_netprobe.cpp +++ b/src/inputs/netprobe/test_netprobe.cpp @@ -168,6 +168,56 @@ TEST_CASE("NetProbe ip_version config", "[netprobe][config][ipv6]") } } +TEST_CASE("NetProbe ip_version config: literal IPv6 target is auto-detected and routed to a probe", "[netprobe][config][ipv6]") +{ + // Design-spec test #3. A ping stream cannot start unprivileged (the shared receiver opens raw + // sockets), so exercise the literal-target parse/route path — shared by ping and tcp — through + // the tcp test type: "::1" must be accepted as an IPv6 literal without any ip_version hint, + // survive start(), and be counted as one target. + NetProbeInputStream stream{"net-probe-test-v6-literal"}; + stream.config_set("test_type", "tcp"); + stream.config_set("interval_msec", 60000); + stream.config_set("timeout_msec", 100); + auto targets = std::make_shared(); + auto target = std::make_shared(); + target->config_set("target", "::1"); + target->config_set("port", 9); + targets->config_set>("v6_literal", target); + stream.config_set>("targets", targets); + + CHECK_NOTHROW(stream.start()); + nlohmann::json j; + stream.info_json(j); + CHECK(j[stream.schema_key()]["current_targets_total"] == 1); + CHECK(j["module"]["config"]["targets"]["v6_literal"]["target"] == "::1"); + CHECK_NOTHROW(stream.stop()); +} + +TEST_CASE("NetProbe ip_version config: DNS target with ip_version 6 round-trips through start and info_json", "[netprobe][config][ipv6]") +{ + // Design-spec test #4: a DNS (non-literal) target may carry ip_version: 6. It must parse, + // reach start() as a DNS-entry probe, and echo the per-target ip_version back via info_json. + // Same tcp-instead-of-ping justification as the literal-IPv6 case above. + NetProbeInputStream stream{"net-probe-test-v6-dns"}; + stream.config_set("test_type", "tcp"); + stream.config_set("interval_msec", 60000); + stream.config_set("timeout_msec", 100); + auto targets = std::make_shared(); + auto target = std::make_shared(); + target->config_set("target", "localhost"); + target->config_set("port", 9); + target->config_set("ip_version", 6); + targets->config_set>("v6_dns", target); + stream.config_set>("targets", targets); + + CHECK_NOTHROW(stream.start()); + nlohmann::json j; + stream.info_json(j); + CHECK(j[stream.schema_key()]["current_targets_total"] == 1); + CHECK(j["module"]["config"]["targets"]["v6_dns"]["ip_version"] == 6); + CHECK_NOTHROW(stream.stop()); +} + TEST_CASE("NetProbe http_method config validates", "[netprobe][config][http]") { // Validates that the http_method key is accepted by validate_configs (no throw before