From b54c65005af95bdd04f5c49baa3f16e08a9301cf Mon Sep 17 00:00:00 2001 From: Dmitry Plotnikov Date: Thu, 29 Feb 2024 10:06:37 -0800 Subject: [PATCH 01/28] Move gRPC/Protobuf logs handling to a dedicated file. --- CMakeLists.txt | 1 + src/grpc_log.cpp | 52 +++++++++++++++++++++++++++++++++++++++++++++ src/grpc_log.hpp | 3 +++ src/http_module.cpp | 34 +++-------------------------- src/ngx.hpp | 7 ++++++ 5 files changed, 66 insertions(+), 31 deletions(-) create mode 100644 src/grpc_log.cpp create mode 100644 src/grpc_log.hpp create mode 100644 src/ngx.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index baca6f8..910300c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -104,6 +104,7 @@ add_compile_options(-Wall -Wtype-limits -Werror) add_library(ngx_otel_module MODULE src/http_module.cpp + src/grpc_log.cpp src/modules.c ${PROTO_SOURCES}) diff --git a/src/grpc_log.cpp b/src/grpc_log.cpp new file mode 100644 index 0000000..d6e1150 --- /dev/null +++ b/src/grpc_log.cpp @@ -0,0 +1,52 @@ +#include "ngx.hpp" + +#include "grpc_log.hpp" + +#include +#include + +class ProtobufLog { +public: + ProtobufLog() { google::protobuf::SetLogHandler(protobufLogHandler); } + ~ProtobufLog() { google::protobuf::SetLogHandler(NULL); } + +private: + static void protobufLogHandler(google::protobuf::LogLevel logLevel, + const char* filename, int line, const std::string& msg) + { + using namespace google::protobuf; + + ngx_uint_t level = logLevel == LOGLEVEL_FATAL ? NGX_LOG_EMERG : + logLevel == LOGLEVEL_ERROR ? NGX_LOG_ERR : + logLevel == LOGLEVEL_WARNING ? NGX_LOG_WARN : + /*LOGLEVEL_INFO*/ NGX_LOG_INFO; + + ngx_log_error(level, ngx_cycle->log, 0, "OTel/protobuf: %s", + msg.c_str()); + } +}; + +class GrpcLog { +public: + GrpcLog() { gpr_set_log_function(grpcLogHandler); } + ~GrpcLog() { gpr_set_log_function(NULL); } + +private: + static void grpcLogHandler(gpr_log_func_args* args) + { + ngx_uint_t level = + args->severity == GPR_LOG_SEVERITY_ERROR ? NGX_LOG_ERR : + args->severity == GPR_LOG_SEVERITY_INFO ? NGX_LOG_INFO : + /*GPR_LOG_SEVERITY_DEBUG*/ NGX_LOG_DEBUG; + + ngx_log_error(level, ngx_cycle->log, 0, "OTel/grpc: %s", + args->message); + } + + ProtobufLog protoLog; +}; + +void initGrpcLog() +{ + static GrpcLog init; +} diff --git a/src/grpc_log.hpp b/src/grpc_log.hpp new file mode 100644 index 0000000..e6da5c9 --- /dev/null +++ b/src/grpc_log.hpp @@ -0,0 +1,3 @@ +#pragma once + +void initGrpcLog(); diff --git a/src/http_module.cpp b/src/http_module.cpp index 5ebe0a0..1ddffc3 100644 --- a/src/http_module.cpp +++ b/src/http_module.cpp @@ -1,11 +1,6 @@ -extern "C" { -#include -#include -#include -} +#include "ngx.hpp" -#include -#include +#include "grpc_log.hpp" #include "str_view.hpp" #include "trace_context.hpp" @@ -504,28 +499,6 @@ ngx_int_t onRequestEnd(ngx_http_request_t* r) return NGX_DECLINED; } -void grpcLogHandler(gpr_log_func_args* args) -{ - ngx_uint_t level = args->severity == GPR_LOG_SEVERITY_ERROR ? NGX_LOG_ERR : - args->severity == GPR_LOG_SEVERITY_INFO ? NGX_LOG_INFO : - /*GPR_LOG_SEVERITY_DEBUG*/ NGX_LOG_DEBUG; - - ngx_log_error(level, ngx_cycle->log, 0, "OTel/grpc: %s", args->message); -} - -void protobufLogHandler(google::protobuf::LogLevel logLevel, - const char* filename, int line, const std::string& msg) -{ - using namespace google::protobuf; - - ngx_uint_t level = logLevel == LOGLEVEL_FATAL ? NGX_LOG_EMERG : - logLevel == LOGLEVEL_ERROR ? NGX_LOG_ERR : - logLevel == LOGLEVEL_WARNING ? NGX_LOG_WARN : - /*LOGLEVEL_INFO*/ NGX_LOG_INFO; - - ngx_log_error(level, ngx_cycle->log, 0, "OTel/protobuf: %s", msg.c_str()); -} - ngx_int_t initModule(ngx_conf_t* cf) { auto cmcf = (ngx_http_core_main_conf_t*)ngx_http_conf_get_module_main_conf( @@ -547,8 +520,7 @@ ngx_int_t initModule(ngx_conf_t* cf) *h = onRequestEnd; - gpr_set_log_function(grpcLogHandler); - google::protobuf::SetLogHandler(protobufLogHandler); + initGrpcLog(); return NGX_OK; } diff --git a/src/ngx.hpp b/src/ngx.hpp new file mode 100644 index 0000000..63351fa --- /dev/null +++ b/src/ngx.hpp @@ -0,0 +1,7 @@ +#pragma once + +extern "C" { +#include +#include +#include +} From 93dc2b1878aa72d7c92dcb8fc8a08a198067b22d Mon Sep 17 00:00:00 2001 From: Dmitry Plotnikov Date: Thu, 29 Feb 2024 10:06:53 -0800 Subject: [PATCH 02/28] Use Abseil logging for Protobuf v22 and above (fix #16). --- src/grpc_log.cpp | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/src/grpc_log.cpp b/src/grpc_log.cpp index d6e1150..84d796c 100644 --- a/src/grpc_log.cpp +++ b/src/grpc_log.cpp @@ -3,6 +3,10 @@ #include "grpc_log.hpp" #include +#include + +#if GOOGLE_PROTOBUF_VERSION < 4022000 + #include class ProtobufLog { @@ -26,6 +30,43 @@ private: } }; +#else + +#include +#include +#include + +class ProtobufLog : absl::LogSink { +public: + ProtobufLog() + { + absl::InitializeLog(); + absl::AddLogSink(this); + // Disable logging to stderr + absl::SetStderrThreshold(static_cast(100)); + } + + ~ProtobufLog() override { absl::RemoveLogSink(this); } + + void Send(const absl::LogEntry& entry) override + { + auto severity = entry.log_severity(); + + ngx_uint_t level = + severity == absl::LogSeverity::kFatal ? NGX_LOG_EMERG : + severity == absl::LogSeverity::kError ? NGX_LOG_ERR : + severity == absl::LogSeverity::kWarning ? NGX_LOG_WARN : + /*absl::LogSeverity::kInfo*/ NGX_LOG_INFO; + + ngx_str_t message { entry.text_message().size(), + (u_char*)entry.text_message().data() }; + + ngx_log_error(level, ngx_cycle->log, 0, "OTel/protobuf: %V", &message); + } +}; + +#endif + class GrpcLog { public: GrpcLog() { gpr_set_log_function(grpcLogHandler); } From fa28f37dab4e7836fe92407940e407b843e41f2e Mon Sep 17 00:00:00 2001 From: Dmitry Plotnikov Date: Fri, 1 Mar 2024 18:48:39 +0000 Subject: [PATCH 03/28] Stop using system provided RE2 for static build. Starting with 2023-06-01 RE2 publicly depends on Abseil, so we can't use system provided RE2 together with gRPC-bundled Abseil. --- CMakeLists.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 910300c..c04c400 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -22,13 +22,12 @@ if(NGX_OTEL_FETCH_DEPS) grpc GIT_REPOSITORY https://github.com/grpc/grpc GIT_TAG 02384e39185f109bd299eb8482306229967dc970 # v1.46.7 - GIT_SUBMODULES third_party/protobuf third_party/abseil-cpp + GIT_SUBMODULES third_party/protobuf third_party/abseil-cpp third_party/re2 GIT_SHALLOW ON) set(gRPC_USE_PROTO_LITE ON CACHE INTERNAL "") set(gRPC_INSTALL OFF CACHE INTERNAL "") set(gRPC_CARES_PROVIDER package CACHE INTERNAL "") - set(gRPC_RE2_PROVIDER package CACHE INTERNAL "") set(gRPC_SSL_PROVIDER package CACHE INTERNAL "") set(gRPC_ZLIB_PROVIDER package CACHE INTERNAL "") From d6d7ce10151429f89b0fc0409d4340566297752d Mon Sep 17 00:00:00 2001 From: Dmitry Plotnikov Date: Tue, 30 Jan 2024 09:51:41 -0800 Subject: [PATCH 04/28] Bump gRPC to v1.49.4 to support building with GCC 13 (fix #13). --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c04c400..ea972ad 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -21,7 +21,7 @@ if(NGX_OTEL_FETCH_DEPS) FetchContent_Declare( grpc GIT_REPOSITORY https://github.com/grpc/grpc - GIT_TAG 02384e39185f109bd299eb8482306229967dc970 # v1.46.7 + GIT_TAG e241f37befe7ba4688effd84bfbf99b0f681a2f7 # v1.49.4 GIT_SUBMODULES third_party/protobuf third_party/abseil-cpp third_party/re2 GIT_SHALLOW ON) From 5a0071f497211ac12c018beeb3f8906981fec7fa Mon Sep 17 00:00:00 2001 From: Dmitry Plotnikov Date: Fri, 1 Mar 2024 06:53:55 +0000 Subject: [PATCH 05/28] Enforce compiler warnings in CI build only. --- .github/workflows/nginx-otel-module-check.yml | 9 +++++---- CMakeLists.txt | 9 ++++++--- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/.github/workflows/nginx-otel-module-check.yml b/.github/workflows/nginx-otel-module-check.yml index 2d8f5ee..bad1a32 100644 --- a/.github/workflows/nginx-otel-module-check.yml +++ b/.github/workflows/nginx-otel-module-check.yml @@ -24,7 +24,8 @@ jobs: - name: Build module working-directory: build run: | - cmake -DNGX_OTEL_NGINX_BUILD_DIR=${PWD}/../nginx/objs .. + cmake -DNGX_OTEL_NGINX_BUILD_DIR=${PWD}/../nginx/objs \ + -DNGX_OTEL_DEV=ON .. make -j 4 strip ngx_otel_module.so - name: Archive module @@ -75,11 +76,11 @@ jobs: working-directory: nginx run: | auto/configure --with-compat --with-debug --with-http_ssl_module \ - --with-http_v2_module --with-http_v3_module + --with-http_v2_module --with-http_v3_module make -j 4 - name: Run tests working-directory: tests run: | PERL5LIB=../nginx-tests/lib TEST_NGINX_UNSAFE=1 \ - TEST_NGINX_VERBOSE=1 TEST_NGINX_GLOBALS="load_module \ - ${PWD}/../build/ngx_otel_module.so;" prove -v . + TEST_NGINX_VERBOSE=1 TEST_NGINX_GLOBALS="load_module \ + ${PWD}/../build/ngx_otel_module.so;" prove -v . diff --git a/CMakeLists.txt b/CMakeLists.txt index ea972ad..50c0c55 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,8 +6,9 @@ set(NGX_OTEL_NGINX_BUILD_DIR "" set(NGX_OTEL_NGINX_DIR "${NGX_OTEL_NGINX_BUILD_DIR}/.." CACHE PATH "Nginx source dir") -set(NGX_OTEL_FETCH_DEPS ON CACHE BOOL "Download dependencies") -set(NGX_OTEL_PROTO_DIR "" CACHE PATH "OTel proto files root") +set(NGX_OTEL_FETCH_DEPS ON CACHE BOOL "Download dependencies") +set(NGX_OTEL_PROTO_DIR "" CACHE PATH "OTel proto files root") +set(NGX_OTEL_DEV OFF CACHE BOOL "Enforce compiler warnings") if(NOT CMAKE_BUILD_TYPE) set(CMAKE_BUILD_TYPE RelWithDebInfo) @@ -99,7 +100,9 @@ add_custom_command( set(CMAKE_CXX_STANDARD 11) set(CMAKE_CXX_EXTENSIONS OFF) -add_compile_options(-Wall -Wtype-limits -Werror) +if (NGX_OTEL_DEV) + add_compile_options(-Wall -Wtype-limits -Werror) +endif() add_library(ngx_otel_module MODULE src/http_module.cpp From 30b9b73546935a72cf3daab44e081e1364a90d2e Mon Sep 17 00:00:00 2001 From: Pavel Pautov Date: Mon, 11 Mar 2024 22:36:34 -0700 Subject: [PATCH 06/28] Disable unnecessary gRPC features to reduce binary size. Incidentally, this also removes RE2 code from the binary. --- CMakeLists.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 50c0c55..4b85113 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -47,6 +47,9 @@ if(NGX_OTEL_FETCH_DEPS) FetchContent_MakeAvailable(grpc otelcpp) + # reconsider once https://github.com/grpc/grpc/issues/36023 is done + target_compile_definitions(grpc PRIVATE GRPC_NO_XDS GRPC_NO_RLS) + set_property(DIRECTORY ${grpc_SOURCE_DIR} PROPERTY EXCLUDE_FROM_ALL YES) set_property(DIRECTORY ${otelcpp_SOURCE_DIR} From d3817136395a326dd9626580d81e0c4703fc1463 Mon Sep 17 00:00:00 2001 From: Dmitry Plotnikov Date: Tue, 12 Mar 2024 12:04:45 -0700 Subject: [PATCH 07/28] Fix undefined symbols error on Mac (fix #38). --- CMakeLists.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4b85113..8588f52 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -119,6 +119,10 @@ set_target_properties(ngx_otel_module PROPERTIES PREFIX "") # can't use OTel's WITH_ABSEIL until cmake 3.24, as it triggers find_package() target_compile_definitions(ngx_otel_module PRIVATE HAVE_ABSEIL) +if (APPLE) + target_link_options(ngx_otel_module PRIVATE -undefined dynamic_lookup) +endif() + target_include_directories(ngx_otel_module PRIVATE ${NGX_OTEL_NGINX_BUILD_DIR} ${NGX_OTEL_NGINX_DIR}/src/core From 01a40c271e4cf5bb4630229db4f228060e2af8a1 Mon Sep 17 00:00:00 2001 From: Pavel Pautov Date: Tue, 12 Mar 2024 23:43:09 -0700 Subject: [PATCH 08/28] Unify CMake and Nginx build system defaults. Provide generic environment variable to adjust CMake settings from Nginx build system. --- config | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/config b/config index 11d3317..75378e2 100644 --- a/config +++ b/config @@ -1,10 +1,9 @@ ngx_addon_name=ngx_otel_module cmake -D NGX_OTEL_NGINX_BUILD_DIR=$NGX_OBJS \ - -D NGX_OTEL_FETCH_DEPS=OFF \ - -D NGX_OTEL_PROTO_DIR=$NGX_OTEL_PROTO_DIR \ -D CMAKE_LIBRARY_OUTPUT_DIRECTORY=$PWD/$NGX_OBJS \ -D "CMAKE_C_FLAGS=$NGX_CC_OPT" \ -D "CMAKE_CXX_FLAGS=$NGX_CC_OPT" \ -D "CMAKE_MODULE_LINKER_FLAGS=$NGX_LD_OPT" \ + $NGX_OTEL_CMAKE_OPTS \ -S $ngx_addon_dir -B $NGX_OBJS/otel || exit 1 From 6ed3910afbfcdec9123238c5a625be60c53d6479 Mon Sep 17 00:00:00 2001 From: Pavel Pautov Date: Mon, 15 Jul 2024 16:12:19 -0700 Subject: [PATCH 09/28] Support custom versions of auto-fetched build dependencies. --- CMakeLists.txt | 46 ++++++++++++++++++++++++++++++---------------- 1 file changed, 30 insertions(+), 16 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8588f52..645c5d5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,7 +6,10 @@ set(NGX_OTEL_NGINX_BUILD_DIR "" set(NGX_OTEL_NGINX_DIR "${NGX_OTEL_NGINX_BUILD_DIR}/.." CACHE PATH "Nginx source dir") -set(NGX_OTEL_FETCH_DEPS ON CACHE BOOL "Download dependencies") +set(NGX_OTEL_GRPC e241f37befe7ba4688effd84bfbf99b0f681a2f7 # v1.49.4 + CACHE STRING "gRPC tag to download or 'package' to use preinstalled") +set(NGX_OTEL_SDK 11d5d9e0d8fd8ba876c8994714cc2647479b6574 # v1.11.0 + CACHE STRING "OTel SDK tag to download or 'package' to use preinstalled") set(NGX_OTEL_PROTO_DIR "" CACHE PATH "OTel proto files root") set(NGX_OTEL_DEV OFF CACHE BOOL "Enforce compiler warnings") @@ -16,13 +19,16 @@ endif() set(CMAKE_CXX_VISIBILITY_PRESET hidden) -if(NGX_OTEL_FETCH_DEPS) +if(NGX_OTEL_GRPC STREQUAL "package") + find_package(protobuf REQUIRED) + find_package(gRPC REQUIRED) +else() include(FetchContent) FetchContent_Declare( grpc GIT_REPOSITORY https://github.com/grpc/grpc - GIT_TAG e241f37befe7ba4688effd84bfbf99b0f681a2f7 # v1.49.4 + GIT_TAG ${NGX_OTEL_GRPC} GIT_SUBMODULES third_party/protobuf third_party/abseil-cpp third_party/re2 GIT_SHALLOW ON) @@ -32,10 +38,29 @@ if(NGX_OTEL_FETCH_DEPS) set(gRPC_SSL_PROVIDER package CACHE INTERNAL "") set(gRPC_ZLIB_PROVIDER package CACHE INTERNAL "") + set(CMAKE_POSITION_INDEPENDENT_CODE ON) + + FetchContent_MakeAvailable(grpc) + + # reconsider once https://github.com/grpc/grpc/issues/36023 is done + target_compile_definitions(grpc PRIVATE GRPC_NO_XDS GRPC_NO_RLS) + + set_property(DIRECTORY ${grpc_SOURCE_DIR} + PROPERTY EXCLUDE_FROM_ALL YES) + + add_library(gRPC::grpc++ ALIAS grpc++) + add_executable(gRPC::grpc_cpp_plugin ALIAS grpc_cpp_plugin) +endif() + +if(NGX_OTEL_SDK STREQUAL "package") + find_package(opentelemetry-cpp REQUIRED) +else() + include(FetchContent) + FetchContent_Declare( otelcpp GIT_REPOSITORY https://github.com/open-telemetry/opentelemetry-cpp - GIT_TAG 11d5d9e0d8fd8ba876c8994714cc2647479b6574 # v1.11.0 + GIT_TAG ${NGX_OTEL_SDK} GIT_SUBMODULES third_party/opentelemetry-proto GIT_SHALLOW ON) @@ -45,13 +70,8 @@ if(NGX_OTEL_FETCH_DEPS) set(CMAKE_POSITION_INDEPENDENT_CODE ON) set(CMAKE_POLICY_DEFAULT_CMP0063 NEW) - FetchContent_MakeAvailable(grpc otelcpp) + FetchContent_MakeAvailable(otelcpp) - # reconsider once https://github.com/grpc/grpc/issues/36023 is done - target_compile_definitions(grpc PRIVATE GRPC_NO_XDS GRPC_NO_RLS) - - set_property(DIRECTORY ${grpc_SOURCE_DIR} - PROPERTY EXCLUDE_FROM_ALL YES) set_property(DIRECTORY ${otelcpp_SOURCE_DIR} PROPERTY EXCLUDE_FROM_ALL YES) @@ -61,12 +81,6 @@ if(NGX_OTEL_FETCH_DEPS) endif() add_library(opentelemetry-cpp::trace ALIAS opentelemetry_trace) - add_library(gRPC::grpc++ ALIAS grpc++) - add_executable(gRPC::grpc_cpp_plugin ALIAS grpc_cpp_plugin) -else() - find_package(opentelemetry-cpp REQUIRED) - find_package(protobuf REQUIRED) - find_package(gRPC REQUIRED) endif() set(PROTO_DIR ${NGX_OTEL_PROTO_DIR}) From 10215eee1b1fa15d701aa3f30a06a71b09f77720 Mon Sep 17 00:00:00 2001 From: Pavel Pautov Date: Tue, 16 Jul 2024 14:13:31 -0700 Subject: [PATCH 10/28] Support building with latest gRPC versions (up to v1.65.0). --- CMakeLists.txt | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 645c5d5..b745342 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -34,10 +34,14 @@ else() set(gRPC_USE_PROTO_LITE ON CACHE INTERNAL "") set(gRPC_INSTALL OFF CACHE INTERNAL "") + set(gRPC_USE_SYSTEMD OFF CACHE INTERNAL "") + set(gRPC_DOWNLOAD_ARCHIVES OFF CACHE INTERNAL "") set(gRPC_CARES_PROVIDER package CACHE INTERNAL "") set(gRPC_SSL_PROVIDER package CACHE INTERNAL "") set(gRPC_ZLIB_PROVIDER package CACHE INTERNAL "") + set(protobuf_INSTALL OFF CACHE INTERNAL "") + set(CMAKE_POSITION_INDEPENDENT_CODE ON) FetchContent_MakeAvailable(grpc) @@ -108,8 +112,8 @@ add_custom_command( --plugin protoc-gen-grpc=$ ${PROTOS} # remove inconsequential UTF8 check during serialization to aid performance - COMMAND sed -i.bak - -e [[/ ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String(/,/);/d]] + COMMAND sed -i.bak -E + -e [[/ ::(PROTOBUF_NAMESPACE_ID|google::protobuf)::internal::WireFormatLite::VerifyUtf8String\(/,/\);/d]] ${PROTO_SOURCES} DEPENDS ${PROTOS} protobuf::protoc gRPC::grpc_cpp_plugin VERBATIM) From 4c24716eef5709489cf6fd61736c0e1beae80ce5 Mon Sep 17 00:00:00 2001 From: Pavel Pautov Date: Tue, 16 Jul 2024 22:45:46 -0700 Subject: [PATCH 11/28] Don't force C++ standard for user builds. This fixes build against C++17 enabled prebuilt dependencies. --- CMakeLists.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b745342..7a74f0a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -118,10 +118,10 @@ add_custom_command( DEPENDS ${PROTOS} protobuf::protoc gRPC::grpc_cpp_plugin VERBATIM) -set(CMAKE_CXX_STANDARD 11) -set(CMAKE_CXX_EXTENSIONS OFF) - if (NGX_OTEL_DEV) + set(CMAKE_CXX_STANDARD 11) + set(CMAKE_CXX_EXTENSIONS OFF) + add_compile_options(-Wall -Wtype-limits -Werror) endif() From 4c841c1c55ea98f22806b19cdb9a7efbab43dc0f Mon Sep 17 00:00:00 2001 From: Pavel Pautov Date: Wed, 17 Jul 2024 16:34:06 -0700 Subject: [PATCH 12/28] Use Abseil logging for gRPC v1.65.0 and above. Original logging method is now deprecated and results in error message on Nginx startup. --- src/grpc_log.cpp | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/src/grpc_log.cpp b/src/grpc_log.cpp index 84d796c..bb83364 100644 --- a/src/grpc_log.cpp +++ b/src/grpc_log.cpp @@ -2,8 +2,8 @@ #include "grpc_log.hpp" -#include #include +#include #if GOOGLE_PROTOBUF_VERSION < 4022000 @@ -36,9 +36,9 @@ private: #include #include -class ProtobufLog : absl::LogSink { +class NgxLogSink : absl::LogSink { public: - ProtobufLog() + NgxLogSink() { absl::InitializeLog(); absl::AddLogSink(this); @@ -46,7 +46,7 @@ public: absl::SetStderrThreshold(static_cast(100)); } - ~ProtobufLog() override { absl::RemoveLogSink(this); } + ~NgxLogSink() override { absl::RemoveLogSink(this); } void Send(const absl::LogEntry& entry) override { @@ -61,12 +61,19 @@ public: ngx_str_t message { entry.text_message().size(), (u_char*)entry.text_message().data() }; - ngx_log_error(level, ngx_cycle->log, 0, "OTel/protobuf: %V", &message); + ngx_log_error(level, ngx_cycle->log, 0, "OTel/grpc: %V", &message); } }; +typedef NgxLogSink ProtobufLog; + #endif +#if (GRPC_CPP_VERSION_MAJOR < 1) || \ + (GRPC_CPP_VERSION_MAJOR == 1 && GRPC_CPP_VERSION_MINOR < 65) + +#include + class GrpcLog { public: GrpcLog() { gpr_set_log_function(grpcLogHandler); } @@ -87,6 +94,13 @@ private: ProtobufLog protoLog; }; +#else + +// newer gRPC implies newer protobuf, and both use Abseil for logging +typedef NgxLogSink GrpcLog; + +#endif + void initGrpcLog() { static GrpcLog init; From 1074d02758156a8c2e5a9e32f16293353226e830 Mon Sep 17 00:00:00 2001 From: Eugene Grebenschikov Date: Fri, 4 Oct 2024 11:21:01 -0700 Subject: [PATCH 13/28] Use github repos for nginx and nginx-tests. --- .github/workflows/nginx-otel-module-check.yml | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/.github/workflows/nginx-otel-module-check.yml b/.github/workflows/nginx-otel-module-check.yml index bad1a32..77a4332 100644 --- a/.github/workflows/nginx-otel-module-check.yml +++ b/.github/workflows/nginx-otel-module-check.yml @@ -15,7 +15,10 @@ jobs: sudo apt-get update sudo apt-get install -y cmake libc-ares-dev libre2-dev - name: Checkout nginx - run: hg clone http://hg.nginx.org/nginx/ + uses: actions/checkout@v4 + with: + repository: nginx/nginx + path: nginx - name: Configure nginx working-directory: nginx run: auto/configure --with-compat @@ -68,16 +71,24 @@ jobs: open-telemetry/opentelemetry-collector-releases/releases/download/\ v0.76.1/otelcol_0.76.1_linux_amd64.tar.gz tar -xzf otelcol_0.76.1_linux_amd64.tar.gz - - name: Checkout nginx and nginx-test - run: | - hg clone http://hg.nginx.org/nginx/ - hg clone http://hg.nginx.org/nginx-tests/ + - name: Checkout nginx + uses: actions/checkout@v4 + with: + repository: nginx/nginx + path: nginx - name: Build nginx working-directory: nginx run: | auto/configure --with-compat --with-debug --with-http_ssl_module \ --with-http_v2_module --with-http_v3_module make -j 4 + - name: Checkout lib from nginx-tests + uses: actions/checkout@v4 + with: + repository: nginx/nginx-tests + sparse-checkout: | + lib + path: nginx-tests - name: Run tests working-directory: tests run: | From b5c8cd8de3e5ea131681d425d0f82e378ed928c5 Mon Sep 17 00:00:00 2001 From: Pavel Pautov Date: Sat, 9 Nov 2024 15:51:33 -0800 Subject: [PATCH 14/28] Support custom resource attributes (fix #32). Now attributes can be set with "otel_resource_attr" directive, e.g. otel_resource_attr my.name "my value"; --- src/batch_exporter.hpp | 11 +++-- src/http_module.cpp | 93 ++++++++++++++++++++++++++++++++---------- 2 files changed, 79 insertions(+), 25 deletions(-) diff --git a/src/batch_exporter.hpp b/src/batch_exporter.hpp index d160d2c..2432fe3 100644 --- a/src/batch_exporter.hpp +++ b/src/batch_exporter.hpp @@ -112,7 +112,8 @@ public: }; BatchExporter(StrView target, - size_t batchSize, size_t batchCount, StrView serviceName) : + size_t batchSize, size_t batchCount, + const std::map& resourceAttrs) : batchSize(batchSize), client(std::string(target)) { free.reserve(batchCount); @@ -120,9 +121,11 @@ public: free.emplace_back(); auto resourceSpans = free.back().add_resource_spans(); - auto attr = resourceSpans->mutable_resource()->add_attributes(); - attr->set_key("service.name"); - attr->mutable_value()->set_string_value(std::string(serviceName)); + for (auto& attr : resourceAttrs) { + auto kv = resourceSpans->mutable_resource()->add_attributes(); + kv->set_key(std::string(attr.first)); + kv->mutable_value()->set_string_value(std::string(attr.second)); + } auto scopeSpans = resourceSpans->add_scope_spans(); scopeSpans->mutable_scope()->set_name("nginx"); diff --git a/src/http_module.cpp b/src/http_module.cpp index 1ddffc3..5c73ef0 100644 --- a/src/http_module.cpp +++ b/src/http_module.cpp @@ -15,7 +15,7 @@ struct OtelCtx { TraceContext current; }; -struct MainConf { +struct MainConfBase { ngx_str_t endpoint; ngx_msec_t interval; size_t batchSize; @@ -24,6 +24,10 @@ struct MainConf { ngx_str_t serviceName; }; +struct MainConf : MainConfBase { + std::map resourceAttrs; +}; + struct SpanAttr { ngx_str_t name; ngx_http_complex_value_t value; @@ -38,6 +42,7 @@ struct LocationConf { }; char* setExporter(ngx_conf_t* cf, ngx_command_t* cmd, void* conf); +char* addResourceAttr(ngx_conf_t* cf, ngx_command_t* cmd, void* conf); char* addSpanAttr(ngx_conf_t* cf, ngx_command_t* cmd, void* conf); namespace Propagation { @@ -59,14 +64,17 @@ ngx_command_t gCommands[] = { { ngx_string("otel_exporter"), NGX_HTTP_MAIN_CONF|NGX_CONF_BLOCK|NGX_CONF_NOARGS, - setExporter, - NGX_HTTP_MAIN_CONF_OFFSET }, + setExporter }, + + { ngx_string("otel_resource_attr"), + NGX_HTTP_MAIN_CONF|NGX_CONF_TAKE2, + addResourceAttr }, { ngx_string("otel_service_name"), NGX_HTTP_MAIN_CONF|NGX_CONF_TAKE1, ngx_conf_set_str_slot, NGX_HTTP_MAIN_CONF_OFFSET, - offsetof(MainConf, serviceName) }, + offsetof(MainConfBase, serviceName) }, { ngx_string("otel_trace"), NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1, @@ -101,25 +109,25 @@ ngx_command_t gExporterCommands[] = { NGX_CONF_TAKE1, ngx_conf_set_str_slot, 0, - offsetof(MainConf, endpoint) }, + offsetof(MainConfBase, endpoint) }, { ngx_string("interval"), NGX_CONF_TAKE1, ngx_conf_set_msec_slot, 0, - offsetof(MainConf, interval) }, + offsetof(MainConfBase, interval) }, { ngx_string("batch_size"), NGX_CONF_TAKE1, ngx_conf_set_size_slot, 0, - offsetof(MainConf, batchSize) }, + offsetof(MainConfBase, batchSize) }, { ngx_string("batch_count"), NGX_CONF_TAKE1, ngx_conf_set_size_slot, 0, - offsetof(MainConf, batchCount) }, + offsetof(MainConfBase, batchCount) }, ngx_null_command }; @@ -136,6 +144,18 @@ ngx_str_t toNgxStr(StrView str) return ngx_str_t{str.size(), (u_char*)str.data()}; } +MainConf* getMainConf(ngx_conf_t* cf) +{ + return static_cast( + (MainConfBase*)ngx_http_conf_get_module_main_conf(cf, gHttpModule)); +} + +MainConf* getMainConf(ngx_cycle_t* cycle) +{ + return static_cast( + (MainConfBase*)ngx_http_cycle_get_module_main_conf(cycle, gHttpModule)); +} + LocationConf* getLocationConf(ngx_http_request_t* r) { return (LocationConf*)ngx_http_get_module_loc_conf(r, gHttpModule); @@ -527,8 +547,7 @@ ngx_int_t initModule(ngx_conf_t* cf) ngx_int_t initWorkerProcess(ngx_cycle_t* cycle) { - auto mcf = (MainConf*)ngx_http_cycle_get_module_main_conf( - cycle, gHttpModule); + auto mcf = getMainConf(cycle); // no 'http' or 'otel_exporter' blocks if (mcf == NULL || mcf->endpoint.len == 0) { @@ -540,7 +559,7 @@ ngx_int_t initWorkerProcess(ngx_cycle_t* cycle) toStrView(mcf->endpoint), mcf->batchSize, mcf->batchCount, - toStrView(mcf->serviceName))); + mcf->resourceAttrs)); } catch (const std::exception& e) { ngx_log_error(NGX_LOG_CRIT, cycle->log, 0, "OTel worker init error: %s", e.what()); @@ -561,8 +580,7 @@ ngx_int_t initWorkerProcess(ngx_cycle_t* cycle) "OTel flush error: %s", e.what()); } - auto mcf = (MainConf*)ngx_http_cycle_get_module_main_conf( - ngx_cycle, gHttpModule); + auto mcf = getMainConf((ngx_cycle_t*)ngx_cycle); ngx_add_timer(ev, mcf->interval); }; @@ -590,7 +608,7 @@ void exitWorkerProcess(ngx_cycle_t* cycle) char* setExporter(ngx_conf_t* cf, ngx_command_t* cmd, void* conf) { - auto mcf = (MainConf*)conf; + auto mcf = getMainConf(cf); if (mcf->endpoint.len) { return (char*)"is duplicate"; @@ -649,31 +667,64 @@ char* setExporter(ngx_conf_t* cf, ngx_command_t* cmd, void* conf) return NGX_CONF_OK; } +char* addResourceAttr(ngx_conf_t* cf, ngx_command_t* cmd, void* conf) +{ + auto mcf = getMainConf(cf); + + try { + auto args = (ngx_str_t*)cf->args->elts; + mcf->resourceAttrs[toStrView(args[1])] = toStrView(args[2]); + } catch (const std::exception& e) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "OTel: %s", e.what()); + return (char*)NGX_CONF_ERROR; + } + + return NGX_CONF_OK; +} + void* createMainConf(ngx_conf_t* cf) { - auto mcf = (MainConf*)ngx_pcalloc(cf->pool, sizeof(MainConf)); - if (mcf == NULL) { + auto cln = ngx_pool_cleanup_add(cf->pool, sizeof(MainConf)); + if (cln == NULL) { return NULL; } + MainConf* mcf; + try { + mcf = new (cln->data) MainConf{}; + } catch (const std::exception& e) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "OTel: %s", e.what()); + return NULL; + } + + cln->handler = [](void* data) { + ((MainConf*)data)->~MainConf(); + }; + mcf->interval = NGX_CONF_UNSET_MSEC; mcf->batchSize = NGX_CONF_UNSET_SIZE; mcf->batchCount = NGX_CONF_UNSET_SIZE; - return mcf; + return static_cast(mcf); } char* initMainConf(ngx_conf_t* cf, void* conf) { auto mcf = (MainConf*)conf; - ngx_conf_init_msec_value(mcf->interval, 5000); ngx_conf_init_size_value(mcf->batchSize, 512); ngx_conf_init_size_value(mcf->batchCount, 4); - if (mcf->serviceName.data == NULL) { - mcf->serviceName = ngx_string("unknown_service:nginx"); + try { + if (mcf->serviceName.data == NULL) { + mcf->resourceAttrs.emplace("service.name", "unknown_service:nginx"); + } else { + mcf->resourceAttrs["service.name"] = toStrView(mcf->serviceName); + } + } catch (const std::exception& e) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "OTel: %s", e.what()); + return (char*)NGX_CONF_ERROR; } return NGX_CONF_OK; @@ -811,7 +862,7 @@ char* mergeLocationConf(ngx_conf_t* cf, void* parent, void* child) conf->spanAttrs = prev->spanAttrs; } - auto mcf = (MainConf*)ngx_http_conf_get_module_main_conf(cf, gHttpModule); + auto mcf = getMainConf(cf); if (mcf->endpoint.len == 0 && conf->trace) { ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, From 668077dbf7cd0e5383c1265c9cecf11f310c7ebf Mon Sep 17 00:00:00 2001 From: Pavel Pautov Date: Tue, 12 Nov 2024 11:49:02 -0800 Subject: [PATCH 15/28] Allow HTTP scheme in endpoint (fix #60). --- src/http_module.cpp | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/http_module.cpp b/src/http_module.cpp index 5c73ef0..d8ae859 100644 --- a/src/http_module.cpp +++ b/src/http_module.cpp @@ -144,6 +144,18 @@ ngx_str_t toNgxStr(StrView str) return ngx_str_t{str.size(), (u_char*)str.data()}; } +bool iremovePrefix(ngx_str_t* str, StrView p) +{ + if (str->len >= p.size() && + ngx_strncasecmp(str->data, (u_char*)p.data(), p.size()) == 0) { + str->data += p.size(); + str->len -= p.size(); + return true; + } + + return false; +} + MainConf* getMainConf(ngx_conf_t* cf) { return static_cast( @@ -658,6 +670,14 @@ char* setExporter(ngx_conf_t* cf, ngx_command_t* cmd, void* conf) return rv; } + if (iremovePrefix(&mcf->endpoint, "https://")) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "\"otel_exporter\" doesn't support \"https\" endpoints"); + return (char*)NGX_CONF_ERROR; + } else { + iremovePrefix(&mcf->endpoint, "http://"); + } + if (mcf->endpoint.len == 0) { ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "\"otel_exporter\" requires \"endpoint\""); From f45b618931e4823bfb8735b3b699b50d65c2e522 Mon Sep 17 00:00:00 2001 From: Pavel Pautov Date: Tue, 19 Nov 2024 14:25:25 -0800 Subject: [PATCH 16/28] Rename CI workflow file to 'ubuntu.yml'. --- .github/workflows/{nginx-otel-module-check.yml => ubuntu.yml} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename .github/workflows/{nginx-otel-module-check.yml => ubuntu.yml} (99%) diff --git a/.github/workflows/nginx-otel-module-check.yml b/.github/workflows/ubuntu.yml similarity index 99% rename from .github/workflows/nginx-otel-module-check.yml rename to .github/workflows/ubuntu.yml index 77a4332..e3bd13e 100644 --- a/.github/workflows/nginx-otel-module-check.yml +++ b/.github/workflows/ubuntu.yml @@ -1,4 +1,4 @@ -name: nginx-otel-module-check +name: Ubuntu build run-name: ${{ github.actor }} is triggering pipeline on: push: From da2e4eb11b0ea70c67aa63db8f462f7923393a55 Mon Sep 17 00:00:00 2001 From: Pavel Pautov Date: Tue, 19 Nov 2024 14:32:15 -0800 Subject: [PATCH 17/28] Trigger CI build on pull requests. Also, use default 'run-name' for more informative message. --- .github/workflows/ubuntu.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index e3bd13e..55c2485 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -1,8 +1,10 @@ name: Ubuntu build -run-name: ${{ github.actor }} is triggering pipeline + on: push: - workflow_dispatch: + branches: + - main + pull_request: jobs: build-module: From 6c1659a20ba946cdde21e9dbc52e7c740b06d968 Mon Sep 17 00:00:00 2001 From: Nikita Vakula Date: Fri, 15 Nov 2024 11:39:30 +0100 Subject: [PATCH 18/28] Support export via TLS (fix #12). --- src/batch_exporter.hpp | 4 ++-- src/http_module.cpp | 45 +++++++++++++++++++++++++++++++++--- src/trace_service_client.hpp | 14 ++++++++--- 3 files changed, 55 insertions(+), 8 deletions(-) diff --git a/src/batch_exporter.hpp b/src/batch_exporter.hpp index 2432fe3..cb3e075 100644 --- a/src/batch_exporter.hpp +++ b/src/batch_exporter.hpp @@ -111,10 +111,10 @@ public: int attrSize{0}; }; - BatchExporter(StrView target, + BatchExporter(StrView target, bool ssl, const std::string& trustedCert, size_t batchSize, size_t batchCount, const std::map& resourceAttrs) : - batchSize(batchSize), client(std::string(target)) + batchSize(batchSize), client(std::string(target), ssl, trustedCert) { free.reserve(batchCount); while (batchCount-- > 0) { diff --git a/src/http_module.cpp b/src/http_module.cpp index d8ae859..df5702e 100644 --- a/src/http_module.cpp +++ b/src/http_module.cpp @@ -6,6 +6,8 @@ #include "trace_context.hpp" #include "batch_exporter.hpp" +#include + extern ngx_module_t gHttpModule; namespace { @@ -26,6 +28,8 @@ struct MainConfBase { struct MainConf : MainConfBase { std::map resourceAttrs; + bool ssl; + std::string trustedCert; }; struct SpanAttr { @@ -44,6 +48,7 @@ struct LocationConf { char* setExporter(ngx_conf_t* cf, ngx_command_t* cmd, void* conf); char* addResourceAttr(ngx_conf_t* cf, ngx_command_t* cmd, void* conf); char* addSpanAttr(ngx_conf_t* cf, ngx_command_t* cmd, void* conf); +char* setTrustedCertificate(ngx_conf_t* cf, ngx_command_t* cmd, void* conf); namespace Propagation { @@ -111,6 +116,10 @@ ngx_command_t gExporterCommands[] = { 0, offsetof(MainConfBase, endpoint) }, + { ngx_string("trusted_certificate"), + NGX_CONF_TAKE1, + setTrustedCertificate }, + { ngx_string("interval"), NGX_CONF_TAKE1, ngx_conf_set_msec_slot, @@ -569,6 +578,8 @@ ngx_int_t initWorkerProcess(ngx_cycle_t* cycle) try { gExporter.reset(new BatchExporter( toStrView(mcf->endpoint), + mcf->ssl, + mcf->trustedCert, mcf->batchSize, mcf->batchCount, mcf->resourceAttrs)); @@ -671,9 +682,7 @@ char* setExporter(ngx_conf_t* cf, ngx_command_t* cmd, void* conf) } if (iremovePrefix(&mcf->endpoint, "https://")) { - ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, - "\"otel_exporter\" doesn't support \"https\" endpoints"); - return (char*)NGX_CONF_ERROR; + mcf->ssl = true; } else { iremovePrefix(&mcf->endpoint, "http://"); } @@ -702,6 +711,36 @@ char* addResourceAttr(ngx_conf_t* cf, ngx_command_t* cmd, void* conf) return NGX_CONF_OK; } +char* setTrustedCertificate(ngx_conf_t* cf, ngx_command_t* cmd, void* conf) { + auto path = ((ngx_str_t*)cf->args->elts)[1]; + auto mcf = getMainConf(cf); + + if (ngx_get_full_name(cf->pool, &cf->cycle->conf_prefix, &path) != NGX_OK) { + return (char*)NGX_CONF_ERROR; + } + + try { + std::ifstream file{(const char*)path.data, std::ios::binary}; + if (!file.is_open()) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, ngx_errno, + "failed to open \"%V\"", &path); + return (char*)NGX_CONF_ERROR; + } + file.exceptions(std::ios::failbit | std::ios::badbit); + file.seekg(0, std::ios::end); + size_t size = file.tellg(); + mcf->trustedCert.resize(size); + file.seekg(0); + file.read(&mcf->trustedCert[0], mcf->trustedCert.size()); + } catch (const std::exception& e) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "failed to read \"%V\": %s", &path, e.what()); + return (char*)NGX_CONF_ERROR; + } + + return NGX_CONF_OK; +} + void* createMainConf(ngx_conf_t* cf) { auto cln = ngx_pool_cleanup_add(cf->pool, sizeof(MainConf)); diff --git a/src/trace_service_client.hpp b/src/trace_service_client.hpp index 6871019..d248f00 100644 --- a/src/trace_service_client.hpp +++ b/src/trace_service_client.hpp @@ -17,10 +17,18 @@ public: typedef std::function ResponseCb; - TraceServiceClient(const std::string& target) + TraceServiceClient(const std::string& target, bool ssl, + const std::string& trustedCert) { - auto channel = grpc::CreateChannel( - target, grpc::InsecureChannelCredentials()); + std::shared_ptr creds; + if (ssl) { + grpc::SslCredentialsOptions options; + options.pem_root_certs = trustedCert; + creds = grpc::SslCredentials(options); + } else { + creds = grpc::InsecureChannelCredentials(); + } + auto channel = grpc::CreateChannel(target, creds); channel->GetState(true); // trigger 'connecting' state stub = TraceService::NewStub(channel); From 1e183a3fa92d1b50bd94e902bce1fd05ccc4cb2b Mon Sep 17 00:00:00 2001 From: Eugene Grebenschikov Date: Tue, 6 Feb 2024 10:43:01 -0800 Subject: [PATCH 19/28] Use pytest for tests. --- .github/workflows/ubuntu.yml | 92 ++---- tests/conftest.py | 100 +++++++ tests/h2_otel.t | 553 ----------------------------------- tests/h3_otel.t | 508 -------------------------------- tests/otel.t | 514 -------------------------------- tests/otel_collector.t | 403 ------------------------- tests/requirements.txt | 6 + tests/test_otel.py | 262 +++++++++++++++++ tests/trace_service.py | 86 ++++++ 9 files changed, 474 insertions(+), 2050 deletions(-) create mode 100644 tests/conftest.py delete mode 100644 tests/h2_otel.t delete mode 100644 tests/h3_otel.t delete mode 100644 tests/otel.t delete mode 100644 tests/otel_collector.t create mode 100644 tests/requirements.txt create mode 100644 tests/test_otel.py create mode 100644 tests/trace_service.py diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 55c2485..1a8edac 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -8,71 +8,14 @@ on: jobs: build-module: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - name: Checkout repository uses: actions/checkout@v4 - name: Install dependencies run: | sudo apt-get update - sudo apt-get install -y cmake libc-ares-dev libre2-dev - - name: Checkout nginx - uses: actions/checkout@v4 - with: - repository: nginx/nginx - path: nginx - - name: Configure nginx - working-directory: nginx - run: auto/configure --with-compat - - name: Create build directory - run: mkdir build - - name: Build module - working-directory: build - run: | - cmake -DNGX_OTEL_NGINX_BUILD_DIR=${PWD}/../nginx/objs \ - -DNGX_OTEL_DEV=ON .. - make -j 4 - strip ngx_otel_module.so - - name: Archive module - uses: actions/upload-artifact@v4 - with: - name: nginx-otel-module - path: build/ngx_otel_module.so - - name: Archive protoc and opentelemetry-proto - uses: actions/upload-artifact@v4 - with: - name: protoc-opentelemetry-proto - path: | - build/_deps/grpc-build/third_party/protobuf/protoc - build/_deps/otelcpp-src/third_party/opentelemetry-proto - test-module: - needs: build-module - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - name: Download module - uses: actions/download-artifact@v4 - with: - name: nginx-otel-module - path: build - - name: Download protoc and opentelemetry-proto - uses: actions/download-artifact@v4 - with: - name: protoc-opentelemetry-proto - path: build/_deps - - name: List files - run: ls -laR . - - name: Fix protoc file permissions - run: chmod +x build/_deps/grpc-build/third_party/protobuf/protoc - - name: Install perl modules - run: sudo cpan IO::Socket::SSL Crypt::Misc - - name: Download otelcol - run: | - curl -LO https://github.com/\ - open-telemetry/opentelemetry-collector-releases/releases/download/\ - v0.76.1/otelcol_0.76.1_linux_amd64.tar.gz - tar -xzf otelcol_0.76.1_linux_amd64.tar.gz + sudo apt-get install -y cmake libc-ares-dev - name: Checkout nginx uses: actions/checkout@v4 with: @@ -83,17 +26,22 @@ jobs: run: | auto/configure --with-compat --with-debug --with-http_ssl_module \ --with-http_v2_module --with-http_v3_module - make -j 4 - - name: Checkout lib from nginx-tests - uses: actions/checkout@v4 - with: - repository: nginx/nginx-tests - sparse-checkout: | - lib - path: nginx-tests - - name: Run tests - working-directory: tests + make -j $(nproc) + - name: Build module run: | - PERL5LIB=../nginx-tests/lib TEST_NGINX_UNSAFE=1 \ - TEST_NGINX_VERBOSE=1 TEST_NGINX_GLOBALS="load_module \ - ${PWD}/../build/ngx_otel_module.so;" prove -v . + mkdir build + cd build + cmake -DNGX_OTEL_NGINX_BUILD_DIR=${PWD}/../nginx/objs \ + -DNGX_OTEL_DEV=ON .. + make -j $(nproc) + - name: Download otelcol + run: | + curl -sLo - https://github.com/\ + open-telemetry/opentelemetry-collector-releases/releases/download/\ + v0.115.1/otelcol_0.115.1_linux_amd64.tar.gz | tar -xzv + - name: Install test dependencies + run: pip install -r tests/requirements.txt + - name: Run tests + run: | + pytest tests --maxfail=10 --nginx=nginx/objs/nginx \ + --module=build/ngx_otel_module.so --otelcol=./otelcol diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..7978759 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,100 @@ +import jinja2 +import logging +from OpenSSL import crypto +import os +import pytest +import subprocess +import time + + +pytest_plugins = [ + "trace_service", +] + + +def pytest_addoption(parser): + parser.addoption("--nginx", required=True) + parser.addoption("--module", required=True) + parser.addoption("--otelcol") + parser.addoption("--globals", default="") + + +def self_signed_cert(test_dir, name): + k = crypto.PKey() + k.generate_key(crypto.TYPE_RSA, 2048) + cert = crypto.X509() + cert.get_subject().CN = name + cert.set_issuer(cert.get_subject()) + cert.gmtime_adj_notBefore(0) + cert.gmtime_adj_notAfter(365 * 86400) # 365 days + cert.set_pubkey(k) + cert.sign(k, "sha512") + (test_dir / f"{name}.key").write_text( + crypto.dump_privatekey(crypto.FILETYPE_PEM, k).decode("utf-8") + ) + (test_dir / f"{name}.crt").write_text( + crypto.dump_certificate(crypto.FILETYPE_PEM, cert).decode("utf-8") + ) + + +@pytest.fixture(scope="session") +def logger(): + logging.basicConfig(level=logging.INFO) + return logging.getLogger(__name__) + + +@pytest.fixture(scope="module") +def testdir(tmp_path_factory): + return tmp_path_factory.mktemp("nginx") + + +@pytest.fixture(scope="module") +def nginx_config(request, pytestconfig, testdir, logger): + tmpl = jinja2.Environment().from_string(request.module.NGINX_CONFIG) + params = getattr(request, "param", {}) + params["globals"] = ( + f"pid {testdir}/nginx.pid;\n" + + "error_log stderr info;\n" + + f"error_log {testdir}/error.log info;\n" + + f"load_module {os.path.abspath(pytestconfig.option.module)};\n" + + pytestconfig.option.globals + ) + params["http_globals"] = f"root {testdir};\n" + "access_log off;\n" + conf = tmpl.render(params) + logger.debug(conf) + return conf + + +@pytest.fixture(scope="module") +def nginx(testdir, pytestconfig, nginx_config, certs, logger, otelcol): + (testdir / "nginx.conf").write_text(nginx_config) + logger.info("Starting nginx...") + proc = subprocess.Popen( + [ + pytestconfig.option.nginx, + "-p", + str(testdir), + "-c", + "nginx.conf", + "-e", + "error.log", + ] + ) + logger.debug(f"args={' '.join(proc.args)}") + logger.debug(f"pid={proc.pid}") + while not (testdir / "nginx.pid").exists(): + time.sleep(0.1) + assert proc.poll() is None, "Can't start nginx" + yield proc + logger.info("Stopping nginx...") + proc.terminate() + try: + proc.wait(timeout=5) + except subprocess.TimeoutExpired: + proc.kill() + assert "[alert]" not in (testdir / "error.log").read_text() + + +@pytest.fixture(scope="module") +def certs(testdir): + self_signed_cert(testdir, "localhost") diff --git a/tests/h2_otel.t b/tests/h2_otel.t deleted file mode 100644 index 24987d7..0000000 --- a/tests/h2_otel.t +++ /dev/null @@ -1,553 +0,0 @@ -#!/usr/bin/perl - -# (C) Nginx, Inc. - -# Tests for OTel exporter in case HTTP/2. - -############################################################################### - -use warnings; -use strict; - -use Test::More; - -BEGIN { use FindBin; chdir($FindBin::Bin); } - -use Test::Nginx; -use Test::Nginx::HTTP2; -use MIME::Base64; - -############################################################################### - -select STDERR; $| = 1; -select STDOUT; $| = 1; - -my $t = Test::Nginx->new() - ->has(qw/http_v2 http_ssl rewrite mirror grpc socket_ssl_alpn/) - ->has_daemon(qw/openssl base64/) - ->write_file_expand('nginx.conf', <<'EOF'); - -%%TEST_GLOBALS%% - -daemon off; - -events { -} - -http { - %%TEST_GLOBALS_HTTP%% - - ssl_certificate_key localhost.key; - ssl_certificate localhost.crt; - - otel_exporter { - endpoint 127.0.0.1:8083; - interval 1s; - batch_size 10; - batch_count 1; - } - - otel_service_name test_server; - otel_trace on; - - server { - listen 127.0.0.1:8080 http2; - listen 127.0.0.1:8081; - listen 127.0.0.1:8082 http2 ssl; - server_name localhost; - - location /trace-on { - otel_trace_context extract; - otel_span_name default_location; - otel_span_attr http.request.header.completion - $request_completion; - otel_span_attr http.response.header.content.type - $sent_http_content_type; - otel_span_attr http.request $request; - add_header "X-Otel-Trace-Id" $otel_trace_id; - add_header "X-Otel-Span-Id" $otel_span_id; - add_header "X-Otel-Parent-Id" $otel_parent_id; - add_header "X-Otel-Parent-Sampled" $otel_parent_sampled; - return 200 "TRACE-ON"; - } - - location /context-ignore { - otel_trace_context ignore; - otel_span_name context_ignore; - add_header "X-Otel-Parent-Id" $otel_parent_id; - proxy_pass http://127.0.0.1:8081/trace-off; - } - - location /context-extract { - otel_trace_context extract; - otel_span_name context_extract; - add_header "X-Otel-Parent-Id" $otel_parent_id; - proxy_pass http://127.0.0.1:8081/trace-off; - } - - location /context-inject { - otel_trace_context inject; - otel_span_name context_inject; - add_header "X-Otel-Parent-Id" $otel_parent_id; - proxy_pass http://127.0.0.1:8081/trace-off; - } - - location /context-propagate { - otel_trace_context propagate; - otel_span_name context_propogate; - add_header "X-Otel-Parent-Id" $otel_parent_id; - proxy_pass http://127.0.0.1:8081/trace-off; - } - - location /trace-off { - otel_trace off; - add_header "X-Otel-Traceparent" $http_traceparent; - add_header "X-Otel-Tracestate" $http_tracestate; - return 200 "TRACE-OFF"; - } - } - - server { - listen 127.0.0.1:8083 http2; - server_name localhost; - otel_trace off; - - location / { - mirror /mirror; - grpc_pass 127.0.0.1:8084; - } - - location /mirror { - internal; - grpc_pass 127.0.0.1:%%PORT_4317%%; - } - } - - server { - listen 127.0.0.1:8084 http2; - server_name localhost; - otel_trace off; - - location / { - add_header content-type application/grpc; - add_header grpc-status 0; - add_header grpc-message ""; - return 200; - } - } - -} - -EOF - -$t->write_file('openssl.conf', <<'EOF'); -[ req ] -default_bits = 2048 -encrypt_key = no -distinguished_name = req_distinguished_name -[ req_distinguished_name ] - -EOF - -my $d = $t->testdir(); - -foreach my $name ('localhost') { - system('openssl req -x509 -new ' - . "-config $d/openssl.conf -subj /CN=$name/ " - . "-out $d/$name.crt -keyout $d/$name.key " - . ">>$d/openssl.out 2>&1") == 0 - or die "Can't create certificate for $name: $!\n"; -} - -$t->try_run('no OTel module')->plan(69); - -############################################################################### - -my $p = port(4317); -my $f = grpc(); - -#do requests -(undef, my $t_off_resp) = http2_get('/trace-off'); - -#batch0 (10 requests) -my ($tp_headers, $tp_resp) = http2_get('/trace-on', trace_headers => 1); -my ($t_headers, $t_resp) = http2_get('/trace-on', port => 8082, ssl => 1); - -(my $t_headers_ignore, undef) = http2_get('/context-ignore'); -(my $tp_headers_ignore, undef) = http2_get('/context-ignore', - trace_headers => 1); -(my $t_headers_extract, undef) = http2_get('/context-extract'); -(my $tp_headers_extract, undef) = http2_get('/context-extract', - trace_headers => 1); -(my $t_headers_inject, undef) = http2_get('/context-inject'); -(my $tp_headers_inject, undef) = http2_get('/context-inject', - trace_headers => 1); -(my $t_headers_propagate, undef) = http2_get('/context-propagate'); -(my $tp_headers_propagate, undef) = - http2_get('/context-propagate', trace_headers => 1); - -my ($frame) = grep { $_->{type} eq "DATA" } @{$f->{http_start}()}; -my $batch0 = to_hash(decode_protobuf(substr $frame->{data}, 8)); -my $spans = $$batch0{scope_spans}; - -#batch1 (5 reqeusts) -http2_get('/trace-on') for (1..5); - -($frame) = grep { $_->{type} eq "DATA" } @{$f->{http_start}()}; -my $batch1 = to_hash(decode_protobuf(substr $frame->{data}, 8)); - -#validate responses -like($tp_resp, qr/TRACE-ON/, 'http request1 - trace on'); -like($t_resp, qr/TRACE-ON/, 'http request2 - trace on'); -like($t_off_resp, qr/TRACE-OFF/, 'http request - trace off'); - -#validate batch size -delete $$spans{scope}; #remove 'scope' entry -is(scalar keys %{$spans}, 10, 'batch0 size - trace on'); -delete $$batch1{scope_spans}{scope}; #remove 'scope' entry -is(scalar keys %{$$batch1{scope_spans}}, 5, 'batch1 size - trace on'); - -#validate general attributes -is(get_attr("service.name", "string_value", - $$batch0{resource}), - 'test_server', 'service.name - trace on'); -is($$spans{span0}{name}, '"default_location"', 'span.name - trace on'); - -#validate http metrics -is(get_attr("http.method", "string_value", $$spans{span0}), 'GET', - 'http.method metric - trace on'); -is(get_attr("http.target", "string_value", $$spans{span0}), '/trace-on', - 'http.target metric - trace on'); -is(get_attr("http.route", "string_value", $$spans{span0}), '/trace-on', - 'http.route metric - trace on'); -is(get_attr("http.scheme", "string_value", $$spans{span0}), 'http', - 'http.scheme metric - trace on'); -is(get_attr("http.flavor", "string_value", $$spans{span0}), '2.0', - 'http.flavor metric - trace on'); -is(get_attr("http.user_agent", "string_value", $$spans{span0}), 'nginx-tests', - 'http.user_agent metric - trace on'); -is(get_attr("http.request_content_length", "int_value", $$spans{span0}), 0, - 'http.request_content_length metric - trace on'); -is(get_attr("http.response_content_length", "int_value", $$spans{span0}), 8, - 'http.response_content_length metric - trace on'); -is(get_attr("http.status_code", "int_value", $$spans{span0}), 200, - 'http.status_code metric - trace on'); -is(get_attr("net.host.name", "string_value", $$spans{span0}), 'localhost', - 'net.host.name metric - trace on'); -is(get_attr("net.host.port", "int_value", $$spans{span0}), 8080, - 'net.host.port metric - trace on'); -is(get_attr("net.sock.peer.addr", "string_value", $$spans{span0}), '127.0.0.1', - 'net.sock.peer.addr metric - trace on'); -like(get_attr("net.sock.peer.port", "int_value", $$spans{span0}), qr/\d+/, - 'net.sock.peer.port metric - trace on'); - -#validate https metrics -is(get_attr("http.method", "string_value", $$spans{span1}), 'GET', - 'http.method metric - trace on (https)'); -is(get_attr("http.target", "string_value", $$spans{span1}), '/trace-on', - 'http.target metric - trace on (https)'); -is(get_attr("http.route", "string_value", $$spans{span1}), '/trace-on', - 'http.route metric - trace on (https)'); -is(get_attr("http.scheme", "string_value", $$spans{span1}), 'https', - 'http.scheme metric - trace on (https)'); -is(get_attr("http.flavor", "string_value", $$spans{span1}), '2.0', - 'http.flavor metric - trace on (https)'); -isnt(get_attr("http.user_agent", "string_value", $$spans{span1}), - 'nginx-tests', 'http.user_agent metric - trace on (https)'); -is(get_attr("http.request_content_length", "int_value", $$spans{span1}), 0, - 'http.request_content_length metric - trace on (https)'); -is(get_attr("http.response_content_length", "int_value", $$spans{span1}), 8, - 'http.response_content_length metric - trace on (https)'); -is(get_attr("http.status_code", "int_value", $$spans{span1}), 200, - 'http.status_code metric - trace on (https)'); -is(get_attr("net.host.name", "string_value", $$spans{span1}), 'localhost', - 'net.host.name metric - trace on (https)'); -is(get_attr("net.host.port", "int_value", $$spans{span1}), 8082, - 'net.host.port metric - trace on (https)'); -is(get_attr("net.sock.peer.addr", "string_value", $$spans{span1}), '127.0.0.1', - 'net.sock.peer.addr metric - trace on (https)'); -like(get_attr("net.sock.peer.port", "int_value", $$spans{span1}), qr/\d+/, - 'net.sock.peer.port metric - trace on (https)'); - -#validate custom http metrics -is(${get_attr("http.request.header.completion", "array_value", $$spans{span0})} - {values}{string_value}, '"OK"', - 'http.request.header.completion metric - trace on'); -is(${get_attr( - "http.response.header.content.type", "array_value", $$spans{span0} - )}{values}{string_value}, '"text/plain"', - 'http.response.header.content.type metric - trace on'); -is(get_attr("http.request", "string_value", $$spans{span0}), - 'GET /trace-on HTTP/2.0', 'http.request metric - trace on'); - -#extract trace info -is($$spans{span0}{parent_span_id}, 'b9c7c989f97918e1', - 'traceparent - trace on'); -is($$spans{span0}{trace_state}, '"congo=ucfJifl5GOE,rojo=00f067aa0ba902b7"', - 'tracestate - trace on'); -is($$spans{span1}{parent_span_id}, undef, 'no traceparent - trace on'); -is($$spans{span1}{trace_state}, undef, 'no tracestate - trace on'); - -#variables -is($tp_headers->{'x-otel-trace-id'}, $$spans{span0}{trace_id}, - '$otel_trace_id variable - trace on'); -is($tp_headers->{'x-otel-span-id'}, $$spans{span0}{span_id}, - '$otel_span_id variable - trace on'); -is($tp_headers->{'x-otel-parent-id'}, $$spans{span0}{parent_span_id}, - '$otel_parent_id variable - trace on'); -is($tp_headers->{'x-otel-parent-sampled'}, 1, - '$otel_parent_sampled variable - trace on'); -is($t_headers->{'x-otel-parent-sampled'}, 0, - '$otel_parent_sampled variable - trace on (no traceparent header)'); - -#trace off -is((scalar grep { - get_attr("http.target", "string_value", $$spans{$_}) eq '/trace-off' - } keys %{$spans}), 0, 'no metric in batch0 - trace off'); -is((scalar grep { - get_attr("http.target", "string_value", $$spans{$_}) eq '/trace-off' - } keys %{$$batch1{scope_spans}}), 0, 'no metric in batch1 - trace off'); - -#trace context: ignore -is($t_headers_ignore->{'x-otel-traceparent'}, undef, - 'no traceparent - trace context ignore (no trace headers)'); -is($t_headers_ignore->{'x-otel-tracestate'}, undef, - 'no tracestate - trace context ignore (no trace headers)'); - -is($tp_headers_ignore->{'x-otel-parent-id'}, undef, - 'no parent span id - trace context ignore (trace headers)'); -is($tp_headers_ignore->{'x-otel-traceparent'}, - '00-0af7651916cd43dd8448eb211c80319c-b9c7c989f97918e1-01', - 'traceparent - trace context ignore (trace headers)'); -is($tp_headers_ignore->{'x-otel-tracestate'}, - 'congo=ucfJifl5GOE,rojo=00f067aa0ba902b7', - 'tracestate - trace context ignore (trace headers)'); - -#trace context: extract -is($t_headers_extract->{'x-otel-traceparent'}, undef, - 'no traceparent - trace context extract (no trace headers)'); -is($t_headers_extract->{'x-otel-tracestate'}, undef, - 'no tracestate - trace context extract (no trace headers)'); - -is($tp_headers_extract->{'x-otel-parent-id'}, 'b9c7c989f97918e1', - 'parent span id - trace context extract (trace headers)'); -is($tp_headers_extract->{'x-otel-traceparent'}, - '00-0af7651916cd43dd8448eb211c80319c-b9c7c989f97918e1-01', - 'traceparent - trace context extract (trace headers)'); -is($tp_headers_extract->{'x-otel-tracestate'}, - 'congo=ucfJifl5GOE,rojo=00f067aa0ba902b7', - 'tracestate - trace context extract (trace headers)'); - -#trace context: inject -is($t_headers_inject->{'x-otel-traceparent'}, - "00-$$spans{span6}{trace_id}-$$spans{span6}{span_id}-01", - 'traceparent - trace context inject (no trace headers)'); -is($t_headers_inject->{'x-otel-tracestate'}, undef, - 'no tracestate - trace context inject (no trace headers)'); - -is($tp_headers_inject->{'x-otel-parent-id'}, undef, - 'no parent span id - trace context inject (trace headers)'); -is($tp_headers_inject->{'x-otel-traceparent'}, - "00-$$spans{span7}{trace_id}-$$spans{span7}{span_id}-01", - 'traceparent - trace context inject (trace headers)'); -is($tp_headers_inject->{'x-otel-tracestate'}, undef, - 'no tracestate - trace context inject (trace headers)'); - -#trace context: propagate -is($t_headers_propagate->{'x-otel-traceparent'}, - "00-$$spans{span8}{trace_id}-$$spans{span8}{span_id}-01", - 'traceparent - trace context propagate (no trace headers)'); -is($t_headers_propagate->{'x-otel-tracestate'}, undef, - 'no tracestate - trace context propagate (no trace headers)'); - -is($tp_headers_propagate->{'x-otel-parent-id'}, 'b9c7c989f97918e1', - 'parent id - trace context propagate (trace headers)'); -is($tp_headers_propagate->{'x-otel-traceparent'}, - "00-0af7651916cd43dd8448eb211c80319c-$$spans{span9}{span_id}-01", - 'traceparent - trace context propagate (trace headers)'); -is($tp_headers_propagate->{'x-otel-tracestate'}, - 'congo=ucfJifl5GOE,rojo=00f067aa0ba902b7', - 'tracestate - trace context propagate (trace headers)'); - -SKIP: { -skip "depends on error log content", 2 unless $ENV{TEST_NGINX_UNSAFE}; - -$t->stop(); -my $log = $t->read_file("error.log"); - -like($log, qr/OTel\/grpc: Error parsing metadata: error=invalid value/, - 'log: error parsing metadata - no protobuf in response'); -unlike($log, qr/OTel export failure: No status received/, - 'log: no export failure'); - -} - -############################################################################### - -sub http2_get { - my ($path, %extra) = @_; - my ($frames, $frame); - - my $port = $extra{port} || 8080; - - my $s = $extra{ssl} - ? Test::Nginx::HTTP2->new( - undef, socket => get_ssl_socket($port, ['h2'])) - : Test::Nginx::HTTP2->new(); - - my $sid = $extra{trace_headers} - ? $s->new_stream({ headers => [ - { name => ':method', value => 'GET' }, - { name => ':scheme', value => 'http' }, - { name => ':path', value => $path }, - { name => ':authority', value => 'localhost' }, - { name => 'user-agent', value => 'nginx-tests', mode => 2 }, - { name => 'traceparent', - value => '00-0af7651916cd43dd8448eb211c80319c-' . - 'b9c7c989f97918e1-01', - mode => 2 - }, - { name => 'tracestate', - value => 'congo=ucfJifl5GOE,rojo=00f067aa0ba902b7', - mode => 2 - }]}) - : $s->new_stream({ path => $path }); - $frames = $s->read(all => [{ sid => $sid, fin => 1 }]); - - ($frame) = grep { $_->{type} eq "HEADERS" } @$frames; - my $headers = $frame->{headers}; - - ($frame) = grep { $_->{type} eq "DATA" } @$frames; - my $data = $frame->{data}; - - return $headers, $data; -} - -sub get_ssl_socket { - my ($port, $alpn) = @_; - - return http( - '', PeerAddr => '127.0.0.1:' . port($port), start => 1, - SSL => 1, - SSL_alpn_protocols => $alpn, - SSL_error_trap => sub { die $_[1] } - ); -} - -sub get_attr { - my($attr, $type, $obj) = @_; - - my ($res) = grep { - $_ =~ /^attribute\d+/ && $$obj{$_}{key} eq '"' . $attr . '"' - } keys %{$obj}; - - if (defined $res) { - $$obj{$res}{value}{$type} =~ s/(^\")|(\"$)//g - if $type eq 'string_value'; - - return $$obj{$res}{value}{$type}; - } - - return undef; -} - -sub decode_protobuf { - my ($protobuf) = @_; - - local $/; - open CMD, "echo '" . encode_base64($protobuf) . "' | base64 -d | " . - '$PWD/../build/_deps/grpc-build/third_party/protobuf/protoc '. - '--decode opentelemetry.proto.trace.v1.ResourceSpans -I ' . - '$PWD/../build/_deps/otelcpp-src/third_party/opentelemetry-proto ' . - 'opentelemetry/proto/collector/trace/v1/trace_service.proto |' - or die "Can't decode protobuf: $!\n"; - my $out = ; - close CMD; - - return $out; -} - -sub decode_bytes { - my ($bytes) = @_; - - my $c = sub { return chr oct(shift) }; - - $bytes =~ s/\\(\d{3})/$c->($1)/eg; - $bytes =~ s/(^\")|(\"$)//g; - $bytes =~ s/\\\\/\\/g; - $bytes =~ s/\\r/\r/g; - $bytes =~ s/\\n/\n/g; - $bytes =~ s/\\t/\t/g; - $bytes =~ s/\\"/\"/g; - $bytes =~ s/\\'/\'/g; - - return unpack("H*", unpack("a*", $bytes)); -} - -sub to_hash { - my ($textdata) = @_; - - my %out = (); - push my @stack, \%out; - my ($attr_count, $span_count) = (0, 0); - for my $line (split /\n/, $textdata) { - $line =~ s/(^\s+)|(\s+$)//g; - if ($line =~ /\:/) { - my ($k, $v) = split /\: /, $line; - $v = decode_bytes($v) if ($k =~ /trace_id|span_id|parent_span_id/); - $stack[$#stack]{$k} = $v; - } elsif ($line =~ /\{/) { - $line =~ s/\s\{//; - $line = 'attribute' . $attr_count++ if ($line eq 'attributes'); - if ($line eq 'spans') { - $line = 'span' . $span_count++; - $attr_count = 0; - } - my %new = (); - $stack[$#stack]{$line} = \%new; - push @stack, \%new; - } elsif ($line =~ /\}/) { - pop @stack; - } - } - - return \%out; -} - -sub grpc { - my ($server, $client, $f, $s, $c, $sid, $csid, $uri); - - $server = IO::Socket::INET->new( - Proto => 'tcp', - LocalHost => '127.0.0.1', - LocalPort => $p, - Listen => 5, - Reuse => 1 - ) or die "Can't create listening socket: $!\n"; - - $f->{http_start} = sub { - if (IO::Select->new($server)->can_read(5)) { - $client = $server->accept(); - } else { - # connection could be unexpectedly reused - goto reused if $client; - return undef; - } - - $client->sysread($_, 24) == 24 or return; # preface - - $c = Test::Nginx::HTTP2->new(1, socket => $client, - pure => 1, preface => "") or return; - -reused: - my $frames = $c->read(all => [{ fin => 1 }]); - - $client->close(); - - return $frames; - }; - - return $f; -} - -############################################################################### diff --git a/tests/h3_otel.t b/tests/h3_otel.t deleted file mode 100644 index ef848cb..0000000 --- a/tests/h3_otel.t +++ /dev/null @@ -1,508 +0,0 @@ -#!/usr/bin/perl - -# (C) Nginx, Inc. - -# Tests for OTel exporter in case HTTP/3. - -############################################################################### - -use warnings; -use strict; - -use Test::More; - -BEGIN { use FindBin; chdir($FindBin::Bin); } - -use Test::Nginx; -use Test::Nginx::HTTP2; -use Test::Nginx::HTTP3; -use MIME::Base64; - -############################################################################### - -select STDERR; $| = 1; -select STDOUT; $| = 1; - -my $t = Test::Nginx->new()->has(qw/http_v2 http_v3 rewrite mirror grpc cryptx/) - ->has_daemon(qw/openssl base64/) - ->write_file_expand('nginx.conf', <<'EOF'); - -%%TEST_GLOBALS%% - -daemon off; - -events { -} - -http { - %%TEST_GLOBALS_HTTP%% - - ssl_certificate_key localhost.key; - ssl_certificate localhost.crt; - ssl_protocols TLSv1.3; - - otel_exporter { - endpoint 127.0.0.1:8082; - interval 1s; - batch_size 10; - batch_count 2; - } - - otel_service_name test_server; - otel_trace on; - - server { - listen 127.0.0.1:%%PORT_8980_UDP%% quic; - listen 127.0.0.1:8081; - server_name localhost; - - location /trace-on { - otel_trace_context extract; - otel_span_name default_location; - otel_span_attr http.request.header.completion - $request_completion; - otel_span_attr http.response.header.content.type - $sent_http_content_type; - otel_span_attr http.request $request; - add_header "X-Otel-Trace-Id" $otel_trace_id; - add_header "X-Otel-Span-Id" $otel_span_id; - add_header "X-Otel-Parent-Id" $otel_parent_id; - add_header "X-Otel-Parent-Sampled" $otel_parent_sampled; - return 200 "TRACE-ON"; - } - - location /context-ignore { - otel_trace_context ignore; - otel_span_name context_ignore; - add_header "X-Otel-Parent-Id" $otel_parent_id; - proxy_pass http://127.0.0.1:8081/trace-off; - } - - location /context-extract { - otel_trace_context extract; - otel_span_name context_extract; - add_header "X-Otel-Parent-Id" $otel_parent_id; - proxy_pass http://127.0.0.1:8081/trace-off; - } - - location /context-inject { - otel_trace_context inject; - otel_span_name context_inject; - add_header "X-Otel-Parent-Id" $otel_parent_id; - proxy_pass http://127.0.0.1:8081/trace-off; - } - - location /context-propagate { - otel_trace_context propagate; - otel_span_name context_propogate; - add_header "X-Otel-Parent-Id" $otel_parent_id; - proxy_pass http://127.0.0.1:8081/trace-off; - } - - location /trace-off { - otel_trace off; - add_header "X-Otel-Traceparent" $http_traceparent; - add_header "X-Otel-Tracestate" $http_tracestate; - return 200 "TRACE-OFF"; - } - } - - server { - listen 127.0.0.1:8082 http2; - server_name localhost; - otel_trace off; - - location / { - mirror /mirror; - grpc_pass 127.0.0.1:8083; - } - - location /mirror { - internal; - grpc_pass 127.0.0.1:%%PORT_4317%%; - } - } - - server { - listen 127.0.0.1:8083 http2; - server_name localhost; - otel_trace off; - - location / { - add_header content-type application/grpc; - add_header grpc-status 0; - add_header grpc-message ""; - return 200; - } - } - -} - -EOF - -$t->write_file('openssl.conf', <<'EOF'); -[ req ] -default_bits = 2048 -encrypt_key = no -distinguished_name = req_distinguished_name -[ req_distinguished_name ] - -EOF - -my $d = $t->testdir(); - -foreach my $name ('localhost') { - system('openssl req -x509 -new ' - . "-config $d/openssl.conf -subj /CN=$name/ " - . "-out $d/$name.crt -keyout $d/$name.key " - . ">>$d/openssl.out 2>&1") == 0 - or die "Can't create certificate for $name: $!\n"; -} - -$t->try_run('no OTel module')->plan(56); - -############################################################################### - -my $p = port(4317); -my $f = grpc(); - -#do requests -(undef, my $t_off_resp) = http3_get('/trace-off'); - -#batch0 (10 requests) -my ($tp_headers, $tp_resp) = http3_get('/trace-on', trace_headers => 1); -my ($t_headers, $t_resp) = http3_get('/trace-on'); - -(my $t_headers_ignore, undef) = http3_get('/context-ignore'); -(my $tp_headers_ignore, undef) = http3_get('/context-ignore', - trace_headers => 1); -(my $t_headers_extract, undef) = http3_get('/context-extract'); -(my $tp_headers_extract, undef) = http3_get('/context-extract', - trace_headers => 1); -(my $t_headers_inject, undef) = http3_get('/context-inject'); -(my $tp_headers_inject, undef) = http3_get('/context-inject', - trace_headers => 1); -(my $t_headers_propagate, undef) = http3_get('/context-propagate'); -(my $tp_headers_propagate, undef) = - http3_get('/context-propagate', trace_headers => 1); - -my ($frame) = grep { $_->{type} eq "DATA" } @{$f->{http_start}()}; -my $batch0 = to_hash(decode_protobuf(substr $frame->{data}, 8)); -my $spans = $$batch0{scope_spans}; - -#batch1 (5 reqeusts) -http3_get('/trace-on') for (1..5); - -($frame) = grep { $_->{type} eq "DATA" } @{$f->{http_start}()}; -my $batch1 = to_hash(decode_protobuf(substr $frame->{data}, 8)); - -#validate responses -like($tp_resp, qr/TRACE-ON/, 'http request1 - trace on'); -like($t_resp, qr/TRACE-ON/, 'http request2 - trace on'); -like($t_off_resp, qr/TRACE-OFF/, 'http request - trace off'); - -#validate batch size -delete $$spans{scope}; #remove 'scope' entry -is(scalar keys %{$spans}, 10, 'batch0 size - trace on'); -delete $$batch1{scope_spans}{scope}; #remove 'scope' entry -is(scalar keys %{$$batch1{scope_spans}}, 5, 'batch1 size - trace on'); - -#validate general attributes -is(get_attr("service.name", "string_value", - $$batch0{resource}), - 'test_server', 'service.name - trace on'); -is($$spans{span0}{name}, '"default_location"', 'span.name - trace on'); - -#validate metrics -is(get_attr("http.method", "string_value", $$spans{span0}), 'GET', - 'http.method metric - trace on'); -is(get_attr("http.target", "string_value", $$spans{span0}), '/trace-on', - 'http.target metric - trace on'); -is(get_attr("http.route", "string_value", $$spans{span0}), '/trace-on', - 'http.route metric - trace on'); -is(get_attr("http.scheme", "string_value", $$spans{span0}), 'https', - 'http.scheme metric - trace on'); -is(get_attr("http.flavor", "string_value", $$spans{span0}), '3.0', - 'http.flavor metric - trace on'); -is(get_attr("http.user_agent", "string_value", $$spans{span0}), 'nginx-tests', - 'http.user_agent metric - trace on'); -is(get_attr("http.request_content_length", "int_value", $$spans{span0}), 0, - 'http.request_content_length metric - trace on'); -is(get_attr("http.response_content_length", "int_value", $$spans{span0}), 8, - 'http.response_content_length metric - trace on'); -is(get_attr("http.status_code", "int_value", $$spans{span0}), 200, - 'http.status_code metric - trace on'); -is(get_attr("net.host.name", "string_value", $$spans{span0}), 'localhost', - 'net.host.name metric - trace on'); -is(get_attr("net.host.port", "int_value", $$spans{span0}), 8980, - 'net.host.port metric - trace on'); -is(get_attr("net.sock.peer.addr", "string_value", $$spans{span0}), '127.0.0.1', - 'net.sock.peer.addr metric - trace on'); -like(get_attr("net.sock.peer.port", "int_value", $$spans{span0}), qr/\d+/, - 'net.sock.peer.port metric - trace on'); - -#validate custom http metrics -is(${get_attr("http.request.header.completion", "array_value", $$spans{span0})} - {values}{string_value}, '"OK"', - 'http.request.header.completion metric - trace on'); -is(${get_attr( - "http.response.header.content.type", "array_value", $$spans{span0} - )}{values}{string_value}, '"text/plain"', - 'http.response.header.content.type metric - trace on'); -is(get_attr("http.request", "string_value", $$spans{span0}), - 'GET /trace-on HTTP/3.0', 'http.request metric - trace on'); - -#extract trace info -is($$spans{span0}{parent_span_id}, 'b9c7c989f97918e1', - 'traceparent - trace on'); -is($$spans{span0}{trace_state}, '"congo=ucfJifl5GOE,rojo=00f067aa0ba902b7"', - 'tracestate - trace on'); -is($$spans{span1}{parent_span_id}, undef, 'no traceparent - trace on'); -is($$spans{span1}{trace_state}, undef, 'no tracestate - trace on'); - -#variables -is($tp_headers->{'x-otel-trace-id'}, $$spans{span0}{trace_id}, - '$otel_trace_id variable - trace on'); -is($tp_headers->{'x-otel-span-id'}, $$spans{span0}{span_id}, - '$otel_span_id variable - trace on'); -is($tp_headers->{'x-otel-parent-id'}, $$spans{span0}{parent_span_id}, - '$otel_parent_id variable - trace on'); -is($tp_headers->{'x-otel-parent-sampled'}, 1, - '$otel_parent_sampled variable - trace on'); -is($t_headers->{'x-otel-parent-sampled'}, 0, - '$otel_parent_sampled variable - trace on (no traceparent header)'); - -#trace off -is((scalar grep { - get_attr("http.target", "string_value", $$spans{$_}) eq '/trace-off' - } keys %{$spans}), 0, 'no metric in batch0 - trace off'); -is((scalar grep { - get_attr("http.target", "string_value", $$spans{$_}) eq '/trace-off' - } keys %{$$batch1{scope_spans}}), 0, 'no metric in batch1 - trace off'); - -#trace context: ignore -is($t_headers_ignore->{'x-otel-traceparent'}, undef, - 'no traceparent - trace context ignore (no trace headers)'); -is($t_headers_ignore->{'x-otel-tracestate'}, undef, - 'no tracestate - trace context ignore (no trace headers)'); - -is($tp_headers_ignore->{'x-otel-parent-id'}, undef, - 'no parent span id - trace context ignore (trace headers)'); -is($tp_headers_ignore->{'x-otel-traceparent'}, - '00-0af7651916cd43dd8448eb211c80319c-b9c7c989f97918e1-01', - 'traceparent - trace context ignore (trace headers)'); -is($tp_headers_ignore->{'x-otel-tracestate'}, - 'congo=ucfJifl5GOE,rojo=00f067aa0ba902b7', - 'tracestate - trace context ignore (trace headers)'); - -#trace context: extract -is($t_headers_extract->{'x-otel-traceparent'}, undef, - 'no traceparent - trace context extract (no trace headers)'); -is($t_headers_extract->{'x-otel-tracestate'}, undef, - 'no tracestate - trace context extract (no trace headers)'); - -is($tp_headers_extract->{'x-otel-parent-id'}, 'b9c7c989f97918e1', - 'parent span id - trace context extract (trace headers)'); -is($tp_headers_extract->{'x-otel-traceparent'}, - '00-0af7651916cd43dd8448eb211c80319c-b9c7c989f97918e1-01', - 'traceparent - trace context extract (trace headers)'); -is($tp_headers_extract->{'x-otel-tracestate'}, - 'congo=ucfJifl5GOE,rojo=00f067aa0ba902b7', - 'tracestate - trace context extract (trace headers)'); - -#trace context: inject -is($t_headers_inject->{'x-otel-traceparent'}, - "00-$$spans{span6}{trace_id}-$$spans{span6}{span_id}-01", - 'traceparent - trace context inject (no trace headers)'); -is($t_headers_inject->{'x-otel-tracestate'}, undef, - 'no tracestate - trace context inject (no trace headers)'); - -is($tp_headers_inject->{'x-otel-parent-id'}, undef, - 'no parent span id - trace context inject (trace headers)'); -is($tp_headers_inject->{'x-otel-traceparent'}, - "00-$$spans{span7}{trace_id}-$$spans{span7}{span_id}-01", - 'traceparent - trace context inject (trace headers)'); -is($tp_headers_inject->{'x-otel-tracestate'}, undef, - 'no tracestate - trace context inject (trace headers)'); - -#trace context: propagate -is($t_headers_propagate->{'x-otel-traceparent'}, - "00-$$spans{span8}{trace_id}-$$spans{span8}{span_id}-01", - 'traceparent - trace context propagate (no trace headers)'); -is($t_headers_propagate->{'x-otel-tracestate'}, undef, - 'no tracestate - trace context propagate (no trace headers)'); - -is($tp_headers_propagate->{'x-otel-parent-id'}, 'b9c7c989f97918e1', - 'parent id - trace context propagate (trace headers)'); -is($tp_headers_propagate->{'x-otel-traceparent'}, - "00-0af7651916cd43dd8448eb211c80319c-$$spans{span9}{span_id}-01", - 'traceparent - trace context propagate (trace headers)'); -is($tp_headers_propagate->{'x-otel-tracestate'}, - 'congo=ucfJifl5GOE,rojo=00f067aa0ba902b7', - 'tracestate - trace context propagate (trace headers)'); - -SKIP: { -skip "depends on error log content", 2 unless $ENV{TEST_NGINX_UNSAFE}; - -$t->stop(); -my $log = $t->read_file("error.log"); - -like($log, qr/OTel\/grpc: Error parsing metadata: error=invalid value/, - 'log: error parsing metadata - no protobuf in response'); -unlike($log, qr/OTel export failure: No status received/, - 'log: no export failure'); - -} - -############################################################################### - -sub http3_get { - my ($path, %extra) = @_; - my ($frames, $frame); - - my $s = Test::Nginx::HTTP3->new(); - - my $sid = $extra{trace_headers} - ? $s->new_stream({ headers => [ - { name => ':method', value => 'GET' }, - { name => ':scheme', value => 'http' }, - { name => ':path', value => $path }, - { name => ':authority', value => 'localhost' }, - { name => 'user-agent', value => 'nginx-tests' }, - { name => 'traceparent', - value => '00-0af7651916cd43dd8448eb211c80319c-' . - 'b9c7c989f97918e1-01' - }, - { name => 'tracestate', - value => 'congo=ucfJifl5GOE,rojo=00f067aa0ba902b7' - }]}) - : $s->new_stream({ path => $path }); - - $frames = $s->read(all => [{ sid => $sid, fin => 1 }]); - - ($frame) = grep { $_->{type} eq "HEADERS" } @$frames; - my $headers = $frame->{headers}; - - ($frame) = grep { $_->{type} eq "DATA" } @$frames; - my $data = $frame->{data}; - - return $headers, $data; -} - -sub get_attr { - my($attr, $type, $obj) = @_; - - my ($res) = grep { - $_ =~ /^attribute\d+/ && $$obj{$_}{key} eq '"' . $attr . '"' - } keys %{$obj}; - - if (defined $res) { - $$obj{$res}{value}{$type} =~ s/(^\")|(\"$)//g - if $type eq 'string_value'; - - return $$obj{$res}{value}{$type}; - } - - return undef; -} - -sub decode_protobuf { - my ($protobuf) = @_; - - local $/; - open CMD, "echo '" . encode_base64($protobuf) . "' | base64 -d | " . - '$PWD/../build/_deps/grpc-build/third_party/protobuf/protoc '. - '--decode opentelemetry.proto.trace.v1.ResourceSpans -I ' . - '$PWD/../build/_deps/otelcpp-src/third_party/opentelemetry-proto ' . - 'opentelemetry/proto/collector/trace/v1/trace_service.proto |' - or die "Can't decode protobuf: $!\n"; - my $out = ; - close CMD; - - return $out; -} - -sub decode_bytes { - my ($bytes) = @_; - - my $c = sub { return chr oct(shift) }; - - $bytes =~ s/\\(\d{3})/$c->($1)/eg; - $bytes =~ s/(^\")|(\"$)//g; - $bytes =~ s/\\\\/\\/g; - $bytes =~ s/\\r/\r/g; - $bytes =~ s/\\n/\n/g; - $bytes =~ s/\\t/\t/g; - $bytes =~ s/\\"/\"/g; - $bytes =~ s/\\'/\'/g; - - return unpack("H*", unpack("a*", $bytes)); -} - -sub to_hash { - my ($textdata) = @_; - - my %out = (); - push my @stack, \%out; - my ($attr_count, $span_count) = (0, 0); - for my $line (split /\n/, $textdata) { - $line =~ s/(^\s+)|(\s+$)//g; - if ($line =~ /\:/) { - my ($k, $v) = split /\: /, $line; - $v = decode_bytes($v) if ($k =~ /trace_id|span_id|parent_span_id/); - $stack[$#stack]{$k} = $v; - } elsif ($line =~ /\{/) { - $line =~ s/\s\{//; - $line = 'attribute' . $attr_count++ if ($line eq 'attributes'); - if ($line eq 'spans') { - $line = 'span' . $span_count++; - $attr_count = 0; - } - my %new = (); - $stack[$#stack]{$line} = \%new; - push @stack, \%new; - } elsif ($line =~ /\}/) { - pop @stack; - } - } - - return \%out; -} - -sub grpc { - my ($server, $client, $f, $s, $c, $sid, $csid, $uri); - - $server = IO::Socket::INET->new( - Proto => 'tcp', - LocalHost => '127.0.0.1', - LocalPort => $p, - Listen => 5, - Reuse => 1 - ) or die "Can't create listening socket: $!\n"; - - $f->{http_start} = sub { - if (IO::Select->new($server)->can_read(5)) { - $client = $server->accept(); - } else { - # connection could be unexpectedly reused - goto reused if $client; - return undef; - } - - $client->sysread($_, 24) == 24 or return; # preface - - $c = Test::Nginx::HTTP2->new(1, socket => $client, - pure => 1, preface => "") or return; - -reused: - my $frames = $c->read(all => [{ fin => 1 }]); - - $client->close(); - - return $frames; - }; - - return $f; -} - -############################################################################### diff --git a/tests/otel.t b/tests/otel.t deleted file mode 100644 index 9887b2f..0000000 --- a/tests/otel.t +++ /dev/null @@ -1,514 +0,0 @@ -#!/usr/bin/perl - -# (C) Nginx, Inc. - -# Tests for OTel exporter in case HTTP. - -############################################################################### - -use warnings; -use strict; - -use Test::More; - -BEGIN { use FindBin; chdir($FindBin::Bin); } - -use Test::Nginx; -use Test::Nginx::HTTP2; -use MIME::Base64; - -############################################################################### - -select STDERR; $| = 1; -select STDOUT; $| = 1; - -my $t = Test::Nginx->new()->has(qw/http http_ssl http_v2 mirror rewrite/) - ->has_daemon(qw/openssl base64/) - ->write_file_expand('nginx.conf', <<'EOF'); - -%%TEST_GLOBALS%% - -daemon off; - -events { -} - -http { - %%TEST_GLOBALS_HTTP%% - - ssl_certificate_key localhost.key; - ssl_certificate localhost.crt; - - otel_exporter { - endpoint 127.0.0.1:8082; - interval 1s; - batch_size 10; - batch_count 2; - } - - otel_service_name test_server; - otel_trace on; - - server { - listen 127.0.0.1:8080; - listen 127.0.0.1:8081 ssl; - server_name localhost; - - location /trace-on { - otel_trace_context extract; - otel_span_name default_location; - otel_span_attr http.request.header.completion - $request_completion; - otel_span_attr http.response.header.content.type - $sent_http_content_type; - otel_span_attr http.request $request; - add_header "X-Otel-Trace-Id" $otel_trace_id; - add_header "X-Otel-Span-Id" $otel_span_id; - add_header "X-Otel-Parent-Id" $otel_parent_id; - add_header "X-Otel-Parent-Sampled" $otel_parent_sampled; - return 200 "TRACE-ON"; - } - - location /context-ignore { - otel_trace_context ignore; - otel_span_name context_ignore; - add_header "X-Otel-Parent-Id" $otel_parent_id; - proxy_pass http://localhost:8080/trace-off; - } - - location /context-extract { - otel_trace_context extract; - otel_span_name context_extract; - add_header "X-Otel-Parent-Id" $otel_parent_id; - proxy_pass http://localhost:8080/trace-off; - } - - location /context-inject { - otel_trace_context inject; - otel_span_name context_inject; - add_header "X-Otel-Parent-Id" $otel_parent_id; - proxy_pass http://localhost:8080/trace-off; - } - - location /context-propagate { - otel_trace_context propagate; - otel_span_name context_propogate; - add_header "X-Otel-Parent-Id" $otel_parent_id; - proxy_pass http://localhost:8080/trace-off; - } - - location /trace-off { - otel_trace off; - add_header "X-Otel-Traceparent" $http_traceparent; - add_header "X-Otel-Tracestate" $http_tracestate; - return 200 "TRACE-OFF"; - } - } - - server { - listen 127.0.0.1:8082 http2; - server_name localhost; - otel_trace off; - - location / { - mirror /mirror; - grpc_pass 127.0.0.1:8083; - } - - location /mirror { - internal; - grpc_pass 127.0.0.1:%%PORT_4317%%; - } - } - - server { - listen 127.0.0.1:8083 http2; - server_name localhost; - otel_trace off; - - location / { - add_header content-type application/grpc; - add_header grpc-status 0; - add_header grpc-message ""; - return 200; - } - } -} - -EOF - -$t->write_file('openssl.conf', <<'EOF'); -[ req ] -default_bits = 2048 -encrypt_key = no -distinguished_name = req_distinguished_name -[ req_distinguished_name ] - -EOF - -my $d = $t->testdir(); - -foreach my $name ('localhost') { - system('openssl req -x509 -new ' - . "-config $d/openssl.conf -subj /CN=$name/ " - . "-out $d/$name.crt -keyout $d/$name.key " - . ">>$d/openssl.out 2>&1") == 0 - or die "Can't create certificate for $name: $!\n"; -} - -$t->try_run('no OTel module')->plan(69); - -############################################################################### - -my $p = port(4317); -my $f = grpc(); - -#do requests -my $t_off_resp = http1_get('/trace-off'); - -#batch0 (10 requests) -my $tp_resp = http1_get('/trace-on', trace_headers => 1); -my $t_resp = http1_get('/trace-on', port => 8081, ssl => 1); - -my $t_resp_ignore = http1_get('/context-ignore'); -my $tp_resp_ignore = http1_get('/context-ignore', trace_headers => 1); -my $t_resp_extract = http1_get('/context-extract'); -my $tp_resp_extract = http1_get('/context-extract', trace_headers => 1); -my $t_resp_inject = http1_get('/context-inject'); -my $tp_resp_inject = http1_get('/context-inject', trace_headers => 1); -my $t_resp_propagate = http1_get('/context-propagate'); -my $tp_resp_propagate = http1_get('/context-propagate', trace_headers => 1); - -my ($frame) = grep { $_->{type} eq "DATA" } @{$f->{http_start}()}; -my $batch0 = to_hash(decode_protobuf(substr $frame->{data}, 8)); -my $spans = $$batch0{scope_spans}; - -#batch1 (5 reqeusts) -http1_get('/trace-on') for (1..5); - -($frame) = grep { $_->{type} eq "DATA" } @{$f->{http_start}()}; -my $batch1 = to_hash(decode_protobuf(substr $frame->{data}, 8)); - -#validate responses -like($tp_resp, qr/TRACE-ON/, 'http request1 - trace on'); -like($t_resp, qr/TRACE-ON/, 'http request2 - trace on'); -like($t_off_resp, qr/TRACE-OFF/, 'http request - trace off'); - -#validate batch size -delete $$spans{scope}; #remove 'scope' entry -is(scalar keys %{$spans}, 10, 'batch0 size - trace on'); -delete $$batch1{scope_spans}{scope}; #remove 'scope' entry -is(scalar keys %{$$batch1{scope_spans}}, 5, 'batch1 size - trace on'); - -#validate general attributes -is(get_attr("service.name", "string_value", - $$batch0{resource}), 'test_server', 'service.name - trace on'); -is($$spans{span0}{name}, '"default_location"', 'span.name - trace on'); - -#validate http metrics -is(get_attr("http.method", "string_value", $$spans{span0}), 'GET', - 'http.method metric - trace on'); -is(get_attr("http.target", "string_value", $$spans{span0}), '/trace-on', - 'http.target metric - trace on'); -is(get_attr("http.route", "string_value", $$spans{span0}), '/trace-on', - 'http.route metric - trace on'); -is(get_attr("http.scheme", "string_value", $$spans{span0}), 'http', - 'http.scheme metric - trace on'); -is(get_attr("http.flavor", "string_value", $$spans{span0}), '1.0', - 'http.flavor metric - trace on'); -is(get_attr("http.user_agent", "string_value", $$spans{span0}), 'nginx-tests', - 'http.user_agent metric - trace on'); -is(get_attr("http.request_content_length", "int_value", $$spans{span0}), 0, - 'http.request_content_length metric - trace on'); -is(get_attr("http.response_content_length", "int_value", $$spans{span0}), 8, - 'http.response_content_length metric - trace on'); -is(get_attr("http.status_code", "int_value", $$spans{span0}), 200, - 'http.status_code metric - trace on'); -is(get_attr("net.host.name", "string_value", $$spans{span0}), 'localhost', - 'net.host.name metric - trace on'); -is(get_attr("net.host.port", "int_value", $$spans{span0}), 8080, - 'net.host.port metric - trace on'); -is(get_attr("net.sock.peer.addr", "string_value", $$spans{span0}), '127.0.0.1', - 'net.sock.peer.addr metric - trace on'); -like(get_attr("net.sock.peer.port", "int_value", $$spans{span0}), qr/\d+/, - 'net.sock.peer.port metric - trace on'); - -#validate https metrics -is(get_attr("http.method", "string_value", $$spans{span1}), 'GET', - 'http.method metric - trace on (https)'); -is(get_attr("http.target", "string_value", $$spans{span1}), '/trace-on', - 'http.target metric - trace on (https)'); -is(get_attr("http.route", "string_value", $$spans{span1}), '/trace-on', - 'http.route metric - trace on (https)'); -is(get_attr("http.scheme", "string_value", $$spans{span1}), 'https', - 'http.scheme metric - trace on (https)'); -is(get_attr("http.flavor", "string_value", $$spans{span1}), '1.0', - 'http.flavor metric - trace on (https)'); -is(get_attr("http.user_agent", "string_value", $$spans{span1}), - 'nginx-tests', 'http.user_agent metric - trace on (https)'); -is(get_attr("http.request_content_length", "int_value", $$spans{span1}), 0, - 'http.request_content_length metric - trace on (https)'); -is(get_attr("http.response_content_length", "int_value", $$spans{span1}), 8, - 'http.response_content_length metric - trace on (https)'); -is(get_attr("http.status_code", "int_value", $$spans{span1}), 200, - 'http.status_code metric - trace on (https)'); -is(get_attr("net.host.name", "string_value", $$spans{span1}), 'localhost', - 'net.host.name metric - trace on (https)'); -is(get_attr("net.host.port", "int_value", $$spans{span1}), 8081, - 'net.host.port metric - trace on (https)'); -is(get_attr("net.sock.peer.addr", "string_value", $$spans{span1}), '127.0.0.1', - 'net.sock.peer.addr metric - trace on (https)'); -like(get_attr("net.sock.peer.port", "int_value", $$spans{span1}), qr/\d+/, - 'net.sock.peer.port metric - trace on (https)'); - -#validate custom http metrics -is(${get_attr("http.request.header.completion", "array_value", $$spans{span0})} - {values}{string_value}, '"OK"', - 'http.request.header.completion metric - trace on'); -is(${get_attr("http.response.header.content.type", - "array_value", $$spans{span0})}{values}{string_value}, '"text/plain"', - 'http.response.header.content.type metric - trace on'); -is(get_attr("http.request", "string_value", $$spans{span0}), - 'GET /trace-on HTTP/1.0', 'http.request metric - trace on'); - -#extract trace info -is($$spans{span0}{parent_span_id}, 'b9c7c989f97918e1', - 'traceparent - trace on'); -is($$spans{span0}{trace_state}, '"congo=ucfJifl5GOE,rojo=00f067aa0ba902b7"', - 'tracestate - trace on'); -is($$spans{span1}{parent_span_id}, undef, 'no traceparent - trace on'); -is($$spans{span1}{trace_state}, undef, 'no tracestate - trace on'); - -#variables -like($tp_resp, qr/X-Otel-Trace-Id: $$spans{span0}{trace_id}/, - '$otel_trace_id variable - trace on'); -like($tp_resp, qr/X-Otel-Span-Id: $$spans{span0}{span_id}/, - '$otel_span_id variable - trace on'); -like($tp_resp, qr/X-Otel-Parent-Id: $$spans{span0}{parent_span_id}/, - '$otel_parent_id variable - trace on'); -like($tp_resp, qr/X-Otel-Parent-Sampled: 1/, - '$otel_parent_sampled variable - trace on'); -like($t_resp, qr/X-Otel-Parent-Sampled: 0/, - '$otel_parent_sampled variable - trace on (no traceparent header)'); - -#trace off -is((scalar grep { - get_attr("http.target", "string_value", $$spans{$_}) eq '/trace-off' - } keys %{$spans}), 0, 'no metric in batch0 - trace off'); -is((scalar grep { - get_attr("http.target", "string_value", $$spans{$_}) eq '/trace-off' - } keys %{$$batch1{scope_spans}}), 0, 'no metric in batch1 - trace off'); - -#trace context: ignore -unlike($t_resp_ignore, qr/X-Otel-Traceparent/, - 'no traceparent - trace context ignore (no trace headers)'); -unlike($t_resp_ignore, qr/X-Otel-Tracestate/, - 'no tracestate - trace context ignore (no trace headers)'); - -unlike($tp_resp_ignore, qr/X-Otel-Parent-Id/, - 'no parent span id - trace context ignore (trace headers)'); -like($tp_resp_ignore, - qr/Traceparent: 00-0af7651916cd43dd8448eb211c80319c-b9c7c989f97918e1-01/, - 'traceparent - trace context ignore (trace headers)'); -like($tp_resp_ignore, - qr/Tracestate: congo=ucfJifl5GOE,rojo=00f067aa0ba902b7/, - 'tracestate - trace context ignore (trace headers)'); - -#trace context: extract -unlike($t_resp_extract, qr/X-Otel-Traceparent/, - 'no traceparent - trace context extract (no trace headers)'); -unlike($t_resp_extract, qr/X-Otel-Tracestate/, - 'no tracestate - trace context extract (no trace headers)'); - -like($tp_resp_extract, qr/X-Otel-Parent-Id: b9c7c989f97918e1/, - 'parent span id - trace context extract (trace headers)'); -like($tp_resp_extract, - qr/Traceparent: 00-0af7651916cd43dd8448eb211c80319c-b9c7c989f97918e1-01/, - 'traceparent - trace context extract (trace headers)'); -like($tp_resp_extract, - qr/Tracestate: congo=ucfJifl5GOE,rojo=00f067aa0ba902b7/, - 'tracestate - trace context extract (trace headers)'); - -#trace context: inject -like($t_resp_inject, - qr/Traceparent: 00-$$spans{span6}{trace_id}-$$spans{span6}{span_id}-01/, - 'traceparent - trace context inject (no trace headers)'); -unlike($t_resp_inject, qr/X-Otel-Tracestate/, - 'no tracestate - trace context inject (no trace headers)'); - -unlike($tp_resp_inject, qr/X-Otel-Parent-Id/, - 'no parent span id - trace context inject (trace headers)'); -like($tp_resp_inject, - qr/Traceparent: 00-$$spans{span7}{trace_id}-$$spans{span7}{span_id}-01/, - 'traceparent - trace context inject (trace headers)'); -unlike($tp_resp_inject, qr/Tracestate:/, - 'no tracestate - trace context inject (trace headers)'); - -#trace context: propagate -like($t_resp_propagate, - qr/Traceparent: 00-$$spans{span8}{trace_id}-$$spans{span8}{span_id}-01/, - 'traceparent - trace context propagate (no trace headers)'); -unlike($t_resp_propagate, qr/X-Otel-Tracestate/, - 'no tracestate - trace context propagate (no trace headers)'); - -like($tp_resp_propagate, qr/X-Otel-Parent-Id: b9c7c989f97918e1/, - 'parent id - trace context propagate (trace headers)'); -like($tp_resp_propagate, - qr/parent: 00-0af7651916cd43dd8448eb211c80319c-$$spans{span9}{span_id}-01/, - 'traceparent - trace context propagate (trace headers)'); -like($tp_resp_propagate, - qr/Tracestate: congo=ucfJifl5GOE,rojo=00f067aa0ba902b7/, - 'tracestate - trace context propagate (trace headers)'); - -SKIP: { -skip "depends on error log contents", 2 unless $ENV{TEST_NGINX_UNSAFE}; - -$t->stop(); -my $log = $t->read_file("error.log"); - -like($log, qr/OTel\/grpc: Error parsing metadata: error=invalid value/, - 'log: error parsing metadata - no protobuf in response'); -unlike($log, qr/OTel export failure: No status received/, - 'log: no export failure'); - -} - -############################################################################### - -sub http1_get { - my ($path, %extra) = @_; - - my $port = $extra{port} || 8080; - - my $r = < '127.0.0.1:' . port($port), - SSL => $extra{ssl}); -} - -sub get_attr { - my($attr, $type, $obj) = @_; - - my ($res) = grep { - $_ =~ /^attribute\d+/ && $$obj{$_}{key} eq '"' . $attr . '"' - } keys %{$obj}; - - if (defined $res) { - $$obj{$res}{value}{$type} =~ s/(^\")|(\"$)//g - if $type eq 'string_value'; - - return $$obj{$res}{value}{$type}; - } - - return undef; -} - -sub decode_protobuf { - my ($protobuf) = @_; - - local $/; - open CMD, "echo '" . encode_base64($protobuf) . "' | base64 -d | " . - '$PWD/../build/_deps/grpc-build/third_party/protobuf/protoc '. - '--decode opentelemetry.proto.trace.v1.ResourceSpans -I ' . - '$PWD/../build/_deps/otelcpp-src/third_party/opentelemetry-proto ' . - 'opentelemetry/proto/collector/trace/v1/trace_service.proto |' - or die "Can't decode protobuf: $!\n"; - my $out = ; - close CMD; - - return $out; -} - -sub decode_bytes { - my ($bytes) = @_; - - my $c = sub { return chr oct(shift) }; - - $bytes =~ s/\\(\d{3})/$c->($1)/eg; - $bytes =~ s/(^\")|(\"$)//g; - $bytes =~ s/\\\\/\\/g; - $bytes =~ s/\\r/\r/g; - $bytes =~ s/\\n/\n/g; - $bytes =~ s/\\t/\t/g; - $bytes =~ s/\\"/\"/g; - $bytes =~ s/\\'/\'/g; - - return unpack("H*", unpack("a*", $bytes)); -} - -sub to_hash { - my ($textdata) = @_; - - my %out = (); - push my @stack, \%out; - my ($attr_count, $span_count) = (0, 0); - for my $line (split /\n/, $textdata) { - $line =~ s/(^\s+)|(\s+$)//g; - if ($line =~ /\:/) { - my ($k, $v) = split /\: /, $line; - $v = decode_bytes($v) if ($k =~ /trace_id|span_id|parent_span_id/); - $stack[$#stack]{$k} = $v; - } elsif ($line =~ /\{/) { - $line =~ s/\s\{//; - $line = 'attribute' . $attr_count++ if ($line eq 'attributes'); - if ($line eq 'spans') { - $line = 'span' . $span_count++; - $attr_count = 0; - } - my %new = (); - $stack[$#stack]{$line} = \%new; - push @stack, \%new; - } elsif ($line =~ /\}/) { - pop @stack; - } - } - - return \%out; -} - -sub grpc { - my ($server, $client, $f, $s, $c, $sid, $csid, $uri); - - $server = IO::Socket::INET->new( - Proto => 'tcp', - LocalHost => '127.0.0.1', - LocalPort => $p, - Listen => 5, - Reuse => 1 - ) or die "Can't create listening socket: $!\n"; - - $f->{http_start} = sub { - if (IO::Select->new($server)->can_read(5)) { - $client = $server->accept(); - } else { - # connection could be unexpectedly reused - goto reused if $client; - return undef; - } - - $client->sysread($_, 24) == 24 or return; # preface - - $c = Test::Nginx::HTTP2->new(1, socket => $client, - pure => 1, preface => "") or return; - -reused: - my $frames = $c->read(all => [{ fin => 1 }]); - - $client->close(); - - return $frames; - }; - - return $f; -} - -############################################################################### diff --git a/tests/otel_collector.t b/tests/otel_collector.t deleted file mode 100644 index 158a2fd..0000000 --- a/tests/otel_collector.t +++ /dev/null @@ -1,403 +0,0 @@ -#!/usr/bin/perl - -# (C) Nginx, Inc. - -# Tests for OTel exporter in case HTTP using otelcol. - -############################################################################### - -use warnings; -use strict; - -use Test::More; - -BEGIN { use FindBin; chdir($FindBin::Bin); } - -use Test::Nginx; - -############################################################################### - -select STDERR; $| = 1; -select STDOUT; $| = 1; - -plan(skip_all => "depends on logs content") unless $ENV{TEST_NGINX_UNSAFE}; - -eval { require JSON::PP; }; -plan(skip_all => "JSON::PP not installed") if $@; - -my $t = Test::Nginx->new()->has(qw/http http_ssl rewrite/) - ->write_file_expand('nginx.conf', <<'EOF'); - -%%TEST_GLOBALS%% - -daemon off; - -events { -} - -http { - %%TEST_GLOBALS_HTTP%% - - ssl_certificate_key localhost.key; - ssl_certificate localhost.crt; - - otel_exporter { - endpoint 127.0.0.1:%%PORT_4317%%; - interval 1s; - batch_size 10; - batch_count 2; - } - - otel_service_name test_server; - otel_trace on; - - server { - listen 127.0.0.1:8080; - listen 127.0.0.1:8081 ssl; - server_name localhost; - - location /trace-on { - otel_trace_context extract; - otel_span_name default_location; - otel_span_attr http.request.header.completion - $request_completion; - otel_span_attr http.response.header.content.type - $sent_http_content_type; - otel_span_attr http.request $request; - add_header "X-Otel-Trace-Id" $otel_trace_id; - add_header "X-Otel-Span-Id" $otel_span_id; - add_header "X-Otel-Parent-Id" $otel_parent_id; - add_header "X-Otel-Parent-Sampled" $otel_parent_sampled; - return 200 "TRACE-ON"; - } - - location /context-ignore { - otel_trace_context ignore; - otel_span_name context_ignore; - add_header "X-Otel-Parent-Id" $otel_parent_id; - proxy_pass http://localhost:8080/trace-off; - } - - location /context-extract { - otel_trace_context extract; - otel_span_name context_extract; - add_header "X-Otel-Parent-Id" $otel_parent_id; - proxy_pass http://localhost:8080/trace-off; - } - - location /context-inject { - otel_trace_context inject; - otel_span_name context_inject; - add_header "X-Otel-Parent-Id" $otel_parent_id; - proxy_pass http://localhost:8080/trace-off; - } - - location /context-propagate { - otel_trace_context propagate; - otel_span_name context_propogate; - add_header "X-Otel-Parent-Id" $otel_parent_id; - proxy_pass http://localhost:8080/trace-off; - } - - location /trace-off { - otel_trace off; - add_header "X-Otel-Traceparent" $http_traceparent; - add_header "X-Otel-Tracestate" $http_tracestate; - return 200 "TRACE-OFF"; - } - } -} - -EOF - -$t->write_file_expand('otel-config.yaml', <testdir() }/otel.json - -service: - pipelines: - traces: - receivers: [otlp] - exporters: [logging, file] - metrics: - receivers: [otlp] - exporters: [logging, file] - -EOF - -$t->write_file('openssl.conf', <<'EOF'); -[ req ] -default_bits = 2048 -encrypt_key = no -distinguished_name = req_distinguished_name -[ req_distinguished_name ] - -EOF - -my $d = $t->testdir(); - -foreach my $name ('localhost') { - system('openssl req -x509 -new ' - . "-config $d/openssl.conf -subj /CN=$name/ " - . "-out $d/$name.crt -keyout $d/$name.key " - . ">>$d/openssl.out 2>&1") == 0 - or die "Can't create certificate for $name: $!\n"; -} - -#suppress otel collector output -open OLDERR, ">&", \*STDERR; -open STDERR, ">>" , $^O eq 'MSWin32' ? 'nul' : '/dev/null'; -$t->run_daemon('../otelcol', '--config', $t->testdir().'/otel-config.yaml'); -open STDERR, ">&", \*OLDERR; -$t->waitforsocket('127.0.0.1:' . port(4317)) or - die 'No otel collector open socket'; - -$t->try_run('no OTel module')->plan(69); - -############################################################################### - -#do requests -my $t_off_resp = http1_get('/trace-off'); - -#batch0 (10 requests) -my $tp_resp = http1_get('/trace-on', trace_headers => 1); -my $t_resp = http1_get('/trace-on', port => 8081, ssl => 1); - -my $t_resp_ignore = http1_get('/context-ignore'); -my $tp_resp_ignore = http1_get('/context-ignore', trace_headers => 1); -my $t_resp_extract = http1_get('/context-extract'); -my $tp_resp_extract = http1_get('/context-extract', trace_headers => 1); -my $t_resp_inject = http1_get('/context-inject'); -my $tp_resp_inject = http1_get('/context-inject', trace_headers => 1); -my $t_resp_propagate = http1_get('/context-propagate'); -my $tp_resp_propagate = http1_get('/context-propagate', trace_headers => 1); - -#batch1 (5 reqeusts) -http1_get('/trace-on') for (1..5); - -#waiting batch1 is sent to collector for 1s -select undef, undef, undef, 1; - -my @batches = split /\n/, $t->read_file('otel.json'); -my $batch_json = JSON::PP::decode_json($batches[0]); -my $spans = $$batch_json{"resourceSpans"}[0]{"scopeSpans"}[0]{"spans"}; - -#validate responses -like($tp_resp, qr/TRACE-ON/, 'http request1 - trace on'); -like($t_resp, qr/TRACE-ON/, 'http request2 - trace on'); -like($t_off_resp, qr/TRACE-OFF/, 'http request - trace off'); - -#validate amount of batches -is(scalar @batches, 2, 'amount of batches - trace on'); - -#validate batch size -is(scalar @{$spans}, 10, 'batch0 size - trace on'); -is(scalar @{${JSON::PP::decode_json($batches[1])}{"resourceSpans"}[0] - {"scopeSpans"}[0]{"spans"}}, 5, 'batch1 size - trace on'); - -#validate general attributes -is(get_attr("service.name", "stringValue", - $$batch_json{resourceSpans}[0]{resource}), - 'test_server', 'service.name - trace on'); -is($$spans[0]{name}, 'default_location', 'span.name - trace on'); - -#validate http metrics -is(get_attr("http.method", "stringValue", $$spans[0]), 'GET', - 'http.method metric - trace on'); -is(get_attr("http.target", "stringValue", $$spans[0]), '/trace-on', - 'http.target metric - trace on'); -is(get_attr("http.route", "stringValue", $$spans[0]), '/trace-on', - 'http.route metric - trace on'); -is(get_attr("http.scheme", "stringValue", $$spans[0]), 'http', - 'http.scheme metric - trace on'); -is(get_attr("http.flavor", "stringValue", $$spans[0]), '1.0', - 'http.flavor metric - trace on'); -is(get_attr("http.user_agent", "stringValue", $$spans[0]), 'nginx-tests', - 'http.user_agent metric - trace on'); -is(get_attr("http.request_content_length", "intValue", $$spans[0]), 0, - 'http.request_content_length metric - trace on'); -is(get_attr("http.response_content_length", "intValue", $$spans[0]), 8, - 'http.response_content_length metric - trace on'); -is(get_attr("http.status_code", "intValue", $$spans[0]), 200, - 'http.status_code metric - trace on'); -is(get_attr("net.host.name", "stringValue", $$spans[0]), 'localhost', - 'net.host.name metric - trace on'); -is(get_attr("net.host.port", "intValue", $$spans[0]), 8080, - 'net.host.port metric - trace on'); -is(get_attr("net.sock.peer.addr", "stringValue", $$spans[0]), '127.0.0.1', - 'net.sock.peer.addr metric - trace on'); -like(get_attr("net.sock.peer.port", "intValue", $$spans[0]), qr/\d+/, - 'net.sock.peer.port metric - trace on'); - -#validate custom http metrics -is(${get_attr("http.request.header.completion", "arrayValue", $$spans[0])} - {values}[0]{stringValue}, 'OK', - 'http.request.header.completion metric - trace on'); -is(${get_attr("http.response.header.content.type", "arrayValue",$$spans[0])} - {values}[0]{stringValue}, 'text/plain', - 'http.response.header.content.type metric - trace on'); -is(get_attr("http.request", "stringValue", $$spans[0]), - 'GET /trace-on HTTP/1.0', 'http.request metric - trace on'); - -#validate https metrics -is(get_attr("http.method", "stringValue", $$spans[1]), 'GET', - 'http.method metric - trace on (https)'); -is(get_attr("http.target", "stringValue", $$spans[1]), '/trace-on', - 'http.target metric - trace on (https)'); -is(get_attr("http.route", "stringValue", $$spans[1]), '/trace-on', - 'http.route metric - trace on (https)'); -is(get_attr("http.scheme", "stringValue", $$spans[1]), 'https', - 'http.scheme metric - trace on (https)'); -is(get_attr("http.flavor", "stringValue", $$spans[1]), '1.0', - 'http.flavor metric - trace on (https)'); -is(get_attr("http.user_agent", "stringValue", $$spans[1]), 'nginx-tests', - 'http.user_agent metric - trace on (https)'); -is(get_attr("http.request_content_length", "intValue", $$spans[1]), 0, - 'http.request_content_length metric - trace on (https)'); -is(get_attr("http.response_content_length", "intValue", $$spans[1]), 8, - 'http.response_content_length metric - trace on (https)'); -is(get_attr("http.status_code", "intValue", $$spans[1]), 200, - 'http.status_code metric - trace on (https)'); -is(get_attr("net.host.name", "stringValue", $$spans[1]), 'localhost', - 'net.host.name metric - trace on (https)'); -is(get_attr("net.host.port", "intValue", $$spans[1]), 8081, - 'net.host.port metric - trace on (https)'); -is(get_attr("net.sock.peer.addr", "stringValue", $$spans[1]), '127.0.0.1', - 'net.sock.peer.addr metric - trace on (https)'); -like(get_attr("net.sock.peer.port", "intValue", $$spans[1]), qr/\d+/, - 'net.sock.peer.port metric - trace on (https)'); - -#extract trace info -is($$spans[0]{parentSpanId}, 'b9c7c989f97918e1', 'traceparent - trace on'); -is($$spans[0]{traceState}, 'congo=ucfJifl5GOE,rojo=00f067aa0ba902b7', - 'tracestate - trace on'); -is($$spans[1]{parentSpanId}, '', 'no traceparent - trace on'); -is($$spans[1]{traceState}, undef, 'no tracestate - trace on'); - -#variables -like($tp_resp, qr/X-Otel-Trace-Id: $$spans[0]{traceId}/, - '$otel_trace_id variable - trace on'); -like($tp_resp, qr/X-Otel-Span-Id: $$spans[0]{spanId}/, - '$otel_span_id variable - trace on'); -like($tp_resp, qr/X-Otel-Parent-Id: $$spans[0]{parentSpanId}/, - '$otel_parent_id variable - trace on'); -like($tp_resp, qr/X-Otel-Parent-Sampled: 1/, - '$otel_parent_sampled variable - trace on'); -like($t_resp, qr/X-Otel-Parent-Sampled: 0/, - '$otel_parent_sampled variable - trace on (no traceparent header)'); - -#trace off -unlike($batches[0].$batches[1], - qr/\Q{"key":"http.target","value":{"stringValue":"\/trace-off"}}\E/, - 'no metrics - trace off'); - -#trace context: ignore -unlike($t_resp_ignore, qr/X-Otel-Traceparent/, - 'no traceparent - trace context ignore (no trace headers)'); -unlike($t_resp_ignore, qr/X-Otel-Tracestate/, - 'no tracestate - trace context ignore (no trace headers)'); - -unlike($tp_resp_ignore, qr/X-Otel-Parent-Id/, - 'no parent span id - trace context ignore (trace headers)'); -like($tp_resp_ignore, - qr/Traceparent: 00-0af7651916cd43dd8448eb211c80319c-b9c7c989f97918e1-01/, - 'traceparent - trace context ignore (trace headers)'); -like($tp_resp_ignore, - qr/Tracestate: congo=ucfJifl5GOE,rojo=00f067aa0ba902b7/, - 'tracestate - trace context ignore (trace headers)'); - -#trace context: extract -unlike($t_resp_extract, qr/X-Otel-Traceparent/, - 'no traceparent - trace context extract (no trace headers)'); -unlike($t_resp_extract, qr/X-Otel-Tracestate/, - 'no tracestate - trace context extract (no trace headers)'); - -like($tp_resp_extract, qr/X-Otel-Parent-Id: b9c7c989f97918e1/, - 'parent span id - trace context extract (trace headers)'); -like($tp_resp_extract, - qr/Traceparent: 00-0af7651916cd43dd8448eb211c80319c-b9c7c989f97918e1-01/, - 'traceparent - trace context extract (trace headers)'); -like($tp_resp_extract, - qr/Tracestate: congo=ucfJifl5GOE,rojo=00f067aa0ba902b7/, - 'tracestate - trace context extract (trace headers)'); - -#trace context: inject -like($t_resp_inject, - qr/X-Otel-Traceparent: 00-$$spans[6]{traceId}-$$spans[6]{spanId}-01/, - 'traceparent - trace context inject (no trace headers)'); -unlike($t_resp_inject, qr/X-Otel-Tracestate/, - 'no tracestate - trace context inject (no trace headers)'); - -unlike($tp_resp_inject, qr/X-Otel-Parent-Id/, - 'no parent span id - trace context inject (trace headers)'); -like($tp_resp_inject, - qr/Traceparent: 00-$$spans[7]{traceId}-$$spans[7]{spanId}-01/, - 'traceparent - trace context inject (trace headers)'); -unlike($tp_resp_inject, qr/Tracestate:/, - 'no tracestate - trace context inject (trace headers)'); - -#trace context: propagate -like($t_resp_propagate, - qr/X-Otel-Traceparent: 00-$$spans[8]{traceId}-$$spans[8]{spanId}-01/, - 'traceparent - trace context propagate (no trace headers)'); -unlike($t_resp_propagate, qr/X-Otel-Tracestate/, - 'no tracestate - trace context propagate (no trace headers)'); - -like($tp_resp_propagate, qr/X-Otel-Parent-Id: b9c7c989f97918e1/, - 'parent id - trace context propagate (trace headers)'); -like($tp_resp_propagate, - qr/Traceparent: 00-0af7651916cd43dd8448eb211c80319c-$$spans[9]{spanId}-01/, - 'traceparent - trace context propagate (trace headers)'); -like($tp_resp_propagate, - qr/Tracestate: congo=ucfJifl5GOE,rojo=00f067aa0ba902b7/, - 'tracestate - trace context propagate (trace headers)'); - -$t->stop(); -my $log = $t->read_file("error.log"); - -unlike($log, qr/OTel\/grpc: Error parsing metadata: error=invalid value/, - 'log: no error parsing metadata'); -unlike($log, qr/OTel export failure: No status received/, - 'log: no export failure'); - -############################################################################### - -sub http1_get { - my ($path, %extra) = @_; - - my $port = $extra{port} || 8080; - - my $r = < '127.0.0.1:' . port($port), - SSL => $extra{ssl}); -} - -sub get_attr { - my($attr, $type, $obj) = @_; - - my ($res) = grep { $$_{"key"} eq $attr } @{$$obj{"attributes"}}; - - return defined $res ? $res->{"value"}{$type} : undef; -} - -############################################################################### diff --git a/tests/requirements.txt b/tests/requirements.txt new file mode 100644 index 0000000..d27c6b1 --- /dev/null +++ b/tests/requirements.txt @@ -0,0 +1,6 @@ +pytest~=8.3 +jinja2~=3.1 +pyopenssl~=24.3 +niquests~=3.11 +grpcio~=1.68 +opentelemetry-proto~=1.28 diff --git a/tests/test_otel.py b/tests/test_otel.py new file mode 100644 index 0000000..5fbac51 --- /dev/null +++ b/tests/test_otel.py @@ -0,0 +1,262 @@ +from collections import namedtuple +import niquests +import pytest +import socket +import time +import urllib3 + + +NGINX_CONFIG = """ +{{ globals }} + +daemon off; + +events { +} + +http { + {{ http_globals }} + + ssl_certificate localhost.crt; + ssl_certificate_key localhost.key; + + otel_exporter { + endpoint 127.0.0.1:14317; + interval {{ interval or "1ms" }}; + batch_size 3; + batch_count 3; + } + + otel_trace on; + otel_service_name test_service; + + server { + listen 127.0.0.1:18443 ssl; + listen 127.0.0.1:18443 quic; + listen 127.0.0.1:18080; + + http2 on; + + server_name localhost; + + location /ok { + return 200 "OK"; + } + + location /err { + return 500 "ERR"; + } + + location /custom { + otel_span_name custom_location; + otel_span_attr http.request.completion + $request_completion; + otel_span_attr http.response.header.content.type + $sent_http_content_type; + otel_span_attr http.request $request; + return 200 "OK"; + } + + location /vars { + otel_trace_context extract; + add_header "X-Otel-Trace-Id" $otel_trace_id; + add_header "X-Otel-Span-Id" $otel_span_id; + add_header "X-Otel-Parent-Id" $otel_parent_id; + add_header "X-Otel-Parent-Sampled" $otel_parent_sampled; + return 204; + } + + location /ignore { + proxy_pass http://127.0.0.1:18080/notrace; + } + + location /extract { + otel_trace_context extract; + proxy_pass http://127.0.0.1:18080/notrace; + } + + location /inject { + otel_trace_context inject; + proxy_pass http://127.0.0.1:18080/notrace; + } + + location /propagate { + otel_trace_context propagate; + proxy_pass http://127.0.0.1:18080/notrace; + } + + location /notrace { + otel_trace off; + add_header "X-Otel-Traceparent" $http_traceparent; + add_header "X-Otel-Tracestate" $http_tracestate; + return 204; + } + } +} + +""" + +TraceContext = namedtuple("TraceContext", ["trace_id", "span_id", "state"]) + +parent_ctx = TraceContext( + trace_id="0af7651916cd43dd8448eb211c80319c", + span_id="b9c7c989f97918e1", + state="congo=ucfJifl5GOE,rojo=00f067aa0ba902b7", +) + + +def trace_headers(ctx): + return ( + { + "Traceparent": f"00-{ctx.trace_id}-{ctx.span_id}-01", + "Tracestate": ctx.state, + } + if ctx + else {"Traceparent": None, "Tracestate": None} + ) + + +def get_attr(span, name): + for value in (a.value for a in span.attributes if a.key == name): + return getattr(value, value.WhichOneof("value")) + + +@pytest.fixture +def client(nginx): + urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + with niquests.Session(multiplexed=True) as s: + yield s + + +def test_http09(trace_service, nginx): + + def get_http09(host, port, path): + with socket.create_connection((host, port)) as sock: + sock.sendall(f"GET {path}\n".encode()) + resp = sock.recv(1024).decode("utf-8") + return resp + + assert get_http09("127.0.0.1", 18080, "/ok") == "OK" + + span = trace_service.get_span() + assert span.name == "/ok" + + +@pytest.mark.parametrize("http_ver", ["1.1", "2.0", "3.0"]) +@pytest.mark.parametrize( + ("path", "status"), + [("/ok", 200), ("/err", 500)], +) +def test_default_attributes(client, trace_service, http_ver, path, status): + scheme, port = ("http", 18080) if http_ver == "1.1" else ("https", 18443) + if http_ver == "3.0": + client.quic_cache_layer.add_domain("127.0.0.1", port) + r = client.get(f"{scheme}://127.0.0.1:{port}{path}", verify=False) + + span = trace_service.get_span() + assert span.name == path + + assert get_attr(span, "http.method") == "GET" + assert get_attr(span, "http.target") == path + assert get_attr(span, "http.route") == path + assert get_attr(span, "http.scheme") == scheme + assert get_attr(span, "http.flavor") == http_ver + assert get_attr(span, "http.user_agent") == ( + f"niquests/{niquests.__version__}" + ) + assert get_attr(span, "http.request_content_length") == 0 + assert get_attr(span, "http.response_content_length") == len(r.text) + assert get_attr(span, "http.status_code") == status + assert get_attr(span, "net.host.name") == "localhost" + assert get_attr(span, "net.host.port") == port + assert get_attr(span, "net.sock.peer.addr") == "127.0.0.1" + assert get_attr(span, "net.sock.peer.port") in range(1024, 65536) + + +def test_custom_attributes(client, trace_service): + assert client.get("http://127.0.0.1:18080/custom").status_code == 200 + + span = trace_service.get_span() + assert span.name == "custom_location" + + assert get_attr(span, "http.request.completion") == "OK" + value = get_attr(span, "http.response.header.content.type") + assert value.values[0].string_value == "text/plain" + assert get_attr(span, "http.request") == "GET /custom HTTP/1.1" + + +def test_trace_off(client, trace_service): + assert client.get("http://127.0.0.1:18080/notrace").status_code == 204 + + time.sleep(0.01) # wait for spans + assert len(trace_service.batches) == 0 + + +@pytest.mark.parametrize("parent", [None, parent_ctx]) +def test_variables(client, trace_service, parent): + r = client.get("http://127.0.0.1:18080/vars", headers=trace_headers(parent)) + + span = trace_service.get_span() + + if parent: + assert span.trace_id.hex() == parent.trace_id + assert span.parent_span_id.hex() == parent.span_id + assert span.trace_state == parent.state + + assert r.headers.get("X-Otel-Trace-Id") == span.trace_id.hex() + assert r.headers.get("X-Otel-Span-Id") == span.span_id.hex() + assert r.headers.get("X-Otel-Parent-Id") or "" == span.parent_span_id.hex() + assert r.headers.get("X-Otel-Parent-Sampled") == ("1" if parent else "0") + + +@pytest.mark.parametrize("parent", [None, parent_ctx]) +@pytest.mark.parametrize( + "path", ["/ignore", "/extract", "/inject", "/propagate"] +) +def test_context(client, trace_service, parent, path): + headers = trace_headers(parent) + + r = client.get(f"http://127.0.0.1:18080{path}", headers=headers) + + span = trace_service.get_span() + + if path in ["/extract", "/propagate"] and parent: + assert span.trace_id.hex() == parent.trace_id + assert span.parent_span_id.hex() == parent.span_id + assert span.trace_state == parent.state + + if path in ["/inject", "/propagate"]: + headers = trace_headers( + TraceContext( + span.trace_id.hex(), + span.span_id.hex(), + span.trace_state or None, + ) + ) + + assert r.headers.get("X-Otel-Traceparent") == headers["Traceparent"] + assert r.headers.get("X-Otel-Tracestate") == headers["Tracestate"] + + +@pytest.mark.parametrize( + "nginx_config", [({"interval": "200ms"})], indirect=True +) +@pytest.mark.parametrize("batch_count", [1, 3]) +def test_batches(client, trace_service, batch_count): + batch_size = 3 + + for _ in range( + batch_count * batch_size + 1 + ): # +1 request to trigger batch sending + assert client.get("http://127.0.0.1:18080/ok").status_code == 200 + + time.sleep(0.01) + + assert len(trace_service.batches) == batch_count + + for batch in trace_service.batches: + assert get_attr(batch[0].resource, "service.name") == "test_service" + assert len(batch[0].scope_spans[0].spans) == batch_size + + time.sleep(0.3) # wait for +1 request to be flushed + trace_service.batches.clear() diff --git a/tests/trace_service.py b/tests/trace_service.py new file mode 100644 index 0000000..9f094f9 --- /dev/null +++ b/tests/trace_service.py @@ -0,0 +1,86 @@ +import concurrent +import grpc +from opentelemetry.proto.collector.trace.v1 import trace_service_pb2 +from opentelemetry.proto.collector.trace.v1 import trace_service_pb2_grpc +import pytest +import subprocess +import time + + +class TraceService(trace_service_pb2_grpc.TraceServiceServicer): + batches = [] + + def Export(self, request, context): + self.batches.append(request.resource_spans) + return trace_service_pb2.ExportTracePartialSuccess() + + def get_span(self): + for _ in range(10): + if len(self.batches): + break + time.sleep(0.001) + + assert len(self.batches) == 1, "No spans received" + span = self.batches[0][0].scope_spans[0].spans.pop() + self.batches.clear() + return span + + +@pytest.fixture(scope="module") +def trace_service(pytestconfig, logger): + server = grpc.server(concurrent.futures.ThreadPoolExecutor()) + trace_service = TraceService() + trace_service_pb2_grpc.add_TraceServiceServicer_to_server( + trace_service, server + ) + listen_addr = f"127.0.0.1:{24317 if pytestconfig.option.otelcol else 14317}" + server.add_insecure_port(listen_addr) + logger.info(f"Starting trace service at {listen_addr}...") + server.start() + yield trace_service + logger.info("Stopping trace service...") + server.stop(grace=None) + + +@pytest.fixture(scope="module") +def otelcol(pytestconfig, testdir, logger, trace_service): + if pytestconfig.option.otelcol is None: + yield + return + + (testdir / "otel-config.yaml").write_text( + """receivers: + otlp: + protocols: + grpc: + endpoint: 127.0.0.1:14317 + +exporters: + otlp: + endpoint: 127.0.0.1:24317 + tls: + insecure: true + +service: + pipelines: + traces: + receivers: [otlp] + exporters: [otlp] + telemetry: + metrics: + # prevent otelcol from opening 8888 port + level: none""" + ) + logger.info("Starting otelcol at 127.0.0.1:14317...") + proc = subprocess.Popen( + [pytestconfig.option.otelcol, "--config", testdir / "otel-config.yaml"] + ) + time.sleep(1) # give some time to get ready + assert proc.poll() is None, "Can't start otelcol" + yield + logger.info("Stopping otelcol...") + proc.terminate() + try: + proc.wait(timeout=5) + except subprocess.TimeoutExpired: + proc.kill() From be30eeffc1761b8ffa9cc5c8b65f6dae6892bac1 Mon Sep 17 00:00:00 2001 From: Eugene Grebenschikov Date: Wed, 18 Dec 2024 16:52:30 -0800 Subject: [PATCH 20/28] Verify HTTP scheme support in endpoint (#60). --- tests/test_otel.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_otel.py b/tests/test_otel.py index 5fbac51..10ce2fe 100644 --- a/tests/test_otel.py +++ b/tests/test_otel.py @@ -21,7 +21,7 @@ http { ssl_certificate_key localhost.key; otel_exporter { - endpoint 127.0.0.1:14317; + endpoint {{ scheme }}127.0.0.1:14317; interval {{ interval or "1ms" }}; batch_size 3; batch_count 3; @@ -239,7 +239,9 @@ def test_context(client, trace_service, parent, path): @pytest.mark.parametrize( - "nginx_config", [({"interval": "200ms"})], indirect=True + "nginx_config", + [({"interval": "200ms", "scheme": "http://"})], + indirect=True, ) @pytest.mark.parametrize("batch_count", [1, 3]) def test_batches(client, trace_service, batch_count): From 1d259542747b41c2fd7f89d6ae17ccfbdfdf6f67 Mon Sep 17 00:00:00 2001 From: Eugene Grebenschikov Date: Wed, 18 Dec 2024 17:23:37 -0800 Subject: [PATCH 21/28] Download the latest otelcol for CI tests. Co-authored-by: Pavel Pautov --- .github/workflows/ubuntu.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 1a8edac..0f328d2 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -36,9 +36,11 @@ jobs: make -j $(nproc) - name: Download otelcol run: | - curl -sLo - https://github.com/\ - open-telemetry/opentelemetry-collector-releases/releases/download/\ - v0.115.1/otelcol_0.115.1_linux_amd64.tar.gz | tar -xzv + LATEST=open-telemetry/opentelemetry-collector-releases/releases/latest + TAG=$(curl -s https://api.github.com/repos/${LATEST} | + jq -r .tag_name) + curl -sLo - https://github.com/${LATEST}/download/\ + otelcol_${TAG:1}_linux_amd64.tar.gz | tar -xzv - name: Install test dependencies run: pip install -r tests/requirements.txt - name: Run tests From c9136f2ec8d76751c35487ba4139e1986074a577 Mon Sep 17 00:00:00 2001 From: Eugene <54681898+jimf5@users.noreply.github.com> Date: Thu, 19 Dec 2024 17:53:38 -0800 Subject: [PATCH 22/28] Verify custom resource attributes support (#32). Co-authored-by: p-pautov <37922380+p-pautov@users.noreply.github.com> --- tests/test_otel.py | 32 +++++++++++++++++++++++++++++--- tests/trace_service.py | 14 +++++++++----- 2 files changed, 38 insertions(+), 8 deletions(-) diff --git a/tests/test_otel.py b/tests/test_otel.py index 10ce2fe..11caa85 100644 --- a/tests/test_otel.py +++ b/tests/test_otel.py @@ -28,7 +28,7 @@ http { } otel_trace on; - otel_service_name test_service; + {{ resource_attrs }} server { listen 127.0.0.1:18443 ssl; @@ -240,7 +240,7 @@ def test_context(client, trace_service, parent, path): @pytest.mark.parametrize( "nginx_config", - [({"interval": "200ms", "scheme": "http://"})], + [{"interval": "200ms", "scheme": "http://"}], indirect=True, ) @pytest.mark.parametrize("batch_count", [1, 3]) @@ -257,8 +257,34 @@ def test_batches(client, trace_service, batch_count): assert len(trace_service.batches) == batch_count for batch in trace_service.batches: - assert get_attr(batch[0].resource, "service.name") == "test_service" + assert ( + get_attr(batch[0].resource, "service.name") + == "unknown_service:nginx" + ) assert len(batch[0].scope_spans[0].spans) == batch_size time.sleep(0.3) # wait for +1 request to be flushed trace_service.batches.clear() + + +@pytest.mark.parametrize( + "nginx_config", + [ + { + "resource_attrs": """ + otel_service_name "test_service"; + otel_resource_attr my.name "my name"; + otel_resource_attr my.service "my service"; + """, + } + ], + indirect=True, +) +def test_custom_resource_attributes(client, trace_service): + assert client.get("http://127.0.0.1:18080/ok").status_code == 200 + + batch = trace_service.get_batch() + + assert get_attr(batch.resource, "service.name") == "test_service" + assert get_attr(batch.resource, "my.name") == "my name" + assert get_attr(batch.resource, "my.service") == "my service" diff --git a/tests/trace_service.py b/tests/trace_service.py index 9f094f9..3b191a1 100644 --- a/tests/trace_service.py +++ b/tests/trace_service.py @@ -14,16 +14,20 @@ class TraceService(trace_service_pb2_grpc.TraceServiceServicer): self.batches.append(request.resource_spans) return trace_service_pb2.ExportTracePartialSuccess() - def get_span(self): + def get_batch(self): for _ in range(10): if len(self.batches): break time.sleep(0.001) + assert len(self.batches) == 1 + assert len(self.batches[0]) == 1 + return self.batches.pop()[0] - assert len(self.batches) == 1, "No spans received" - span = self.batches[0][0].scope_spans[0].spans.pop() - self.batches.clear() - return span + def get_span(self): + batch = self.get_batch() + assert len(batch.scope_spans) == 1 + assert len(batch.scope_spans[0].spans) == 1 + return batch.scope_spans[0].spans.pop() @pytest.fixture(scope="module") From f633a8eef23cdc5f4c4e980605b981cf75595a14 Mon Sep 17 00:00:00 2001 From: Pavel Pautov Date: Thu, 21 Nov 2024 13:57:43 -0800 Subject: [PATCH 23/28] Fail early if "trusted_certificate" is a directory. Previously, the error was caused by enormous std::string allocation. --- src/http_module.cpp | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/http_module.cpp b/src/http_module.cpp index df5702e..bc08b23 100644 --- a/src/http_module.cpp +++ b/src/http_module.cpp @@ -711,7 +711,8 @@ char* addResourceAttr(ngx_conf_t* cf, ngx_command_t* cmd, void* conf) return NGX_CONF_OK; } -char* setTrustedCertificate(ngx_conf_t* cf, ngx_command_t* cmd, void* conf) { +char* setTrustedCertificate(ngx_conf_t* cf, ngx_command_t* cmd, void* conf) +{ auto path = ((ngx_str_t*)cf->args->elts)[1]; auto mcf = getMainConf(cf); @@ -727,11 +728,13 @@ char* setTrustedCertificate(ngx_conf_t* cf, ngx_command_t* cmd, void* conf) { return (char*)NGX_CONF_ERROR; } file.exceptions(std::ios::failbit | std::ios::badbit); - file.seekg(0, std::ios::end); - size_t size = file.tellg(); - mcf->trustedCert.resize(size); + file.peek(); // trigger early error for dirs + + size_t size = file.seekg(0, std::ios::end).tellg(); file.seekg(0); - file.read(&mcf->trustedCert[0], mcf->trustedCert.size()); + + mcf->trustedCert.resize(size); + file.read(&mcf->trustedCert[0], size); } catch (const std::exception& e) { ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "failed to read \"%V\": %s", &path, e.what()); From 88a64bb2c3027be87057efedaa84d0c75d68610e Mon Sep 17 00:00:00 2001 From: Pavel Pautov Date: Thu, 21 Nov 2024 21:46:01 -0800 Subject: [PATCH 24/28] Consolidate transport related parameters into a struct. Also, replace leftover cast with getMainConf(). --- src/batch_exporter.hpp | 4 ++-- src/http_module.cpp | 11 +++++++---- src/trace_service_client.hpp | 16 +++++++++++----- 3 files changed, 20 insertions(+), 11 deletions(-) diff --git a/src/batch_exporter.hpp b/src/batch_exporter.hpp index cb3e075..a2e65b1 100644 --- a/src/batch_exporter.hpp +++ b/src/batch_exporter.hpp @@ -111,10 +111,10 @@ public: int attrSize{0}; }; - BatchExporter(StrView target, bool ssl, const std::string& trustedCert, + BatchExporter(const Target& target, size_t batchSize, size_t batchCount, const std::map& resourceAttrs) : - batchSize(batchSize), client(std::string(target), ssl, trustedCert) + batchSize(batchSize), client(target) { free.reserve(batchCount); while (batchCount-- > 0) { diff --git a/src/http_module.cpp b/src/http_module.cpp index bc08b23..93898b1 100644 --- a/src/http_module.cpp +++ b/src/http_module.cpp @@ -576,10 +576,13 @@ ngx_int_t initWorkerProcess(ngx_cycle_t* cycle) } try { + Target target; + target.endpoint = std::string(toStrView(mcf->endpoint)); + target.ssl = mcf->ssl; + target.trustedCert = mcf->trustedCert; + gExporter.reset(new BatchExporter( - toStrView(mcf->endpoint), - mcf->ssl, - mcf->trustedCert, + target, mcf->batchSize, mcf->batchCount, mcf->resourceAttrs)); @@ -772,7 +775,7 @@ void* createMainConf(ngx_conf_t* cf) char* initMainConf(ngx_conf_t* cf, void* conf) { - auto mcf = (MainConf*)conf; + auto mcf = getMainConf(cf); ngx_conf_init_msec_value(mcf->interval, 5000); ngx_conf_init_size_value(mcf->batchSize, 512); diff --git a/src/trace_service_client.hpp b/src/trace_service_client.hpp index d248f00..485143c 100644 --- a/src/trace_service_client.hpp +++ b/src/trace_service_client.hpp @@ -8,6 +8,12 @@ namespace otel_proto_trace = opentelemetry::proto::collector::trace::v1; +struct Target { + std::string endpoint; + bool ssl; + std::string trustedCert; +}; + class TraceServiceClient { public: typedef otel_proto_trace::ExportTraceServiceRequest Request; @@ -17,18 +23,18 @@ public: typedef std::function ResponseCb; - TraceServiceClient(const std::string& target, bool ssl, - const std::string& trustedCert) + TraceServiceClient(const Target& target) { std::shared_ptr creds; - if (ssl) { + if (target.ssl) { grpc::SslCredentialsOptions options; - options.pem_root_certs = trustedCert; + options.pem_root_certs = target.trustedCert; + creds = grpc::SslCredentials(options); } else { creds = grpc::InsecureChannelCredentials(); } - auto channel = grpc::CreateChannel(target, creds); + auto channel = grpc::CreateChannel(target.endpoint, creds); channel->GetState(true); // trigger 'connecting' state stub = TraceService::NewStub(channel); From a45a594801fbd57657fd821bdd298a39d4575175 Mon Sep 17 00:00:00 2001 From: Pavel Pautov Date: Mon, 2 Dec 2024 21:07:38 -0800 Subject: [PATCH 25/28] Support sending custom headers to export endpoint (fix #62). The headers are configured by "header" directive in "otel_exporter" block, e.g. otel_exporter { endpoint localhost:4317; header X-API-Token "token value"; } --- src/http_module.cpp | 36 +++++++++++++++++++++++++++++++++++- src/trace_service_client.hpp | 23 ++++++++++++++++++++++- 2 files changed, 57 insertions(+), 2 deletions(-) diff --git a/src/http_module.cpp b/src/http_module.cpp index 93898b1..ef77ffb 100644 --- a/src/http_module.cpp +++ b/src/http_module.cpp @@ -30,6 +30,7 @@ struct MainConf : MainConfBase { std::map resourceAttrs; bool ssl; std::string trustedCert; + Target::HeaderVec headers; }; struct SpanAttr { @@ -49,6 +50,7 @@ char* setExporter(ngx_conf_t* cf, ngx_command_t* cmd, void* conf); char* addResourceAttr(ngx_conf_t* cf, ngx_command_t* cmd, void* conf); char* addSpanAttr(ngx_conf_t* cf, ngx_command_t* cmd, void* conf); char* setTrustedCertificate(ngx_conf_t* cf, ngx_command_t* cmd, void* conf); +char* addExporterHeader(ngx_conf_t* cf, ngx_command_t* cmd, void* conf); namespace Propagation { @@ -120,6 +122,10 @@ ngx_command_t gExporterCommands[] = { NGX_CONF_TAKE1, setTrustedCertificate }, + { ngx_string("header"), + NGX_CONF_TAKE2, + addExporterHeader }, + { ngx_string("interval"), NGX_CONF_TAKE1, ngx_conf_set_msec_slot, @@ -580,6 +586,7 @@ ngx_int_t initWorkerProcess(ngx_cycle_t* cycle) target.endpoint = std::string(toStrView(mcf->endpoint)); target.ssl = mcf->ssl; target.trustedCert = mcf->trustedCert; + target.headers = mcf->headers; gExporter.reset(new BatchExporter( target, @@ -651,7 +658,7 @@ char* setExporter(ngx_conf_t* cf, ngx_command_t* cmd, void* conf) continue; } - if (cf->args->nelts != 2) { + if (cf->args->nelts != static_cast(ffs(cmd->type))) { ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "invalid number of arguments in \"%V\" " "directive of \"otel_exporter\"", name); @@ -747,6 +754,33 @@ char* setTrustedCertificate(ngx_conf_t* cf, ngx_command_t* cmd, void* conf) return NGX_CONF_OK; } +char* addExporterHeader(ngx_conf_t* cf, ngx_command_t* cmd, void* conf) +{ + auto args = (ngx_str_t*)cf->args->elts; + + // don't force on users lower case name requirement of gRPC + ngx_strlow(args[1].data, args[1].data, args[1].len); + + try { + // validate header here to avoid runtime assert failure in gRPC + auto name = toStrView(args[1]); + if (!Target::validateHeaderName(name)) { + return (char*)"has invalid header name"; + } + auto value = toStrView(args[2]); + if (!Target::validateHeaderValue(value)) { + return (char*)"has invalid header value"; + } + + getMainConf(cf)->headers.emplace_back(name, value); + } catch (const std::exception& e) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "OTel: %s", e.what()); + return (char*)NGX_CONF_ERROR; + } + + return NGX_CONF_OK; +} + void* createMainConf(ngx_conf_t* cf) { auto cln = ngx_pool_cleanup_add(cf->pool, sizeof(MainConf)); diff --git a/src/trace_service_client.hpp b/src/trace_service_client.hpp index 485143c..4ed92e7 100644 --- a/src/trace_service_client.hpp +++ b/src/trace_service_client.hpp @@ -9,9 +9,24 @@ namespace otel_proto_trace = opentelemetry::proto::collector::trace::v1; struct Target { + typedef std::vector> HeaderVec; + std::string endpoint; bool ssl; std::string trustedCert; + HeaderVec headers; + + static bool validateHeaderName(StrView name) + { + return grpc_header_key_is_legal( + grpc_slice_from_static_buffer(name.data(), name.size())); + } + + static bool validateHeaderValue(StrView value) + { + return grpc_header_nonbin_value_is_legal( + grpc_slice_from_static_buffer(value.data(), value.size())); + } }; class TraceServiceClient { @@ -23,7 +38,7 @@ public: typedef std::function ResponseCb; - TraceServiceClient(const Target& target) + TraceServiceClient(const Target& target) : headers(target.headers) { std::shared_ptr creds; if (target.ssl) { @@ -44,6 +59,10 @@ public: { std::unique_ptr call{new ActiveCall{}}; + for (auto& header : headers) { + call->context.AddMetadata(header.first, header.second); + } + call->request = std::move(req); call->cb = std::move(cb); @@ -113,6 +132,8 @@ private: ResponseCb cb; }; + Target::HeaderVec headers; + std::unique_ptr stub; grpc::CompletionQueue queue; From 9dc4dc2803cc2e7e37f5c73fee7f8d342101aa41 Mon Sep 17 00:00:00 2001 From: Pavel Pautov Date: Thu, 19 Dec 2024 21:28:17 -0800 Subject: [PATCH 26/28] Verify custom exporter headers support (#62). --- tests/test_otel.py | 25 +++++++++++++++++++++++++ tests/trace_service.py | 11 ++++++++--- 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/tests/test_otel.py b/tests/test_otel.py index 11caa85..513c1f1 100644 --- a/tests/test_otel.py +++ b/tests/test_otel.py @@ -25,6 +25,8 @@ http { interval {{ interval or "1ms" }}; batch_size 3; batch_count 3; + + {{ exporter_opts }} } otel_trace on; @@ -288,3 +290,26 @@ def test_custom_resource_attributes(client, trace_service): assert get_attr(batch.resource, "service.name") == "test_service" assert get_attr(batch.resource, "my.name") == "my name" assert get_attr(batch.resource, "my.service") == "my service" + + +@pytest.mark.parametrize( + "nginx_config", + [ + { + "exporter_opts": """ + header X-API-TOKEN api.value; + header Authorization "Basic value"; + """, + } + ], + indirect=True, +) +@pytest.mark.parametrize("trace_service", ["skip_otelcol"], indirect=True) +def test_exporter_headers(client, trace_service): + assert client.get("http://127.0.0.1:18080/ok").status_code == 200 + + assert trace_service.get_span().name == "/ok" + + headers = dict(trace_service.last_metadata) + assert headers["x-api-token"] == "api.value" + assert headers["authorization"] == "Basic value" diff --git a/tests/trace_service.py b/tests/trace_service.py index 3b191a1..f47b104 100644 --- a/tests/trace_service.py +++ b/tests/trace_service.py @@ -12,6 +12,7 @@ class TraceService(trace_service_pb2_grpc.TraceServiceServicer): def Export(self, request, context): self.batches.append(request.resource_spans) + self.last_metadata = context.invocation_metadata() return trace_service_pb2.ExportTracePartialSuccess() def get_batch(self): @@ -31,13 +32,17 @@ class TraceService(trace_service_pb2_grpc.TraceServiceServicer): @pytest.fixture(scope="module") -def trace_service(pytestconfig, logger): +def trace_service(request, pytestconfig, logger): server = grpc.server(concurrent.futures.ThreadPoolExecutor()) trace_service = TraceService() trace_service_pb2_grpc.add_TraceServiceServicer_to_server( trace_service, server ) - listen_addr = f"127.0.0.1:{24317 if pytestconfig.option.otelcol else 14317}" + trace_service.use_otelcol = ( + pytestconfig.option.otelcol + and getattr(request, "param", "") != "skip_otelcol" + ) + listen_addr = f"127.0.0.1:{24317 if trace_service.use_otelcol else 14317}" server.add_insecure_port(listen_addr) logger.info(f"Starting trace service at {listen_addr}...") server.start() @@ -48,7 +53,7 @@ def trace_service(pytestconfig, logger): @pytest.fixture(scope="module") def otelcol(pytestconfig, testdir, logger, trace_service): - if pytestconfig.option.otelcol is None: + if not trace_service.use_otelcol: yield return From f578402f196499edd2a65c31020570fc37e1bdbf Mon Sep 17 00:00:00 2001 From: Eugene Grebenschikov Date: Fri, 20 Dec 2024 17:24:27 -0800 Subject: [PATCH 27/28] Verify export via TLS (#12). Co-authored-by: Pavel Pautov --- tests/conftest.py | 19 ++++++++++--------- tests/test_otel.py | 20 ++++++++++++++++++-- tests/trace_service.py | 20 ++++++++++++++++---- 3 files changed, 44 insertions(+), 15 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 7978759..25933e4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -19,7 +19,7 @@ def pytest_addoption(parser): parser.addoption("--globals", default="") -def self_signed_cert(test_dir, name): +def self_signed_cert(name): k = crypto.PKey() k.generate_key(crypto.TYPE_RSA, 2048) cert = crypto.X509() @@ -29,11 +29,9 @@ def self_signed_cert(test_dir, name): cert.gmtime_adj_notAfter(365 * 86400) # 365 days cert.set_pubkey(k) cert.sign(k, "sha512") - (test_dir / f"{name}.key").write_text( - crypto.dump_privatekey(crypto.FILETYPE_PEM, k).decode("utf-8") - ) - (test_dir / f"{name}.crt").write_text( - crypto.dump_certificate(crypto.FILETYPE_PEM, cert).decode("utf-8") + return ( + crypto.dump_privatekey(crypto.FILETYPE_PEM, k), + crypto.dump_certificate(crypto.FILETYPE_PEM, cert), ) @@ -66,7 +64,7 @@ def nginx_config(request, pytestconfig, testdir, logger): @pytest.fixture(scope="module") -def nginx(testdir, pytestconfig, nginx_config, certs, logger, otelcol): +def nginx(testdir, pytestconfig, nginx_config, cert, logger, otelcol): (testdir / "nginx.conf").write_text(nginx_config) logger.info("Starting nginx...") proc = subprocess.Popen( @@ -96,5 +94,8 @@ def nginx(testdir, pytestconfig, nginx_config, certs, logger, otelcol): @pytest.fixture(scope="module") -def certs(testdir): - self_signed_cert(testdir, "localhost") +def cert(testdir): + key, cert = self_signed_cert("localhost") + (testdir / "localhost.key").write_text(key.decode("utf-8")) + (testdir / "localhost.crt").write_text(cert.decode("utf-8")) + yield (key, cert) diff --git a/tests/test_otel.py b/tests/test_otel.py index 513c1f1..fef771a 100644 --- a/tests/test_otel.py +++ b/tests/test_otel.py @@ -21,7 +21,7 @@ http { ssl_certificate_key localhost.key; otel_exporter { - endpoint {{ scheme }}127.0.0.1:14317; + endpoint {{ endpoint or "127.0.0.1:14317" }}; interval {{ interval or "1ms" }}; batch_size 3; batch_count 3; @@ -242,7 +242,7 @@ def test_context(client, trace_service, parent, path): @pytest.mark.parametrize( "nginx_config", - [{"interval": "200ms", "scheme": "http://"}], + [{"interval": "200ms", "endpoint": "http://127.0.0.1:14317"}], indirect=True, ) @pytest.mark.parametrize("batch_count", [1, 3]) @@ -313,3 +313,19 @@ def test_exporter_headers(client, trace_service): headers = dict(trace_service.last_metadata) assert headers["x-api-token"] == "api.value" assert headers["authorization"] == "Basic value" + + +@pytest.mark.parametrize( + "nginx_config", + [ + { + "endpoint": "https://localhost:14318", + "exporter_opts": "trusted_certificate localhost.crt;", + } + ], + indirect=True, +) +def test_tls_export(client, trace_service): + assert client.get("http://127.0.0.1:18080/ok").status_code == 200 + + assert trace_service.get_span().name == "/ok" diff --git a/tests/trace_service.py b/tests/trace_service.py index f47b104..5ef2bc6 100644 --- a/tests/trace_service.py +++ b/tests/trace_service.py @@ -32,7 +32,7 @@ class TraceService(trace_service_pb2_grpc.TraceServiceServicer): @pytest.fixture(scope="module") -def trace_service(request, pytestconfig, logger): +def trace_service(request, pytestconfig, logger, cert): server = grpc.server(concurrent.futures.ThreadPoolExecutor()) trace_service = TraceService() trace_service_pb2_grpc.add_TraceServiceServicer_to_server( @@ -44,6 +44,10 @@ def trace_service(request, pytestconfig, logger): ) listen_addr = f"127.0.0.1:{24317 if trace_service.use_otelcol else 14317}" server.add_insecure_port(listen_addr) + if not trace_service.use_otelcol: + creds = grpc.ssl_server_credentials([cert]) + server.add_secure_port("127.0.0.1:14318", creds) + listen_addr += " and 127.0.0.1:14318" logger.info(f"Starting trace service at {listen_addr}...") server.start() yield trace_service @@ -52,18 +56,26 @@ def trace_service(request, pytestconfig, logger): @pytest.fixture(scope="module") -def otelcol(pytestconfig, testdir, logger, trace_service): +def otelcol(pytestconfig, testdir, logger, trace_service, cert): if not trace_service.use_otelcol: yield return (testdir / "otel-config.yaml").write_text( - """receivers: + f"""receivers: otlp: protocols: grpc: endpoint: 127.0.0.1:14317 + otlp/tls: + protocols: + grpc: + endpoint: 127.0.0.1:14318 + tls: + cert_file: {testdir}/localhost.crt + key_file: {testdir}/localhost.key + exporters: otlp: endpoint: 127.0.0.1:24317 @@ -73,7 +85,7 @@ exporters: service: pipelines: traces: - receivers: [otlp] + receivers: [otlp, otlp/tls] exporters: [otlp] telemetry: metrics: From 72d8eed53af4c2cd6f3e30a2efe0e38d66f5e176 Mon Sep 17 00:00:00 2001 From: Pavel Pautov Date: Tue, 21 Jan 2025 23:02:39 -0800 Subject: [PATCH 28/28] Fix build against Nginx 1.22 (fix #85). --- src/http_module.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/http_module.cpp b/src/http_module.cpp index ef77ffb..78a5e89 100644 --- a/src/http_module.cpp +++ b/src/http_module.cpp @@ -296,10 +296,10 @@ ngx_int_t setHeader(ngx_http_request_t* r, StrView name, StrView value) return NGX_ERROR; } + *header = {}; header->hash = hash; header->key = toNgxStr(name); header->lowcase_key = header->key.data; - header->next = NULL; } header->value = toNgxStr(value);