diff --git a/.github/workflows/nginx-otel-module-check.yml b/.github/workflows/nginx-otel-module-check.yml new file mode 100644 index 0000000..2d8f5ee --- /dev/null +++ b/.github/workflows/nginx-otel-module-check.yml @@ -0,0 +1,85 @@ +name: nginx-otel-module-check +run-name: ${{ github.actor }} is triggering pipeline +on: + push: + workflow_dispatch: + +jobs: + build-module: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y cmake libc-ares-dev libre2-dev + - name: Checkout nginx + run: hg clone http://hg.nginx.org/nginx/ + - name: Configure nginx + working-directory: nginx + run: auto/configure --with-compat + - name: Create build directory + run: mkdir build + - name: Build module + working-directory: build + run: | + cmake -DNGX_OTEL_NGINX_BUILD_DIR=${PWD}/../nginx/objs .. + make -j 4 + strip ngx_otel_module.so + - name: Archive module + uses: actions/upload-artifact@v4 + with: + name: nginx-otel-module + path: build/ngx_otel_module.so + - name: Archive protoc and opentelemetry-proto + uses: actions/upload-artifact@v4 + with: + name: protoc-opentelemetry-proto + path: | + build/_deps/grpc-build/third_party/protobuf/protoc + build/_deps/otelcpp-src/third_party/opentelemetry-proto + test-module: + needs: build-module + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Download module + uses: actions/download-artifact@v4 + with: + name: nginx-otel-module + path: build + - name: Download protoc and opentelemetry-proto + uses: actions/download-artifact@v4 + with: + name: protoc-opentelemetry-proto + path: build/_deps + - name: List files + run: ls -laR . + - name: Fix protoc file permissions + run: chmod +x build/_deps/grpc-build/third_party/protobuf/protoc + - name: Install perl modules + run: sudo cpan IO::Socket::SSL Crypt::Misc + - name: Download otelcol + run: | + curl -LO https://github.com/\ + open-telemetry/opentelemetry-collector-releases/releases/download/\ + v0.76.1/otelcol_0.76.1_linux_amd64.tar.gz + tar -xzf otelcol_0.76.1_linux_amd64.tar.gz + - name: Checkout nginx and nginx-test + run: | + hg clone http://hg.nginx.org/nginx/ + hg clone http://hg.nginx.org/nginx-tests/ + - name: Build nginx + working-directory: nginx + run: | + auto/configure --with-compat --with-debug --with-http_ssl_module \ + --with-http_v2_module --with-http_v3_module + make -j 4 + - name: Run tests + working-directory: tests + run: | + PERL5LIB=../nginx-tests/lib TEST_NGINX_UNSAFE=1 \ + TEST_NGINX_VERBOSE=1 TEST_NGINX_GLOBALS="load_module \ + ${PWD}/../build/ngx_otel_module.so;" prove -v . diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml deleted file mode 100644 index 0f328d2..0000000 --- a/.github/workflows/ubuntu.yml +++ /dev/null @@ -1,49 +0,0 @@ -name: Ubuntu build - -on: - push: - branches: - - main - pull_request: - -jobs: - build-module: - runs-on: ubuntu-22.04 - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - name: Install dependencies - run: | - sudo apt-get update - sudo apt-get install -y cmake libc-ares-dev - - name: Checkout nginx - uses: actions/checkout@v4 - with: - repository: nginx/nginx - path: nginx - - name: Build nginx - working-directory: nginx - run: | - auto/configure --with-compat --with-debug --with-http_ssl_module \ - --with-http_v2_module --with-http_v3_module - make -j $(nproc) - - name: Build module - run: | - mkdir build - cd build - cmake -DNGX_OTEL_NGINX_BUILD_DIR=${PWD}/../nginx/objs \ - -DNGX_OTEL_DEV=ON .. - make -j $(nproc) - - name: Download otelcol - run: | - LATEST=open-telemetry/opentelemetry-collector-releases/releases/latest - TAG=$(curl -s https://api.github.com/repos/${LATEST} | - jq -r .tag_name) - curl -sLo - https://github.com/${LATEST}/download/\ - otelcol_${TAG:1}_linux_amd64.tar.gz | tar -xzv - - name: Install test dependencies - run: pip install -r tests/requirements.txt - - name: Run tests - run: | - pytest tests --maxfail=10 --nginx=nginx/objs/nginx \ - --module=build/ngx_otel_module.so --otelcol=./otelcol diff --git a/CMakeLists.txt b/CMakeLists.txt index 7a74f0a..baca6f8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,12 +6,8 @@ set(NGX_OTEL_NGINX_BUILD_DIR "" set(NGX_OTEL_NGINX_DIR "${NGX_OTEL_NGINX_BUILD_DIR}/.." CACHE PATH "Nginx source dir") -set(NGX_OTEL_GRPC e241f37befe7ba4688effd84bfbf99b0f681a2f7 # v1.49.4 - CACHE STRING "gRPC tag to download or 'package' to use preinstalled") -set(NGX_OTEL_SDK 11d5d9e0d8fd8ba876c8994714cc2647479b6574 # v1.11.0 - CACHE STRING "OTel SDK tag to download or 'package' to use preinstalled") -set(NGX_OTEL_PROTO_DIR "" CACHE PATH "OTel proto files root") -set(NGX_OTEL_DEV OFF CACHE BOOL "Enforce compiler warnings") +set(NGX_OTEL_FETCH_DEPS ON CACHE BOOL "Download dependencies") +set(NGX_OTEL_PROTO_DIR "" CACHE PATH "OTel proto files root") if(NOT CMAKE_BUILD_TYPE) set(CMAKE_BUILD_TYPE RelWithDebInfo) @@ -19,52 +15,27 @@ endif() set(CMAKE_CXX_VISIBILITY_PRESET hidden) -if(NGX_OTEL_GRPC STREQUAL "package") - find_package(protobuf REQUIRED) - find_package(gRPC REQUIRED) -else() +if(NGX_OTEL_FETCH_DEPS) include(FetchContent) FetchContent_Declare( grpc GIT_REPOSITORY https://github.com/grpc/grpc - GIT_TAG ${NGX_OTEL_GRPC} - GIT_SUBMODULES third_party/protobuf third_party/abseil-cpp third_party/re2 + GIT_TAG 02384e39185f109bd299eb8482306229967dc970 # v1.46.7 + GIT_SUBMODULES third_party/protobuf third_party/abseil-cpp GIT_SHALLOW ON) set(gRPC_USE_PROTO_LITE ON CACHE INTERNAL "") set(gRPC_INSTALL OFF CACHE INTERNAL "") - set(gRPC_USE_SYSTEMD OFF CACHE INTERNAL "") - set(gRPC_DOWNLOAD_ARCHIVES OFF CACHE INTERNAL "") set(gRPC_CARES_PROVIDER package CACHE INTERNAL "") + set(gRPC_RE2_PROVIDER package CACHE INTERNAL "") set(gRPC_SSL_PROVIDER package CACHE INTERNAL "") set(gRPC_ZLIB_PROVIDER package CACHE INTERNAL "") - set(protobuf_INSTALL OFF CACHE INTERNAL "") - - set(CMAKE_POSITION_INDEPENDENT_CODE ON) - - FetchContent_MakeAvailable(grpc) - - # reconsider once https://github.com/grpc/grpc/issues/36023 is done - target_compile_definitions(grpc PRIVATE GRPC_NO_XDS GRPC_NO_RLS) - - set_property(DIRECTORY ${grpc_SOURCE_DIR} - PROPERTY EXCLUDE_FROM_ALL YES) - - add_library(gRPC::grpc++ ALIAS grpc++) - add_executable(gRPC::grpc_cpp_plugin ALIAS grpc_cpp_plugin) -endif() - -if(NGX_OTEL_SDK STREQUAL "package") - find_package(opentelemetry-cpp REQUIRED) -else() - include(FetchContent) - FetchContent_Declare( otelcpp GIT_REPOSITORY https://github.com/open-telemetry/opentelemetry-cpp - GIT_TAG ${NGX_OTEL_SDK} + GIT_TAG 11d5d9e0d8fd8ba876c8994714cc2647479b6574 # v1.11.0 GIT_SUBMODULES third_party/opentelemetry-proto GIT_SHALLOW ON) @@ -74,8 +45,10 @@ else() set(CMAKE_POSITION_INDEPENDENT_CODE ON) set(CMAKE_POLICY_DEFAULT_CMP0063 NEW) - FetchContent_MakeAvailable(otelcpp) + FetchContent_MakeAvailable(grpc otelcpp) + set_property(DIRECTORY ${grpc_SOURCE_DIR} + PROPERTY EXCLUDE_FROM_ALL YES) set_property(DIRECTORY ${otelcpp_SOURCE_DIR} PROPERTY EXCLUDE_FROM_ALL YES) @@ -85,6 +58,12 @@ else() endif() add_library(opentelemetry-cpp::trace ALIAS opentelemetry_trace) + add_library(gRPC::grpc++ ALIAS grpc++) + add_executable(gRPC::grpc_cpp_plugin ALIAS grpc_cpp_plugin) +else() + find_package(opentelemetry-cpp REQUIRED) + find_package(protobuf REQUIRED) + find_package(gRPC REQUIRED) endif() set(PROTO_DIR ${NGX_OTEL_PROTO_DIR}) @@ -112,22 +91,19 @@ add_custom_command( --plugin protoc-gen-grpc=$ ${PROTOS} # remove inconsequential UTF8 check during serialization to aid performance - COMMAND sed -i.bak -E - -e [[/ ::(PROTOBUF_NAMESPACE_ID|google::protobuf)::internal::WireFormatLite::VerifyUtf8String\(/,/\);/d]] + COMMAND sed -i.bak + -e [[/ ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String(/,/);/d]] ${PROTO_SOURCES} DEPENDS ${PROTOS} protobuf::protoc gRPC::grpc_cpp_plugin VERBATIM) -if (NGX_OTEL_DEV) - set(CMAKE_CXX_STANDARD 11) - set(CMAKE_CXX_EXTENSIONS OFF) +set(CMAKE_CXX_STANDARD 11) +set(CMAKE_CXX_EXTENSIONS OFF) - add_compile_options(-Wall -Wtype-limits -Werror) -endif() +add_compile_options(-Wall -Wtype-limits -Werror) add_library(ngx_otel_module MODULE src/http_module.cpp - src/grpc_log.cpp src/modules.c ${PROTO_SOURCES}) @@ -137,10 +113,6 @@ set_target_properties(ngx_otel_module PROPERTIES PREFIX "") # can't use OTel's WITH_ABSEIL until cmake 3.24, as it triggers find_package() target_compile_definitions(ngx_otel_module PRIVATE HAVE_ABSEIL) -if (APPLE) - target_link_options(ngx_otel_module PRIVATE -undefined dynamic_lookup) -endif() - target_include_directories(ngx_otel_module PRIVATE ${NGX_OTEL_NGINX_BUILD_DIR} ${NGX_OTEL_NGINX_DIR}/src/core diff --git a/config b/config index 75378e2..11d3317 100644 --- a/config +++ b/config @@ -1,9 +1,10 @@ ngx_addon_name=ngx_otel_module cmake -D NGX_OTEL_NGINX_BUILD_DIR=$NGX_OBJS \ + -D NGX_OTEL_FETCH_DEPS=OFF \ + -D NGX_OTEL_PROTO_DIR=$NGX_OTEL_PROTO_DIR \ -D CMAKE_LIBRARY_OUTPUT_DIRECTORY=$PWD/$NGX_OBJS \ -D "CMAKE_C_FLAGS=$NGX_CC_OPT" \ -D "CMAKE_CXX_FLAGS=$NGX_CC_OPT" \ -D "CMAKE_MODULE_LINKER_FLAGS=$NGX_LD_OPT" \ - $NGX_OTEL_CMAKE_OPTS \ -S $ngx_addon_dir -B $NGX_OBJS/otel || exit 1 diff --git a/src/batch_exporter.hpp b/src/batch_exporter.hpp index a2e65b1..d160d2c 100644 --- a/src/batch_exporter.hpp +++ b/src/batch_exporter.hpp @@ -111,21 +111,18 @@ public: int attrSize{0}; }; - BatchExporter(const Target& target, - size_t batchSize, size_t batchCount, - const std::map& resourceAttrs) : - batchSize(batchSize), client(target) + BatchExporter(StrView target, + size_t batchSize, size_t batchCount, StrView serviceName) : + batchSize(batchSize), client(std::string(target)) { free.reserve(batchCount); while (batchCount-- > 0) { free.emplace_back(); auto resourceSpans = free.back().add_resource_spans(); - for (auto& attr : resourceAttrs) { - auto kv = resourceSpans->mutable_resource()->add_attributes(); - kv->set_key(std::string(attr.first)); - kv->mutable_value()->set_string_value(std::string(attr.second)); - } + auto attr = resourceSpans->mutable_resource()->add_attributes(); + attr->set_key("service.name"); + attr->mutable_value()->set_string_value(std::string(serviceName)); auto scopeSpans = resourceSpans->add_scope_spans(); scopeSpans->mutable_scope()->set_name("nginx"); diff --git a/src/grpc_log.cpp b/src/grpc_log.cpp deleted file mode 100644 index bb83364..0000000 --- a/src/grpc_log.cpp +++ /dev/null @@ -1,107 +0,0 @@ -#include "ngx.hpp" - -#include "grpc_log.hpp" - -#include -#include - -#if GOOGLE_PROTOBUF_VERSION < 4022000 - -#include - -class ProtobufLog { -public: - ProtobufLog() { google::protobuf::SetLogHandler(protobufLogHandler); } - ~ProtobufLog() { google::protobuf::SetLogHandler(NULL); } - -private: - static void protobufLogHandler(google::protobuf::LogLevel logLevel, - const char* filename, int line, const std::string& msg) - { - using namespace google::protobuf; - - ngx_uint_t level = logLevel == LOGLEVEL_FATAL ? NGX_LOG_EMERG : - logLevel == LOGLEVEL_ERROR ? NGX_LOG_ERR : - logLevel == LOGLEVEL_WARNING ? NGX_LOG_WARN : - /*LOGLEVEL_INFO*/ NGX_LOG_INFO; - - ngx_log_error(level, ngx_cycle->log, 0, "OTel/protobuf: %s", - msg.c_str()); - } -}; - -#else - -#include -#include -#include - -class NgxLogSink : absl::LogSink { -public: - NgxLogSink() - { - absl::InitializeLog(); - absl::AddLogSink(this); - // Disable logging to stderr - absl::SetStderrThreshold(static_cast(100)); - } - - ~NgxLogSink() override { absl::RemoveLogSink(this); } - - void Send(const absl::LogEntry& entry) override - { - auto severity = entry.log_severity(); - - ngx_uint_t level = - severity == absl::LogSeverity::kFatal ? NGX_LOG_EMERG : - severity == absl::LogSeverity::kError ? NGX_LOG_ERR : - severity == absl::LogSeverity::kWarning ? NGX_LOG_WARN : - /*absl::LogSeverity::kInfo*/ NGX_LOG_INFO; - - ngx_str_t message { entry.text_message().size(), - (u_char*)entry.text_message().data() }; - - ngx_log_error(level, ngx_cycle->log, 0, "OTel/grpc: %V", &message); - } -}; - -typedef NgxLogSink ProtobufLog; - -#endif - -#if (GRPC_CPP_VERSION_MAJOR < 1) || \ - (GRPC_CPP_VERSION_MAJOR == 1 && GRPC_CPP_VERSION_MINOR < 65) - -#include - -class GrpcLog { -public: - GrpcLog() { gpr_set_log_function(grpcLogHandler); } - ~GrpcLog() { gpr_set_log_function(NULL); } - -private: - static void grpcLogHandler(gpr_log_func_args* args) - { - ngx_uint_t level = - args->severity == GPR_LOG_SEVERITY_ERROR ? NGX_LOG_ERR : - args->severity == GPR_LOG_SEVERITY_INFO ? NGX_LOG_INFO : - /*GPR_LOG_SEVERITY_DEBUG*/ NGX_LOG_DEBUG; - - ngx_log_error(level, ngx_cycle->log, 0, "OTel/grpc: %s", - args->message); - } - - ProtobufLog protoLog; -}; - -#else - -// newer gRPC implies newer protobuf, and both use Abseil for logging -typedef NgxLogSink GrpcLog; - -#endif - -void initGrpcLog() -{ - static GrpcLog init; -} diff --git a/src/grpc_log.hpp b/src/grpc_log.hpp deleted file mode 100644 index e6da5c9..0000000 --- a/src/grpc_log.hpp +++ /dev/null @@ -1,3 +0,0 @@ -#pragma once - -void initGrpcLog(); diff --git a/src/http_module.cpp b/src/http_module.cpp index 78a5e89..5ebe0a0 100644 --- a/src/http_module.cpp +++ b/src/http_module.cpp @@ -1,13 +1,16 @@ -#include "ngx.hpp" +extern "C" { +#include +#include +#include +} -#include "grpc_log.hpp" +#include +#include #include "str_view.hpp" #include "trace_context.hpp" #include "batch_exporter.hpp" -#include - extern ngx_module_t gHttpModule; namespace { @@ -17,7 +20,7 @@ struct OtelCtx { TraceContext current; }; -struct MainConfBase { +struct MainConf { ngx_str_t endpoint; ngx_msec_t interval; size_t batchSize; @@ -26,13 +29,6 @@ struct MainConfBase { ngx_str_t serviceName; }; -struct MainConf : MainConfBase { - std::map resourceAttrs; - bool ssl; - std::string trustedCert; - Target::HeaderVec headers; -}; - struct SpanAttr { ngx_str_t name; ngx_http_complex_value_t value; @@ -47,10 +43,7 @@ struct LocationConf { }; char* setExporter(ngx_conf_t* cf, ngx_command_t* cmd, void* conf); -char* addResourceAttr(ngx_conf_t* cf, ngx_command_t* cmd, void* conf); char* addSpanAttr(ngx_conf_t* cf, ngx_command_t* cmd, void* conf); -char* setTrustedCertificate(ngx_conf_t* cf, ngx_command_t* cmd, void* conf); -char* addExporterHeader(ngx_conf_t* cf, ngx_command_t* cmd, void* conf); namespace Propagation { @@ -71,17 +64,14 @@ ngx_command_t gCommands[] = { { ngx_string("otel_exporter"), NGX_HTTP_MAIN_CONF|NGX_CONF_BLOCK|NGX_CONF_NOARGS, - setExporter }, - - { ngx_string("otel_resource_attr"), - NGX_HTTP_MAIN_CONF|NGX_CONF_TAKE2, - addResourceAttr }, + setExporter, + NGX_HTTP_MAIN_CONF_OFFSET }, { ngx_string("otel_service_name"), NGX_HTTP_MAIN_CONF|NGX_CONF_TAKE1, ngx_conf_set_str_slot, NGX_HTTP_MAIN_CONF_OFFSET, - offsetof(MainConfBase, serviceName) }, + offsetof(MainConf, serviceName) }, { ngx_string("otel_trace"), NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1, @@ -116,33 +106,25 @@ ngx_command_t gExporterCommands[] = { NGX_CONF_TAKE1, ngx_conf_set_str_slot, 0, - offsetof(MainConfBase, endpoint) }, - - { ngx_string("trusted_certificate"), - NGX_CONF_TAKE1, - setTrustedCertificate }, - - { ngx_string("header"), - NGX_CONF_TAKE2, - addExporterHeader }, + offsetof(MainConf, endpoint) }, { ngx_string("interval"), NGX_CONF_TAKE1, ngx_conf_set_msec_slot, 0, - offsetof(MainConfBase, interval) }, + offsetof(MainConf, interval) }, { ngx_string("batch_size"), NGX_CONF_TAKE1, ngx_conf_set_size_slot, 0, - offsetof(MainConfBase, batchSize) }, + offsetof(MainConf, batchSize) }, { ngx_string("batch_count"), NGX_CONF_TAKE1, ngx_conf_set_size_slot, 0, - offsetof(MainConfBase, batchCount) }, + offsetof(MainConf, batchCount) }, ngx_null_command }; @@ -159,30 +141,6 @@ ngx_str_t toNgxStr(StrView str) return ngx_str_t{str.size(), (u_char*)str.data()}; } -bool iremovePrefix(ngx_str_t* str, StrView p) -{ - if (str->len >= p.size() && - ngx_strncasecmp(str->data, (u_char*)p.data(), p.size()) == 0) { - str->data += p.size(); - str->len -= p.size(); - return true; - } - - return false; -} - -MainConf* getMainConf(ngx_conf_t* cf) -{ - return static_cast( - (MainConfBase*)ngx_http_conf_get_module_main_conf(cf, gHttpModule)); -} - -MainConf* getMainConf(ngx_cycle_t* cycle) -{ - return static_cast( - (MainConfBase*)ngx_http_cycle_get_module_main_conf(cycle, gHttpModule)); -} - LocationConf* getLocationConf(ngx_http_request_t* r) { return (LocationConf*)ngx_http_get_module_loc_conf(r, gHttpModule); @@ -296,10 +254,10 @@ ngx_int_t setHeader(ngx_http_request_t* r, StrView name, StrView value) return NGX_ERROR; } - *header = {}; header->hash = hash; header->key = toNgxStr(name); header->lowcase_key = header->key.data; + header->next = NULL; } header->value = toNgxStr(value); @@ -546,6 +504,28 @@ ngx_int_t onRequestEnd(ngx_http_request_t* r) return NGX_DECLINED; } +void grpcLogHandler(gpr_log_func_args* args) +{ + ngx_uint_t level = args->severity == GPR_LOG_SEVERITY_ERROR ? NGX_LOG_ERR : + args->severity == GPR_LOG_SEVERITY_INFO ? NGX_LOG_INFO : + /*GPR_LOG_SEVERITY_DEBUG*/ NGX_LOG_DEBUG; + + ngx_log_error(level, ngx_cycle->log, 0, "OTel/grpc: %s", args->message); +} + +void protobufLogHandler(google::protobuf::LogLevel logLevel, + const char* filename, int line, const std::string& msg) +{ + using namespace google::protobuf; + + ngx_uint_t level = logLevel == LOGLEVEL_FATAL ? NGX_LOG_EMERG : + logLevel == LOGLEVEL_ERROR ? NGX_LOG_ERR : + logLevel == LOGLEVEL_WARNING ? NGX_LOG_WARN : + /*LOGLEVEL_INFO*/ NGX_LOG_INFO; + + ngx_log_error(level, ngx_cycle->log, 0, "OTel/protobuf: %s", msg.c_str()); +} + ngx_int_t initModule(ngx_conf_t* cf) { auto cmcf = (ngx_http_core_main_conf_t*)ngx_http_conf_get_module_main_conf( @@ -567,14 +547,16 @@ ngx_int_t initModule(ngx_conf_t* cf) *h = onRequestEnd; - initGrpcLog(); + gpr_set_log_function(grpcLogHandler); + google::protobuf::SetLogHandler(protobufLogHandler); return NGX_OK; } ngx_int_t initWorkerProcess(ngx_cycle_t* cycle) { - auto mcf = getMainConf(cycle); + auto mcf = (MainConf*)ngx_http_cycle_get_module_main_conf( + cycle, gHttpModule); // no 'http' or 'otel_exporter' blocks if (mcf == NULL || mcf->endpoint.len == 0) { @@ -582,17 +564,11 @@ ngx_int_t initWorkerProcess(ngx_cycle_t* cycle) } try { - Target target; - target.endpoint = std::string(toStrView(mcf->endpoint)); - target.ssl = mcf->ssl; - target.trustedCert = mcf->trustedCert; - target.headers = mcf->headers; - gExporter.reset(new BatchExporter( - target, + toStrView(mcf->endpoint), mcf->batchSize, mcf->batchCount, - mcf->resourceAttrs)); + toStrView(mcf->serviceName))); } catch (const std::exception& e) { ngx_log_error(NGX_LOG_CRIT, cycle->log, 0, "OTel worker init error: %s", e.what()); @@ -613,7 +589,8 @@ ngx_int_t initWorkerProcess(ngx_cycle_t* cycle) "OTel flush error: %s", e.what()); } - auto mcf = getMainConf((ngx_cycle_t*)ngx_cycle); + auto mcf = (MainConf*)ngx_http_cycle_get_module_main_conf( + ngx_cycle, gHttpModule); ngx_add_timer(ev, mcf->interval); }; @@ -641,7 +618,7 @@ void exitWorkerProcess(ngx_cycle_t* cycle) char* setExporter(ngx_conf_t* cf, ngx_command_t* cmd, void* conf) { - auto mcf = getMainConf(cf); + auto mcf = (MainConf*)conf; if (mcf->endpoint.len) { return (char*)"is duplicate"; @@ -658,7 +635,7 @@ char* setExporter(ngx_conf_t* cf, ngx_command_t* cmd, void* conf) continue; } - if (cf->args->nelts != static_cast(ffs(cmd->type))) { + if (cf->args->nelts != 2) { ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "invalid number of arguments in \"%V\" " "directive of \"otel_exporter\"", name); @@ -691,12 +668,6 @@ char* setExporter(ngx_conf_t* cf, ngx_command_t* cmd, void* conf) return rv; } - if (iremovePrefix(&mcf->endpoint, "https://")) { - mcf->ssl = true; - } else { - iremovePrefix(&mcf->endpoint, "http://"); - } - if (mcf->endpoint.len == 0) { ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "\"otel_exporter\" requires \"endpoint\""); @@ -706,124 +677,31 @@ char* setExporter(ngx_conf_t* cf, ngx_command_t* cmd, void* conf) return NGX_CONF_OK; } -char* addResourceAttr(ngx_conf_t* cf, ngx_command_t* cmd, void* conf) -{ - auto mcf = getMainConf(cf); - - try { - auto args = (ngx_str_t*)cf->args->elts; - mcf->resourceAttrs[toStrView(args[1])] = toStrView(args[2]); - } catch (const std::exception& e) { - ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "OTel: %s", e.what()); - return (char*)NGX_CONF_ERROR; - } - - return NGX_CONF_OK; -} - -char* setTrustedCertificate(ngx_conf_t* cf, ngx_command_t* cmd, void* conf) -{ - auto path = ((ngx_str_t*)cf->args->elts)[1]; - auto mcf = getMainConf(cf); - - if (ngx_get_full_name(cf->pool, &cf->cycle->conf_prefix, &path) != NGX_OK) { - return (char*)NGX_CONF_ERROR; - } - - try { - std::ifstream file{(const char*)path.data, std::ios::binary}; - if (!file.is_open()) { - ngx_conf_log_error(NGX_LOG_EMERG, cf, ngx_errno, - "failed to open \"%V\"", &path); - return (char*)NGX_CONF_ERROR; - } - file.exceptions(std::ios::failbit | std::ios::badbit); - file.peek(); // trigger early error for dirs - - size_t size = file.seekg(0, std::ios::end).tellg(); - file.seekg(0); - - mcf->trustedCert.resize(size); - file.read(&mcf->trustedCert[0], size); - } catch (const std::exception& e) { - ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, - "failed to read \"%V\": %s", &path, e.what()); - return (char*)NGX_CONF_ERROR; - } - - return NGX_CONF_OK; -} - -char* addExporterHeader(ngx_conf_t* cf, ngx_command_t* cmd, void* conf) -{ - auto args = (ngx_str_t*)cf->args->elts; - - // don't force on users lower case name requirement of gRPC - ngx_strlow(args[1].data, args[1].data, args[1].len); - - try { - // validate header here to avoid runtime assert failure in gRPC - auto name = toStrView(args[1]); - if (!Target::validateHeaderName(name)) { - return (char*)"has invalid header name"; - } - auto value = toStrView(args[2]); - if (!Target::validateHeaderValue(value)) { - return (char*)"has invalid header value"; - } - - getMainConf(cf)->headers.emplace_back(name, value); - } catch (const std::exception& e) { - ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "OTel: %s", e.what()); - return (char*)NGX_CONF_ERROR; - } - - return NGX_CONF_OK; -} - void* createMainConf(ngx_conf_t* cf) { - auto cln = ngx_pool_cleanup_add(cf->pool, sizeof(MainConf)); - if (cln == NULL) { + auto mcf = (MainConf*)ngx_pcalloc(cf->pool, sizeof(MainConf)); + if (mcf == NULL) { return NULL; } - MainConf* mcf; - try { - mcf = new (cln->data) MainConf{}; - } catch (const std::exception& e) { - ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "OTel: %s", e.what()); - return NULL; - } - - cln->handler = [](void* data) { - ((MainConf*)data)->~MainConf(); - }; - mcf->interval = NGX_CONF_UNSET_MSEC; mcf->batchSize = NGX_CONF_UNSET_SIZE; mcf->batchCount = NGX_CONF_UNSET_SIZE; - return static_cast(mcf); + return mcf; } char* initMainConf(ngx_conf_t* cf, void* conf) { - auto mcf = getMainConf(cf); + auto mcf = (MainConf*)conf; + ngx_conf_init_msec_value(mcf->interval, 5000); ngx_conf_init_size_value(mcf->batchSize, 512); ngx_conf_init_size_value(mcf->batchCount, 4); - try { - if (mcf->serviceName.data == NULL) { - mcf->resourceAttrs.emplace("service.name", "unknown_service:nginx"); - } else { - mcf->resourceAttrs["service.name"] = toStrView(mcf->serviceName); - } - } catch (const std::exception& e) { - ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "OTel: %s", e.what()); - return (char*)NGX_CONF_ERROR; + if (mcf->serviceName.data == NULL) { + mcf->serviceName = ngx_string("unknown_service:nginx"); } return NGX_CONF_OK; @@ -961,7 +839,7 @@ char* mergeLocationConf(ngx_conf_t* cf, void* parent, void* child) conf->spanAttrs = prev->spanAttrs; } - auto mcf = getMainConf(cf); + auto mcf = (MainConf*)ngx_http_conf_get_module_main_conf(cf, gHttpModule); if (mcf->endpoint.len == 0 && conf->trace) { ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, diff --git a/src/ngx.hpp b/src/ngx.hpp deleted file mode 100644 index 63351fa..0000000 --- a/src/ngx.hpp +++ /dev/null @@ -1,7 +0,0 @@ -#pragma once - -extern "C" { -#include -#include -#include -} diff --git a/src/trace_service_client.hpp b/src/trace_service_client.hpp index 4ed92e7..6871019 100644 --- a/src/trace_service_client.hpp +++ b/src/trace_service_client.hpp @@ -8,27 +8,6 @@ namespace otel_proto_trace = opentelemetry::proto::collector::trace::v1; -struct Target { - typedef std::vector> HeaderVec; - - std::string endpoint; - bool ssl; - std::string trustedCert; - HeaderVec headers; - - static bool validateHeaderName(StrView name) - { - return grpc_header_key_is_legal( - grpc_slice_from_static_buffer(name.data(), name.size())); - } - - static bool validateHeaderValue(StrView value) - { - return grpc_header_nonbin_value_is_legal( - grpc_slice_from_static_buffer(value.data(), value.size())); - } -}; - class TraceServiceClient { public: typedef otel_proto_trace::ExportTraceServiceRequest Request; @@ -38,18 +17,10 @@ public: typedef std::function ResponseCb; - TraceServiceClient(const Target& target) : headers(target.headers) + TraceServiceClient(const std::string& target) { - std::shared_ptr creds; - if (target.ssl) { - grpc::SslCredentialsOptions options; - options.pem_root_certs = target.trustedCert; - - creds = grpc::SslCredentials(options); - } else { - creds = grpc::InsecureChannelCredentials(); - } - auto channel = grpc::CreateChannel(target.endpoint, creds); + auto channel = grpc::CreateChannel( + target, grpc::InsecureChannelCredentials()); channel->GetState(true); // trigger 'connecting' state stub = TraceService::NewStub(channel); @@ -59,10 +30,6 @@ public: { std::unique_ptr call{new ActiveCall{}}; - for (auto& header : headers) { - call->context.AddMetadata(header.first, header.second); - } - call->request = std::move(req); call->cb = std::move(cb); @@ -132,8 +99,6 @@ private: ResponseCb cb; }; - Target::HeaderVec headers; - std::unique_ptr stub; grpc::CompletionQueue queue; diff --git a/tests/conftest.py b/tests/conftest.py deleted file mode 100644 index 25933e4..0000000 --- a/tests/conftest.py +++ /dev/null @@ -1,101 +0,0 @@ -import jinja2 -import logging -from OpenSSL import crypto -import os -import pytest -import subprocess -import time - - -pytest_plugins = [ - "trace_service", -] - - -def pytest_addoption(parser): - parser.addoption("--nginx", required=True) - parser.addoption("--module", required=True) - parser.addoption("--otelcol") - parser.addoption("--globals", default="") - - -def self_signed_cert(name): - k = crypto.PKey() - k.generate_key(crypto.TYPE_RSA, 2048) - cert = crypto.X509() - cert.get_subject().CN = name - cert.set_issuer(cert.get_subject()) - cert.gmtime_adj_notBefore(0) - cert.gmtime_adj_notAfter(365 * 86400) # 365 days - cert.set_pubkey(k) - cert.sign(k, "sha512") - return ( - crypto.dump_privatekey(crypto.FILETYPE_PEM, k), - crypto.dump_certificate(crypto.FILETYPE_PEM, cert), - ) - - -@pytest.fixture(scope="session") -def logger(): - logging.basicConfig(level=logging.INFO) - return logging.getLogger(__name__) - - -@pytest.fixture(scope="module") -def testdir(tmp_path_factory): - return tmp_path_factory.mktemp("nginx") - - -@pytest.fixture(scope="module") -def nginx_config(request, pytestconfig, testdir, logger): - tmpl = jinja2.Environment().from_string(request.module.NGINX_CONFIG) - params = getattr(request, "param", {}) - params["globals"] = ( - f"pid {testdir}/nginx.pid;\n" - + "error_log stderr info;\n" - + f"error_log {testdir}/error.log info;\n" - + f"load_module {os.path.abspath(pytestconfig.option.module)};\n" - + pytestconfig.option.globals - ) - params["http_globals"] = f"root {testdir};\n" + "access_log off;\n" - conf = tmpl.render(params) - logger.debug(conf) - return conf - - -@pytest.fixture(scope="module") -def nginx(testdir, pytestconfig, nginx_config, cert, logger, otelcol): - (testdir / "nginx.conf").write_text(nginx_config) - logger.info("Starting nginx...") - proc = subprocess.Popen( - [ - pytestconfig.option.nginx, - "-p", - str(testdir), - "-c", - "nginx.conf", - "-e", - "error.log", - ] - ) - logger.debug(f"args={' '.join(proc.args)}") - logger.debug(f"pid={proc.pid}") - while not (testdir / "nginx.pid").exists(): - time.sleep(0.1) - assert proc.poll() is None, "Can't start nginx" - yield proc - logger.info("Stopping nginx...") - proc.terminate() - try: - proc.wait(timeout=5) - except subprocess.TimeoutExpired: - proc.kill() - assert "[alert]" not in (testdir / "error.log").read_text() - - -@pytest.fixture(scope="module") -def cert(testdir): - key, cert = self_signed_cert("localhost") - (testdir / "localhost.key").write_text(key.decode("utf-8")) - (testdir / "localhost.crt").write_text(cert.decode("utf-8")) - yield (key, cert) diff --git a/tests/h2_otel.t b/tests/h2_otel.t new file mode 100644 index 0000000..24987d7 --- /dev/null +++ b/tests/h2_otel.t @@ -0,0 +1,553 @@ +#!/usr/bin/perl + +# (C) Nginx, Inc. + +# Tests for OTel exporter in case HTTP/2. + +############################################################################### + +use warnings; +use strict; + +use Test::More; + +BEGIN { use FindBin; chdir($FindBin::Bin); } + +use Test::Nginx; +use Test::Nginx::HTTP2; +use MIME::Base64; + +############################################################################### + +select STDERR; $| = 1; +select STDOUT; $| = 1; + +my $t = Test::Nginx->new() + ->has(qw/http_v2 http_ssl rewrite mirror grpc socket_ssl_alpn/) + ->has_daemon(qw/openssl base64/) + ->write_file_expand('nginx.conf', <<'EOF'); + +%%TEST_GLOBALS%% + +daemon off; + +events { +} + +http { + %%TEST_GLOBALS_HTTP%% + + ssl_certificate_key localhost.key; + ssl_certificate localhost.crt; + + otel_exporter { + endpoint 127.0.0.1:8083; + interval 1s; + batch_size 10; + batch_count 1; + } + + otel_service_name test_server; + otel_trace on; + + server { + listen 127.0.0.1:8080 http2; + listen 127.0.0.1:8081; + listen 127.0.0.1:8082 http2 ssl; + server_name localhost; + + location /trace-on { + otel_trace_context extract; + otel_span_name default_location; + otel_span_attr http.request.header.completion + $request_completion; + otel_span_attr http.response.header.content.type + $sent_http_content_type; + otel_span_attr http.request $request; + add_header "X-Otel-Trace-Id" $otel_trace_id; + add_header "X-Otel-Span-Id" $otel_span_id; + add_header "X-Otel-Parent-Id" $otel_parent_id; + add_header "X-Otel-Parent-Sampled" $otel_parent_sampled; + return 200 "TRACE-ON"; + } + + location /context-ignore { + otel_trace_context ignore; + otel_span_name context_ignore; + add_header "X-Otel-Parent-Id" $otel_parent_id; + proxy_pass http://127.0.0.1:8081/trace-off; + } + + location /context-extract { + otel_trace_context extract; + otel_span_name context_extract; + add_header "X-Otel-Parent-Id" $otel_parent_id; + proxy_pass http://127.0.0.1:8081/trace-off; + } + + location /context-inject { + otel_trace_context inject; + otel_span_name context_inject; + add_header "X-Otel-Parent-Id" $otel_parent_id; + proxy_pass http://127.0.0.1:8081/trace-off; + } + + location /context-propagate { + otel_trace_context propagate; + otel_span_name context_propogate; + add_header "X-Otel-Parent-Id" $otel_parent_id; + proxy_pass http://127.0.0.1:8081/trace-off; + } + + location /trace-off { + otel_trace off; + add_header "X-Otel-Traceparent" $http_traceparent; + add_header "X-Otel-Tracestate" $http_tracestate; + return 200 "TRACE-OFF"; + } + } + + server { + listen 127.0.0.1:8083 http2; + server_name localhost; + otel_trace off; + + location / { + mirror /mirror; + grpc_pass 127.0.0.1:8084; + } + + location /mirror { + internal; + grpc_pass 127.0.0.1:%%PORT_4317%%; + } + } + + server { + listen 127.0.0.1:8084 http2; + server_name localhost; + otel_trace off; + + location / { + add_header content-type application/grpc; + add_header grpc-status 0; + add_header grpc-message ""; + return 200; + } + } + +} + +EOF + +$t->write_file('openssl.conf', <<'EOF'); +[ req ] +default_bits = 2048 +encrypt_key = no +distinguished_name = req_distinguished_name +[ req_distinguished_name ] + +EOF + +my $d = $t->testdir(); + +foreach my $name ('localhost') { + system('openssl req -x509 -new ' + . "-config $d/openssl.conf -subj /CN=$name/ " + . "-out $d/$name.crt -keyout $d/$name.key " + . ">>$d/openssl.out 2>&1") == 0 + or die "Can't create certificate for $name: $!\n"; +} + +$t->try_run('no OTel module')->plan(69); + +############################################################################### + +my $p = port(4317); +my $f = grpc(); + +#do requests +(undef, my $t_off_resp) = http2_get('/trace-off'); + +#batch0 (10 requests) +my ($tp_headers, $tp_resp) = http2_get('/trace-on', trace_headers => 1); +my ($t_headers, $t_resp) = http2_get('/trace-on', port => 8082, ssl => 1); + +(my $t_headers_ignore, undef) = http2_get('/context-ignore'); +(my $tp_headers_ignore, undef) = http2_get('/context-ignore', + trace_headers => 1); +(my $t_headers_extract, undef) = http2_get('/context-extract'); +(my $tp_headers_extract, undef) = http2_get('/context-extract', + trace_headers => 1); +(my $t_headers_inject, undef) = http2_get('/context-inject'); +(my $tp_headers_inject, undef) = http2_get('/context-inject', + trace_headers => 1); +(my $t_headers_propagate, undef) = http2_get('/context-propagate'); +(my $tp_headers_propagate, undef) = + http2_get('/context-propagate', trace_headers => 1); + +my ($frame) = grep { $_->{type} eq "DATA" } @{$f->{http_start}()}; +my $batch0 = to_hash(decode_protobuf(substr $frame->{data}, 8)); +my $spans = $$batch0{scope_spans}; + +#batch1 (5 reqeusts) +http2_get('/trace-on') for (1..5); + +($frame) = grep { $_->{type} eq "DATA" } @{$f->{http_start}()}; +my $batch1 = to_hash(decode_protobuf(substr $frame->{data}, 8)); + +#validate responses +like($tp_resp, qr/TRACE-ON/, 'http request1 - trace on'); +like($t_resp, qr/TRACE-ON/, 'http request2 - trace on'); +like($t_off_resp, qr/TRACE-OFF/, 'http request - trace off'); + +#validate batch size +delete $$spans{scope}; #remove 'scope' entry +is(scalar keys %{$spans}, 10, 'batch0 size - trace on'); +delete $$batch1{scope_spans}{scope}; #remove 'scope' entry +is(scalar keys %{$$batch1{scope_spans}}, 5, 'batch1 size - trace on'); + +#validate general attributes +is(get_attr("service.name", "string_value", + $$batch0{resource}), + 'test_server', 'service.name - trace on'); +is($$spans{span0}{name}, '"default_location"', 'span.name - trace on'); + +#validate http metrics +is(get_attr("http.method", "string_value", $$spans{span0}), 'GET', + 'http.method metric - trace on'); +is(get_attr("http.target", "string_value", $$spans{span0}), '/trace-on', + 'http.target metric - trace on'); +is(get_attr("http.route", "string_value", $$spans{span0}), '/trace-on', + 'http.route metric - trace on'); +is(get_attr("http.scheme", "string_value", $$spans{span0}), 'http', + 'http.scheme metric - trace on'); +is(get_attr("http.flavor", "string_value", $$spans{span0}), '2.0', + 'http.flavor metric - trace on'); +is(get_attr("http.user_agent", "string_value", $$spans{span0}), 'nginx-tests', + 'http.user_agent metric - trace on'); +is(get_attr("http.request_content_length", "int_value", $$spans{span0}), 0, + 'http.request_content_length metric - trace on'); +is(get_attr("http.response_content_length", "int_value", $$spans{span0}), 8, + 'http.response_content_length metric - trace on'); +is(get_attr("http.status_code", "int_value", $$spans{span0}), 200, + 'http.status_code metric - trace on'); +is(get_attr("net.host.name", "string_value", $$spans{span0}), 'localhost', + 'net.host.name metric - trace on'); +is(get_attr("net.host.port", "int_value", $$spans{span0}), 8080, + 'net.host.port metric - trace on'); +is(get_attr("net.sock.peer.addr", "string_value", $$spans{span0}), '127.0.0.1', + 'net.sock.peer.addr metric - trace on'); +like(get_attr("net.sock.peer.port", "int_value", $$spans{span0}), qr/\d+/, + 'net.sock.peer.port metric - trace on'); + +#validate https metrics +is(get_attr("http.method", "string_value", $$spans{span1}), 'GET', + 'http.method metric - trace on (https)'); +is(get_attr("http.target", "string_value", $$spans{span1}), '/trace-on', + 'http.target metric - trace on (https)'); +is(get_attr("http.route", "string_value", $$spans{span1}), '/trace-on', + 'http.route metric - trace on (https)'); +is(get_attr("http.scheme", "string_value", $$spans{span1}), 'https', + 'http.scheme metric - trace on (https)'); +is(get_attr("http.flavor", "string_value", $$spans{span1}), '2.0', + 'http.flavor metric - trace on (https)'); +isnt(get_attr("http.user_agent", "string_value", $$spans{span1}), + 'nginx-tests', 'http.user_agent metric - trace on (https)'); +is(get_attr("http.request_content_length", "int_value", $$spans{span1}), 0, + 'http.request_content_length metric - trace on (https)'); +is(get_attr("http.response_content_length", "int_value", $$spans{span1}), 8, + 'http.response_content_length metric - trace on (https)'); +is(get_attr("http.status_code", "int_value", $$spans{span1}), 200, + 'http.status_code metric - trace on (https)'); +is(get_attr("net.host.name", "string_value", $$spans{span1}), 'localhost', + 'net.host.name metric - trace on (https)'); +is(get_attr("net.host.port", "int_value", $$spans{span1}), 8082, + 'net.host.port metric - trace on (https)'); +is(get_attr("net.sock.peer.addr", "string_value", $$spans{span1}), '127.0.0.1', + 'net.sock.peer.addr metric - trace on (https)'); +like(get_attr("net.sock.peer.port", "int_value", $$spans{span1}), qr/\d+/, + 'net.sock.peer.port metric - trace on (https)'); + +#validate custom http metrics +is(${get_attr("http.request.header.completion", "array_value", $$spans{span0})} + {values}{string_value}, '"OK"', + 'http.request.header.completion metric - trace on'); +is(${get_attr( + "http.response.header.content.type", "array_value", $$spans{span0} + )}{values}{string_value}, '"text/plain"', + 'http.response.header.content.type metric - trace on'); +is(get_attr("http.request", "string_value", $$spans{span0}), + 'GET /trace-on HTTP/2.0', 'http.request metric - trace on'); + +#extract trace info +is($$spans{span0}{parent_span_id}, 'b9c7c989f97918e1', + 'traceparent - trace on'); +is($$spans{span0}{trace_state}, '"congo=ucfJifl5GOE,rojo=00f067aa0ba902b7"', + 'tracestate - trace on'); +is($$spans{span1}{parent_span_id}, undef, 'no traceparent - trace on'); +is($$spans{span1}{trace_state}, undef, 'no tracestate - trace on'); + +#variables +is($tp_headers->{'x-otel-trace-id'}, $$spans{span0}{trace_id}, + '$otel_trace_id variable - trace on'); +is($tp_headers->{'x-otel-span-id'}, $$spans{span0}{span_id}, + '$otel_span_id variable - trace on'); +is($tp_headers->{'x-otel-parent-id'}, $$spans{span0}{parent_span_id}, + '$otel_parent_id variable - trace on'); +is($tp_headers->{'x-otel-parent-sampled'}, 1, + '$otel_parent_sampled variable - trace on'); +is($t_headers->{'x-otel-parent-sampled'}, 0, + '$otel_parent_sampled variable - trace on (no traceparent header)'); + +#trace off +is((scalar grep { + get_attr("http.target", "string_value", $$spans{$_}) eq '/trace-off' + } keys %{$spans}), 0, 'no metric in batch0 - trace off'); +is((scalar grep { + get_attr("http.target", "string_value", $$spans{$_}) eq '/trace-off' + } keys %{$$batch1{scope_spans}}), 0, 'no metric in batch1 - trace off'); + +#trace context: ignore +is($t_headers_ignore->{'x-otel-traceparent'}, undef, + 'no traceparent - trace context ignore (no trace headers)'); +is($t_headers_ignore->{'x-otel-tracestate'}, undef, + 'no tracestate - trace context ignore (no trace headers)'); + +is($tp_headers_ignore->{'x-otel-parent-id'}, undef, + 'no parent span id - trace context ignore (trace headers)'); +is($tp_headers_ignore->{'x-otel-traceparent'}, + '00-0af7651916cd43dd8448eb211c80319c-b9c7c989f97918e1-01', + 'traceparent - trace context ignore (trace headers)'); +is($tp_headers_ignore->{'x-otel-tracestate'}, + 'congo=ucfJifl5GOE,rojo=00f067aa0ba902b7', + 'tracestate - trace context ignore (trace headers)'); + +#trace context: extract +is($t_headers_extract->{'x-otel-traceparent'}, undef, + 'no traceparent - trace context extract (no trace headers)'); +is($t_headers_extract->{'x-otel-tracestate'}, undef, + 'no tracestate - trace context extract (no trace headers)'); + +is($tp_headers_extract->{'x-otel-parent-id'}, 'b9c7c989f97918e1', + 'parent span id - trace context extract (trace headers)'); +is($tp_headers_extract->{'x-otel-traceparent'}, + '00-0af7651916cd43dd8448eb211c80319c-b9c7c989f97918e1-01', + 'traceparent - trace context extract (trace headers)'); +is($tp_headers_extract->{'x-otel-tracestate'}, + 'congo=ucfJifl5GOE,rojo=00f067aa0ba902b7', + 'tracestate - trace context extract (trace headers)'); + +#trace context: inject +is($t_headers_inject->{'x-otel-traceparent'}, + "00-$$spans{span6}{trace_id}-$$spans{span6}{span_id}-01", + 'traceparent - trace context inject (no trace headers)'); +is($t_headers_inject->{'x-otel-tracestate'}, undef, + 'no tracestate - trace context inject (no trace headers)'); + +is($tp_headers_inject->{'x-otel-parent-id'}, undef, + 'no parent span id - trace context inject (trace headers)'); +is($tp_headers_inject->{'x-otel-traceparent'}, + "00-$$spans{span7}{trace_id}-$$spans{span7}{span_id}-01", + 'traceparent - trace context inject (trace headers)'); +is($tp_headers_inject->{'x-otel-tracestate'}, undef, + 'no tracestate - trace context inject (trace headers)'); + +#trace context: propagate +is($t_headers_propagate->{'x-otel-traceparent'}, + "00-$$spans{span8}{trace_id}-$$spans{span8}{span_id}-01", + 'traceparent - trace context propagate (no trace headers)'); +is($t_headers_propagate->{'x-otel-tracestate'}, undef, + 'no tracestate - trace context propagate (no trace headers)'); + +is($tp_headers_propagate->{'x-otel-parent-id'}, 'b9c7c989f97918e1', + 'parent id - trace context propagate (trace headers)'); +is($tp_headers_propagate->{'x-otel-traceparent'}, + "00-0af7651916cd43dd8448eb211c80319c-$$spans{span9}{span_id}-01", + 'traceparent - trace context propagate (trace headers)'); +is($tp_headers_propagate->{'x-otel-tracestate'}, + 'congo=ucfJifl5GOE,rojo=00f067aa0ba902b7', + 'tracestate - trace context propagate (trace headers)'); + +SKIP: { +skip "depends on error log content", 2 unless $ENV{TEST_NGINX_UNSAFE}; + +$t->stop(); +my $log = $t->read_file("error.log"); + +like($log, qr/OTel\/grpc: Error parsing metadata: error=invalid value/, + 'log: error parsing metadata - no protobuf in response'); +unlike($log, qr/OTel export failure: No status received/, + 'log: no export failure'); + +} + +############################################################################### + +sub http2_get { + my ($path, %extra) = @_; + my ($frames, $frame); + + my $port = $extra{port} || 8080; + + my $s = $extra{ssl} + ? Test::Nginx::HTTP2->new( + undef, socket => get_ssl_socket($port, ['h2'])) + : Test::Nginx::HTTP2->new(); + + my $sid = $extra{trace_headers} + ? $s->new_stream({ headers => [ + { name => ':method', value => 'GET' }, + { name => ':scheme', value => 'http' }, + { name => ':path', value => $path }, + { name => ':authority', value => 'localhost' }, + { name => 'user-agent', value => 'nginx-tests', mode => 2 }, + { name => 'traceparent', + value => '00-0af7651916cd43dd8448eb211c80319c-' . + 'b9c7c989f97918e1-01', + mode => 2 + }, + { name => 'tracestate', + value => 'congo=ucfJifl5GOE,rojo=00f067aa0ba902b7', + mode => 2 + }]}) + : $s->new_stream({ path => $path }); + $frames = $s->read(all => [{ sid => $sid, fin => 1 }]); + + ($frame) = grep { $_->{type} eq "HEADERS" } @$frames; + my $headers = $frame->{headers}; + + ($frame) = grep { $_->{type} eq "DATA" } @$frames; + my $data = $frame->{data}; + + return $headers, $data; +} + +sub get_ssl_socket { + my ($port, $alpn) = @_; + + return http( + '', PeerAddr => '127.0.0.1:' . port($port), start => 1, + SSL => 1, + SSL_alpn_protocols => $alpn, + SSL_error_trap => sub { die $_[1] } + ); +} + +sub get_attr { + my($attr, $type, $obj) = @_; + + my ($res) = grep { + $_ =~ /^attribute\d+/ && $$obj{$_}{key} eq '"' . $attr . '"' + } keys %{$obj}; + + if (defined $res) { + $$obj{$res}{value}{$type} =~ s/(^\")|(\"$)//g + if $type eq 'string_value'; + + return $$obj{$res}{value}{$type}; + } + + return undef; +} + +sub decode_protobuf { + my ($protobuf) = @_; + + local $/; + open CMD, "echo '" . encode_base64($protobuf) . "' | base64 -d | " . + '$PWD/../build/_deps/grpc-build/third_party/protobuf/protoc '. + '--decode opentelemetry.proto.trace.v1.ResourceSpans -I ' . + '$PWD/../build/_deps/otelcpp-src/third_party/opentelemetry-proto ' . + 'opentelemetry/proto/collector/trace/v1/trace_service.proto |' + or die "Can't decode protobuf: $!\n"; + my $out = ; + close CMD; + + return $out; +} + +sub decode_bytes { + my ($bytes) = @_; + + my $c = sub { return chr oct(shift) }; + + $bytes =~ s/\\(\d{3})/$c->($1)/eg; + $bytes =~ s/(^\")|(\"$)//g; + $bytes =~ s/\\\\/\\/g; + $bytes =~ s/\\r/\r/g; + $bytes =~ s/\\n/\n/g; + $bytes =~ s/\\t/\t/g; + $bytes =~ s/\\"/\"/g; + $bytes =~ s/\\'/\'/g; + + return unpack("H*", unpack("a*", $bytes)); +} + +sub to_hash { + my ($textdata) = @_; + + my %out = (); + push my @stack, \%out; + my ($attr_count, $span_count) = (0, 0); + for my $line (split /\n/, $textdata) { + $line =~ s/(^\s+)|(\s+$)//g; + if ($line =~ /\:/) { + my ($k, $v) = split /\: /, $line; + $v = decode_bytes($v) if ($k =~ /trace_id|span_id|parent_span_id/); + $stack[$#stack]{$k} = $v; + } elsif ($line =~ /\{/) { + $line =~ s/\s\{//; + $line = 'attribute' . $attr_count++ if ($line eq 'attributes'); + if ($line eq 'spans') { + $line = 'span' . $span_count++; + $attr_count = 0; + } + my %new = (); + $stack[$#stack]{$line} = \%new; + push @stack, \%new; + } elsif ($line =~ /\}/) { + pop @stack; + } + } + + return \%out; +} + +sub grpc { + my ($server, $client, $f, $s, $c, $sid, $csid, $uri); + + $server = IO::Socket::INET->new( + Proto => 'tcp', + LocalHost => '127.0.0.1', + LocalPort => $p, + Listen => 5, + Reuse => 1 + ) or die "Can't create listening socket: $!\n"; + + $f->{http_start} = sub { + if (IO::Select->new($server)->can_read(5)) { + $client = $server->accept(); + } else { + # connection could be unexpectedly reused + goto reused if $client; + return undef; + } + + $client->sysread($_, 24) == 24 or return; # preface + + $c = Test::Nginx::HTTP2->new(1, socket => $client, + pure => 1, preface => "") or return; + +reused: + my $frames = $c->read(all => [{ fin => 1 }]); + + $client->close(); + + return $frames; + }; + + return $f; +} + +############################################################################### diff --git a/tests/h3_otel.t b/tests/h3_otel.t new file mode 100644 index 0000000..ef848cb --- /dev/null +++ b/tests/h3_otel.t @@ -0,0 +1,508 @@ +#!/usr/bin/perl + +# (C) Nginx, Inc. + +# Tests for OTel exporter in case HTTP/3. + +############################################################################### + +use warnings; +use strict; + +use Test::More; + +BEGIN { use FindBin; chdir($FindBin::Bin); } + +use Test::Nginx; +use Test::Nginx::HTTP2; +use Test::Nginx::HTTP3; +use MIME::Base64; + +############################################################################### + +select STDERR; $| = 1; +select STDOUT; $| = 1; + +my $t = Test::Nginx->new()->has(qw/http_v2 http_v3 rewrite mirror grpc cryptx/) + ->has_daemon(qw/openssl base64/) + ->write_file_expand('nginx.conf', <<'EOF'); + +%%TEST_GLOBALS%% + +daemon off; + +events { +} + +http { + %%TEST_GLOBALS_HTTP%% + + ssl_certificate_key localhost.key; + ssl_certificate localhost.crt; + ssl_protocols TLSv1.3; + + otel_exporter { + endpoint 127.0.0.1:8082; + interval 1s; + batch_size 10; + batch_count 2; + } + + otel_service_name test_server; + otel_trace on; + + server { + listen 127.0.0.1:%%PORT_8980_UDP%% quic; + listen 127.0.0.1:8081; + server_name localhost; + + location /trace-on { + otel_trace_context extract; + otel_span_name default_location; + otel_span_attr http.request.header.completion + $request_completion; + otel_span_attr http.response.header.content.type + $sent_http_content_type; + otel_span_attr http.request $request; + add_header "X-Otel-Trace-Id" $otel_trace_id; + add_header "X-Otel-Span-Id" $otel_span_id; + add_header "X-Otel-Parent-Id" $otel_parent_id; + add_header "X-Otel-Parent-Sampled" $otel_parent_sampled; + return 200 "TRACE-ON"; + } + + location /context-ignore { + otel_trace_context ignore; + otel_span_name context_ignore; + add_header "X-Otel-Parent-Id" $otel_parent_id; + proxy_pass http://127.0.0.1:8081/trace-off; + } + + location /context-extract { + otel_trace_context extract; + otel_span_name context_extract; + add_header "X-Otel-Parent-Id" $otel_parent_id; + proxy_pass http://127.0.0.1:8081/trace-off; + } + + location /context-inject { + otel_trace_context inject; + otel_span_name context_inject; + add_header "X-Otel-Parent-Id" $otel_parent_id; + proxy_pass http://127.0.0.1:8081/trace-off; + } + + location /context-propagate { + otel_trace_context propagate; + otel_span_name context_propogate; + add_header "X-Otel-Parent-Id" $otel_parent_id; + proxy_pass http://127.0.0.1:8081/trace-off; + } + + location /trace-off { + otel_trace off; + add_header "X-Otel-Traceparent" $http_traceparent; + add_header "X-Otel-Tracestate" $http_tracestate; + return 200 "TRACE-OFF"; + } + } + + server { + listen 127.0.0.1:8082 http2; + server_name localhost; + otel_trace off; + + location / { + mirror /mirror; + grpc_pass 127.0.0.1:8083; + } + + location /mirror { + internal; + grpc_pass 127.0.0.1:%%PORT_4317%%; + } + } + + server { + listen 127.0.0.1:8083 http2; + server_name localhost; + otel_trace off; + + location / { + add_header content-type application/grpc; + add_header grpc-status 0; + add_header grpc-message ""; + return 200; + } + } + +} + +EOF + +$t->write_file('openssl.conf', <<'EOF'); +[ req ] +default_bits = 2048 +encrypt_key = no +distinguished_name = req_distinguished_name +[ req_distinguished_name ] + +EOF + +my $d = $t->testdir(); + +foreach my $name ('localhost') { + system('openssl req -x509 -new ' + . "-config $d/openssl.conf -subj /CN=$name/ " + . "-out $d/$name.crt -keyout $d/$name.key " + . ">>$d/openssl.out 2>&1") == 0 + or die "Can't create certificate for $name: $!\n"; +} + +$t->try_run('no OTel module')->plan(56); + +############################################################################### + +my $p = port(4317); +my $f = grpc(); + +#do requests +(undef, my $t_off_resp) = http3_get('/trace-off'); + +#batch0 (10 requests) +my ($tp_headers, $tp_resp) = http3_get('/trace-on', trace_headers => 1); +my ($t_headers, $t_resp) = http3_get('/trace-on'); + +(my $t_headers_ignore, undef) = http3_get('/context-ignore'); +(my $tp_headers_ignore, undef) = http3_get('/context-ignore', + trace_headers => 1); +(my $t_headers_extract, undef) = http3_get('/context-extract'); +(my $tp_headers_extract, undef) = http3_get('/context-extract', + trace_headers => 1); +(my $t_headers_inject, undef) = http3_get('/context-inject'); +(my $tp_headers_inject, undef) = http3_get('/context-inject', + trace_headers => 1); +(my $t_headers_propagate, undef) = http3_get('/context-propagate'); +(my $tp_headers_propagate, undef) = + http3_get('/context-propagate', trace_headers => 1); + +my ($frame) = grep { $_->{type} eq "DATA" } @{$f->{http_start}()}; +my $batch0 = to_hash(decode_protobuf(substr $frame->{data}, 8)); +my $spans = $$batch0{scope_spans}; + +#batch1 (5 reqeusts) +http3_get('/trace-on') for (1..5); + +($frame) = grep { $_->{type} eq "DATA" } @{$f->{http_start}()}; +my $batch1 = to_hash(decode_protobuf(substr $frame->{data}, 8)); + +#validate responses +like($tp_resp, qr/TRACE-ON/, 'http request1 - trace on'); +like($t_resp, qr/TRACE-ON/, 'http request2 - trace on'); +like($t_off_resp, qr/TRACE-OFF/, 'http request - trace off'); + +#validate batch size +delete $$spans{scope}; #remove 'scope' entry +is(scalar keys %{$spans}, 10, 'batch0 size - trace on'); +delete $$batch1{scope_spans}{scope}; #remove 'scope' entry +is(scalar keys %{$$batch1{scope_spans}}, 5, 'batch1 size - trace on'); + +#validate general attributes +is(get_attr("service.name", "string_value", + $$batch0{resource}), + 'test_server', 'service.name - trace on'); +is($$spans{span0}{name}, '"default_location"', 'span.name - trace on'); + +#validate metrics +is(get_attr("http.method", "string_value", $$spans{span0}), 'GET', + 'http.method metric - trace on'); +is(get_attr("http.target", "string_value", $$spans{span0}), '/trace-on', + 'http.target metric - trace on'); +is(get_attr("http.route", "string_value", $$spans{span0}), '/trace-on', + 'http.route metric - trace on'); +is(get_attr("http.scheme", "string_value", $$spans{span0}), 'https', + 'http.scheme metric - trace on'); +is(get_attr("http.flavor", "string_value", $$spans{span0}), '3.0', + 'http.flavor metric - trace on'); +is(get_attr("http.user_agent", "string_value", $$spans{span0}), 'nginx-tests', + 'http.user_agent metric - trace on'); +is(get_attr("http.request_content_length", "int_value", $$spans{span0}), 0, + 'http.request_content_length metric - trace on'); +is(get_attr("http.response_content_length", "int_value", $$spans{span0}), 8, + 'http.response_content_length metric - trace on'); +is(get_attr("http.status_code", "int_value", $$spans{span0}), 200, + 'http.status_code metric - trace on'); +is(get_attr("net.host.name", "string_value", $$spans{span0}), 'localhost', + 'net.host.name metric - trace on'); +is(get_attr("net.host.port", "int_value", $$spans{span0}), 8980, + 'net.host.port metric - trace on'); +is(get_attr("net.sock.peer.addr", "string_value", $$spans{span0}), '127.0.0.1', + 'net.sock.peer.addr metric - trace on'); +like(get_attr("net.sock.peer.port", "int_value", $$spans{span0}), qr/\d+/, + 'net.sock.peer.port metric - trace on'); + +#validate custom http metrics +is(${get_attr("http.request.header.completion", "array_value", $$spans{span0})} + {values}{string_value}, '"OK"', + 'http.request.header.completion metric - trace on'); +is(${get_attr( + "http.response.header.content.type", "array_value", $$spans{span0} + )}{values}{string_value}, '"text/plain"', + 'http.response.header.content.type metric - trace on'); +is(get_attr("http.request", "string_value", $$spans{span0}), + 'GET /trace-on HTTP/3.0', 'http.request metric - trace on'); + +#extract trace info +is($$spans{span0}{parent_span_id}, 'b9c7c989f97918e1', + 'traceparent - trace on'); +is($$spans{span0}{trace_state}, '"congo=ucfJifl5GOE,rojo=00f067aa0ba902b7"', + 'tracestate - trace on'); +is($$spans{span1}{parent_span_id}, undef, 'no traceparent - trace on'); +is($$spans{span1}{trace_state}, undef, 'no tracestate - trace on'); + +#variables +is($tp_headers->{'x-otel-trace-id'}, $$spans{span0}{trace_id}, + '$otel_trace_id variable - trace on'); +is($tp_headers->{'x-otel-span-id'}, $$spans{span0}{span_id}, + '$otel_span_id variable - trace on'); +is($tp_headers->{'x-otel-parent-id'}, $$spans{span0}{parent_span_id}, + '$otel_parent_id variable - trace on'); +is($tp_headers->{'x-otel-parent-sampled'}, 1, + '$otel_parent_sampled variable - trace on'); +is($t_headers->{'x-otel-parent-sampled'}, 0, + '$otel_parent_sampled variable - trace on (no traceparent header)'); + +#trace off +is((scalar grep { + get_attr("http.target", "string_value", $$spans{$_}) eq '/trace-off' + } keys %{$spans}), 0, 'no metric in batch0 - trace off'); +is((scalar grep { + get_attr("http.target", "string_value", $$spans{$_}) eq '/trace-off' + } keys %{$$batch1{scope_spans}}), 0, 'no metric in batch1 - trace off'); + +#trace context: ignore +is($t_headers_ignore->{'x-otel-traceparent'}, undef, + 'no traceparent - trace context ignore (no trace headers)'); +is($t_headers_ignore->{'x-otel-tracestate'}, undef, + 'no tracestate - trace context ignore (no trace headers)'); + +is($tp_headers_ignore->{'x-otel-parent-id'}, undef, + 'no parent span id - trace context ignore (trace headers)'); +is($tp_headers_ignore->{'x-otel-traceparent'}, + '00-0af7651916cd43dd8448eb211c80319c-b9c7c989f97918e1-01', + 'traceparent - trace context ignore (trace headers)'); +is($tp_headers_ignore->{'x-otel-tracestate'}, + 'congo=ucfJifl5GOE,rojo=00f067aa0ba902b7', + 'tracestate - trace context ignore (trace headers)'); + +#trace context: extract +is($t_headers_extract->{'x-otel-traceparent'}, undef, + 'no traceparent - trace context extract (no trace headers)'); +is($t_headers_extract->{'x-otel-tracestate'}, undef, + 'no tracestate - trace context extract (no trace headers)'); + +is($tp_headers_extract->{'x-otel-parent-id'}, 'b9c7c989f97918e1', + 'parent span id - trace context extract (trace headers)'); +is($tp_headers_extract->{'x-otel-traceparent'}, + '00-0af7651916cd43dd8448eb211c80319c-b9c7c989f97918e1-01', + 'traceparent - trace context extract (trace headers)'); +is($tp_headers_extract->{'x-otel-tracestate'}, + 'congo=ucfJifl5GOE,rojo=00f067aa0ba902b7', + 'tracestate - trace context extract (trace headers)'); + +#trace context: inject +is($t_headers_inject->{'x-otel-traceparent'}, + "00-$$spans{span6}{trace_id}-$$spans{span6}{span_id}-01", + 'traceparent - trace context inject (no trace headers)'); +is($t_headers_inject->{'x-otel-tracestate'}, undef, + 'no tracestate - trace context inject (no trace headers)'); + +is($tp_headers_inject->{'x-otel-parent-id'}, undef, + 'no parent span id - trace context inject (trace headers)'); +is($tp_headers_inject->{'x-otel-traceparent'}, + "00-$$spans{span7}{trace_id}-$$spans{span7}{span_id}-01", + 'traceparent - trace context inject (trace headers)'); +is($tp_headers_inject->{'x-otel-tracestate'}, undef, + 'no tracestate - trace context inject (trace headers)'); + +#trace context: propagate +is($t_headers_propagate->{'x-otel-traceparent'}, + "00-$$spans{span8}{trace_id}-$$spans{span8}{span_id}-01", + 'traceparent - trace context propagate (no trace headers)'); +is($t_headers_propagate->{'x-otel-tracestate'}, undef, + 'no tracestate - trace context propagate (no trace headers)'); + +is($tp_headers_propagate->{'x-otel-parent-id'}, 'b9c7c989f97918e1', + 'parent id - trace context propagate (trace headers)'); +is($tp_headers_propagate->{'x-otel-traceparent'}, + "00-0af7651916cd43dd8448eb211c80319c-$$spans{span9}{span_id}-01", + 'traceparent - trace context propagate (trace headers)'); +is($tp_headers_propagate->{'x-otel-tracestate'}, + 'congo=ucfJifl5GOE,rojo=00f067aa0ba902b7', + 'tracestate - trace context propagate (trace headers)'); + +SKIP: { +skip "depends on error log content", 2 unless $ENV{TEST_NGINX_UNSAFE}; + +$t->stop(); +my $log = $t->read_file("error.log"); + +like($log, qr/OTel\/grpc: Error parsing metadata: error=invalid value/, + 'log: error parsing metadata - no protobuf in response'); +unlike($log, qr/OTel export failure: No status received/, + 'log: no export failure'); + +} + +############################################################################### + +sub http3_get { + my ($path, %extra) = @_; + my ($frames, $frame); + + my $s = Test::Nginx::HTTP3->new(); + + my $sid = $extra{trace_headers} + ? $s->new_stream({ headers => [ + { name => ':method', value => 'GET' }, + { name => ':scheme', value => 'http' }, + { name => ':path', value => $path }, + { name => ':authority', value => 'localhost' }, + { name => 'user-agent', value => 'nginx-tests' }, + { name => 'traceparent', + value => '00-0af7651916cd43dd8448eb211c80319c-' . + 'b9c7c989f97918e1-01' + }, + { name => 'tracestate', + value => 'congo=ucfJifl5GOE,rojo=00f067aa0ba902b7' + }]}) + : $s->new_stream({ path => $path }); + + $frames = $s->read(all => [{ sid => $sid, fin => 1 }]); + + ($frame) = grep { $_->{type} eq "HEADERS" } @$frames; + my $headers = $frame->{headers}; + + ($frame) = grep { $_->{type} eq "DATA" } @$frames; + my $data = $frame->{data}; + + return $headers, $data; +} + +sub get_attr { + my($attr, $type, $obj) = @_; + + my ($res) = grep { + $_ =~ /^attribute\d+/ && $$obj{$_}{key} eq '"' . $attr . '"' + } keys %{$obj}; + + if (defined $res) { + $$obj{$res}{value}{$type} =~ s/(^\")|(\"$)//g + if $type eq 'string_value'; + + return $$obj{$res}{value}{$type}; + } + + return undef; +} + +sub decode_protobuf { + my ($protobuf) = @_; + + local $/; + open CMD, "echo '" . encode_base64($protobuf) . "' | base64 -d | " . + '$PWD/../build/_deps/grpc-build/third_party/protobuf/protoc '. + '--decode opentelemetry.proto.trace.v1.ResourceSpans -I ' . + '$PWD/../build/_deps/otelcpp-src/third_party/opentelemetry-proto ' . + 'opentelemetry/proto/collector/trace/v1/trace_service.proto |' + or die "Can't decode protobuf: $!\n"; + my $out = ; + close CMD; + + return $out; +} + +sub decode_bytes { + my ($bytes) = @_; + + my $c = sub { return chr oct(shift) }; + + $bytes =~ s/\\(\d{3})/$c->($1)/eg; + $bytes =~ s/(^\")|(\"$)//g; + $bytes =~ s/\\\\/\\/g; + $bytes =~ s/\\r/\r/g; + $bytes =~ s/\\n/\n/g; + $bytes =~ s/\\t/\t/g; + $bytes =~ s/\\"/\"/g; + $bytes =~ s/\\'/\'/g; + + return unpack("H*", unpack("a*", $bytes)); +} + +sub to_hash { + my ($textdata) = @_; + + my %out = (); + push my @stack, \%out; + my ($attr_count, $span_count) = (0, 0); + for my $line (split /\n/, $textdata) { + $line =~ s/(^\s+)|(\s+$)//g; + if ($line =~ /\:/) { + my ($k, $v) = split /\: /, $line; + $v = decode_bytes($v) if ($k =~ /trace_id|span_id|parent_span_id/); + $stack[$#stack]{$k} = $v; + } elsif ($line =~ /\{/) { + $line =~ s/\s\{//; + $line = 'attribute' . $attr_count++ if ($line eq 'attributes'); + if ($line eq 'spans') { + $line = 'span' . $span_count++; + $attr_count = 0; + } + my %new = (); + $stack[$#stack]{$line} = \%new; + push @stack, \%new; + } elsif ($line =~ /\}/) { + pop @stack; + } + } + + return \%out; +} + +sub grpc { + my ($server, $client, $f, $s, $c, $sid, $csid, $uri); + + $server = IO::Socket::INET->new( + Proto => 'tcp', + LocalHost => '127.0.0.1', + LocalPort => $p, + Listen => 5, + Reuse => 1 + ) or die "Can't create listening socket: $!\n"; + + $f->{http_start} = sub { + if (IO::Select->new($server)->can_read(5)) { + $client = $server->accept(); + } else { + # connection could be unexpectedly reused + goto reused if $client; + return undef; + } + + $client->sysread($_, 24) == 24 or return; # preface + + $c = Test::Nginx::HTTP2->new(1, socket => $client, + pure => 1, preface => "") or return; + +reused: + my $frames = $c->read(all => [{ fin => 1 }]); + + $client->close(); + + return $frames; + }; + + return $f; +} + +############################################################################### diff --git a/tests/otel.t b/tests/otel.t new file mode 100644 index 0000000..9887b2f --- /dev/null +++ b/tests/otel.t @@ -0,0 +1,514 @@ +#!/usr/bin/perl + +# (C) Nginx, Inc. + +# Tests for OTel exporter in case HTTP. + +############################################################################### + +use warnings; +use strict; + +use Test::More; + +BEGIN { use FindBin; chdir($FindBin::Bin); } + +use Test::Nginx; +use Test::Nginx::HTTP2; +use MIME::Base64; + +############################################################################### + +select STDERR; $| = 1; +select STDOUT; $| = 1; + +my $t = Test::Nginx->new()->has(qw/http http_ssl http_v2 mirror rewrite/) + ->has_daemon(qw/openssl base64/) + ->write_file_expand('nginx.conf', <<'EOF'); + +%%TEST_GLOBALS%% + +daemon off; + +events { +} + +http { + %%TEST_GLOBALS_HTTP%% + + ssl_certificate_key localhost.key; + ssl_certificate localhost.crt; + + otel_exporter { + endpoint 127.0.0.1:8082; + interval 1s; + batch_size 10; + batch_count 2; + } + + otel_service_name test_server; + otel_trace on; + + server { + listen 127.0.0.1:8080; + listen 127.0.0.1:8081 ssl; + server_name localhost; + + location /trace-on { + otel_trace_context extract; + otel_span_name default_location; + otel_span_attr http.request.header.completion + $request_completion; + otel_span_attr http.response.header.content.type + $sent_http_content_type; + otel_span_attr http.request $request; + add_header "X-Otel-Trace-Id" $otel_trace_id; + add_header "X-Otel-Span-Id" $otel_span_id; + add_header "X-Otel-Parent-Id" $otel_parent_id; + add_header "X-Otel-Parent-Sampled" $otel_parent_sampled; + return 200 "TRACE-ON"; + } + + location /context-ignore { + otel_trace_context ignore; + otel_span_name context_ignore; + add_header "X-Otel-Parent-Id" $otel_parent_id; + proxy_pass http://localhost:8080/trace-off; + } + + location /context-extract { + otel_trace_context extract; + otel_span_name context_extract; + add_header "X-Otel-Parent-Id" $otel_parent_id; + proxy_pass http://localhost:8080/trace-off; + } + + location /context-inject { + otel_trace_context inject; + otel_span_name context_inject; + add_header "X-Otel-Parent-Id" $otel_parent_id; + proxy_pass http://localhost:8080/trace-off; + } + + location /context-propagate { + otel_trace_context propagate; + otel_span_name context_propogate; + add_header "X-Otel-Parent-Id" $otel_parent_id; + proxy_pass http://localhost:8080/trace-off; + } + + location /trace-off { + otel_trace off; + add_header "X-Otel-Traceparent" $http_traceparent; + add_header "X-Otel-Tracestate" $http_tracestate; + return 200 "TRACE-OFF"; + } + } + + server { + listen 127.0.0.1:8082 http2; + server_name localhost; + otel_trace off; + + location / { + mirror /mirror; + grpc_pass 127.0.0.1:8083; + } + + location /mirror { + internal; + grpc_pass 127.0.0.1:%%PORT_4317%%; + } + } + + server { + listen 127.0.0.1:8083 http2; + server_name localhost; + otel_trace off; + + location / { + add_header content-type application/grpc; + add_header grpc-status 0; + add_header grpc-message ""; + return 200; + } + } +} + +EOF + +$t->write_file('openssl.conf', <<'EOF'); +[ req ] +default_bits = 2048 +encrypt_key = no +distinguished_name = req_distinguished_name +[ req_distinguished_name ] + +EOF + +my $d = $t->testdir(); + +foreach my $name ('localhost') { + system('openssl req -x509 -new ' + . "-config $d/openssl.conf -subj /CN=$name/ " + . "-out $d/$name.crt -keyout $d/$name.key " + . ">>$d/openssl.out 2>&1") == 0 + or die "Can't create certificate for $name: $!\n"; +} + +$t->try_run('no OTel module')->plan(69); + +############################################################################### + +my $p = port(4317); +my $f = grpc(); + +#do requests +my $t_off_resp = http1_get('/trace-off'); + +#batch0 (10 requests) +my $tp_resp = http1_get('/trace-on', trace_headers => 1); +my $t_resp = http1_get('/trace-on', port => 8081, ssl => 1); + +my $t_resp_ignore = http1_get('/context-ignore'); +my $tp_resp_ignore = http1_get('/context-ignore', trace_headers => 1); +my $t_resp_extract = http1_get('/context-extract'); +my $tp_resp_extract = http1_get('/context-extract', trace_headers => 1); +my $t_resp_inject = http1_get('/context-inject'); +my $tp_resp_inject = http1_get('/context-inject', trace_headers => 1); +my $t_resp_propagate = http1_get('/context-propagate'); +my $tp_resp_propagate = http1_get('/context-propagate', trace_headers => 1); + +my ($frame) = grep { $_->{type} eq "DATA" } @{$f->{http_start}()}; +my $batch0 = to_hash(decode_protobuf(substr $frame->{data}, 8)); +my $spans = $$batch0{scope_spans}; + +#batch1 (5 reqeusts) +http1_get('/trace-on') for (1..5); + +($frame) = grep { $_->{type} eq "DATA" } @{$f->{http_start}()}; +my $batch1 = to_hash(decode_protobuf(substr $frame->{data}, 8)); + +#validate responses +like($tp_resp, qr/TRACE-ON/, 'http request1 - trace on'); +like($t_resp, qr/TRACE-ON/, 'http request2 - trace on'); +like($t_off_resp, qr/TRACE-OFF/, 'http request - trace off'); + +#validate batch size +delete $$spans{scope}; #remove 'scope' entry +is(scalar keys %{$spans}, 10, 'batch0 size - trace on'); +delete $$batch1{scope_spans}{scope}; #remove 'scope' entry +is(scalar keys %{$$batch1{scope_spans}}, 5, 'batch1 size - trace on'); + +#validate general attributes +is(get_attr("service.name", "string_value", + $$batch0{resource}), 'test_server', 'service.name - trace on'); +is($$spans{span0}{name}, '"default_location"', 'span.name - trace on'); + +#validate http metrics +is(get_attr("http.method", "string_value", $$spans{span0}), 'GET', + 'http.method metric - trace on'); +is(get_attr("http.target", "string_value", $$spans{span0}), '/trace-on', + 'http.target metric - trace on'); +is(get_attr("http.route", "string_value", $$spans{span0}), '/trace-on', + 'http.route metric - trace on'); +is(get_attr("http.scheme", "string_value", $$spans{span0}), 'http', + 'http.scheme metric - trace on'); +is(get_attr("http.flavor", "string_value", $$spans{span0}), '1.0', + 'http.flavor metric - trace on'); +is(get_attr("http.user_agent", "string_value", $$spans{span0}), 'nginx-tests', + 'http.user_agent metric - trace on'); +is(get_attr("http.request_content_length", "int_value", $$spans{span0}), 0, + 'http.request_content_length metric - trace on'); +is(get_attr("http.response_content_length", "int_value", $$spans{span0}), 8, + 'http.response_content_length metric - trace on'); +is(get_attr("http.status_code", "int_value", $$spans{span0}), 200, + 'http.status_code metric - trace on'); +is(get_attr("net.host.name", "string_value", $$spans{span0}), 'localhost', + 'net.host.name metric - trace on'); +is(get_attr("net.host.port", "int_value", $$spans{span0}), 8080, + 'net.host.port metric - trace on'); +is(get_attr("net.sock.peer.addr", "string_value", $$spans{span0}), '127.0.0.1', + 'net.sock.peer.addr metric - trace on'); +like(get_attr("net.sock.peer.port", "int_value", $$spans{span0}), qr/\d+/, + 'net.sock.peer.port metric - trace on'); + +#validate https metrics +is(get_attr("http.method", "string_value", $$spans{span1}), 'GET', + 'http.method metric - trace on (https)'); +is(get_attr("http.target", "string_value", $$spans{span1}), '/trace-on', + 'http.target metric - trace on (https)'); +is(get_attr("http.route", "string_value", $$spans{span1}), '/trace-on', + 'http.route metric - trace on (https)'); +is(get_attr("http.scheme", "string_value", $$spans{span1}), 'https', + 'http.scheme metric - trace on (https)'); +is(get_attr("http.flavor", "string_value", $$spans{span1}), '1.0', + 'http.flavor metric - trace on (https)'); +is(get_attr("http.user_agent", "string_value", $$spans{span1}), + 'nginx-tests', 'http.user_agent metric - trace on (https)'); +is(get_attr("http.request_content_length", "int_value", $$spans{span1}), 0, + 'http.request_content_length metric - trace on (https)'); +is(get_attr("http.response_content_length", "int_value", $$spans{span1}), 8, + 'http.response_content_length metric - trace on (https)'); +is(get_attr("http.status_code", "int_value", $$spans{span1}), 200, + 'http.status_code metric - trace on (https)'); +is(get_attr("net.host.name", "string_value", $$spans{span1}), 'localhost', + 'net.host.name metric - trace on (https)'); +is(get_attr("net.host.port", "int_value", $$spans{span1}), 8081, + 'net.host.port metric - trace on (https)'); +is(get_attr("net.sock.peer.addr", "string_value", $$spans{span1}), '127.0.0.1', + 'net.sock.peer.addr metric - trace on (https)'); +like(get_attr("net.sock.peer.port", "int_value", $$spans{span1}), qr/\d+/, + 'net.sock.peer.port metric - trace on (https)'); + +#validate custom http metrics +is(${get_attr("http.request.header.completion", "array_value", $$spans{span0})} + {values}{string_value}, '"OK"', + 'http.request.header.completion metric - trace on'); +is(${get_attr("http.response.header.content.type", + "array_value", $$spans{span0})}{values}{string_value}, '"text/plain"', + 'http.response.header.content.type metric - trace on'); +is(get_attr("http.request", "string_value", $$spans{span0}), + 'GET /trace-on HTTP/1.0', 'http.request metric - trace on'); + +#extract trace info +is($$spans{span0}{parent_span_id}, 'b9c7c989f97918e1', + 'traceparent - trace on'); +is($$spans{span0}{trace_state}, '"congo=ucfJifl5GOE,rojo=00f067aa0ba902b7"', + 'tracestate - trace on'); +is($$spans{span1}{parent_span_id}, undef, 'no traceparent - trace on'); +is($$spans{span1}{trace_state}, undef, 'no tracestate - trace on'); + +#variables +like($tp_resp, qr/X-Otel-Trace-Id: $$spans{span0}{trace_id}/, + '$otel_trace_id variable - trace on'); +like($tp_resp, qr/X-Otel-Span-Id: $$spans{span0}{span_id}/, + '$otel_span_id variable - trace on'); +like($tp_resp, qr/X-Otel-Parent-Id: $$spans{span0}{parent_span_id}/, + '$otel_parent_id variable - trace on'); +like($tp_resp, qr/X-Otel-Parent-Sampled: 1/, + '$otel_parent_sampled variable - trace on'); +like($t_resp, qr/X-Otel-Parent-Sampled: 0/, + '$otel_parent_sampled variable - trace on (no traceparent header)'); + +#trace off +is((scalar grep { + get_attr("http.target", "string_value", $$spans{$_}) eq '/trace-off' + } keys %{$spans}), 0, 'no metric in batch0 - trace off'); +is((scalar grep { + get_attr("http.target", "string_value", $$spans{$_}) eq '/trace-off' + } keys %{$$batch1{scope_spans}}), 0, 'no metric in batch1 - trace off'); + +#trace context: ignore +unlike($t_resp_ignore, qr/X-Otel-Traceparent/, + 'no traceparent - trace context ignore (no trace headers)'); +unlike($t_resp_ignore, qr/X-Otel-Tracestate/, + 'no tracestate - trace context ignore (no trace headers)'); + +unlike($tp_resp_ignore, qr/X-Otel-Parent-Id/, + 'no parent span id - trace context ignore (trace headers)'); +like($tp_resp_ignore, + qr/Traceparent: 00-0af7651916cd43dd8448eb211c80319c-b9c7c989f97918e1-01/, + 'traceparent - trace context ignore (trace headers)'); +like($tp_resp_ignore, + qr/Tracestate: congo=ucfJifl5GOE,rojo=00f067aa0ba902b7/, + 'tracestate - trace context ignore (trace headers)'); + +#trace context: extract +unlike($t_resp_extract, qr/X-Otel-Traceparent/, + 'no traceparent - trace context extract (no trace headers)'); +unlike($t_resp_extract, qr/X-Otel-Tracestate/, + 'no tracestate - trace context extract (no trace headers)'); + +like($tp_resp_extract, qr/X-Otel-Parent-Id: b9c7c989f97918e1/, + 'parent span id - trace context extract (trace headers)'); +like($tp_resp_extract, + qr/Traceparent: 00-0af7651916cd43dd8448eb211c80319c-b9c7c989f97918e1-01/, + 'traceparent - trace context extract (trace headers)'); +like($tp_resp_extract, + qr/Tracestate: congo=ucfJifl5GOE,rojo=00f067aa0ba902b7/, + 'tracestate - trace context extract (trace headers)'); + +#trace context: inject +like($t_resp_inject, + qr/Traceparent: 00-$$spans{span6}{trace_id}-$$spans{span6}{span_id}-01/, + 'traceparent - trace context inject (no trace headers)'); +unlike($t_resp_inject, qr/X-Otel-Tracestate/, + 'no tracestate - trace context inject (no trace headers)'); + +unlike($tp_resp_inject, qr/X-Otel-Parent-Id/, + 'no parent span id - trace context inject (trace headers)'); +like($tp_resp_inject, + qr/Traceparent: 00-$$spans{span7}{trace_id}-$$spans{span7}{span_id}-01/, + 'traceparent - trace context inject (trace headers)'); +unlike($tp_resp_inject, qr/Tracestate:/, + 'no tracestate - trace context inject (trace headers)'); + +#trace context: propagate +like($t_resp_propagate, + qr/Traceparent: 00-$$spans{span8}{trace_id}-$$spans{span8}{span_id}-01/, + 'traceparent - trace context propagate (no trace headers)'); +unlike($t_resp_propagate, qr/X-Otel-Tracestate/, + 'no tracestate - trace context propagate (no trace headers)'); + +like($tp_resp_propagate, qr/X-Otel-Parent-Id: b9c7c989f97918e1/, + 'parent id - trace context propagate (trace headers)'); +like($tp_resp_propagate, + qr/parent: 00-0af7651916cd43dd8448eb211c80319c-$$spans{span9}{span_id}-01/, + 'traceparent - trace context propagate (trace headers)'); +like($tp_resp_propagate, + qr/Tracestate: congo=ucfJifl5GOE,rojo=00f067aa0ba902b7/, + 'tracestate - trace context propagate (trace headers)'); + +SKIP: { +skip "depends on error log contents", 2 unless $ENV{TEST_NGINX_UNSAFE}; + +$t->stop(); +my $log = $t->read_file("error.log"); + +like($log, qr/OTel\/grpc: Error parsing metadata: error=invalid value/, + 'log: error parsing metadata - no protobuf in response'); +unlike($log, qr/OTel export failure: No status received/, + 'log: no export failure'); + +} + +############################################################################### + +sub http1_get { + my ($path, %extra) = @_; + + my $port = $extra{port} || 8080; + + my $r = < '127.0.0.1:' . port($port), + SSL => $extra{ssl}); +} + +sub get_attr { + my($attr, $type, $obj) = @_; + + my ($res) = grep { + $_ =~ /^attribute\d+/ && $$obj{$_}{key} eq '"' . $attr . '"' + } keys %{$obj}; + + if (defined $res) { + $$obj{$res}{value}{$type} =~ s/(^\")|(\"$)//g + if $type eq 'string_value'; + + return $$obj{$res}{value}{$type}; + } + + return undef; +} + +sub decode_protobuf { + my ($protobuf) = @_; + + local $/; + open CMD, "echo '" . encode_base64($protobuf) . "' | base64 -d | " . + '$PWD/../build/_deps/grpc-build/third_party/protobuf/protoc '. + '--decode opentelemetry.proto.trace.v1.ResourceSpans -I ' . + '$PWD/../build/_deps/otelcpp-src/third_party/opentelemetry-proto ' . + 'opentelemetry/proto/collector/trace/v1/trace_service.proto |' + or die "Can't decode protobuf: $!\n"; + my $out = ; + close CMD; + + return $out; +} + +sub decode_bytes { + my ($bytes) = @_; + + my $c = sub { return chr oct(shift) }; + + $bytes =~ s/\\(\d{3})/$c->($1)/eg; + $bytes =~ s/(^\")|(\"$)//g; + $bytes =~ s/\\\\/\\/g; + $bytes =~ s/\\r/\r/g; + $bytes =~ s/\\n/\n/g; + $bytes =~ s/\\t/\t/g; + $bytes =~ s/\\"/\"/g; + $bytes =~ s/\\'/\'/g; + + return unpack("H*", unpack("a*", $bytes)); +} + +sub to_hash { + my ($textdata) = @_; + + my %out = (); + push my @stack, \%out; + my ($attr_count, $span_count) = (0, 0); + for my $line (split /\n/, $textdata) { + $line =~ s/(^\s+)|(\s+$)//g; + if ($line =~ /\:/) { + my ($k, $v) = split /\: /, $line; + $v = decode_bytes($v) if ($k =~ /trace_id|span_id|parent_span_id/); + $stack[$#stack]{$k} = $v; + } elsif ($line =~ /\{/) { + $line =~ s/\s\{//; + $line = 'attribute' . $attr_count++ if ($line eq 'attributes'); + if ($line eq 'spans') { + $line = 'span' . $span_count++; + $attr_count = 0; + } + my %new = (); + $stack[$#stack]{$line} = \%new; + push @stack, \%new; + } elsif ($line =~ /\}/) { + pop @stack; + } + } + + return \%out; +} + +sub grpc { + my ($server, $client, $f, $s, $c, $sid, $csid, $uri); + + $server = IO::Socket::INET->new( + Proto => 'tcp', + LocalHost => '127.0.0.1', + LocalPort => $p, + Listen => 5, + Reuse => 1 + ) or die "Can't create listening socket: $!\n"; + + $f->{http_start} = sub { + if (IO::Select->new($server)->can_read(5)) { + $client = $server->accept(); + } else { + # connection could be unexpectedly reused + goto reused if $client; + return undef; + } + + $client->sysread($_, 24) == 24 or return; # preface + + $c = Test::Nginx::HTTP2->new(1, socket => $client, + pure => 1, preface => "") or return; + +reused: + my $frames = $c->read(all => [{ fin => 1 }]); + + $client->close(); + + return $frames; + }; + + return $f; +} + +############################################################################### diff --git a/tests/otel_collector.t b/tests/otel_collector.t new file mode 100644 index 0000000..158a2fd --- /dev/null +++ b/tests/otel_collector.t @@ -0,0 +1,403 @@ +#!/usr/bin/perl + +# (C) Nginx, Inc. + +# Tests for OTel exporter in case HTTP using otelcol. + +############################################################################### + +use warnings; +use strict; + +use Test::More; + +BEGIN { use FindBin; chdir($FindBin::Bin); } + +use Test::Nginx; + +############################################################################### + +select STDERR; $| = 1; +select STDOUT; $| = 1; + +plan(skip_all => "depends on logs content") unless $ENV{TEST_NGINX_UNSAFE}; + +eval { require JSON::PP; }; +plan(skip_all => "JSON::PP not installed") if $@; + +my $t = Test::Nginx->new()->has(qw/http http_ssl rewrite/) + ->write_file_expand('nginx.conf', <<'EOF'); + +%%TEST_GLOBALS%% + +daemon off; + +events { +} + +http { + %%TEST_GLOBALS_HTTP%% + + ssl_certificate_key localhost.key; + ssl_certificate localhost.crt; + + otel_exporter { + endpoint 127.0.0.1:%%PORT_4317%%; + interval 1s; + batch_size 10; + batch_count 2; + } + + otel_service_name test_server; + otel_trace on; + + server { + listen 127.0.0.1:8080; + listen 127.0.0.1:8081 ssl; + server_name localhost; + + location /trace-on { + otel_trace_context extract; + otel_span_name default_location; + otel_span_attr http.request.header.completion + $request_completion; + otel_span_attr http.response.header.content.type + $sent_http_content_type; + otel_span_attr http.request $request; + add_header "X-Otel-Trace-Id" $otel_trace_id; + add_header "X-Otel-Span-Id" $otel_span_id; + add_header "X-Otel-Parent-Id" $otel_parent_id; + add_header "X-Otel-Parent-Sampled" $otel_parent_sampled; + return 200 "TRACE-ON"; + } + + location /context-ignore { + otel_trace_context ignore; + otel_span_name context_ignore; + add_header "X-Otel-Parent-Id" $otel_parent_id; + proxy_pass http://localhost:8080/trace-off; + } + + location /context-extract { + otel_trace_context extract; + otel_span_name context_extract; + add_header "X-Otel-Parent-Id" $otel_parent_id; + proxy_pass http://localhost:8080/trace-off; + } + + location /context-inject { + otel_trace_context inject; + otel_span_name context_inject; + add_header "X-Otel-Parent-Id" $otel_parent_id; + proxy_pass http://localhost:8080/trace-off; + } + + location /context-propagate { + otel_trace_context propagate; + otel_span_name context_propogate; + add_header "X-Otel-Parent-Id" $otel_parent_id; + proxy_pass http://localhost:8080/trace-off; + } + + location /trace-off { + otel_trace off; + add_header "X-Otel-Traceparent" $http_traceparent; + add_header "X-Otel-Tracestate" $http_tracestate; + return 200 "TRACE-OFF"; + } + } +} + +EOF + +$t->write_file_expand('otel-config.yaml', <testdir() }/otel.json + +service: + pipelines: + traces: + receivers: [otlp] + exporters: [logging, file] + metrics: + receivers: [otlp] + exporters: [logging, file] + +EOF + +$t->write_file('openssl.conf', <<'EOF'); +[ req ] +default_bits = 2048 +encrypt_key = no +distinguished_name = req_distinguished_name +[ req_distinguished_name ] + +EOF + +my $d = $t->testdir(); + +foreach my $name ('localhost') { + system('openssl req -x509 -new ' + . "-config $d/openssl.conf -subj /CN=$name/ " + . "-out $d/$name.crt -keyout $d/$name.key " + . ">>$d/openssl.out 2>&1") == 0 + or die "Can't create certificate for $name: $!\n"; +} + +#suppress otel collector output +open OLDERR, ">&", \*STDERR; +open STDERR, ">>" , $^O eq 'MSWin32' ? 'nul' : '/dev/null'; +$t->run_daemon('../otelcol', '--config', $t->testdir().'/otel-config.yaml'); +open STDERR, ">&", \*OLDERR; +$t->waitforsocket('127.0.0.1:' . port(4317)) or + die 'No otel collector open socket'; + +$t->try_run('no OTel module')->plan(69); + +############################################################################### + +#do requests +my $t_off_resp = http1_get('/trace-off'); + +#batch0 (10 requests) +my $tp_resp = http1_get('/trace-on', trace_headers => 1); +my $t_resp = http1_get('/trace-on', port => 8081, ssl => 1); + +my $t_resp_ignore = http1_get('/context-ignore'); +my $tp_resp_ignore = http1_get('/context-ignore', trace_headers => 1); +my $t_resp_extract = http1_get('/context-extract'); +my $tp_resp_extract = http1_get('/context-extract', trace_headers => 1); +my $t_resp_inject = http1_get('/context-inject'); +my $tp_resp_inject = http1_get('/context-inject', trace_headers => 1); +my $t_resp_propagate = http1_get('/context-propagate'); +my $tp_resp_propagate = http1_get('/context-propagate', trace_headers => 1); + +#batch1 (5 reqeusts) +http1_get('/trace-on') for (1..5); + +#waiting batch1 is sent to collector for 1s +select undef, undef, undef, 1; + +my @batches = split /\n/, $t->read_file('otel.json'); +my $batch_json = JSON::PP::decode_json($batches[0]); +my $spans = $$batch_json{"resourceSpans"}[0]{"scopeSpans"}[0]{"spans"}; + +#validate responses +like($tp_resp, qr/TRACE-ON/, 'http request1 - trace on'); +like($t_resp, qr/TRACE-ON/, 'http request2 - trace on'); +like($t_off_resp, qr/TRACE-OFF/, 'http request - trace off'); + +#validate amount of batches +is(scalar @batches, 2, 'amount of batches - trace on'); + +#validate batch size +is(scalar @{$spans}, 10, 'batch0 size - trace on'); +is(scalar @{${JSON::PP::decode_json($batches[1])}{"resourceSpans"}[0] + {"scopeSpans"}[0]{"spans"}}, 5, 'batch1 size - trace on'); + +#validate general attributes +is(get_attr("service.name", "stringValue", + $$batch_json{resourceSpans}[0]{resource}), + 'test_server', 'service.name - trace on'); +is($$spans[0]{name}, 'default_location', 'span.name - trace on'); + +#validate http metrics +is(get_attr("http.method", "stringValue", $$spans[0]), 'GET', + 'http.method metric - trace on'); +is(get_attr("http.target", "stringValue", $$spans[0]), '/trace-on', + 'http.target metric - trace on'); +is(get_attr("http.route", "stringValue", $$spans[0]), '/trace-on', + 'http.route metric - trace on'); +is(get_attr("http.scheme", "stringValue", $$spans[0]), 'http', + 'http.scheme metric - trace on'); +is(get_attr("http.flavor", "stringValue", $$spans[0]), '1.0', + 'http.flavor metric - trace on'); +is(get_attr("http.user_agent", "stringValue", $$spans[0]), 'nginx-tests', + 'http.user_agent metric - trace on'); +is(get_attr("http.request_content_length", "intValue", $$spans[0]), 0, + 'http.request_content_length metric - trace on'); +is(get_attr("http.response_content_length", "intValue", $$spans[0]), 8, + 'http.response_content_length metric - trace on'); +is(get_attr("http.status_code", "intValue", $$spans[0]), 200, + 'http.status_code metric - trace on'); +is(get_attr("net.host.name", "stringValue", $$spans[0]), 'localhost', + 'net.host.name metric - trace on'); +is(get_attr("net.host.port", "intValue", $$spans[0]), 8080, + 'net.host.port metric - trace on'); +is(get_attr("net.sock.peer.addr", "stringValue", $$spans[0]), '127.0.0.1', + 'net.sock.peer.addr metric - trace on'); +like(get_attr("net.sock.peer.port", "intValue", $$spans[0]), qr/\d+/, + 'net.sock.peer.port metric - trace on'); + +#validate custom http metrics +is(${get_attr("http.request.header.completion", "arrayValue", $$spans[0])} + {values}[0]{stringValue}, 'OK', + 'http.request.header.completion metric - trace on'); +is(${get_attr("http.response.header.content.type", "arrayValue",$$spans[0])} + {values}[0]{stringValue}, 'text/plain', + 'http.response.header.content.type metric - trace on'); +is(get_attr("http.request", "stringValue", $$spans[0]), + 'GET /trace-on HTTP/1.0', 'http.request metric - trace on'); + +#validate https metrics +is(get_attr("http.method", "stringValue", $$spans[1]), 'GET', + 'http.method metric - trace on (https)'); +is(get_attr("http.target", "stringValue", $$spans[1]), '/trace-on', + 'http.target metric - trace on (https)'); +is(get_attr("http.route", "stringValue", $$spans[1]), '/trace-on', + 'http.route metric - trace on (https)'); +is(get_attr("http.scheme", "stringValue", $$spans[1]), 'https', + 'http.scheme metric - trace on (https)'); +is(get_attr("http.flavor", "stringValue", $$spans[1]), '1.0', + 'http.flavor metric - trace on (https)'); +is(get_attr("http.user_agent", "stringValue", $$spans[1]), 'nginx-tests', + 'http.user_agent metric - trace on (https)'); +is(get_attr("http.request_content_length", "intValue", $$spans[1]), 0, + 'http.request_content_length metric - trace on (https)'); +is(get_attr("http.response_content_length", "intValue", $$spans[1]), 8, + 'http.response_content_length metric - trace on (https)'); +is(get_attr("http.status_code", "intValue", $$spans[1]), 200, + 'http.status_code metric - trace on (https)'); +is(get_attr("net.host.name", "stringValue", $$spans[1]), 'localhost', + 'net.host.name metric - trace on (https)'); +is(get_attr("net.host.port", "intValue", $$spans[1]), 8081, + 'net.host.port metric - trace on (https)'); +is(get_attr("net.sock.peer.addr", "stringValue", $$spans[1]), '127.0.0.1', + 'net.sock.peer.addr metric - trace on (https)'); +like(get_attr("net.sock.peer.port", "intValue", $$spans[1]), qr/\d+/, + 'net.sock.peer.port metric - trace on (https)'); + +#extract trace info +is($$spans[0]{parentSpanId}, 'b9c7c989f97918e1', 'traceparent - trace on'); +is($$spans[0]{traceState}, 'congo=ucfJifl5GOE,rojo=00f067aa0ba902b7', + 'tracestate - trace on'); +is($$spans[1]{parentSpanId}, '', 'no traceparent - trace on'); +is($$spans[1]{traceState}, undef, 'no tracestate - trace on'); + +#variables +like($tp_resp, qr/X-Otel-Trace-Id: $$spans[0]{traceId}/, + '$otel_trace_id variable - trace on'); +like($tp_resp, qr/X-Otel-Span-Id: $$spans[0]{spanId}/, + '$otel_span_id variable - trace on'); +like($tp_resp, qr/X-Otel-Parent-Id: $$spans[0]{parentSpanId}/, + '$otel_parent_id variable - trace on'); +like($tp_resp, qr/X-Otel-Parent-Sampled: 1/, + '$otel_parent_sampled variable - trace on'); +like($t_resp, qr/X-Otel-Parent-Sampled: 0/, + '$otel_parent_sampled variable - trace on (no traceparent header)'); + +#trace off +unlike($batches[0].$batches[1], + qr/\Q{"key":"http.target","value":{"stringValue":"\/trace-off"}}\E/, + 'no metrics - trace off'); + +#trace context: ignore +unlike($t_resp_ignore, qr/X-Otel-Traceparent/, + 'no traceparent - trace context ignore (no trace headers)'); +unlike($t_resp_ignore, qr/X-Otel-Tracestate/, + 'no tracestate - trace context ignore (no trace headers)'); + +unlike($tp_resp_ignore, qr/X-Otel-Parent-Id/, + 'no parent span id - trace context ignore (trace headers)'); +like($tp_resp_ignore, + qr/Traceparent: 00-0af7651916cd43dd8448eb211c80319c-b9c7c989f97918e1-01/, + 'traceparent - trace context ignore (trace headers)'); +like($tp_resp_ignore, + qr/Tracestate: congo=ucfJifl5GOE,rojo=00f067aa0ba902b7/, + 'tracestate - trace context ignore (trace headers)'); + +#trace context: extract +unlike($t_resp_extract, qr/X-Otel-Traceparent/, + 'no traceparent - trace context extract (no trace headers)'); +unlike($t_resp_extract, qr/X-Otel-Tracestate/, + 'no tracestate - trace context extract (no trace headers)'); + +like($tp_resp_extract, qr/X-Otel-Parent-Id: b9c7c989f97918e1/, + 'parent span id - trace context extract (trace headers)'); +like($tp_resp_extract, + qr/Traceparent: 00-0af7651916cd43dd8448eb211c80319c-b9c7c989f97918e1-01/, + 'traceparent - trace context extract (trace headers)'); +like($tp_resp_extract, + qr/Tracestate: congo=ucfJifl5GOE,rojo=00f067aa0ba902b7/, + 'tracestate - trace context extract (trace headers)'); + +#trace context: inject +like($t_resp_inject, + qr/X-Otel-Traceparent: 00-$$spans[6]{traceId}-$$spans[6]{spanId}-01/, + 'traceparent - trace context inject (no trace headers)'); +unlike($t_resp_inject, qr/X-Otel-Tracestate/, + 'no tracestate - trace context inject (no trace headers)'); + +unlike($tp_resp_inject, qr/X-Otel-Parent-Id/, + 'no parent span id - trace context inject (trace headers)'); +like($tp_resp_inject, + qr/Traceparent: 00-$$spans[7]{traceId}-$$spans[7]{spanId}-01/, + 'traceparent - trace context inject (trace headers)'); +unlike($tp_resp_inject, qr/Tracestate:/, + 'no tracestate - trace context inject (trace headers)'); + +#trace context: propagate +like($t_resp_propagate, + qr/X-Otel-Traceparent: 00-$$spans[8]{traceId}-$$spans[8]{spanId}-01/, + 'traceparent - trace context propagate (no trace headers)'); +unlike($t_resp_propagate, qr/X-Otel-Tracestate/, + 'no tracestate - trace context propagate (no trace headers)'); + +like($tp_resp_propagate, qr/X-Otel-Parent-Id: b9c7c989f97918e1/, + 'parent id - trace context propagate (trace headers)'); +like($tp_resp_propagate, + qr/Traceparent: 00-0af7651916cd43dd8448eb211c80319c-$$spans[9]{spanId}-01/, + 'traceparent - trace context propagate (trace headers)'); +like($tp_resp_propagate, + qr/Tracestate: congo=ucfJifl5GOE,rojo=00f067aa0ba902b7/, + 'tracestate - trace context propagate (trace headers)'); + +$t->stop(); +my $log = $t->read_file("error.log"); + +unlike($log, qr/OTel\/grpc: Error parsing metadata: error=invalid value/, + 'log: no error parsing metadata'); +unlike($log, qr/OTel export failure: No status received/, + 'log: no export failure'); + +############################################################################### + +sub http1_get { + my ($path, %extra) = @_; + + my $port = $extra{port} || 8080; + + my $r = < '127.0.0.1:' . port($port), + SSL => $extra{ssl}); +} + +sub get_attr { + my($attr, $type, $obj) = @_; + + my ($res) = grep { $$_{"key"} eq $attr } @{$$obj{"attributes"}}; + + return defined $res ? $res->{"value"}{$type} : undef; +} + +############################################################################### diff --git a/tests/requirements.txt b/tests/requirements.txt deleted file mode 100644 index d27c6b1..0000000 --- a/tests/requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ -pytest~=8.3 -jinja2~=3.1 -pyopenssl~=24.3 -niquests~=3.11 -grpcio~=1.68 -opentelemetry-proto~=1.28 diff --git a/tests/test_otel.py b/tests/test_otel.py deleted file mode 100644 index fef771a..0000000 --- a/tests/test_otel.py +++ /dev/null @@ -1,331 +0,0 @@ -from collections import namedtuple -import niquests -import pytest -import socket -import time -import urllib3 - - -NGINX_CONFIG = """ -{{ globals }} - -daemon off; - -events { -} - -http { - {{ http_globals }} - - ssl_certificate localhost.crt; - ssl_certificate_key localhost.key; - - otel_exporter { - endpoint {{ endpoint or "127.0.0.1:14317" }}; - interval {{ interval or "1ms" }}; - batch_size 3; - batch_count 3; - - {{ exporter_opts }} - } - - otel_trace on; - {{ resource_attrs }} - - server { - listen 127.0.0.1:18443 ssl; - listen 127.0.0.1:18443 quic; - listen 127.0.0.1:18080; - - http2 on; - - server_name localhost; - - location /ok { - return 200 "OK"; - } - - location /err { - return 500 "ERR"; - } - - location /custom { - otel_span_name custom_location; - otel_span_attr http.request.completion - $request_completion; - otel_span_attr http.response.header.content.type - $sent_http_content_type; - otel_span_attr http.request $request; - return 200 "OK"; - } - - location /vars { - otel_trace_context extract; - add_header "X-Otel-Trace-Id" $otel_trace_id; - add_header "X-Otel-Span-Id" $otel_span_id; - add_header "X-Otel-Parent-Id" $otel_parent_id; - add_header "X-Otel-Parent-Sampled" $otel_parent_sampled; - return 204; - } - - location /ignore { - proxy_pass http://127.0.0.1:18080/notrace; - } - - location /extract { - otel_trace_context extract; - proxy_pass http://127.0.0.1:18080/notrace; - } - - location /inject { - otel_trace_context inject; - proxy_pass http://127.0.0.1:18080/notrace; - } - - location /propagate { - otel_trace_context propagate; - proxy_pass http://127.0.0.1:18080/notrace; - } - - location /notrace { - otel_trace off; - add_header "X-Otel-Traceparent" $http_traceparent; - add_header "X-Otel-Tracestate" $http_tracestate; - return 204; - } - } -} - -""" - -TraceContext = namedtuple("TraceContext", ["trace_id", "span_id", "state"]) - -parent_ctx = TraceContext( - trace_id="0af7651916cd43dd8448eb211c80319c", - span_id="b9c7c989f97918e1", - state="congo=ucfJifl5GOE,rojo=00f067aa0ba902b7", -) - - -def trace_headers(ctx): - return ( - { - "Traceparent": f"00-{ctx.trace_id}-{ctx.span_id}-01", - "Tracestate": ctx.state, - } - if ctx - else {"Traceparent": None, "Tracestate": None} - ) - - -def get_attr(span, name): - for value in (a.value for a in span.attributes if a.key == name): - return getattr(value, value.WhichOneof("value")) - - -@pytest.fixture -def client(nginx): - urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) - with niquests.Session(multiplexed=True) as s: - yield s - - -def test_http09(trace_service, nginx): - - def get_http09(host, port, path): - with socket.create_connection((host, port)) as sock: - sock.sendall(f"GET {path}\n".encode()) - resp = sock.recv(1024).decode("utf-8") - return resp - - assert get_http09("127.0.0.1", 18080, "/ok") == "OK" - - span = trace_service.get_span() - assert span.name == "/ok" - - -@pytest.mark.parametrize("http_ver", ["1.1", "2.0", "3.0"]) -@pytest.mark.parametrize( - ("path", "status"), - [("/ok", 200), ("/err", 500)], -) -def test_default_attributes(client, trace_service, http_ver, path, status): - scheme, port = ("http", 18080) if http_ver == "1.1" else ("https", 18443) - if http_ver == "3.0": - client.quic_cache_layer.add_domain("127.0.0.1", port) - r = client.get(f"{scheme}://127.0.0.1:{port}{path}", verify=False) - - span = trace_service.get_span() - assert span.name == path - - assert get_attr(span, "http.method") == "GET" - assert get_attr(span, "http.target") == path - assert get_attr(span, "http.route") == path - assert get_attr(span, "http.scheme") == scheme - assert get_attr(span, "http.flavor") == http_ver - assert get_attr(span, "http.user_agent") == ( - f"niquests/{niquests.__version__}" - ) - assert get_attr(span, "http.request_content_length") == 0 - assert get_attr(span, "http.response_content_length") == len(r.text) - assert get_attr(span, "http.status_code") == status - assert get_attr(span, "net.host.name") == "localhost" - assert get_attr(span, "net.host.port") == port - assert get_attr(span, "net.sock.peer.addr") == "127.0.0.1" - assert get_attr(span, "net.sock.peer.port") in range(1024, 65536) - - -def test_custom_attributes(client, trace_service): - assert client.get("http://127.0.0.1:18080/custom").status_code == 200 - - span = trace_service.get_span() - assert span.name == "custom_location" - - assert get_attr(span, "http.request.completion") == "OK" - value = get_attr(span, "http.response.header.content.type") - assert value.values[0].string_value == "text/plain" - assert get_attr(span, "http.request") == "GET /custom HTTP/1.1" - - -def test_trace_off(client, trace_service): - assert client.get("http://127.0.0.1:18080/notrace").status_code == 204 - - time.sleep(0.01) # wait for spans - assert len(trace_service.batches) == 0 - - -@pytest.mark.parametrize("parent", [None, parent_ctx]) -def test_variables(client, trace_service, parent): - r = client.get("http://127.0.0.1:18080/vars", headers=trace_headers(parent)) - - span = trace_service.get_span() - - if parent: - assert span.trace_id.hex() == parent.trace_id - assert span.parent_span_id.hex() == parent.span_id - assert span.trace_state == parent.state - - assert r.headers.get("X-Otel-Trace-Id") == span.trace_id.hex() - assert r.headers.get("X-Otel-Span-Id") == span.span_id.hex() - assert r.headers.get("X-Otel-Parent-Id") or "" == span.parent_span_id.hex() - assert r.headers.get("X-Otel-Parent-Sampled") == ("1" if parent else "0") - - -@pytest.mark.parametrize("parent", [None, parent_ctx]) -@pytest.mark.parametrize( - "path", ["/ignore", "/extract", "/inject", "/propagate"] -) -def test_context(client, trace_service, parent, path): - headers = trace_headers(parent) - - r = client.get(f"http://127.0.0.1:18080{path}", headers=headers) - - span = trace_service.get_span() - - if path in ["/extract", "/propagate"] and parent: - assert span.trace_id.hex() == parent.trace_id - assert span.parent_span_id.hex() == parent.span_id - assert span.trace_state == parent.state - - if path in ["/inject", "/propagate"]: - headers = trace_headers( - TraceContext( - span.trace_id.hex(), - span.span_id.hex(), - span.trace_state or None, - ) - ) - - assert r.headers.get("X-Otel-Traceparent") == headers["Traceparent"] - assert r.headers.get("X-Otel-Tracestate") == headers["Tracestate"] - - -@pytest.mark.parametrize( - "nginx_config", - [{"interval": "200ms", "endpoint": "http://127.0.0.1:14317"}], - indirect=True, -) -@pytest.mark.parametrize("batch_count", [1, 3]) -def test_batches(client, trace_service, batch_count): - batch_size = 3 - - for _ in range( - batch_count * batch_size + 1 - ): # +1 request to trigger batch sending - assert client.get("http://127.0.0.1:18080/ok").status_code == 200 - - time.sleep(0.01) - - assert len(trace_service.batches) == batch_count - - for batch in trace_service.batches: - assert ( - get_attr(batch[0].resource, "service.name") - == "unknown_service:nginx" - ) - assert len(batch[0].scope_spans[0].spans) == batch_size - - time.sleep(0.3) # wait for +1 request to be flushed - trace_service.batches.clear() - - -@pytest.mark.parametrize( - "nginx_config", - [ - { - "resource_attrs": """ - otel_service_name "test_service"; - otel_resource_attr my.name "my name"; - otel_resource_attr my.service "my service"; - """, - } - ], - indirect=True, -) -def test_custom_resource_attributes(client, trace_service): - assert client.get("http://127.0.0.1:18080/ok").status_code == 200 - - batch = trace_service.get_batch() - - assert get_attr(batch.resource, "service.name") == "test_service" - assert get_attr(batch.resource, "my.name") == "my name" - assert get_attr(batch.resource, "my.service") == "my service" - - -@pytest.mark.parametrize( - "nginx_config", - [ - { - "exporter_opts": """ - header X-API-TOKEN api.value; - header Authorization "Basic value"; - """, - } - ], - indirect=True, -) -@pytest.mark.parametrize("trace_service", ["skip_otelcol"], indirect=True) -def test_exporter_headers(client, trace_service): - assert client.get("http://127.0.0.1:18080/ok").status_code == 200 - - assert trace_service.get_span().name == "/ok" - - headers = dict(trace_service.last_metadata) - assert headers["x-api-token"] == "api.value" - assert headers["authorization"] == "Basic value" - - -@pytest.mark.parametrize( - "nginx_config", - [ - { - "endpoint": "https://localhost:14318", - "exporter_opts": "trusted_certificate localhost.crt;", - } - ], - indirect=True, -) -def test_tls_export(client, trace_service): - assert client.get("http://127.0.0.1:18080/ok").status_code == 200 - - assert trace_service.get_span().name == "/ok" diff --git a/tests/trace_service.py b/tests/trace_service.py deleted file mode 100644 index 5ef2bc6..0000000 --- a/tests/trace_service.py +++ /dev/null @@ -1,107 +0,0 @@ -import concurrent -import grpc -from opentelemetry.proto.collector.trace.v1 import trace_service_pb2 -from opentelemetry.proto.collector.trace.v1 import trace_service_pb2_grpc -import pytest -import subprocess -import time - - -class TraceService(trace_service_pb2_grpc.TraceServiceServicer): - batches = [] - - def Export(self, request, context): - self.batches.append(request.resource_spans) - self.last_metadata = context.invocation_metadata() - return trace_service_pb2.ExportTracePartialSuccess() - - def get_batch(self): - for _ in range(10): - if len(self.batches): - break - time.sleep(0.001) - assert len(self.batches) == 1 - assert len(self.batches[0]) == 1 - return self.batches.pop()[0] - - def get_span(self): - batch = self.get_batch() - assert len(batch.scope_spans) == 1 - assert len(batch.scope_spans[0].spans) == 1 - return batch.scope_spans[0].spans.pop() - - -@pytest.fixture(scope="module") -def trace_service(request, pytestconfig, logger, cert): - server = grpc.server(concurrent.futures.ThreadPoolExecutor()) - trace_service = TraceService() - trace_service_pb2_grpc.add_TraceServiceServicer_to_server( - trace_service, server - ) - trace_service.use_otelcol = ( - pytestconfig.option.otelcol - and getattr(request, "param", "") != "skip_otelcol" - ) - listen_addr = f"127.0.0.1:{24317 if trace_service.use_otelcol else 14317}" - server.add_insecure_port(listen_addr) - if not trace_service.use_otelcol: - creds = grpc.ssl_server_credentials([cert]) - server.add_secure_port("127.0.0.1:14318", creds) - listen_addr += " and 127.0.0.1:14318" - logger.info(f"Starting trace service at {listen_addr}...") - server.start() - yield trace_service - logger.info("Stopping trace service...") - server.stop(grace=None) - - -@pytest.fixture(scope="module") -def otelcol(pytestconfig, testdir, logger, trace_service, cert): - if not trace_service.use_otelcol: - yield - return - - (testdir / "otel-config.yaml").write_text( - f"""receivers: - otlp: - protocols: - grpc: - endpoint: 127.0.0.1:14317 - - otlp/tls: - protocols: - grpc: - endpoint: 127.0.0.1:14318 - tls: - cert_file: {testdir}/localhost.crt - key_file: {testdir}/localhost.key - -exporters: - otlp: - endpoint: 127.0.0.1:24317 - tls: - insecure: true - -service: - pipelines: - traces: - receivers: [otlp, otlp/tls] - exporters: [otlp] - telemetry: - metrics: - # prevent otelcol from opening 8888 port - level: none""" - ) - logger.info("Starting otelcol at 127.0.0.1:14317...") - proc = subprocess.Popen( - [pytestconfig.option.otelcol, "--config", testdir / "otel-config.yaml"] - ) - time.sleep(1) # give some time to get ready - assert proc.poll() is None, "Can't start otelcol" - yield - logger.info("Stopping otelcol...") - proc.terminate() - try: - proc.wait(timeout=5) - except subprocess.TimeoutExpired: - proc.kill()