Fix build against Nginx 1.22 (fix #85).

Co-authored-by: Pavel Pautov <p.pautov@f5.com>

.github/workflows/nginx-otel-module-check.yml

@@ -1,85 +0,0 @@
-name: nginx-otel-module-check
-run-name: ${{ github.actor }} is triggering pipeline
-on:
-  push:
-  workflow_dispatch:
-
-jobs:
-  build-module:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-      - name: Install dependencies
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y cmake libc-ares-dev libre2-dev
-      - name: Checkout nginx
-        run: hg clone http://hg.nginx.org/nginx/
-      - name: Configure nginx
-        working-directory: nginx
-        run: auto/configure --with-compat
-      - name: Create build directory
-        run: mkdir build
-      - name: Build module
-        working-directory: build
-        run: |
-          cmake -DNGX_OTEL_NGINX_BUILD_DIR=${PWD}/../nginx/objs ..
-          make -j 4
-          strip ngx_otel_module.so
-      - name: Archive module
-        uses: actions/upload-artifact@v4
-        with:
-          name: nginx-otel-module
-          path: build/ngx_otel_module.so
-      - name: Archive protoc and opentelemetry-proto
-        uses: actions/upload-artifact@v4
-        with:
-          name: protoc-opentelemetry-proto
-          path: |
-            build/_deps/grpc-build/third_party/protobuf/protoc
-            build/_deps/otelcpp-src/third_party/opentelemetry-proto
-  test-module:
-    needs: build-module
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-      - name: Download module
-        uses: actions/download-artifact@v4
-        with:
-          name: nginx-otel-module
-          path: build
-      - name: Download protoc and opentelemetry-proto
-        uses: actions/download-artifact@v4
-        with:
-          name: protoc-opentelemetry-proto
-          path: build/_deps
-      - name: List files
-        run: ls -laR .
-      - name: Fix protoc file permissions
-        run: chmod +x build/_deps/grpc-build/third_party/protobuf/protoc
-      - name: Install perl modules
-        run: sudo cpan IO::Socket::SSL Crypt::Misc
-      - name: Download otelcol
-        run: |
-          curl -LO https://github.com/\
-          open-telemetry/opentelemetry-collector-releases/releases/download/\
-          v0.76.1/otelcol_0.76.1_linux_amd64.tar.gz
-          tar -xzf otelcol_0.76.1_linux_amd64.tar.gz
-      - name: Checkout nginx and nginx-test
-        run: |
-          hg clone http://hg.nginx.org/nginx/
-          hg clone http://hg.nginx.org/nginx-tests/
-      - name: Build nginx
-        working-directory: nginx
-        run: |
-          auto/configure --with-compat --with-debug --with-http_ssl_module \
-          --with-http_v2_module --with-http_v3_module
-          make -j 4
-      - name: Run tests
-        working-directory: tests
-        run: |
-          PERL5LIB=../nginx-tests/lib TEST_NGINX_UNSAFE=1 \
-          TEST_NGINX_VERBOSE=1 TEST_NGINX_GLOBALS="load_module \
-          ${PWD}/../build/ngx_otel_module.so;" prove -v .

.github/workflows/ubuntu.yml

@@ -0,0 +1,49 @@
+name: Ubuntu build
+
+on:
+  push:
+    branches:
+    - main
+  pull_request:
+
+jobs:
+  build-module:
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Install dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y cmake libc-ares-dev
+      - name: Checkout nginx
+        uses: actions/checkout@v4
+        with:
+          repository: nginx/nginx
+          path: nginx
+      - name: Build nginx
+        working-directory: nginx
+        run: |
+          auto/configure --with-compat --with-debug --with-http_ssl_module \
+                         --with-http_v2_module --with-http_v3_module
+          make -j $(nproc)
+      - name: Build module
+        run: |
+          mkdir build
+          cd build
+          cmake -DNGX_OTEL_NGINX_BUILD_DIR=${PWD}/../nginx/objs \
+                -DNGX_OTEL_DEV=ON ..
+          make -j $(nproc)
+      - name: Download otelcol
+        run: |
+          LATEST=open-telemetry/opentelemetry-collector-releases/releases/latest
+          TAG=$(curl -s https://api.github.com/repos/${LATEST} |
+                jq -r .tag_name)
+          curl -sLo - https://github.com/${LATEST}/download/\
+          otelcol_${TAG:1}_linux_amd64.tar.gz | tar -xzv
+      - name: Install test dependencies
+        run: pip install -r tests/requirements.txt
+      - name: Run tests
+        run: |
+          pytest tests --maxfail=10 --nginx=nginx/objs/nginx \
+                       --module=build/ngx_otel_module.so --otelcol=./otelcol

CMakeLists.txt

@@ -6,8 +6,12 @@ set(NGX_OTEL_NGINX_BUILD_DIR ""
 set(NGX_OTEL_NGINX_DIR "${NGX_OTEL_NGINX_BUILD_DIR}/.."
     CACHE PATH "Nginx source dir")
 
-set(NGX_OTEL_FETCH_DEPS ON CACHE BOOL "Download dependencies")
+set(NGX_OTEL_GRPC e241f37befe7ba4688effd84bfbf99b0f681a2f7 # v1.49.4
+    CACHE STRING "gRPC tag to download or 'package' to use preinstalled")
+set(NGX_OTEL_SDK  11d5d9e0d8fd8ba876c8994714cc2647479b6574 # v1.11.0
+    CACHE STRING "OTel SDK tag to download or 'package' to use preinstalled")
 set(NGX_OTEL_PROTO_DIR  ""  CACHE PATH "OTel proto files root")
+set(NGX_OTEL_DEV        OFF CACHE BOOL "Enforce compiler warnings")
 
 if(NOT CMAKE_BUILD_TYPE)
     set(CMAKE_BUILD_TYPE RelWithDebInfo)
@@ -15,27 +19,52 @@ endif()
 
 set(CMAKE_CXX_VISIBILITY_PRESET hidden)
 
-if(NGX_OTEL_FETCH_DEPS)
+if(NGX_OTEL_GRPC STREQUAL "package")
+    find_package(protobuf REQUIRED)
+    find_package(gRPC REQUIRED)
+else()
     include(FetchContent)
 
     FetchContent_Declare(
         grpc
         GIT_REPOSITORY https://github.com/grpc/grpc
-        GIT_TAG        02384e39185f109bd299eb8482306229967dc970 # v1.46.7
+        GIT_TAG        ${NGX_OTEL_GRPC}
-        GIT_SUBMODULES third_party/protobuf third_party/abseil-cpp
+        GIT_SUBMODULES third_party/protobuf third_party/abseil-cpp third_party/re2
         GIT_SHALLOW    ON)
 
     set(gRPC_USE_PROTO_LITE ON  CACHE INTERNAL "")
     set(gRPC_INSTALL        OFF CACHE INTERNAL "")
+    set(gRPC_USE_SYSTEMD    OFF CACHE INTERNAL "")
+    set(gRPC_DOWNLOAD_ARCHIVES OFF CACHE INTERNAL "")
     set(gRPC_CARES_PROVIDER package CACHE INTERNAL "")
-    set(gRPC_RE2_PROVIDER   package CACHE INTERNAL "")
     set(gRPC_SSL_PROVIDER   package CACHE INTERNAL "")
     set(gRPC_ZLIB_PROVIDER  package CACHE INTERNAL "")
 
+    set(protobuf_INSTALL OFF CACHE INTERNAL "")
+
+    set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+
+    FetchContent_MakeAvailable(grpc)
+
+    # reconsider once https://github.com/grpc/grpc/issues/36023 is done
+    target_compile_definitions(grpc PRIVATE GRPC_NO_XDS GRPC_NO_RLS)
+
+    set_property(DIRECTORY ${grpc_SOURCE_DIR}
+                 PROPERTY EXCLUDE_FROM_ALL YES)
+
+    add_library(gRPC::grpc++ ALIAS grpc++)
+    add_executable(gRPC::grpc_cpp_plugin ALIAS grpc_cpp_plugin)
+endif()
+
+if(NGX_OTEL_SDK STREQUAL "package")
+    find_package(opentelemetry-cpp REQUIRED)
+else()
+    include(FetchContent)
+
     FetchContent_Declare(
         otelcpp
         GIT_REPOSITORY https://github.com/open-telemetry/opentelemetry-cpp
-        GIT_TAG        11d5d9e0d8fd8ba876c8994714cc2647479b6574 # v1.11.0
+        GIT_TAG        ${NGX_OTEL_SDK}
         GIT_SUBMODULES third_party/opentelemetry-proto
         GIT_SHALLOW    ON)
 
@@ -45,10 +74,8 @@ if(NGX_OTEL_FETCH_DEPS)
     set(CMAKE_POSITION_INDEPENDENT_CODE ON)
     set(CMAKE_POLICY_DEFAULT_CMP0063 NEW)
 
-    FetchContent_MakeAvailable(grpc otelcpp)
+    FetchContent_MakeAvailable(otelcpp)
 
-    set_property(DIRECTORY ${grpc_SOURCE_DIR}
-                 PROPERTY EXCLUDE_FROM_ALL YES)
     set_property(DIRECTORY ${otelcpp_SOURCE_DIR}
                  PROPERTY EXCLUDE_FROM_ALL YES)
 
@@ -58,12 +85,6 @@ if(NGX_OTEL_FETCH_DEPS)
     endif()
 
     add_library(opentelemetry-cpp::trace ALIAS opentelemetry_trace)
-    add_library(gRPC::grpc++ ALIAS grpc++)
-    add_executable(gRPC::grpc_cpp_plugin ALIAS grpc_cpp_plugin)
-else()
-    find_package(opentelemetry-cpp REQUIRED)
-    find_package(protobuf REQUIRED)
-    find_package(gRPC REQUIRED)
 endif()
 
 set(PROTO_DIR ${NGX_OTEL_PROTO_DIR})
@@ -91,19 +112,22 @@ add_custom_command(
         --plugin protoc-gen-grpc=$<TARGET_FILE:gRPC::grpc_cpp_plugin>
         ${PROTOS}
     # remove inconsequential UTF8 check during serialization to aid performance
-    COMMAND sed -i.bak
+    COMMAND sed -i.bak -E
-        -e [[/    ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String(/,/);/d]]
+        -e [[/    ::(PROTOBUF_NAMESPACE_ID|google::protobuf)::internal::WireFormatLite::VerifyUtf8String\(/,/\);/d]]
         ${PROTO_SOURCES}
     DEPENDS ${PROTOS} protobuf::protoc gRPC::grpc_cpp_plugin
     VERBATIM)
 
-set(CMAKE_CXX_STANDARD 11)
+if (NGX_OTEL_DEV)
-set(CMAKE_CXX_EXTENSIONS OFF)
+    set(CMAKE_CXX_STANDARD 11)
+    set(CMAKE_CXX_EXTENSIONS OFF)
 
-add_compile_options(-Wall -Wtype-limits -Werror)
+    add_compile_options(-Wall -Wtype-limits -Werror)
+endif()
 
 add_library(ngx_otel_module MODULE
     src/http_module.cpp
+    src/grpc_log.cpp
     src/modules.c
     ${PROTO_SOURCES})
 
@@ -113,6 +137,10 @@ set_target_properties(ngx_otel_module PROPERTIES PREFIX "")
 # can't use OTel's WITH_ABSEIL until cmake 3.24, as it triggers find_package()
 target_compile_definitions(ngx_otel_module PRIVATE HAVE_ABSEIL)
 
+if (APPLE)
+    target_link_options(ngx_otel_module PRIVATE -undefined dynamic_lookup)
+endif()
+
 target_include_directories(ngx_otel_module PRIVATE
     ${NGX_OTEL_NGINX_BUILD_DIR}
     ${NGX_OTEL_NGINX_DIR}/src/core

config

@@ -1,10 +1,9 @@
 ngx_addon_name=ngx_otel_module
 
 cmake -D NGX_OTEL_NGINX_BUILD_DIR=$NGX_OBJS \
-      -D NGX_OTEL_FETCH_DEPS=OFF \
-      -D NGX_OTEL_PROTO_DIR=$NGX_OTEL_PROTO_DIR \
       -D CMAKE_LIBRARY_OUTPUT_DIRECTORY=$PWD/$NGX_OBJS \
       -D "CMAKE_C_FLAGS=$NGX_CC_OPT" \
       -D "CMAKE_CXX_FLAGS=$NGX_CC_OPT" \
       -D "CMAKE_MODULE_LINKER_FLAGS=$NGX_LD_OPT" \
+      $NGX_OTEL_CMAKE_OPTS \
       -S $ngx_addon_dir -B $NGX_OBJS/otel || exit 1

src/batch_exporter.hpp

@@ -111,18 +111,21 @@ public:
         int attrSize{0};
     };
 
-    BatchExporter(StrView target,
+    BatchExporter(const Target& target,
-            size_t batchSize, size_t batchCount, StrView serviceName) :
+            size_t batchSize, size_t batchCount,
-        batchSize(batchSize), client(std::string(target))
+            const std::map<StrView, StrView>& resourceAttrs) :
+        batchSize(batchSize), client(target)
     {
         free.reserve(batchCount);
         while (batchCount-- > 0) {
             free.emplace_back();
             auto resourceSpans = free.back().add_resource_spans();
 
-            auto attr = resourceSpans->mutable_resource()->add_attributes();
+            for (auto& attr : resourceAttrs) {
-            attr->set_key("service.name");
+                auto kv = resourceSpans->mutable_resource()->add_attributes();
-            attr->mutable_value()->set_string_value(std::string(serviceName));
+                kv->set_key(std::string(attr.first));
+                kv->mutable_value()->set_string_value(std::string(attr.second));
+            }
 
             auto scopeSpans = resourceSpans->add_scope_spans();
             scopeSpans->mutable_scope()->set_name("nginx");

src/grpc_log.cpp

@@ -0,0 +1,107 @@
+#include "ngx.hpp"
+
+#include "grpc_log.hpp"
+
+#include <google/protobuf/stubs/common.h>
+#include <grpcpp/grpcpp.h>
+
+#if GOOGLE_PROTOBUF_VERSION < 4022000
+
+#include <google/protobuf/stubs/logging.h>
+
+class ProtobufLog {
+public:
+    ProtobufLog() { google::protobuf::SetLogHandler(protobufLogHandler); }
+    ~ProtobufLog() { google::protobuf::SetLogHandler(NULL); }
+
+private:
+    static void protobufLogHandler(google::protobuf::LogLevel logLevel,
+        const char* filename, int line, const std::string& msg)
+    {
+        using namespace google::protobuf;
+
+        ngx_uint_t level = logLevel == LOGLEVEL_FATAL   ? NGX_LOG_EMERG :
+                           logLevel == LOGLEVEL_ERROR   ? NGX_LOG_ERR :
+                           logLevel == LOGLEVEL_WARNING ? NGX_LOG_WARN :
+                                     /*LOGLEVEL_INFO*/    NGX_LOG_INFO;
+
+        ngx_log_error(level, ngx_cycle->log, 0, "OTel/protobuf: %s",
+            msg.c_str());
+    }
+};
+
+#else
+
+#include <absl/log/globals.h>
+#include <absl/log/initialize.h>
+#include <absl/log/log_sink_registry.h>
+
+class NgxLogSink : absl::LogSink {
+public:
+    NgxLogSink()
+    {
+        absl::InitializeLog();
+        absl::AddLogSink(this);
+        // Disable logging to stderr
+        absl::SetStderrThreshold(static_cast<absl::LogSeverity>(100));
+    }
+
+    ~NgxLogSink() override { absl::RemoveLogSink(this); }
+
+    void Send(const absl::LogEntry& entry) override
+    {
+        auto severity = entry.log_severity();
+
+        ngx_uint_t level =
+            severity == absl::LogSeverity::kFatal   ? NGX_LOG_EMERG :
+            severity == absl::LogSeverity::kError   ? NGX_LOG_ERR :
+            severity == absl::LogSeverity::kWarning ? NGX_LOG_WARN :
+                      /*absl::LogSeverity::kInfo*/    NGX_LOG_INFO;
+
+        ngx_str_t message { entry.text_message().size(),
+                            (u_char*)entry.text_message().data() };
+
+        ngx_log_error(level, ngx_cycle->log, 0, "OTel/grpc: %V", &message);
+    }
+};
+
+typedef NgxLogSink ProtobufLog;
+
+#endif
+
+#if (GRPC_CPP_VERSION_MAJOR < 1) || \
+    (GRPC_CPP_VERSION_MAJOR == 1 && GRPC_CPP_VERSION_MINOR < 65)
+
+#include <grpc/support/log.h>
+
+class GrpcLog {
+public:
+    GrpcLog() { gpr_set_log_function(grpcLogHandler); }
+    ~GrpcLog() { gpr_set_log_function(NULL); }
+
+private:
+    static void grpcLogHandler(gpr_log_func_args* args)
+    {
+        ngx_uint_t level =
+            args->severity == GPR_LOG_SEVERITY_ERROR ? NGX_LOG_ERR :
+            args->severity == GPR_LOG_SEVERITY_INFO  ? NGX_LOG_INFO :
+                            /*GPR_LOG_SEVERITY_DEBUG*/ NGX_LOG_DEBUG;
+
+        ngx_log_error(level, ngx_cycle->log, 0, "OTel/grpc: %s",
+            args->message);
+    }
+
+    ProtobufLog protoLog;
+};
+
+#else
+
+// newer gRPC implies newer protobuf, and both use Abseil for logging
+typedef NgxLogSink GrpcLog;
+
+#endif
+
+void initGrpcLog()
+{
+    static GrpcLog init;
+}

src/grpc_log.hpp

@@ -0,0 +1,3 @@
+#pragma once
+
+void initGrpcLog();

src/http_module.cpp

@@ -1,16 +1,13 @@
-extern "C" {
+#include "ngx.hpp"
-#include <ngx_config.h>
-#include <ngx_core.h>
-#include <ngx_http.h>
-}
 
-#include <grpc/support/log.h>
+#include "grpc_log.hpp"
-#include <google/protobuf/stubs/logging.h>
 
 #include "str_view.hpp"
 #include "trace_context.hpp"
 #include "batch_exporter.hpp"
 
+#include <fstream>
+
 extern ngx_module_t gHttpModule;
 
 namespace {
@@ -20,7 +17,7 @@ struct OtelCtx {
     TraceContext current;
 };
 
-struct MainConf {
+struct MainConfBase {
     ngx_str_t endpoint;
     ngx_msec_t interval;
     size_t batchSize;
@@ -29,6 +26,13 @@ struct MainConf {
     ngx_str_t serviceName;
 };
 
+struct MainConf : MainConfBase {
+    std::map<StrView, StrView> resourceAttrs;
+    bool ssl;
+    std::string trustedCert;
+    Target::HeaderVec headers;
+};
+
 struct SpanAttr {
     ngx_str_t name;
     ngx_http_complex_value_t value;
@@ -43,7 +47,10 @@ struct LocationConf {
 };
 
 char* setExporter(ngx_conf_t* cf, ngx_command_t* cmd, void* conf);
+char* addResourceAttr(ngx_conf_t* cf, ngx_command_t* cmd, void* conf);
 char* addSpanAttr(ngx_conf_t* cf, ngx_command_t* cmd, void* conf);
+char* setTrustedCertificate(ngx_conf_t* cf, ngx_command_t* cmd, void* conf);
+char* addExporterHeader(ngx_conf_t* cf, ngx_command_t* cmd, void* conf);
 
 namespace Propagation {
 
@@ -64,14 +71,17 @@ ngx_command_t gCommands[] = {
 
     { ngx_string("otel_exporter"),
       NGX_HTTP_MAIN_CONF|NGX_CONF_BLOCK|NGX_CONF_NOARGS,
-      setExporter,
+      setExporter },
-      NGX_HTTP_MAIN_CONF_OFFSET },
+
+    { ngx_string("otel_resource_attr"),
+      NGX_HTTP_MAIN_CONF|NGX_CONF_TAKE2,
+      addResourceAttr },
 
     { ngx_string("otel_service_name"),
       NGX_HTTP_MAIN_CONF|NGX_CONF_TAKE1,
       ngx_conf_set_str_slot,
       NGX_HTTP_MAIN_CONF_OFFSET,
-      offsetof(MainConf, serviceName) },
+      offsetof(MainConfBase, serviceName) },
 
     { ngx_string("otel_trace"),
       NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1,
@@ -106,25 +116,33 @@ ngx_command_t gExporterCommands[] = {
       NGX_CONF_TAKE1,
       ngx_conf_set_str_slot,
       0,
-      offsetof(MainConf, endpoint) },
+      offsetof(MainConfBase, endpoint) },
+
+    { ngx_string("trusted_certificate"),
+      NGX_CONF_TAKE1,
+      setTrustedCertificate },
+
+    { ngx_string("header"),
+      NGX_CONF_TAKE2,
+      addExporterHeader },
 
     { ngx_string("interval"),
       NGX_CONF_TAKE1,
       ngx_conf_set_msec_slot,
       0,
-      offsetof(MainConf, interval) },
+      offsetof(MainConfBase, interval) },
 
     { ngx_string("batch_size"),
       NGX_CONF_TAKE1,
       ngx_conf_set_size_slot,
       0,
-      offsetof(MainConf, batchSize) },
+      offsetof(MainConfBase, batchSize) },
 
     { ngx_string("batch_count"),
       NGX_CONF_TAKE1,
       ngx_conf_set_size_slot,
       0,
-      offsetof(MainConf, batchCount) },
+      offsetof(MainConfBase, batchCount) },
 
       ngx_null_command
 };
@@ -141,6 +159,30 @@ ngx_str_t toNgxStr(StrView str)
     return ngx_str_t{str.size(), (u_char*)str.data()};
 }
 
+bool iremovePrefix(ngx_str_t* str, StrView p)
+{
+    if (str->len >= p.size() &&
+            ngx_strncasecmp(str->data, (u_char*)p.data(), p.size()) == 0) {
+        str->data += p.size();
+        str->len -= p.size();
+        return true;
+    }
+
+    return false;
+}
+
+MainConf* getMainConf(ngx_conf_t* cf)
+{
+    return static_cast<MainConf*>(
+        (MainConfBase*)ngx_http_conf_get_module_main_conf(cf, gHttpModule));
+}
+
+MainConf* getMainConf(ngx_cycle_t* cycle)
+{
+    return static_cast<MainConf*>(
+        (MainConfBase*)ngx_http_cycle_get_module_main_conf(cycle, gHttpModule));
+}
+
 LocationConf* getLocationConf(ngx_http_request_t* r)
 {
     return (LocationConf*)ngx_http_get_module_loc_conf(r, gHttpModule);
@@ -254,10 +296,10 @@ ngx_int_t setHeader(ngx_http_request_t* r, StrView name, StrView value)
             return NGX_ERROR;
         }
 
+        *header = {};
         header->hash = hash;
         header->key = toNgxStr(name);
         header->lowcase_key = header->key.data;
-        header->next = NULL;
     }
 
     header->value = toNgxStr(value);
@@ -504,28 +546,6 @@ ngx_int_t onRequestEnd(ngx_http_request_t* r)
     return NGX_DECLINED;
 }
 
-void grpcLogHandler(gpr_log_func_args* args)
-{
-    ngx_uint_t level = args->severity == GPR_LOG_SEVERITY_ERROR ? NGX_LOG_ERR :
-                       args->severity == GPR_LOG_SEVERITY_INFO  ? NGX_LOG_INFO :
-                                       /*GPR_LOG_SEVERITY_DEBUG*/ NGX_LOG_DEBUG;
-
-    ngx_log_error(level, ngx_cycle->log, 0, "OTel/grpc: %s", args->message);
-}
-
-void protobufLogHandler(google::protobuf::LogLevel logLevel,
-    const char* filename, int line, const std::string& msg)
-{
-    using namespace google::protobuf;
-
-    ngx_uint_t level = logLevel == LOGLEVEL_FATAL   ? NGX_LOG_EMERG :
-                       logLevel == LOGLEVEL_ERROR   ? NGX_LOG_ERR :
-                       logLevel == LOGLEVEL_WARNING ? NGX_LOG_WARN :
-                                 /*LOGLEVEL_INFO*/    NGX_LOG_INFO;
-
-    ngx_log_error(level, ngx_cycle->log, 0, "OTel/protobuf: %s", msg.c_str());
-}
-
 ngx_int_t initModule(ngx_conf_t* cf)
 {
     auto cmcf = (ngx_http_core_main_conf_t*)ngx_http_conf_get_module_main_conf(
@@ -547,16 +567,14 @@ ngx_int_t initModule(ngx_conf_t* cf)
 
     *h = onRequestEnd;
 
-    gpr_set_log_function(grpcLogHandler);
+    initGrpcLog();
-    google::protobuf::SetLogHandler(protobufLogHandler);
 
     return NGX_OK;
 }
 
 ngx_int_t initWorkerProcess(ngx_cycle_t* cycle)
 {
-    auto mcf = (MainConf*)ngx_http_cycle_get_module_main_conf(
+    auto mcf = getMainConf(cycle);
-        cycle, gHttpModule);
 
     // no 'http' or 'otel_exporter' blocks
     if (mcf == NULL || mcf->endpoint.len == 0) {
@@ -564,11 +582,17 @@ ngx_int_t initWorkerProcess(ngx_cycle_t* cycle)
     }
 
     try {
+        Target target;
+        target.endpoint = std::string(toStrView(mcf->endpoint));
+        target.ssl = mcf->ssl;
+        target.trustedCert = mcf->trustedCert;
+        target.headers = mcf->headers;
+
         gExporter.reset(new BatchExporter(
-            toStrView(mcf->endpoint),
+            target,
             mcf->batchSize,
             mcf->batchCount,
-            toStrView(mcf->serviceName)));
+            mcf->resourceAttrs));
     } catch (const std::exception& e) {
         ngx_log_error(NGX_LOG_CRIT, cycle->log, 0,
             "OTel worker init error: %s", e.what());
@@ -589,8 +613,7 @@ ngx_int_t initWorkerProcess(ngx_cycle_t* cycle)
                 "OTel flush error: %s", e.what());
         }
 
-        auto mcf = (MainConf*)ngx_http_cycle_get_module_main_conf(
+        auto mcf = getMainConf((ngx_cycle_t*)ngx_cycle);
-            ngx_cycle, gHttpModule);
 
         ngx_add_timer(ev, mcf->interval);
     };
@@ -618,7 +641,7 @@ void exitWorkerProcess(ngx_cycle_t* cycle)
 
 char* setExporter(ngx_conf_t* cf, ngx_command_t* cmd, void* conf)
 {
-    auto mcf = (MainConf*)conf;
+    auto mcf = getMainConf(cf);
 
     if (mcf->endpoint.len) {
         return (char*)"is duplicate";
@@ -635,7 +658,7 @@ char* setExporter(ngx_conf_t* cf, ngx_command_t* cmd, void* conf)
                 continue;
             }
 
-            if (cf->args->nelts != 2) {
+            if (cf->args->nelts != static_cast<unsigned>(ffs(cmd->type))) {
                 ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
                     "invalid number of arguments in \"%V\" "
                     "directive of \"otel_exporter\"", name);
@@ -668,6 +691,12 @@ char* setExporter(ngx_conf_t* cf, ngx_command_t* cmd, void* conf)
         return rv;
     }
 
+    if (iremovePrefix(&mcf->endpoint, "https://")) {
+        mcf->ssl = true;
+    } else {
+        iremovePrefix(&mcf->endpoint, "http://");
+    }
+
     if (mcf->endpoint.len == 0) {
         ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
             "\"otel_exporter\" requires \"endpoint\"");
@@ -677,31 +706,124 @@ char* setExporter(ngx_conf_t* cf, ngx_command_t* cmd, void* conf)
     return NGX_CONF_OK;
 }
 
+char* addResourceAttr(ngx_conf_t* cf, ngx_command_t* cmd, void* conf)
+{
+    auto mcf = getMainConf(cf);
+
+    try {
+        auto args = (ngx_str_t*)cf->args->elts;
+        mcf->resourceAttrs[toStrView(args[1])] = toStrView(args[2]);
+    } catch (const std::exception& e) {
+        ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "OTel: %s", e.what());
+        return (char*)NGX_CONF_ERROR;
+    }
+
+    return NGX_CONF_OK;
+}
+
+char* setTrustedCertificate(ngx_conf_t* cf, ngx_command_t* cmd, void* conf)
+{
+    auto path = ((ngx_str_t*)cf->args->elts)[1];
+    auto mcf = getMainConf(cf);
+
+    if (ngx_get_full_name(cf->pool, &cf->cycle->conf_prefix, &path) != NGX_OK) {
+        return (char*)NGX_CONF_ERROR;
+    }
+
+    try {
+        std::ifstream file{(const char*)path.data, std::ios::binary};
+        if (!file.is_open()) {
+            ngx_conf_log_error(NGX_LOG_EMERG, cf, ngx_errno,
+                "failed to open \"%V\"", &path);
+            return (char*)NGX_CONF_ERROR;
+        }
+        file.exceptions(std::ios::failbit | std::ios::badbit);
+        file.peek(); // trigger early error for dirs
+
+        size_t size = file.seekg(0, std::ios::end).tellg();
+        file.seekg(0);
+
+        mcf->trustedCert.resize(size);
+        file.read(&mcf->trustedCert[0], size);
+    } catch (const std::exception& e) {
+        ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
+            "failed to read \"%V\": %s", &path, e.what());
+        return (char*)NGX_CONF_ERROR;
+    }
+
+    return NGX_CONF_OK;
+}
+
+char* addExporterHeader(ngx_conf_t* cf, ngx_command_t* cmd, void* conf)
+{
+    auto args = (ngx_str_t*)cf->args->elts;
+
+    // don't force on users lower case name requirement of gRPC
+    ngx_strlow(args[1].data, args[1].data, args[1].len);
+
+    try {
+        // validate header here to avoid runtime assert failure in gRPC
+        auto name = toStrView(args[1]);
+        if (!Target::validateHeaderName(name)) {
+            return (char*)"has invalid header name";
+        }
+        auto value = toStrView(args[2]);
+        if (!Target::validateHeaderValue(value)) {
+            return (char*)"has invalid header value";
+        }
+
+        getMainConf(cf)->headers.emplace_back(name, value);
+    } catch (const std::exception& e) {
+        ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "OTel: %s", e.what());
+        return (char*)NGX_CONF_ERROR;
+    }
+
+    return NGX_CONF_OK;
+}
+
 void* createMainConf(ngx_conf_t* cf)
 {
-    auto mcf = (MainConf*)ngx_pcalloc(cf->pool, sizeof(MainConf));
+    auto cln = ngx_pool_cleanup_add(cf->pool, sizeof(MainConf));
-    if (mcf == NULL) {
+    if (cln == NULL) {
         return NULL;
     }
 
+    MainConf* mcf;
+    try {
+        mcf = new (cln->data) MainConf{};
+    } catch (const std::exception& e) {
+        ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "OTel: %s", e.what());
+        return NULL;
+    }
+
+    cln->handler = [](void* data) {
+        ((MainConf*)data)->~MainConf();
+    };
+
     mcf->interval = NGX_CONF_UNSET_MSEC;
     mcf->batchSize = NGX_CONF_UNSET_SIZE;
     mcf->batchCount = NGX_CONF_UNSET_SIZE;
 
-    return mcf;
+    return static_cast<MainConfBase*>(mcf);
 }
 
 char* initMainConf(ngx_conf_t* cf, void* conf)
 {
-    auto mcf = (MainConf*)conf;
+    auto mcf = getMainConf(cf);
-
 
     ngx_conf_init_msec_value(mcf->interval, 5000);
     ngx_conf_init_size_value(mcf->batchSize, 512);
     ngx_conf_init_size_value(mcf->batchCount, 4);
 
+    try {
         if (mcf->serviceName.data == NULL) {
-        mcf->serviceName = ngx_string("unknown_service:nginx");
+            mcf->resourceAttrs.emplace("service.name", "unknown_service:nginx");
+        } else {
+            mcf->resourceAttrs["service.name"] = toStrView(mcf->serviceName);
+        }
+    } catch (const std::exception& e) {
+        ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "OTel: %s", e.what());
+        return (char*)NGX_CONF_ERROR;
     }
 
     return NGX_CONF_OK;
@@ -839,7 +961,7 @@ char* mergeLocationConf(ngx_conf_t* cf, void* parent, void* child)
         conf->spanAttrs = prev->spanAttrs;
     }
 
-    auto mcf = (MainConf*)ngx_http_conf_get_module_main_conf(cf, gHttpModule);
+    auto mcf = getMainConf(cf);
 
     if (mcf->endpoint.len == 0 && conf->trace) {
         ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,

src/ngx.hpp

@@ -0,0 +1,7 @@
+#pragma once
+
+extern "C" {
+#include <ngx_config.h>
+#include <ngx_core.h>
+#include <ngx_http.h>
+}

src/trace_service_client.hpp

@@ -8,6 +8,27 @@
 
 namespace otel_proto_trace = opentelemetry::proto::collector::trace::v1;
 
+struct Target {
+    typedef std::vector<std::pair<std::string, std::string>> HeaderVec;
+
+    std::string endpoint;
+    bool ssl;
+    std::string trustedCert;
+    HeaderVec headers;
+
+    static bool validateHeaderName(StrView name)
+    {
+        return grpc_header_key_is_legal(
+            grpc_slice_from_static_buffer(name.data(), name.size()));
+    }
+
+    static bool validateHeaderValue(StrView value)
+    {
+        return grpc_header_nonbin_value_is_legal(
+            grpc_slice_from_static_buffer(value.data(), value.size()));
+    }
+};
+
 class TraceServiceClient {
 public:
     typedef otel_proto_trace::ExportTraceServiceRequest Request;
@@ -17,10 +38,18 @@ public:
     typedef std::function<void (Request, Response, grpc::Status)>
         ResponseCb;
 
-    TraceServiceClient(const std::string& target)
+    TraceServiceClient(const Target& target) : headers(target.headers)
     {
-        auto channel = grpc::CreateChannel(
+        std::shared_ptr<grpc::ChannelCredentials> creds;
-            target, grpc::InsecureChannelCredentials());
+        if (target.ssl) {
+            grpc::SslCredentialsOptions options;
+            options.pem_root_certs = target.trustedCert;
+
+            creds = grpc::SslCredentials(options);
+        } else {
+            creds = grpc::InsecureChannelCredentials();
+        }
+        auto channel = grpc::CreateChannel(target.endpoint, creds);
         channel->GetState(true); // trigger 'connecting' state
 
         stub = TraceService::NewStub(channel);
@@ -30,6 +59,10 @@ public:
     {
         std::unique_ptr<ActiveCall> call{new ActiveCall{}};
 
+        for (auto& header : headers) {
+            call->context.AddMetadata(header.first, header.second);
+        }
+
         call->request = std::move(req);
         call->cb = std::move(cb);
 
@@ -99,6 +132,8 @@ private:
         ResponseCb cb;
     };
 
+    Target::HeaderVec headers;
+
     std::unique_ptr<TraceService::Stub> stub;
     grpc::CompletionQueue queue;
 

tests/conftest.py

@@ -0,0 +1,101 @@
+import jinja2
+import logging
+from OpenSSL import crypto
+import os
+import pytest
+import subprocess
+import time
+
+
+pytest_plugins = [
+    "trace_service",
+]
+
+
+def pytest_addoption(parser):
+    parser.addoption("--nginx", required=True)
+    parser.addoption("--module", required=True)
+    parser.addoption("--otelcol")
+    parser.addoption("--globals", default="")
+
+
+def self_signed_cert(name):
+    k = crypto.PKey()
+    k.generate_key(crypto.TYPE_RSA, 2048)
+    cert = crypto.X509()
+    cert.get_subject().CN = name
+    cert.set_issuer(cert.get_subject())
+    cert.gmtime_adj_notBefore(0)
+    cert.gmtime_adj_notAfter(365 * 86400)  # 365 days
+    cert.set_pubkey(k)
+    cert.sign(k, "sha512")
+    return (
+        crypto.dump_privatekey(crypto.FILETYPE_PEM, k),
+        crypto.dump_certificate(crypto.FILETYPE_PEM, cert),
+    )
+
+
+@pytest.fixture(scope="session")
+def logger():
+    logging.basicConfig(level=logging.INFO)
+    return logging.getLogger(__name__)
+
+
+@pytest.fixture(scope="module")
+def testdir(tmp_path_factory):
+    return tmp_path_factory.mktemp("nginx")
+
+
+@pytest.fixture(scope="module")
+def nginx_config(request, pytestconfig, testdir, logger):
+    tmpl = jinja2.Environment().from_string(request.module.NGINX_CONFIG)
+    params = getattr(request, "param", {})
+    params["globals"] = (
+        f"pid {testdir}/nginx.pid;\n"
+        + "error_log stderr info;\n"
+        + f"error_log {testdir}/error.log info;\n"
+        + f"load_module {os.path.abspath(pytestconfig.option.module)};\n"
+        + pytestconfig.option.globals
+    )
+    params["http_globals"] = f"root {testdir};\n" + "access_log off;\n"
+    conf = tmpl.render(params)
+    logger.debug(conf)
+    return conf
+
+
+@pytest.fixture(scope="module")
+def nginx(testdir, pytestconfig, nginx_config, cert, logger, otelcol):
+    (testdir / "nginx.conf").write_text(nginx_config)
+    logger.info("Starting nginx...")
+    proc = subprocess.Popen(
+        [
+            pytestconfig.option.nginx,
+            "-p",
+            str(testdir),
+            "-c",
+            "nginx.conf",
+            "-e",
+            "error.log",
+        ]
+    )
+    logger.debug(f"args={' '.join(proc.args)}")
+    logger.debug(f"pid={proc.pid}")
+    while not (testdir / "nginx.pid").exists():
+        time.sleep(0.1)
+        assert proc.poll() is None, "Can't start nginx"
+    yield proc
+    logger.info("Stopping nginx...")
+    proc.terminate()
+    try:
+        proc.wait(timeout=5)
+    except subprocess.TimeoutExpired:
+        proc.kill()
+    assert "[alert]" not in (testdir / "error.log").read_text()
+
+
+@pytest.fixture(scope="module")
+def cert(testdir):
+    key, cert = self_signed_cert("localhost")
+    (testdir / "localhost.key").write_text(key.decode("utf-8"))
+    (testdir / "localhost.crt").write_text(cert.decode("utf-8"))
+    yield (key, cert)

tests/h2_otel.t

@@ -1,553 +0,0 @@
-#!/usr/bin/perl
-
-# (C) Nginx, Inc.
-
-# Tests for OTel exporter in case HTTP/2.
-
-###############################################################################
-
-use warnings;
-use strict;
-
-use Test::More;
-
-BEGIN { use FindBin; chdir($FindBin::Bin); }
-
-use Test::Nginx;
-use Test::Nginx::HTTP2;
-use MIME::Base64;
-
-###############################################################################
-
-select STDERR; $| = 1;
-select STDOUT; $| = 1;
-
-my $t = Test::Nginx->new()
-	->has(qw/http_v2 http_ssl rewrite mirror grpc socket_ssl_alpn/)
-	->has_daemon(qw/openssl base64/)
-	->write_file_expand('nginx.conf', <<'EOF');
-
-%%TEST_GLOBALS%%
-
-daemon off;
-
-events {
-}
-
-http {
-    %%TEST_GLOBALS_HTTP%%
-
-    ssl_certificate_key localhost.key;
-    ssl_certificate localhost.crt;
-
-    otel_exporter {
-        endpoint 127.0.0.1:8083;
-        interval 1s;
-        batch_size 10;
-        batch_count 1;
-    }
-
-    otel_service_name test_server;
-    otel_trace on;
-
-    server {
-        listen       127.0.0.1:8080 http2;
-        listen       127.0.0.1:8081;
-        listen       127.0.0.1:8082 http2 ssl;
-        server_name  localhost;
-
-        location /trace-on {
-            otel_trace_context extract;
-            otel_span_name default_location;
-            otel_span_attr http.request.header.completion
-                $request_completion;
-            otel_span_attr http.response.header.content.type
-                $sent_http_content_type;
-            otel_span_attr http.request $request;
-            add_header "X-Otel-Trace-Id" $otel_trace_id;
-            add_header "X-Otel-Span-Id" $otel_span_id;
-            add_header "X-Otel-Parent-Id" $otel_parent_id;
-            add_header "X-Otel-Parent-Sampled" $otel_parent_sampled;
-            return 200 "TRACE-ON";
-        }
-
-        location /context-ignore {
-            otel_trace_context ignore;
-            otel_span_name context_ignore;
-            add_header "X-Otel-Parent-Id" $otel_parent_id;
-            proxy_pass http://127.0.0.1:8081/trace-off;
-        }
-
-        location /context-extract {
-            otel_trace_context extract;
-            otel_span_name context_extract;
-            add_header "X-Otel-Parent-Id" $otel_parent_id;
-            proxy_pass http://127.0.0.1:8081/trace-off;
-        }
-
-        location /context-inject {
-            otel_trace_context inject;
-            otel_span_name context_inject;
-            add_header "X-Otel-Parent-Id" $otel_parent_id;
-            proxy_pass http://127.0.0.1:8081/trace-off;
-        }
-
-        location /context-propagate {
-            otel_trace_context propagate;
-            otel_span_name context_propogate;
-            add_header "X-Otel-Parent-Id" $otel_parent_id;
-            proxy_pass http://127.0.0.1:8081/trace-off;
-        }
-
-        location /trace-off {
-            otel_trace off;
-            add_header "X-Otel-Traceparent" $http_traceparent;
-            add_header "X-Otel-Tracestate" $http_tracestate;
-            return 200 "TRACE-OFF";
-        }
-    }
-
-    server {
-        listen       127.0.0.1:8083 http2;
-        server_name  localhost;
-        otel_trace off;
-
-        location / {
-            mirror /mirror;
-            grpc_pass 127.0.0.1:8084;
-        }
-
-        location /mirror {
-            internal;
-            grpc_pass 127.0.0.1:%%PORT_4317%%;
-        }
-    }
-
-    server {
-        listen       127.0.0.1:8084 http2;
-        server_name  localhost;
-        otel_trace off;
-
-        location / {
-            add_header content-type application/grpc;
-            add_header grpc-status 0;
-            add_header grpc-message "";
-            return 200;
-        }
-    }
-
-}
-
-EOF
-
-$t->write_file('openssl.conf', <<'EOF');
-[ req ]
-default_bits = 2048
-encrypt_key = no
-distinguished_name = req_distinguished_name
-[ req_distinguished_name ]
-
-EOF
-
-my $d = $t->testdir();
-
-foreach my $name ('localhost') {
-	system('openssl req -x509 -new '
-		. "-config $d/openssl.conf -subj /CN=$name/ "
-		. "-out $d/$name.crt -keyout $d/$name.key "
-		. ">>$d/openssl.out 2>&1") == 0
-		or die "Can't create certificate for $name: $!\n";
-}
-
-$t->try_run('no OTel module')->plan(69);
-
-###############################################################################
-
-my $p = port(4317);
-my $f = grpc();
-
-#do requests
-(undef, my $t_off_resp) = http2_get('/trace-off');
-
-#batch0 (10 requests)
-my ($tp_headers, $tp_resp) = http2_get('/trace-on', trace_headers => 1);
-my ($t_headers, $t_resp) = http2_get('/trace-on', port => 8082, ssl => 1);
-
-(my $t_headers_ignore, undef) = http2_get('/context-ignore');
-(my $tp_headers_ignore, undef) = http2_get('/context-ignore',
-	trace_headers => 1);
-(my $t_headers_extract, undef) = http2_get('/context-extract');
-(my $tp_headers_extract, undef) = http2_get('/context-extract',
-	trace_headers => 1);
-(my $t_headers_inject, undef) = http2_get('/context-inject');
-(my $tp_headers_inject, undef) = http2_get('/context-inject',
-	trace_headers => 1);
-(my $t_headers_propagate, undef) = http2_get('/context-propagate');
-(my $tp_headers_propagate, undef) =
-	http2_get('/context-propagate', trace_headers => 1);
-
-my ($frame) = grep { $_->{type} eq "DATA" } @{$f->{http_start}()};
-my $batch0 = to_hash(decode_protobuf(substr $frame->{data}, 8));
-my $spans = $$batch0{scope_spans};
-
-#batch1 (5 reqeusts)
-http2_get('/trace-on') for (1..5);
-
-($frame) = grep { $_->{type} eq "DATA" } @{$f->{http_start}()};
-my $batch1 = to_hash(decode_protobuf(substr $frame->{data}, 8));
-
-#validate responses
-like($tp_resp, qr/TRACE-ON/, 'http request1 - trace on');
-like($t_resp, qr/TRACE-ON/, 'http request2 - trace on');
-like($t_off_resp, qr/TRACE-OFF/, 'http request - trace off');
-
-#validate batch size
-delete $$spans{scope};  #remove 'scope' entry
-is(scalar keys %{$spans}, 10, 'batch0 size - trace on');
-delete $$batch1{scope_spans}{scope};  #remove 'scope' entry
-is(scalar keys %{$$batch1{scope_spans}}, 5, 'batch1 size - trace on');
-
-#validate general attributes
-is(get_attr("service.name", "string_value",
-	$$batch0{resource}),
-	'test_server', 'service.name - trace on');
-is($$spans{span0}{name}, '"default_location"', 'span.name - trace on');
-
-#validate http metrics
-is(get_attr("http.method", "string_value", $$spans{span0}), 'GET',
-	'http.method metric - trace on');
-is(get_attr("http.target", "string_value", $$spans{span0}), '/trace-on',
-	'http.target metric - trace on');
-is(get_attr("http.route", "string_value", $$spans{span0}), '/trace-on',
-	'http.route metric - trace on');
-is(get_attr("http.scheme", "string_value", $$spans{span0}), 'http',
-	'http.scheme metric - trace on');
-is(get_attr("http.flavor", "string_value", $$spans{span0}), '2.0',
-	'http.flavor metric - trace on');
-is(get_attr("http.user_agent", "string_value", $$spans{span0}), 'nginx-tests',
-	'http.user_agent metric - trace on');
-is(get_attr("http.request_content_length", "int_value", $$spans{span0}), 0,
-	'http.request_content_length metric - trace on');
-is(get_attr("http.response_content_length", "int_value", $$spans{span0}), 8,
-	'http.response_content_length metric - trace on');
-is(get_attr("http.status_code", "int_value", $$spans{span0}), 200,
-	'http.status_code metric - trace on');
-is(get_attr("net.host.name", "string_value", $$spans{span0}), 'localhost',
-	'net.host.name metric - trace on');
-is(get_attr("net.host.port", "int_value", $$spans{span0}), 8080,
-	'net.host.port metric - trace on');
-is(get_attr("net.sock.peer.addr", "string_value", $$spans{span0}), '127.0.0.1',
-	'net.sock.peer.addr metric - trace on');
-like(get_attr("net.sock.peer.port", "int_value", $$spans{span0}), qr/\d+/,
-	'net.sock.peer.port metric - trace on');
-
-#validate https metrics
-is(get_attr("http.method", "string_value", $$spans{span1}), 'GET',
-	'http.method metric - trace on (https)');
-is(get_attr("http.target", "string_value", $$spans{span1}), '/trace-on',
-	'http.target metric - trace on (https)');
-is(get_attr("http.route", "string_value", $$spans{span1}), '/trace-on',
-	'http.route metric - trace on (https)');
-is(get_attr("http.scheme", "string_value", $$spans{span1}), 'https',
-	'http.scheme metric - trace on (https)');
-is(get_attr("http.flavor", "string_value", $$spans{span1}), '2.0',
-	'http.flavor metric - trace on (https)');
-isnt(get_attr("http.user_agent", "string_value", $$spans{span1}),
-	'nginx-tests', 'http.user_agent metric - trace on (https)');
-is(get_attr("http.request_content_length", "int_value", $$spans{span1}), 0,
-	'http.request_content_length metric - trace on (https)');
-is(get_attr("http.response_content_length", "int_value", $$spans{span1}), 8,
-	'http.response_content_length metric - trace on (https)');
-is(get_attr("http.status_code", "int_value", $$spans{span1}), 200,
-	'http.status_code metric - trace on (https)');
-is(get_attr("net.host.name", "string_value", $$spans{span1}), 'localhost',
-	'net.host.name metric - trace on (https)');
-is(get_attr("net.host.port", "int_value", $$spans{span1}), 8082,
-	'net.host.port metric - trace on (https)');
-is(get_attr("net.sock.peer.addr", "string_value", $$spans{span1}), '127.0.0.1',
-	'net.sock.peer.addr metric - trace on (https)');
-like(get_attr("net.sock.peer.port", "int_value", $$spans{span1}), qr/\d+/,
-	'net.sock.peer.port metric - trace on (https)');
-
-#validate custom http metrics
-is(${get_attr("http.request.header.completion", "array_value", $$spans{span0})}
-	{values}{string_value}, '"OK"',
-	'http.request.header.completion metric - trace on');
-is(${get_attr(
-		"http.response.header.content.type", "array_value", $$spans{span0}
-	)}{values}{string_value}, '"text/plain"',
-	'http.response.header.content.type metric - trace on');
-is(get_attr("http.request", "string_value", $$spans{span0}),
-	'GET /trace-on HTTP/2.0', 'http.request metric - trace on');
-
-#extract trace info
-is($$spans{span0}{parent_span_id}, 'b9c7c989f97918e1',
-	'traceparent - trace on');
-is($$spans{span0}{trace_state}, '"congo=ucfJifl5GOE,rojo=00f067aa0ba902b7"',
-	'tracestate - trace on');
-is($$spans{span1}{parent_span_id}, undef, 'no traceparent - trace on');
-is($$spans{span1}{trace_state}, undef, 'no tracestate - trace on');
-
-#variables
-is($tp_headers->{'x-otel-trace-id'}, $$spans{span0}{trace_id},
-	'$otel_trace_id variable - trace on');
-is($tp_headers->{'x-otel-span-id'}, $$spans{span0}{span_id},
-	'$otel_span_id variable - trace on');
-is($tp_headers->{'x-otel-parent-id'}, $$spans{span0}{parent_span_id},
-	'$otel_parent_id variable - trace on');
-is($tp_headers->{'x-otel-parent-sampled'}, 1,
-	'$otel_parent_sampled variable - trace on');
-is($t_headers->{'x-otel-parent-sampled'}, 0,
-	'$otel_parent_sampled variable - trace on (no traceparent header)');
-
-#trace off
-is((scalar grep {
-		get_attr("http.target", "string_value", $$spans{$_}) eq '/trace-off'
-	} keys %{$spans}), 0, 'no metric in batch0 - trace off');
-is((scalar grep {
-		get_attr("http.target", "string_value", $$spans{$_}) eq '/trace-off'
-	} keys %{$$batch1{scope_spans}}), 0, 'no metric in batch1 - trace off');
-
-#trace context: ignore
-is($t_headers_ignore->{'x-otel-traceparent'}, undef,
-	'no traceparent - trace context ignore (no trace headers)');
-is($t_headers_ignore->{'x-otel-tracestate'}, undef,
-	'no tracestate - trace context ignore (no trace headers)');
-
-is($tp_headers_ignore->{'x-otel-parent-id'}, undef,
-	'no parent span id - trace context ignore (trace headers)');
-is($tp_headers_ignore->{'x-otel-traceparent'},
-	'00-0af7651916cd43dd8448eb211c80319c-b9c7c989f97918e1-01',
-	'traceparent - trace context ignore (trace headers)');
-is($tp_headers_ignore->{'x-otel-tracestate'},
-	'congo=ucfJifl5GOE,rojo=00f067aa0ba902b7',
-	'tracestate - trace context ignore (trace headers)');
-
-#trace context: extract
-is($t_headers_extract->{'x-otel-traceparent'}, undef,
-	'no traceparent - trace context extract (no trace headers)');
-is($t_headers_extract->{'x-otel-tracestate'}, undef,
-	'no tracestate - trace context extract (no trace headers)');
-
-is($tp_headers_extract->{'x-otel-parent-id'}, 'b9c7c989f97918e1',
-	'parent span id - trace context extract (trace headers)');
-is($tp_headers_extract->{'x-otel-traceparent'},
-	'00-0af7651916cd43dd8448eb211c80319c-b9c7c989f97918e1-01',
-	'traceparent - trace context extract (trace headers)');
-is($tp_headers_extract->{'x-otel-tracestate'},
-	'congo=ucfJifl5GOE,rojo=00f067aa0ba902b7',
-	'tracestate - trace context extract (trace headers)');
-
-#trace context: inject
-is($t_headers_inject->{'x-otel-traceparent'},
-	"00-$$spans{span6}{trace_id}-$$spans{span6}{span_id}-01",
-	'traceparent - trace context inject (no trace headers)');
-is($t_headers_inject->{'x-otel-tracestate'}, undef,
-	'no tracestate - trace context inject (no trace headers)');
-
-is($tp_headers_inject->{'x-otel-parent-id'}, undef,
-	'no parent span id - trace context inject (trace headers)');
-is($tp_headers_inject->{'x-otel-traceparent'},
-	"00-$$spans{span7}{trace_id}-$$spans{span7}{span_id}-01",
-	'traceparent - trace context inject (trace headers)');
-is($tp_headers_inject->{'x-otel-tracestate'}, undef,
-	'no tracestate - trace context inject (trace headers)');
-
-#trace context: propagate
-is($t_headers_propagate->{'x-otel-traceparent'},
-	"00-$$spans{span8}{trace_id}-$$spans{span8}{span_id}-01",
-	'traceparent - trace context propagate (no trace headers)');
-is($t_headers_propagate->{'x-otel-tracestate'}, undef,
-	'no tracestate - trace context propagate (no trace headers)');
-
-is($tp_headers_propagate->{'x-otel-parent-id'}, 'b9c7c989f97918e1',
-	'parent id - trace context propagate (trace headers)');
-is($tp_headers_propagate->{'x-otel-traceparent'},
-	"00-0af7651916cd43dd8448eb211c80319c-$$spans{span9}{span_id}-01",
-	'traceparent - trace context propagate (trace headers)');
-is($tp_headers_propagate->{'x-otel-tracestate'},
-	'congo=ucfJifl5GOE,rojo=00f067aa0ba902b7',
-	'tracestate - trace context propagate (trace headers)');
-
-SKIP: {
-skip "depends on error log content", 2 unless $ENV{TEST_NGINX_UNSAFE};
-
-$t->stop();
-my $log = $t->read_file("error.log");
-
-like($log, qr/OTel\/grpc: Error parsing metadata: error=invalid value/,
-	'log: error parsing metadata - no protobuf in response');
-unlike($log, qr/OTel export failure: No status received/,
-	'log: no export failure');
-
-}
-
-###############################################################################
-
-sub http2_get {
-	my ($path, %extra) = @_;
-	my ($frames, $frame);
-
-	my $port = $extra{port} || 8080;
-
-	my $s = $extra{ssl}
-		? Test::Nginx::HTTP2->new(
-			undef, socket => get_ssl_socket($port, ['h2']))
-		: Test::Nginx::HTTP2->new();
-
-	my $sid = $extra{trace_headers}
-		? $s->new_stream({ headers => [
-			{ name => ':method', value => 'GET' },
-			{ name => ':scheme', value => 'http' },
-			{ name => ':path', value => $path },
-			{ name => ':authority', value => 'localhost' },
-			{ name => 'user-agent', value => 'nginx-tests', mode => 2 },
-			{ name => 'traceparent',
-				value => '00-0af7651916cd43dd8448eb211c80319c-' .
-					'b9c7c989f97918e1-01',
-				mode => 2
-			},
-			{ name => 'tracestate',
-				value => 'congo=ucfJifl5GOE,rojo=00f067aa0ba902b7',
-				mode => 2
-			}]})
-		: $s->new_stream({ path => $path });
-	$frames = $s->read(all => [{ sid => $sid, fin => 1 }]);
-
-	($frame) = grep { $_->{type} eq "HEADERS" } @$frames;
-	my $headers = $frame->{headers};
-
-	($frame) = grep { $_->{type} eq "DATA" } @$frames;
-	my $data = $frame->{data};
-
-	return $headers, $data;
-}
-
-sub get_ssl_socket {
-	my ($port, $alpn) = @_;
-
-	return http(
-		'', PeerAddr => '127.0.0.1:' . port($port), start => 1,
-		SSL => 1,
-		SSL_alpn_protocols => $alpn,
-		SSL_error_trap => sub { die $_[1] }
-	);
-}
-
-sub get_attr {
-	my($attr, $type, $obj) = @_;
-
-	my ($res) = grep {
-			$_ =~ /^attribute\d+/ && $$obj{$_}{key} eq '"' . $attr . '"'
-		} keys %{$obj};
-
-	if (defined $res) {
-		$$obj{$res}{value}{$type} =~ s/(^\")|(\"$)//g
-			if $type eq 'string_value';
-
-		return $$obj{$res}{value}{$type};
-	}
-
-	return undef;
-}
-
-sub decode_protobuf {
-	my ($protobuf) = @_;
-
-	local $/;
-	open CMD, "echo '" . encode_base64($protobuf) . "' | base64 -d | " .
-		'$PWD/../build/_deps/grpc-build/third_party/protobuf/protoc '.
-		'--decode opentelemetry.proto.trace.v1.ResourceSpans -I ' .
-		'$PWD/../build/_deps/otelcpp-src/third_party/opentelemetry-proto ' .
-		'opentelemetry/proto/collector/trace/v1/trace_service.proto |'
-		or die "Can't decode protobuf: $!\n";
-	my $out = <CMD>;
-	close CMD;
-
-	return $out;
-}
-
-sub decode_bytes {
-	my ($bytes) = @_;
-
-	my $c = sub { return chr oct(shift) };
-
-	$bytes =~ s/\\(\d{3})/$c->($1)/eg;
-	$bytes =~ s/(^\")|(\"$)//g;
-	$bytes =~ s/\\\\/\\/g;
-	$bytes =~ s/\\r/\r/g;
-	$bytes =~ s/\\n/\n/g;
-	$bytes =~ s/\\t/\t/g;
-	$bytes =~ s/\\"/\"/g;
-	$bytes =~ s/\\'/\'/g;
-
-	return unpack("H*", unpack("a*", $bytes));
-}
-
-sub to_hash {
-	my ($textdata) = @_;
-
-	my %out = ();
-	push my @stack, \%out;
-	my ($attr_count, $span_count) = (0, 0);
-	for my $line (split /\n/, $textdata) {
-		$line =~ s/(^\s+)|(\s+$)//g;
-		if ($line =~ /\:/) {
-			my ($k, $v) = split /\: /, $line;
-			$v = decode_bytes($v) if ($k =~ /trace_id|span_id|parent_span_id/);
-			$stack[$#stack]{$k} = $v;
-		} elsif ($line =~ /\{/) {
-			$line =~ s/\s\{//;
-			$line = 'attribute' . $attr_count++ if ($line eq 'attributes');
-			if ($line eq 'spans') {
-				$line = 'span' . $span_count++;
-				$attr_count = 0;
-			}
-			my %new = ();
-			$stack[$#stack]{$line} = \%new;
-			push @stack, \%new;
-		} elsif ($line =~ /\}/) {
-			pop @stack;
-		}
-	}
-
-	return \%out;
-}
-
-sub grpc {
-	my ($server, $client, $f, $s, $c, $sid, $csid, $uri);
-
-	$server = IO::Socket::INET->new(
-		Proto => 'tcp',
-		LocalHost => '127.0.0.1',
-		LocalPort => $p,
-		Listen => 5,
-		Reuse => 1
-	) or die "Can't create listening socket: $!\n";
-
-	$f->{http_start} = sub {
-		if (IO::Select->new($server)->can_read(5)) {
-			$client = $server->accept();
-		} else {
-			# connection could be unexpectedly reused
-			goto reused if $client;
-			return undef;
-		}
-
-		$client->sysread($_, 24) == 24 or return; # preface
-
-		$c = Test::Nginx::HTTP2->new(1, socket => $client,
-			pure => 1, preface => "") or return;
-
-reused:
-		my $frames = $c->read(all => [{ fin => 1 }]);
-
-		$client->close();
-
-		return $frames;
-	};
-
-	return $f;
-}
-
-###############################################################################

tests/h3_otel.t

@@ -1,508 +0,0 @@
-#!/usr/bin/perl
-
-# (C) Nginx, Inc.
-
-# Tests for OTel exporter in case HTTP/3.
-
-###############################################################################
-
-use warnings;
-use strict;
-
-use Test::More;
-
-BEGIN { use FindBin; chdir($FindBin::Bin); }
-
-use Test::Nginx;
-use Test::Nginx::HTTP2;
-use Test::Nginx::HTTP3;
-use MIME::Base64;
-
-###############################################################################
-
-select STDERR; $| = 1;
-select STDOUT; $| = 1;
-
-my $t = Test::Nginx->new()->has(qw/http_v2 http_v3 rewrite mirror grpc cryptx/)
-	->has_daemon(qw/openssl base64/)
-	->write_file_expand('nginx.conf', <<'EOF');
-
-%%TEST_GLOBALS%%
-
-daemon off;
-
-events {
-}
-
-http {
-    %%TEST_GLOBALS_HTTP%%
-
-    ssl_certificate_key localhost.key;
-    ssl_certificate localhost.crt;
-    ssl_protocols TLSv1.3;
-
-    otel_exporter {
-        endpoint 127.0.0.1:8082;
-        interval 1s;
-        batch_size 10;
-        batch_count 2;
-    }
-
-    otel_service_name test_server;
-    otel_trace on;
-
-    server {
-        listen       127.0.0.1:%%PORT_8980_UDP%% quic;
-        listen       127.0.0.1:8081;
-        server_name  localhost;
-
-        location /trace-on {
-            otel_trace_context extract;
-            otel_span_name default_location;
-            otel_span_attr http.request.header.completion
-                $request_completion;
-            otel_span_attr http.response.header.content.type
-                $sent_http_content_type;
-            otel_span_attr http.request $request;
-            add_header "X-Otel-Trace-Id" $otel_trace_id;
-            add_header "X-Otel-Span-Id" $otel_span_id;
-            add_header "X-Otel-Parent-Id" $otel_parent_id;
-            add_header "X-Otel-Parent-Sampled" $otel_parent_sampled;
-            return 200 "TRACE-ON";
-        }
-
-        location /context-ignore {
-            otel_trace_context ignore;
-            otel_span_name context_ignore;
-            add_header "X-Otel-Parent-Id" $otel_parent_id;
-            proxy_pass http://127.0.0.1:8081/trace-off;
-        }
-
-        location /context-extract {
-            otel_trace_context extract;
-            otel_span_name context_extract;
-            add_header "X-Otel-Parent-Id" $otel_parent_id;
-            proxy_pass http://127.0.0.1:8081/trace-off;
-        }
-
-        location /context-inject {
-            otel_trace_context inject;
-            otel_span_name context_inject;
-            add_header "X-Otel-Parent-Id" $otel_parent_id;
-            proxy_pass http://127.0.0.1:8081/trace-off;
-        }
-
-        location /context-propagate {
-            otel_trace_context propagate;
-            otel_span_name context_propogate;
-            add_header "X-Otel-Parent-Id" $otel_parent_id;
-            proxy_pass http://127.0.0.1:8081/trace-off;
-        }
-
-        location /trace-off {
-            otel_trace off;
-            add_header "X-Otel-Traceparent" $http_traceparent;
-            add_header "X-Otel-Tracestate" $http_tracestate;
-            return 200 "TRACE-OFF";
-        }
-    }
-
-    server {
-        listen       127.0.0.1:8082 http2;
-        server_name  localhost;
-        otel_trace off;
-
-        location / {
-            mirror /mirror;
-            grpc_pass 127.0.0.1:8083;
-        }
-
-        location /mirror {
-            internal;
-            grpc_pass 127.0.0.1:%%PORT_4317%%;
-        }
-    }
-
-    server {
-        listen       127.0.0.1:8083 http2;
-        server_name  localhost;
-        otel_trace off;
-
-        location / {
-            add_header content-type application/grpc;
-            add_header grpc-status 0;
-            add_header grpc-message "";
-            return 200;
-        }
-    }
-
-}
-
-EOF
-
-$t->write_file('openssl.conf', <<'EOF');
-[ req ]
-default_bits = 2048
-encrypt_key = no
-distinguished_name = req_distinguished_name
-[ req_distinguished_name ]
-
-EOF
-
-my $d = $t->testdir();
-
-foreach my $name ('localhost') {
-	system('openssl req -x509 -new '
-		. "-config $d/openssl.conf -subj /CN=$name/ "
-		. "-out $d/$name.crt -keyout $d/$name.key "
-		. ">>$d/openssl.out 2>&1") == 0
-		or die "Can't create certificate for $name: $!\n";
-}
-
-$t->try_run('no OTel module')->plan(56);
-
-###############################################################################
-
-my $p = port(4317);
-my $f = grpc();
-
-#do requests
-(undef, my $t_off_resp) = http3_get('/trace-off');
-
-#batch0 (10 requests)
-my ($tp_headers, $tp_resp) = http3_get('/trace-on', trace_headers => 1);
-my ($t_headers, $t_resp) = http3_get('/trace-on');
-
-(my $t_headers_ignore, undef) = http3_get('/context-ignore');
-(my $tp_headers_ignore, undef) = http3_get('/context-ignore',
-	trace_headers => 1);
-(my $t_headers_extract, undef) = http3_get('/context-extract');
-(my $tp_headers_extract, undef) = http3_get('/context-extract',
-	trace_headers => 1);
-(my $t_headers_inject, undef) = http3_get('/context-inject');
-(my $tp_headers_inject, undef) = http3_get('/context-inject',
-	trace_headers => 1);
-(my $t_headers_propagate, undef) = http3_get('/context-propagate');
-(my $tp_headers_propagate, undef) =
-	http3_get('/context-propagate', trace_headers => 1);
-
-my ($frame) = grep { $_->{type} eq "DATA" } @{$f->{http_start}()};
-my $batch0 = to_hash(decode_protobuf(substr $frame->{data}, 8));
-my $spans = $$batch0{scope_spans};
-
-#batch1 (5 reqeusts)
-http3_get('/trace-on') for (1..5);
-
-($frame) = grep { $_->{type} eq "DATA" } @{$f->{http_start}()};
-my $batch1 = to_hash(decode_protobuf(substr $frame->{data}, 8));
-
-#validate responses
-like($tp_resp, qr/TRACE-ON/, 'http request1 - trace on');
-like($t_resp, qr/TRACE-ON/, 'http request2 - trace on');
-like($t_off_resp, qr/TRACE-OFF/, 'http request - trace off');
-
-#validate batch size
-delete $$spans{scope};  #remove 'scope' entry
-is(scalar keys %{$spans}, 10, 'batch0 size - trace on');
-delete $$batch1{scope_spans}{scope};  #remove 'scope' entry
-is(scalar keys %{$$batch1{scope_spans}}, 5, 'batch1 size - trace on');
-
-#validate general attributes
-is(get_attr("service.name", "string_value",
-	$$batch0{resource}),
-	'test_server', 'service.name - trace on');
-is($$spans{span0}{name}, '"default_location"', 'span.name - trace on');
-
-#validate metrics
-is(get_attr("http.method", "string_value", $$spans{span0}), 'GET',
-	'http.method metric - trace on');
-is(get_attr("http.target", "string_value", $$spans{span0}), '/trace-on',
-	'http.target metric - trace on');
-is(get_attr("http.route", "string_value", $$spans{span0}), '/trace-on',
-	'http.route metric - trace on');
-is(get_attr("http.scheme", "string_value", $$spans{span0}), 'https',
-	'http.scheme metric - trace on');
-is(get_attr("http.flavor", "string_value", $$spans{span0}), '3.0',
-	'http.flavor metric - trace on');
-is(get_attr("http.user_agent", "string_value", $$spans{span0}), 'nginx-tests',
-	'http.user_agent metric - trace on');
-is(get_attr("http.request_content_length", "int_value", $$spans{span0}), 0,
-	'http.request_content_length metric - trace on');
-is(get_attr("http.response_content_length", "int_value", $$spans{span0}), 8,
-	'http.response_content_length metric - trace on');
-is(get_attr("http.status_code", "int_value", $$spans{span0}), 200,
-	'http.status_code metric - trace on');
-is(get_attr("net.host.name", "string_value", $$spans{span0}), 'localhost',
-	'net.host.name metric - trace on');
-is(get_attr("net.host.port", "int_value", $$spans{span0}), 8980,
-	'net.host.port metric - trace on');
-is(get_attr("net.sock.peer.addr", "string_value", $$spans{span0}), '127.0.0.1',
-	'net.sock.peer.addr metric - trace on');
-like(get_attr("net.sock.peer.port", "int_value", $$spans{span0}), qr/\d+/,
-	'net.sock.peer.port metric - trace on');
-
-#validate custom http metrics
-is(${get_attr("http.request.header.completion", "array_value", $$spans{span0})}
-	{values}{string_value}, '"OK"',
-	'http.request.header.completion metric - trace on');
-is(${get_attr(
-		"http.response.header.content.type", "array_value", $$spans{span0}
-	)}{values}{string_value}, '"text/plain"',
-	'http.response.header.content.type metric - trace on');
-is(get_attr("http.request", "string_value", $$spans{span0}),
-	'GET /trace-on HTTP/3.0', 'http.request metric - trace on');
-
-#extract trace info
-is($$spans{span0}{parent_span_id}, 'b9c7c989f97918e1',
-	'traceparent - trace on');
-is($$spans{span0}{trace_state}, '"congo=ucfJifl5GOE,rojo=00f067aa0ba902b7"',
-	'tracestate - trace on');
-is($$spans{span1}{parent_span_id}, undef, 'no traceparent - trace on');
-is($$spans{span1}{trace_state}, undef, 'no tracestate - trace on');
-
-#variables
-is($tp_headers->{'x-otel-trace-id'}, $$spans{span0}{trace_id},
-	'$otel_trace_id variable - trace on');
-is($tp_headers->{'x-otel-span-id'}, $$spans{span0}{span_id},
-	'$otel_span_id variable - trace on');
-is($tp_headers->{'x-otel-parent-id'}, $$spans{span0}{parent_span_id},
-	'$otel_parent_id variable - trace on');
-is($tp_headers->{'x-otel-parent-sampled'}, 1,
-	'$otel_parent_sampled variable - trace on');
-is($t_headers->{'x-otel-parent-sampled'}, 0,
-	'$otel_parent_sampled variable - trace on (no traceparent header)');
-
-#trace off
-is((scalar grep {
-		get_attr("http.target", "string_value", $$spans{$_}) eq '/trace-off'
-	} keys %{$spans}), 0, 'no metric in batch0 - trace off');
-is((scalar grep {
-		get_attr("http.target", "string_value", $$spans{$_}) eq '/trace-off'
-	} keys %{$$batch1{scope_spans}}), 0, 'no metric in batch1 - trace off');
-
-#trace context: ignore
-is($t_headers_ignore->{'x-otel-traceparent'}, undef,
-	'no traceparent - trace context ignore (no trace headers)');
-is($t_headers_ignore->{'x-otel-tracestate'}, undef,
-	'no tracestate - trace context ignore (no trace headers)');
-
-is($tp_headers_ignore->{'x-otel-parent-id'}, undef,
-	'no parent span id - trace context ignore (trace headers)');
-is($tp_headers_ignore->{'x-otel-traceparent'},
-	'00-0af7651916cd43dd8448eb211c80319c-b9c7c989f97918e1-01',
-	'traceparent - trace context ignore (trace headers)');
-is($tp_headers_ignore->{'x-otel-tracestate'},
-	'congo=ucfJifl5GOE,rojo=00f067aa0ba902b7',
-	'tracestate - trace context ignore (trace headers)');
-
-#trace context: extract
-is($t_headers_extract->{'x-otel-traceparent'}, undef,
-	'no traceparent - trace context extract (no trace headers)');
-is($t_headers_extract->{'x-otel-tracestate'}, undef,
-	'no tracestate - trace context extract (no trace headers)');
-
-is($tp_headers_extract->{'x-otel-parent-id'}, 'b9c7c989f97918e1',
-	'parent span id - trace context extract (trace headers)');
-is($tp_headers_extract->{'x-otel-traceparent'},
-	'00-0af7651916cd43dd8448eb211c80319c-b9c7c989f97918e1-01',
-	'traceparent - trace context extract (trace headers)');
-is($tp_headers_extract->{'x-otel-tracestate'},
-	'congo=ucfJifl5GOE,rojo=00f067aa0ba902b7',
-	'tracestate - trace context extract (trace headers)');
-
-#trace context: inject
-is($t_headers_inject->{'x-otel-traceparent'},
-	"00-$$spans{span6}{trace_id}-$$spans{span6}{span_id}-01",
-	'traceparent - trace context inject (no trace headers)');
-is($t_headers_inject->{'x-otel-tracestate'}, undef,
-	'no tracestate - trace context inject (no trace headers)');
-
-is($tp_headers_inject->{'x-otel-parent-id'}, undef,
-	'no parent span id - trace context inject (trace headers)');
-is($tp_headers_inject->{'x-otel-traceparent'},
-	"00-$$spans{span7}{trace_id}-$$spans{span7}{span_id}-01",
-	'traceparent - trace context inject (trace headers)');
-is($tp_headers_inject->{'x-otel-tracestate'}, undef,
-	'no tracestate - trace context inject (trace headers)');
-
-#trace context: propagate
-is($t_headers_propagate->{'x-otel-traceparent'},
-	"00-$$spans{span8}{trace_id}-$$spans{span8}{span_id}-01",
-	'traceparent - trace context propagate (no trace headers)');
-is($t_headers_propagate->{'x-otel-tracestate'}, undef,
-	'no tracestate - trace context propagate (no trace headers)');
-
-is($tp_headers_propagate->{'x-otel-parent-id'}, 'b9c7c989f97918e1',
-	'parent id - trace context propagate (trace headers)');
-is($tp_headers_propagate->{'x-otel-traceparent'},
-	"00-0af7651916cd43dd8448eb211c80319c-$$spans{span9}{span_id}-01",
-	'traceparent - trace context propagate (trace headers)');
-is($tp_headers_propagate->{'x-otel-tracestate'},
-	'congo=ucfJifl5GOE,rojo=00f067aa0ba902b7',
-	'tracestate - trace context propagate (trace headers)');
-
-SKIP: {
-skip "depends on error log content", 2 unless $ENV{TEST_NGINX_UNSAFE};
-
-$t->stop();
-my $log = $t->read_file("error.log");
-
-like($log, qr/OTel\/grpc: Error parsing metadata: error=invalid value/,
-	'log: error parsing metadata - no protobuf in response');
-unlike($log, qr/OTel export failure: No status received/,
-	'log: no export failure');
-
-}
-
-###############################################################################
-
-sub http3_get {
-	my ($path, %extra) = @_;
-	my ($frames, $frame);
-
-	my $s = Test::Nginx::HTTP3->new();
-
-	my $sid = $extra{trace_headers}
-		? $s->new_stream({ headers => [
-			{ name => ':method', value => 'GET' },
-			{ name => ':scheme', value => 'http' },
-			{ name => ':path', value => $path },
-			{ name => ':authority', value => 'localhost' },
-			{ name => 'user-agent', value => 'nginx-tests' },
-			{ name => 'traceparent',
-				value => '00-0af7651916cd43dd8448eb211c80319c-' .
-					'b9c7c989f97918e1-01'
-			},
-			{ name => 'tracestate',
-				value => 'congo=ucfJifl5GOE,rojo=00f067aa0ba902b7'
-			}]})
-		: $s->new_stream({ path => $path });
-
-	$frames = $s->read(all => [{ sid => $sid, fin => 1 }]);
-
-	($frame) = grep { $_->{type} eq "HEADERS" } @$frames;
-	my $headers = $frame->{headers};
-
-	($frame) = grep { $_->{type} eq "DATA" } @$frames;
-	my $data = $frame->{data};
-
-	return $headers, $data;
-}
-
-sub get_attr {
-	my($attr, $type, $obj) = @_;
-
-	my ($res) = grep {
-			$_ =~ /^attribute\d+/ && $$obj{$_}{key} eq '"' . $attr . '"'
-		} keys %{$obj};
-
-	if (defined $res) {
-		$$obj{$res}{value}{$type} =~ s/(^\")|(\"$)//g
-			if $type eq 'string_value';
-
-		return $$obj{$res}{value}{$type};
-	}
-
-	return undef;
-}
-
-sub decode_protobuf {
-	my ($protobuf) = @_;
-
-	local $/;
-	open CMD, "echo '" . encode_base64($protobuf) . "' | base64 -d | " .
-		'$PWD/../build/_deps/grpc-build/third_party/protobuf/protoc '.
-		'--decode opentelemetry.proto.trace.v1.ResourceSpans -I ' .
-		'$PWD/../build/_deps/otelcpp-src/third_party/opentelemetry-proto ' .
-		'opentelemetry/proto/collector/trace/v1/trace_service.proto |'
-		or die "Can't decode protobuf: $!\n";
-	my $out = <CMD>;
-	close CMD;
-
-	return $out;
-}
-
-sub decode_bytes {
-	my ($bytes) = @_;
-
-	my $c = sub { return chr oct(shift) };
-
-	$bytes =~ s/\\(\d{3})/$c->($1)/eg;
-	$bytes =~ s/(^\")|(\"$)//g;
-	$bytes =~ s/\\\\/\\/g;
-	$bytes =~ s/\\r/\r/g;
-	$bytes =~ s/\\n/\n/g;
-	$bytes =~ s/\\t/\t/g;
-	$bytes =~ s/\\"/\"/g;
-	$bytes =~ s/\\'/\'/g;
-
-	return unpack("H*", unpack("a*", $bytes));
-}
-
-sub to_hash {
-	my ($textdata) = @_;
-
-	my %out = ();
-	push my @stack, \%out;
-	my ($attr_count, $span_count) = (0, 0);
-	for my $line (split /\n/, $textdata) {
-		$line =~ s/(^\s+)|(\s+$)//g;
-		if ($line =~ /\:/) {
-			my ($k, $v) = split /\: /, $line;
-			$v = decode_bytes($v) if ($k =~ /trace_id|span_id|parent_span_id/);
-			$stack[$#stack]{$k} = $v;
-		} elsif ($line =~ /\{/) {
-			$line =~ s/\s\{//;
-			$line = 'attribute' . $attr_count++ if ($line eq 'attributes');
-			if ($line eq 'spans') {
-				$line = 'span' . $span_count++;
-				$attr_count = 0;
-			}
-			my %new = ();
-			$stack[$#stack]{$line} = \%new;
-			push @stack, \%new;
-		} elsif ($line =~ /\}/) {
-			pop @stack;
-		}
-	}
-
-	return \%out;
-}
-
-sub grpc {
-	my ($server, $client, $f, $s, $c, $sid, $csid, $uri);
-
-	$server = IO::Socket::INET->new(
-		Proto => 'tcp',
-		LocalHost => '127.0.0.1',
-		LocalPort => $p,
-		Listen => 5,
-		Reuse => 1
-	) or die "Can't create listening socket: $!\n";
-
-	$f->{http_start} = sub {
-		if (IO::Select->new($server)->can_read(5)) {
-			$client = $server->accept();
-		} else {
-			# connection could be unexpectedly reused
-			goto reused if $client;
-			return undef;
-		}
-
-		$client->sysread($_, 24) == 24 or return; # preface
-
-		$c = Test::Nginx::HTTP2->new(1, socket => $client,
-			pure => 1, preface => "") or return;
-
-reused:
-		my $frames = $c->read(all => [{ fin => 1 }]);
-
-		$client->close();
-
-		return $frames;
-	};
-
-	return $f;
-}
-
-###############################################################################

tests/otel.t

@@ -1,514 +0,0 @@
-#!/usr/bin/perl
-
-# (C) Nginx, Inc.
-
-# Tests for OTel exporter in case HTTP.
-
-###############################################################################
-
-use warnings;
-use strict;
-
-use Test::More;
-
-BEGIN { use FindBin; chdir($FindBin::Bin); }
-
-use Test::Nginx;
-use Test::Nginx::HTTP2;
-use MIME::Base64;
-
-###############################################################################
-
-select STDERR; $| = 1;
-select STDOUT; $| = 1;
-
-my $t = Test::Nginx->new()->has(qw/http http_ssl http_v2 mirror rewrite/)
-	->has_daemon(qw/openssl base64/)
-	->write_file_expand('nginx.conf', <<'EOF');
-
-%%TEST_GLOBALS%%
-
-daemon off;
-
-events {
-}
-
-http {
-    %%TEST_GLOBALS_HTTP%%
-
-    ssl_certificate_key localhost.key;
-    ssl_certificate localhost.crt;
-
-    otel_exporter {
-        endpoint 127.0.0.1:8082;
-        interval 1s;
-        batch_size 10;
-        batch_count 2;
-    }
-
-    otel_service_name test_server;
-    otel_trace on;
-
-    server {
-        listen       127.0.0.1:8080;
-        listen       127.0.0.1:8081 ssl;
-        server_name  localhost;
-
-        location /trace-on {
-            otel_trace_context extract;
-            otel_span_name default_location;
-            otel_span_attr http.request.header.completion
-                $request_completion;
-            otel_span_attr http.response.header.content.type
-                $sent_http_content_type;
-            otel_span_attr http.request $request;
-            add_header "X-Otel-Trace-Id" $otel_trace_id;
-            add_header "X-Otel-Span-Id" $otel_span_id;
-            add_header "X-Otel-Parent-Id" $otel_parent_id;
-            add_header "X-Otel-Parent-Sampled" $otel_parent_sampled;
-            return 200 "TRACE-ON";
-        }
-
-        location /context-ignore {
-            otel_trace_context ignore;
-            otel_span_name context_ignore;
-            add_header "X-Otel-Parent-Id" $otel_parent_id;
-            proxy_pass http://localhost:8080/trace-off;
-        }
-
-        location /context-extract {
-            otel_trace_context extract;
-            otel_span_name context_extract;
-            add_header "X-Otel-Parent-Id" $otel_parent_id;
-            proxy_pass http://localhost:8080/trace-off;
-        }
-
-        location /context-inject {
-            otel_trace_context inject;
-            otel_span_name context_inject;
-            add_header "X-Otel-Parent-Id" $otel_parent_id;
-            proxy_pass http://localhost:8080/trace-off;
-        }
-
-        location /context-propagate {
-            otel_trace_context propagate;
-            otel_span_name context_propogate;
-            add_header "X-Otel-Parent-Id" $otel_parent_id;
-            proxy_pass http://localhost:8080/trace-off;
-        }
-
-        location /trace-off {
-            otel_trace off;
-            add_header "X-Otel-Traceparent" $http_traceparent;
-            add_header "X-Otel-Tracestate" $http_tracestate;
-            return 200 "TRACE-OFF";
-        }
-    }
-
-    server {
-        listen       127.0.0.1:8082 http2;
-        server_name  localhost;
-        otel_trace off;
-
-        location / {
-            mirror /mirror;
-            grpc_pass 127.0.0.1:8083;
-        }
-
-        location /mirror {
-            internal;
-            grpc_pass 127.0.0.1:%%PORT_4317%%;
-        }
-    }
-
-    server {
-        listen       127.0.0.1:8083 http2;
-        server_name  localhost;
-        otel_trace off;
-
-        location / {
-            add_header content-type application/grpc;
-            add_header grpc-status 0;
-            add_header grpc-message "";
-            return 200;
-        }
-    }
-}
-
-EOF
-
-$t->write_file('openssl.conf', <<'EOF');
-[ req ]
-default_bits = 2048
-encrypt_key = no
-distinguished_name = req_distinguished_name
-[ req_distinguished_name ]
-
-EOF
-
-my $d = $t->testdir();
-
-foreach my $name ('localhost') {
-	system('openssl req -x509 -new '
-		. "-config $d/openssl.conf -subj /CN=$name/ "
-		. "-out $d/$name.crt -keyout $d/$name.key "
-		. ">>$d/openssl.out 2>&1") == 0
-		or die "Can't create certificate for $name: $!\n";
-}
-
-$t->try_run('no OTel module')->plan(69);
-
-###############################################################################
-
-my $p = port(4317);
-my $f = grpc();
-
-#do requests
-my $t_off_resp = http1_get('/trace-off');
-
-#batch0 (10 requests)
-my $tp_resp = http1_get('/trace-on', trace_headers => 1);
-my $t_resp = http1_get('/trace-on', port => 8081, ssl => 1);
-
-my $t_resp_ignore = http1_get('/context-ignore');
-my $tp_resp_ignore = http1_get('/context-ignore', trace_headers => 1);
-my $t_resp_extract = http1_get('/context-extract');
-my $tp_resp_extract = http1_get('/context-extract', trace_headers => 1);
-my $t_resp_inject = http1_get('/context-inject');
-my $tp_resp_inject = http1_get('/context-inject', trace_headers => 1);
-my $t_resp_propagate = http1_get('/context-propagate');
-my $tp_resp_propagate = http1_get('/context-propagate', trace_headers => 1);
-
-my ($frame) = grep { $_->{type} eq "DATA" } @{$f->{http_start}()};
-my $batch0 = to_hash(decode_protobuf(substr $frame->{data}, 8));
-my $spans = $$batch0{scope_spans};
-
-#batch1 (5 reqeusts)
-http1_get('/trace-on') for (1..5);
-
-($frame) = grep { $_->{type} eq "DATA" } @{$f->{http_start}()};
-my $batch1 = to_hash(decode_protobuf(substr $frame->{data}, 8));
-
-#validate responses
-like($tp_resp, qr/TRACE-ON/, 'http request1 - trace on');
-like($t_resp, qr/TRACE-ON/, 'http request2 - trace on');
-like($t_off_resp, qr/TRACE-OFF/, 'http request - trace off');
-
-#validate batch size
-delete $$spans{scope};  #remove 'scope' entry
-is(scalar keys %{$spans}, 10, 'batch0 size - trace on');
-delete $$batch1{scope_spans}{scope};  #remove 'scope' entry
-is(scalar keys %{$$batch1{scope_spans}}, 5, 'batch1 size - trace on');
-
-#validate general attributes
-is(get_attr("service.name", "string_value",
-	$$batch0{resource}), 'test_server', 'service.name - trace on');
-is($$spans{span0}{name}, '"default_location"', 'span.name - trace on');
-
-#validate http metrics
-is(get_attr("http.method", "string_value", $$spans{span0}), 'GET',
-	'http.method metric - trace on');
-is(get_attr("http.target", "string_value", $$spans{span0}), '/trace-on',
-	'http.target metric - trace on');
-is(get_attr("http.route", "string_value", $$spans{span0}), '/trace-on',
-	'http.route metric - trace on');
-is(get_attr("http.scheme", "string_value", $$spans{span0}), 'http',
-	'http.scheme metric - trace on');
-is(get_attr("http.flavor", "string_value", $$spans{span0}), '1.0',
-	'http.flavor metric - trace on');
-is(get_attr("http.user_agent", "string_value", $$spans{span0}), 'nginx-tests',
-	'http.user_agent metric - trace on');
-is(get_attr("http.request_content_length", "int_value", $$spans{span0}), 0,
-	'http.request_content_length metric - trace on');
-is(get_attr("http.response_content_length", "int_value", $$spans{span0}), 8,
-	'http.response_content_length metric - trace on');
-is(get_attr("http.status_code", "int_value", $$spans{span0}), 200,
-	'http.status_code metric - trace on');
-is(get_attr("net.host.name", "string_value", $$spans{span0}), 'localhost',
-	'net.host.name metric - trace on');
-is(get_attr("net.host.port", "int_value", $$spans{span0}), 8080,
-	'net.host.port metric - trace on');
-is(get_attr("net.sock.peer.addr", "string_value", $$spans{span0}), '127.0.0.1',
-	'net.sock.peer.addr metric - trace on');
-like(get_attr("net.sock.peer.port", "int_value", $$spans{span0}), qr/\d+/,
-	'net.sock.peer.port metric - trace on');
-
-#validate https metrics
-is(get_attr("http.method", "string_value", $$spans{span1}), 'GET',
-	'http.method metric - trace on (https)');
-is(get_attr("http.target", "string_value", $$spans{span1}), '/trace-on',
-	'http.target metric - trace on (https)');
-is(get_attr("http.route", "string_value", $$spans{span1}), '/trace-on',
-	'http.route metric - trace on (https)');
-is(get_attr("http.scheme", "string_value", $$spans{span1}), 'https',
-	'http.scheme metric - trace on (https)');
-is(get_attr("http.flavor", "string_value", $$spans{span1}), '1.0',
-	'http.flavor metric - trace on (https)');
-is(get_attr("http.user_agent", "string_value", $$spans{span1}),
-	'nginx-tests', 'http.user_agent metric - trace on (https)');
-is(get_attr("http.request_content_length", "int_value", $$spans{span1}), 0,
-	'http.request_content_length metric - trace on (https)');
-is(get_attr("http.response_content_length", "int_value", $$spans{span1}), 8,
-	'http.response_content_length metric - trace on (https)');
-is(get_attr("http.status_code", "int_value", $$spans{span1}), 200,
-	'http.status_code metric - trace on (https)');
-is(get_attr("net.host.name", "string_value", $$spans{span1}), 'localhost',
-	'net.host.name metric - trace on (https)');
-is(get_attr("net.host.port", "int_value", $$spans{span1}), 8081,
-	'net.host.port metric - trace on (https)');
-is(get_attr("net.sock.peer.addr", "string_value", $$spans{span1}), '127.0.0.1',
-	'net.sock.peer.addr metric - trace on (https)');
-like(get_attr("net.sock.peer.port", "int_value", $$spans{span1}), qr/\d+/,
-	'net.sock.peer.port metric - trace on (https)');
-
-#validate custom http metrics
-is(${get_attr("http.request.header.completion", "array_value", $$spans{span0})}
-	{values}{string_value}, '"OK"',
-	'http.request.header.completion metric - trace on');
-is(${get_attr("http.response.header.content.type",
-		"array_value", $$spans{span0})}{values}{string_value}, '"text/plain"',
-	'http.response.header.content.type metric - trace on');
-is(get_attr("http.request", "string_value", $$spans{span0}),
-	'GET /trace-on HTTP/1.0', 'http.request metric - trace on');
-
-#extract trace info
-is($$spans{span0}{parent_span_id}, 'b9c7c989f97918e1',
-	'traceparent - trace on');
-is($$spans{span0}{trace_state}, '"congo=ucfJifl5GOE,rojo=00f067aa0ba902b7"',
-	'tracestate - trace on');
-is($$spans{span1}{parent_span_id}, undef, 'no traceparent - trace on');
-is($$spans{span1}{trace_state}, undef, 'no tracestate - trace on');
-
-#variables
-like($tp_resp, qr/X-Otel-Trace-Id: $$spans{span0}{trace_id}/,
-	'$otel_trace_id variable - trace on');
-like($tp_resp, qr/X-Otel-Span-Id: $$spans{span0}{span_id}/,
-	'$otel_span_id variable - trace on');
-like($tp_resp, qr/X-Otel-Parent-Id: $$spans{span0}{parent_span_id}/,
-	'$otel_parent_id variable - trace on');
-like($tp_resp, qr/X-Otel-Parent-Sampled: 1/,
-	'$otel_parent_sampled variable - trace on');
-like($t_resp, qr/X-Otel-Parent-Sampled: 0/,
-	'$otel_parent_sampled variable - trace on (no traceparent header)');
-
-#trace off
-is((scalar grep {
-		get_attr("http.target", "string_value", $$spans{$_}) eq '/trace-off'
-	} keys %{$spans}), 0, 'no metric in batch0 - trace off');
-is((scalar grep {
-		get_attr("http.target", "string_value", $$spans{$_}) eq '/trace-off'
-	} keys %{$$batch1{scope_spans}}), 0, 'no metric in batch1 - trace off');
-
-#trace context: ignore
-unlike($t_resp_ignore, qr/X-Otel-Traceparent/,
-	'no traceparent - trace context ignore (no trace headers)');
-unlike($t_resp_ignore, qr/X-Otel-Tracestate/,
-	'no tracestate - trace context ignore (no trace headers)');
-
-unlike($tp_resp_ignore, qr/X-Otel-Parent-Id/,
-	'no parent span id - trace context ignore (trace headers)');
-like($tp_resp_ignore,
-	qr/Traceparent: 00-0af7651916cd43dd8448eb211c80319c-b9c7c989f97918e1-01/,
-	'traceparent - trace context ignore (trace headers)');
-like($tp_resp_ignore,
-	qr/Tracestate: congo=ucfJifl5GOE,rojo=00f067aa0ba902b7/,
-	'tracestate - trace context ignore (trace headers)');
-
-#trace context: extract
-unlike($t_resp_extract, qr/X-Otel-Traceparent/,
-	'no traceparent - trace context extract (no trace headers)');
-unlike($t_resp_extract, qr/X-Otel-Tracestate/,
-	'no tracestate - trace context extract (no trace headers)');
-
-like($tp_resp_extract, qr/X-Otel-Parent-Id: b9c7c989f97918e1/,
-	'parent span id - trace context extract (trace headers)');
-like($tp_resp_extract,
-	qr/Traceparent: 00-0af7651916cd43dd8448eb211c80319c-b9c7c989f97918e1-01/,
-	'traceparent - trace context extract (trace headers)');
-like($tp_resp_extract,
-	qr/Tracestate: congo=ucfJifl5GOE,rojo=00f067aa0ba902b7/,
-	'tracestate - trace context extract (trace headers)');
-
-#trace context: inject
-like($t_resp_inject,
-	qr/Traceparent: 00-$$spans{span6}{trace_id}-$$spans{span6}{span_id}-01/,
-	'traceparent - trace context inject (no trace headers)');
-unlike($t_resp_inject, qr/X-Otel-Tracestate/,
-	'no tracestate - trace context inject (no trace headers)');
-
-unlike($tp_resp_inject, qr/X-Otel-Parent-Id/,
-	'no parent span id - trace context inject (trace headers)');
-like($tp_resp_inject,
-	qr/Traceparent: 00-$$spans{span7}{trace_id}-$$spans{span7}{span_id}-01/,
-	'traceparent - trace context inject (trace headers)');
-unlike($tp_resp_inject, qr/Tracestate:/,
-	'no tracestate - trace context inject (trace headers)');
-
-#trace context: propagate
-like($t_resp_propagate,
-	qr/Traceparent: 00-$$spans{span8}{trace_id}-$$spans{span8}{span_id}-01/,
-	'traceparent - trace context propagate (no trace headers)');
-unlike($t_resp_propagate, qr/X-Otel-Tracestate/,
-	'no tracestate - trace context propagate (no trace headers)');
-
-like($tp_resp_propagate, qr/X-Otel-Parent-Id: b9c7c989f97918e1/,
-	'parent id - trace context propagate (trace headers)');
-like($tp_resp_propagate,
-	qr/parent: 00-0af7651916cd43dd8448eb211c80319c-$$spans{span9}{span_id}-01/,
-	'traceparent - trace context propagate (trace headers)');
-like($tp_resp_propagate,
-	qr/Tracestate: congo=ucfJifl5GOE,rojo=00f067aa0ba902b7/,
-	'tracestate - trace context propagate (trace headers)');
-
-SKIP: {
-skip "depends on error log contents", 2 unless $ENV{TEST_NGINX_UNSAFE};
-
-$t->stop();
-my $log = $t->read_file("error.log");
-
-like($log, qr/OTel\/grpc: Error parsing metadata: error=invalid value/,
-	'log: error parsing metadata - no protobuf in response');
-unlike($log, qr/OTel export failure: No status received/,
-	'log: no export failure');
-
-}
-
-###############################################################################
-
-sub http1_get {
-	my ($path, %extra) = @_;
-
-	my $port = $extra{port} || 8080;
-
-	my $r = <<EOF;
-GET $path HTTP/1.0
-Host: localhost
-User-agent: nginx-tests
-EOF
-
-	$r .= <<EOF if $extra{trace_headers};
-Traceparent: 00-0af7651916cd43dd8448eb211c80319c-b9c7c989f97918e1-01
-Tracestate: congo=ucfJifl5GOE,rojo=00f067aa0ba902b7
-EOF
-
-	return http($r . "\n", PeerAddr => '127.0.0.1:' . port($port),
-		SSL => $extra{ssl});
-}
-
-sub get_attr {
-	my($attr, $type, $obj) = @_;
-
-	my ($res) = grep {
-			$_ =~ /^attribute\d+/ && $$obj{$_}{key} eq '"' . $attr . '"'
-		} keys %{$obj};
-
-	if (defined $res) {
-		$$obj{$res}{value}{$type} =~ s/(^\")|(\"$)//g
-			if $type eq 'string_value';
-
-		return $$obj{$res}{value}{$type};
-	}
-
-	return undef;
-}
-
-sub decode_protobuf {
-	my ($protobuf) = @_;
-
-	local $/;
-	open CMD, "echo '" . encode_base64($protobuf) . "' | base64 -d | " .
-		'$PWD/../build/_deps/grpc-build/third_party/protobuf/protoc '.
-		'--decode opentelemetry.proto.trace.v1.ResourceSpans -I ' .
-		'$PWD/../build/_deps/otelcpp-src/third_party/opentelemetry-proto ' .
-		'opentelemetry/proto/collector/trace/v1/trace_service.proto |'
-		or die "Can't decode protobuf: $!\n";
-	my $out = <CMD>;
-	close CMD;
-
-	return $out;
-}
-
-sub decode_bytes {
-	my ($bytes) = @_;
-
-	my $c = sub { return chr oct(shift) };
-
-	$bytes =~ s/\\(\d{3})/$c->($1)/eg;
-	$bytes =~ s/(^\")|(\"$)//g;
-	$bytes =~ s/\\\\/\\/g;
-	$bytes =~ s/\\r/\r/g;
-	$bytes =~ s/\\n/\n/g;
-	$bytes =~ s/\\t/\t/g;
-	$bytes =~ s/\\"/\"/g;
-	$bytes =~ s/\\'/\'/g;
-
-	return unpack("H*", unpack("a*", $bytes));
-}
-
-sub to_hash {
-	my ($textdata) = @_;
-
-	my %out = ();
-	push my @stack, \%out;
-	my ($attr_count, $span_count) = (0, 0);
-	for my $line (split /\n/, $textdata) {
-		$line =~ s/(^\s+)|(\s+$)//g;
-		if ($line =~ /\:/) {
-			my ($k, $v) = split /\: /, $line;
-			$v = decode_bytes($v) if ($k =~ /trace_id|span_id|parent_span_id/);
-			$stack[$#stack]{$k} = $v;
-		} elsif ($line =~ /\{/) {
-			$line =~ s/\s\{//;
-			$line = 'attribute' . $attr_count++ if ($line eq 'attributes');
-			if ($line eq 'spans') {
-				$line = 'span' . $span_count++;
-				$attr_count = 0;
-			}
-			my %new = ();
-			$stack[$#stack]{$line} = \%new;
-			push @stack, \%new;
-		} elsif ($line =~ /\}/) {
-			pop @stack;
-		}
-	}
-
-	return \%out;
-}
-
-sub grpc {
-	my ($server, $client, $f, $s, $c, $sid, $csid, $uri);
-
-	$server = IO::Socket::INET->new(
-		Proto => 'tcp',
-		LocalHost => '127.0.0.1',
-		LocalPort => $p,
-		Listen => 5,
-		Reuse => 1
-	) or die "Can't create listening socket: $!\n";
-
-	$f->{http_start} = sub {
-		if (IO::Select->new($server)->can_read(5)) {
-			$client = $server->accept();
-		} else {
-			# connection could be unexpectedly reused
-			goto reused if $client;
-			return undef;
-		}
-
-		$client->sysread($_, 24) == 24 or return; # preface
-
-		$c = Test::Nginx::HTTP2->new(1, socket => $client,
-			pure => 1, preface => "") or return;
-
-reused:
-		my $frames = $c->read(all => [{ fin => 1 }]);
-
-		$client->close();
-
-		return $frames;
-	};
-
-	return $f;
-}
-
-###############################################################################

tests/otel_collector.t

@@ -1,403 +0,0 @@
-#!/usr/bin/perl
-
-# (C) Nginx, Inc.
-
-# Tests for OTel exporter in case HTTP using otelcol.
-
-###############################################################################
-
-use warnings;
-use strict;
-
-use Test::More;
-
-BEGIN { use FindBin; chdir($FindBin::Bin); }
-
-use Test::Nginx;
-
-###############################################################################
-
-select STDERR; $| = 1;
-select STDOUT; $| = 1;
-
-plan(skip_all => "depends on logs content") unless $ENV{TEST_NGINX_UNSAFE};
-
-eval { require JSON::PP; };
-plan(skip_all => "JSON::PP not installed") if $@;
-
-my $t = Test::Nginx->new()->has(qw/http http_ssl rewrite/)
-	->write_file_expand('nginx.conf', <<'EOF');
-
-%%TEST_GLOBALS%%
-
-daemon off;
-
-events {
-}
-
-http {
-    %%TEST_GLOBALS_HTTP%%
-
-    ssl_certificate_key localhost.key;
-    ssl_certificate localhost.crt;
-
-    otel_exporter {
-        endpoint 127.0.0.1:%%PORT_4317%%;
-        interval 1s;
-        batch_size 10;
-        batch_count 2;
-    }
-
-    otel_service_name test_server;
-    otel_trace on;
-
-    server {
-        listen       127.0.0.1:8080;
-        listen       127.0.0.1:8081 ssl;
-        server_name  localhost;
-
-        location /trace-on {
-            otel_trace_context extract;
-            otel_span_name default_location;
-            otel_span_attr http.request.header.completion
-                $request_completion;
-            otel_span_attr http.response.header.content.type
-                $sent_http_content_type;
-            otel_span_attr http.request $request;
-            add_header "X-Otel-Trace-Id" $otel_trace_id;
-            add_header "X-Otel-Span-Id" $otel_span_id;
-            add_header "X-Otel-Parent-Id" $otel_parent_id;
-            add_header "X-Otel-Parent-Sampled" $otel_parent_sampled;
-            return 200 "TRACE-ON";
-        }
-
-        location /context-ignore {
-            otel_trace_context ignore;
-            otel_span_name context_ignore;
-            add_header "X-Otel-Parent-Id" $otel_parent_id;
-            proxy_pass http://localhost:8080/trace-off;
-        }
-
-        location /context-extract {
-            otel_trace_context extract;
-            otel_span_name context_extract;
-            add_header "X-Otel-Parent-Id" $otel_parent_id;
-            proxy_pass http://localhost:8080/trace-off;
-        }
-
-        location /context-inject {
-            otel_trace_context inject;
-            otel_span_name context_inject;
-            add_header "X-Otel-Parent-Id" $otel_parent_id;
-            proxy_pass http://localhost:8080/trace-off;
-        }
-
-        location /context-propagate {
-            otel_trace_context propagate;
-            otel_span_name context_propogate;
-            add_header "X-Otel-Parent-Id" $otel_parent_id;
-            proxy_pass http://localhost:8080/trace-off;
-        }
-
-        location /trace-off {
-            otel_trace off;
-            add_header "X-Otel-Traceparent" $http_traceparent;
-            add_header "X-Otel-Tracestate" $http_tracestate;
-            return 200 "TRACE-OFF";
-        }
-    }
-}
-
-EOF
-
-$t->write_file_expand('otel-config.yaml', <<EOF);
-
-receivers:
-  otlp:
-    protocols:
-      grpc:
-          endpoint: 127.0.0.1:%%PORT_4317%%
-
-exporters:
-  logging:
-    loglevel: debug
-  file:
-    path: ${\ $t->testdir() }/otel.json
-
-service:
-  pipelines:
-    traces:
-      receivers: [otlp]
-      exporters: [logging, file]
-    metrics:
-      receivers: [otlp]
-      exporters: [logging, file]
-
-EOF
-
-$t->write_file('openssl.conf', <<'EOF');
-[ req ]
-default_bits = 2048
-encrypt_key = no
-distinguished_name = req_distinguished_name
-[ req_distinguished_name ]
-
-EOF
-
-my $d = $t->testdir();
-
-foreach my $name ('localhost') {
-	system('openssl req -x509 -new '
-		. "-config $d/openssl.conf -subj /CN=$name/ "
-		. "-out $d/$name.crt -keyout $d/$name.key "
-		. ">>$d/openssl.out 2>&1") == 0
-		or die "Can't create certificate for $name: $!\n";
-}
-
-#suppress otel collector output
-open OLDERR, ">&", \*STDERR;
-open STDERR, ">>" , $^O eq 'MSWin32' ? 'nul' : '/dev/null';
-$t->run_daemon('../otelcol', '--config', $t->testdir().'/otel-config.yaml');
-open STDERR, ">&", \*OLDERR;
-$t->waitforsocket('127.0.0.1:' . port(4317)) or
-	die 'No otel collector open socket';
-
-$t->try_run('no OTel module')->plan(69);
-
-###############################################################################
-
-#do requests
-my $t_off_resp = http1_get('/trace-off');
-
-#batch0 (10 requests)
-my $tp_resp = http1_get('/trace-on', trace_headers => 1);
-my $t_resp = http1_get('/trace-on', port => 8081, ssl => 1);
-
-my $t_resp_ignore = http1_get('/context-ignore');
-my $tp_resp_ignore = http1_get('/context-ignore', trace_headers => 1);
-my $t_resp_extract = http1_get('/context-extract');
-my $tp_resp_extract = http1_get('/context-extract', trace_headers => 1);
-my $t_resp_inject = http1_get('/context-inject');
-my $tp_resp_inject = http1_get('/context-inject', trace_headers => 1);
-my $t_resp_propagate = http1_get('/context-propagate');
-my $tp_resp_propagate = http1_get('/context-propagate', trace_headers => 1);
-
-#batch1 (5 reqeusts)
-http1_get('/trace-on') for (1..5);
-
-#waiting batch1 is sent to collector for 1s
-select undef, undef, undef, 1;
-
-my @batches = split /\n/, $t->read_file('otel.json');
-my $batch_json = JSON::PP::decode_json($batches[0]);
-my $spans = $$batch_json{"resourceSpans"}[0]{"scopeSpans"}[0]{"spans"};
-
-#validate responses
-like($tp_resp, qr/TRACE-ON/, 'http request1 - trace on');
-like($t_resp, qr/TRACE-ON/, 'http request2 - trace on');
-like($t_off_resp, qr/TRACE-OFF/, 'http request - trace off');
-
-#validate amount of batches
-is(scalar @batches, 2, 'amount of batches - trace on');
-
-#validate batch size
-is(scalar @{$spans}, 10, 'batch0 size - trace on');
-is(scalar @{${JSON::PP::decode_json($batches[1])}{"resourceSpans"}[0]
-	{"scopeSpans"}[0]{"spans"}}, 5, 'batch1 size - trace on');
-
-#validate general attributes
-is(get_attr("service.name", "stringValue",
-	$$batch_json{resourceSpans}[0]{resource}),
-	'test_server', 'service.name - trace on');
-is($$spans[0]{name}, 'default_location', 'span.name - trace on');
-
-#validate http metrics
-is(get_attr("http.method", "stringValue", $$spans[0]), 'GET',
-	'http.method metric - trace on');
-is(get_attr("http.target", "stringValue", $$spans[0]), '/trace-on',
-	'http.target metric - trace on');
-is(get_attr("http.route", "stringValue", $$spans[0]), '/trace-on',
-	'http.route metric - trace on');
-is(get_attr("http.scheme", "stringValue", $$spans[0]), 'http',
-	'http.scheme metric - trace on');
-is(get_attr("http.flavor", "stringValue", $$spans[0]), '1.0',
-	'http.flavor metric - trace on');
-is(get_attr("http.user_agent", "stringValue", $$spans[0]), 'nginx-tests',
-	'http.user_agent metric - trace on');
-is(get_attr("http.request_content_length", "intValue", $$spans[0]), 0,
-	'http.request_content_length metric - trace on');
-is(get_attr("http.response_content_length", "intValue", $$spans[0]), 8,
-	'http.response_content_length metric - trace on');
-is(get_attr("http.status_code", "intValue", $$spans[0]), 200,
-	'http.status_code metric - trace on');
-is(get_attr("net.host.name", "stringValue", $$spans[0]), 'localhost',
-	'net.host.name metric - trace on');
-is(get_attr("net.host.port", "intValue", $$spans[0]), 8080,
-	'net.host.port metric - trace on');
-is(get_attr("net.sock.peer.addr", "stringValue", $$spans[0]), '127.0.0.1',
-	'net.sock.peer.addr metric - trace on');
-like(get_attr("net.sock.peer.port", "intValue", $$spans[0]), qr/\d+/,
-	'net.sock.peer.port metric - trace on');
-
-#validate custom http metrics
-is(${get_attr("http.request.header.completion", "arrayValue", $$spans[0])}
-	{values}[0]{stringValue}, 'OK',
-	'http.request.header.completion metric - trace on');
-is(${get_attr("http.response.header.content.type", "arrayValue",$$spans[0])}
-	{values}[0]{stringValue}, 'text/plain',
-	'http.response.header.content.type metric - trace on');
-is(get_attr("http.request", "stringValue", $$spans[0]),
-	'GET /trace-on HTTP/1.0', 'http.request metric - trace on');
-
-#validate https metrics
-is(get_attr("http.method", "stringValue", $$spans[1]), 'GET',
-	'http.method metric - trace on (https)');
-is(get_attr("http.target", "stringValue", $$spans[1]), '/trace-on',
-	'http.target metric - trace on (https)');
-is(get_attr("http.route", "stringValue", $$spans[1]), '/trace-on',
-	'http.route metric - trace on (https)');
-is(get_attr("http.scheme", "stringValue", $$spans[1]), 'https',
-	'http.scheme metric - trace on (https)');
-is(get_attr("http.flavor", "stringValue", $$spans[1]), '1.0',
-	'http.flavor metric - trace on (https)');
-is(get_attr("http.user_agent", "stringValue", $$spans[1]), 'nginx-tests',
-	'http.user_agent metric - trace on (https)');
-is(get_attr("http.request_content_length", "intValue", $$spans[1]), 0,
-	'http.request_content_length metric - trace on (https)');
-is(get_attr("http.response_content_length", "intValue", $$spans[1]), 8,
-	'http.response_content_length metric - trace on (https)');
-is(get_attr("http.status_code", "intValue", $$spans[1]), 200,
-	'http.status_code metric - trace on (https)');
-is(get_attr("net.host.name", "stringValue", $$spans[1]), 'localhost',
-	'net.host.name metric - trace on (https)');
-is(get_attr("net.host.port", "intValue", $$spans[1]), 8081,
-	'net.host.port metric - trace on (https)');
-is(get_attr("net.sock.peer.addr", "stringValue", $$spans[1]), '127.0.0.1',
-	'net.sock.peer.addr metric - trace on (https)');
-like(get_attr("net.sock.peer.port", "intValue", $$spans[1]), qr/\d+/,
-	'net.sock.peer.port metric - trace on (https)');
-
-#extract trace info
-is($$spans[0]{parentSpanId}, 'b9c7c989f97918e1', 'traceparent - trace on');
-is($$spans[0]{traceState}, 'congo=ucfJifl5GOE,rojo=00f067aa0ba902b7',
-	'tracestate - trace on');
-is($$spans[1]{parentSpanId}, '', 'no traceparent - trace on');
-is($$spans[1]{traceState}, undef, 'no tracestate - trace on');
-
-#variables
-like($tp_resp, qr/X-Otel-Trace-Id: $$spans[0]{traceId}/,
-	'$otel_trace_id variable - trace on');
-like($tp_resp, qr/X-Otel-Span-Id: $$spans[0]{spanId}/,
-	'$otel_span_id variable - trace on');
-like($tp_resp, qr/X-Otel-Parent-Id: $$spans[0]{parentSpanId}/,
-	'$otel_parent_id variable - trace on');
-like($tp_resp, qr/X-Otel-Parent-Sampled: 1/,
-	'$otel_parent_sampled variable - trace on');
-like($t_resp, qr/X-Otel-Parent-Sampled: 0/,
-	'$otel_parent_sampled variable - trace on (no traceparent header)');
-
-#trace off
-unlike($batches[0].$batches[1],
-	qr/\Q{"key":"http.target","value":{"stringValue":"\/trace-off"}}\E/,
-	'no metrics - trace off');
-
-#trace context: ignore
-unlike($t_resp_ignore, qr/X-Otel-Traceparent/,
-	'no traceparent - trace context ignore (no trace headers)');
-unlike($t_resp_ignore, qr/X-Otel-Tracestate/,
-	'no tracestate - trace context ignore (no trace headers)');
-
-unlike($tp_resp_ignore, qr/X-Otel-Parent-Id/,
-	'no parent span id - trace context ignore (trace headers)');
-like($tp_resp_ignore,
-	qr/Traceparent: 00-0af7651916cd43dd8448eb211c80319c-b9c7c989f97918e1-01/,
-	'traceparent - trace context ignore (trace headers)');
-like($tp_resp_ignore,
-	qr/Tracestate: congo=ucfJifl5GOE,rojo=00f067aa0ba902b7/,
-	'tracestate - trace context ignore (trace headers)');
-
-#trace context: extract
-unlike($t_resp_extract, qr/X-Otel-Traceparent/,
-	'no traceparent - trace context extract (no trace headers)');
-unlike($t_resp_extract, qr/X-Otel-Tracestate/,
-	'no tracestate - trace context extract (no trace headers)');
-
-like($tp_resp_extract, qr/X-Otel-Parent-Id: b9c7c989f97918e1/,
-	'parent span id - trace context extract (trace headers)');
-like($tp_resp_extract,
-	qr/Traceparent: 00-0af7651916cd43dd8448eb211c80319c-b9c7c989f97918e1-01/,
-	'traceparent - trace context extract (trace headers)');
-like($tp_resp_extract,
-	qr/Tracestate: congo=ucfJifl5GOE,rojo=00f067aa0ba902b7/,
-	'tracestate - trace context extract (trace headers)');
-
-#trace context: inject
-like($t_resp_inject,
-	qr/X-Otel-Traceparent: 00-$$spans[6]{traceId}-$$spans[6]{spanId}-01/,
-	'traceparent - trace context inject (no trace headers)');
-unlike($t_resp_inject, qr/X-Otel-Tracestate/,
-	'no tracestate - trace context inject (no trace headers)');
-
-unlike($tp_resp_inject, qr/X-Otel-Parent-Id/,
-	'no parent span id - trace context inject (trace headers)');
-like($tp_resp_inject,
-	qr/Traceparent: 00-$$spans[7]{traceId}-$$spans[7]{spanId}-01/,
-	'traceparent - trace context inject (trace headers)');
-unlike($tp_resp_inject, qr/Tracestate:/,
-	'no tracestate - trace context inject (trace headers)');
-
-#trace context: propagate
-like($t_resp_propagate,
-	qr/X-Otel-Traceparent: 00-$$spans[8]{traceId}-$$spans[8]{spanId}-01/,
-	'traceparent - trace context propagate (no trace headers)');
-unlike($t_resp_propagate, qr/X-Otel-Tracestate/,
-	'no tracestate - trace context propagate (no trace headers)');
-
-like($tp_resp_propagate, qr/X-Otel-Parent-Id: b9c7c989f97918e1/,
-	'parent id - trace context propagate (trace headers)');
-like($tp_resp_propagate,
-	qr/Traceparent: 00-0af7651916cd43dd8448eb211c80319c-$$spans[9]{spanId}-01/,
-	'traceparent - trace context propagate (trace headers)');
-like($tp_resp_propagate,
-	qr/Tracestate: congo=ucfJifl5GOE,rojo=00f067aa0ba902b7/,
-	'tracestate - trace context propagate (trace headers)');
-
-$t->stop();
-my $log = $t->read_file("error.log");
-
-unlike($log, qr/OTel\/grpc: Error parsing metadata: error=invalid value/,
-	'log: no error parsing metadata');
-unlike($log, qr/OTel export failure: No status received/,
-	'log: no export failure');
-
-###############################################################################
-
-sub http1_get {
-	my ($path, %extra) = @_;
-
-	my $port = $extra{port} || 8080;
-
-	my $r = <<EOF;
-GET $path HTTP/1.0
-Host: localhost
-User-agent: nginx-tests
-EOF
-
-	$r .= <<EOF if $extra{trace_headers};
-Traceparent: 00-0af7651916cd43dd8448eb211c80319c-b9c7c989f97918e1-01
-Tracestate: congo=ucfJifl5GOE,rojo=00f067aa0ba902b7
-EOF
-
-	return http($r . "\n", PeerAddr => '127.0.0.1:' . port($port),
-		SSL => $extra{ssl});
-}
-
-sub get_attr {
-	my($attr, $type, $obj) = @_;
-
-	my ($res) = grep { $$_{"key"} eq $attr } @{$$obj{"attributes"}};
-
-	return defined $res ? $res->{"value"}{$type} : undef;
-}
-
-###############################################################################

tests/requirements.txt

@@ -0,0 +1,6 @@
+pytest~=8.3
+jinja2~=3.1
+pyopenssl~=24.3
+niquests~=3.11
+grpcio~=1.68
+opentelemetry-proto~=1.28

tests/test_otel.py

@@ -0,0 +1,331 @@
+from collections import namedtuple
+import niquests
+import pytest
+import socket
+import time
+import urllib3
+
+
+NGINX_CONFIG = """
+{{ globals }}
+
+daemon off;
+
+events {
+}
+
+http {
+    {{ http_globals }}
+
+    ssl_certificate localhost.crt;
+    ssl_certificate_key localhost.key;
+
+    otel_exporter {
+        endpoint {{ endpoint or "127.0.0.1:14317" }};
+        interval {{ interval or "1ms" }};
+        batch_size 3;
+        batch_count 3;
+
+        {{ exporter_opts }}
+    }
+
+    otel_trace on;
+    {{ resource_attrs }}
+
+    server {
+        listen       127.0.0.1:18443 ssl;
+        listen       127.0.0.1:18443 quic;
+        listen       127.0.0.1:18080;
+
+        http2 on;
+
+        server_name  localhost;
+
+        location /ok {
+            return 200 "OK";
+        }
+
+        location /err {
+            return 500 "ERR";
+        }
+
+        location /custom {
+            otel_span_name custom_location;
+            otel_span_attr http.request.completion
+                $request_completion;
+            otel_span_attr http.response.header.content.type
+                $sent_http_content_type;
+            otel_span_attr http.request $request;
+            return 200 "OK";
+        }
+
+        location /vars {
+            otel_trace_context extract;
+            add_header "X-Otel-Trace-Id" $otel_trace_id;
+            add_header "X-Otel-Span-Id" $otel_span_id;
+            add_header "X-Otel-Parent-Id" $otel_parent_id;
+            add_header "X-Otel-Parent-Sampled" $otel_parent_sampled;
+            return 204;
+        }
+
+        location /ignore {
+            proxy_pass http://127.0.0.1:18080/notrace;
+        }
+
+        location /extract {
+            otel_trace_context extract;
+            proxy_pass http://127.0.0.1:18080/notrace;
+        }
+
+        location /inject {
+            otel_trace_context inject;
+            proxy_pass http://127.0.0.1:18080/notrace;
+        }
+
+        location /propagate {
+            otel_trace_context propagate;
+            proxy_pass http://127.0.0.1:18080/notrace;
+        }
+
+        location /notrace {
+            otel_trace off;
+            add_header "X-Otel-Traceparent" $http_traceparent;
+            add_header "X-Otel-Tracestate" $http_tracestate;
+            return 204;
+        }
+    }
+}
+
+"""
+
+TraceContext = namedtuple("TraceContext", ["trace_id", "span_id", "state"])
+
+parent_ctx = TraceContext(
+    trace_id="0af7651916cd43dd8448eb211c80319c",
+    span_id="b9c7c989f97918e1",
+    state="congo=ucfJifl5GOE,rojo=00f067aa0ba902b7",
+)
+
+
+def trace_headers(ctx):
+    return (
+        {
+            "Traceparent": f"00-{ctx.trace_id}-{ctx.span_id}-01",
+            "Tracestate": ctx.state,
+        }
+        if ctx
+        else {"Traceparent": None, "Tracestate": None}
+    )
+
+
+def get_attr(span, name):
+    for value in (a.value for a in span.attributes if a.key == name):
+        return getattr(value, value.WhichOneof("value"))
+
+
+@pytest.fixture
+def client(nginx):
+    urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+    with niquests.Session(multiplexed=True) as s:
+        yield s
+
+
+def test_http09(trace_service, nginx):
+
+    def get_http09(host, port, path):
+        with socket.create_connection((host, port)) as sock:
+            sock.sendall(f"GET {path}\n".encode())
+            resp = sock.recv(1024).decode("utf-8")
+        return resp
+
+    assert get_http09("127.0.0.1", 18080, "/ok") == "OK"
+
+    span = trace_service.get_span()
+    assert span.name == "/ok"
+
+
+@pytest.mark.parametrize("http_ver", ["1.1", "2.0", "3.0"])
+@pytest.mark.parametrize(
+    ("path", "status"),
+    [("/ok", 200), ("/err", 500)],
+)
+def test_default_attributes(client, trace_service, http_ver, path, status):
+    scheme, port = ("http", 18080) if http_ver == "1.1" else ("https", 18443)
+    if http_ver == "3.0":
+        client.quic_cache_layer.add_domain("127.0.0.1", port)
+    r = client.get(f"{scheme}://127.0.0.1:{port}{path}", verify=False)
+
+    span = trace_service.get_span()
+    assert span.name == path
+
+    assert get_attr(span, "http.method") == "GET"
+    assert get_attr(span, "http.target") == path
+    assert get_attr(span, "http.route") == path
+    assert get_attr(span, "http.scheme") == scheme
+    assert get_attr(span, "http.flavor") == http_ver
+    assert get_attr(span, "http.user_agent") == (
+        f"niquests/{niquests.__version__}"
+    )
+    assert get_attr(span, "http.request_content_length") == 0
+    assert get_attr(span, "http.response_content_length") == len(r.text)
+    assert get_attr(span, "http.status_code") == status
+    assert get_attr(span, "net.host.name") == "localhost"
+    assert get_attr(span, "net.host.port") == port
+    assert get_attr(span, "net.sock.peer.addr") == "127.0.0.1"
+    assert get_attr(span, "net.sock.peer.port") in range(1024, 65536)
+
+
+def test_custom_attributes(client, trace_service):
+    assert client.get("http://127.0.0.1:18080/custom").status_code == 200
+
+    span = trace_service.get_span()
+    assert span.name == "custom_location"
+
+    assert get_attr(span, "http.request.completion") == "OK"
+    value = get_attr(span, "http.response.header.content.type")
+    assert value.values[0].string_value == "text/plain"
+    assert get_attr(span, "http.request") == "GET /custom HTTP/1.1"
+
+
+def test_trace_off(client, trace_service):
+    assert client.get("http://127.0.0.1:18080/notrace").status_code == 204
+
+    time.sleep(0.01)  # wait for spans
+    assert len(trace_service.batches) == 0
+
+
+@pytest.mark.parametrize("parent", [None, parent_ctx])
+def test_variables(client, trace_service, parent):
+    r = client.get("http://127.0.0.1:18080/vars", headers=trace_headers(parent))
+
+    span = trace_service.get_span()
+
+    if parent:
+        assert span.trace_id.hex() == parent.trace_id
+        assert span.parent_span_id.hex() == parent.span_id
+        assert span.trace_state == parent.state
+
+    assert r.headers.get("X-Otel-Trace-Id") == span.trace_id.hex()
+    assert r.headers.get("X-Otel-Span-Id") == span.span_id.hex()
+    assert r.headers.get("X-Otel-Parent-Id") or "" == span.parent_span_id.hex()
+    assert r.headers.get("X-Otel-Parent-Sampled") == ("1" if parent else "0")
+
+
+@pytest.mark.parametrize("parent", [None, parent_ctx])
+@pytest.mark.parametrize(
+    "path", ["/ignore", "/extract", "/inject", "/propagate"]
+)
+def test_context(client, trace_service, parent, path):
+    headers = trace_headers(parent)
+
+    r = client.get(f"http://127.0.0.1:18080{path}", headers=headers)
+
+    span = trace_service.get_span()
+
+    if path in ["/extract", "/propagate"] and parent:
+        assert span.trace_id.hex() == parent.trace_id
+        assert span.parent_span_id.hex() == parent.span_id
+        assert span.trace_state == parent.state
+
+    if path in ["/inject", "/propagate"]:
+        headers = trace_headers(
+            TraceContext(
+                span.trace_id.hex(),
+                span.span_id.hex(),
+                span.trace_state or None,
+            )
+        )
+
+    assert r.headers.get("X-Otel-Traceparent") == headers["Traceparent"]
+    assert r.headers.get("X-Otel-Tracestate") == headers["Tracestate"]
+
+
+@pytest.mark.parametrize(
+    "nginx_config",
+    [{"interval": "200ms", "endpoint": "http://127.0.0.1:14317"}],
+    indirect=True,
+)
+@pytest.mark.parametrize("batch_count", [1, 3])
+def test_batches(client, trace_service, batch_count):
+    batch_size = 3
+
+    for _ in range(
+        batch_count * batch_size + 1
+    ):  # +1 request to trigger batch sending
+        assert client.get("http://127.0.0.1:18080/ok").status_code == 200
+
+    time.sleep(0.01)
+
+    assert len(trace_service.batches) == batch_count
+
+    for batch in trace_service.batches:
+        assert (
+            get_attr(batch[0].resource, "service.name")
+            == "unknown_service:nginx"
+        )
+        assert len(batch[0].scope_spans[0].spans) == batch_size
+
+    time.sleep(0.3)  # wait for +1 request to be flushed
+    trace_service.batches.clear()
+
+
+@pytest.mark.parametrize(
+    "nginx_config",
+    [
+        {
+            "resource_attrs": """
+                otel_service_name "test_service";
+                otel_resource_attr my.name "my name";
+                otel_resource_attr my.service "my service";
+            """,
+        }
+    ],
+    indirect=True,
+)
+def test_custom_resource_attributes(client, trace_service):
+    assert client.get("http://127.0.0.1:18080/ok").status_code == 200
+
+    batch = trace_service.get_batch()
+
+    assert get_attr(batch.resource, "service.name") == "test_service"
+    assert get_attr(batch.resource, "my.name") == "my name"
+    assert get_attr(batch.resource, "my.service") == "my service"
+
+
+@pytest.mark.parametrize(
+    "nginx_config",
+    [
+        {
+            "exporter_opts": """
+                header X-API-TOKEN api.value;
+                header Authorization "Basic value";
+            """,
+        }
+    ],
+    indirect=True,
+)
+@pytest.mark.parametrize("trace_service", ["skip_otelcol"], indirect=True)
+def test_exporter_headers(client, trace_service):
+    assert client.get("http://127.0.0.1:18080/ok").status_code == 200
+
+    assert trace_service.get_span().name == "/ok"
+
+    headers = dict(trace_service.last_metadata)
+    assert headers["x-api-token"] == "api.value"
+    assert headers["authorization"] == "Basic value"
+
+
+@pytest.mark.parametrize(
+    "nginx_config",
+    [
+        {
+            "endpoint": "https://localhost:14318",
+            "exporter_opts": "trusted_certificate localhost.crt;",
+        }
+    ],
+    indirect=True,
+)
+def test_tls_export(client, trace_service):
+    assert client.get("http://127.0.0.1:18080/ok").status_code == 200
+
+    assert trace_service.get_span().name == "/ok"

tests/trace_service.py

@@ -0,0 +1,107 @@
+import concurrent
+import grpc
+from opentelemetry.proto.collector.trace.v1 import trace_service_pb2
+from opentelemetry.proto.collector.trace.v1 import trace_service_pb2_grpc
+import pytest
+import subprocess
+import time
+
+
+class TraceService(trace_service_pb2_grpc.TraceServiceServicer):
+    batches = []
+
+    def Export(self, request, context):
+        self.batches.append(request.resource_spans)
+        self.last_metadata = context.invocation_metadata()
+        return trace_service_pb2.ExportTracePartialSuccess()
+
+    def get_batch(self):
+        for _ in range(10):
+            if len(self.batches):
+                break
+            time.sleep(0.001)
+        assert len(self.batches) == 1
+        assert len(self.batches[0]) == 1
+        return self.batches.pop()[0]
+
+    def get_span(self):
+        batch = self.get_batch()
+        assert len(batch.scope_spans) == 1
+        assert len(batch.scope_spans[0].spans) == 1
+        return batch.scope_spans[0].spans.pop()
+
+
+@pytest.fixture(scope="module")
+def trace_service(request, pytestconfig, logger, cert):
+    server = grpc.server(concurrent.futures.ThreadPoolExecutor())
+    trace_service = TraceService()
+    trace_service_pb2_grpc.add_TraceServiceServicer_to_server(
+        trace_service, server
+    )
+    trace_service.use_otelcol = (
+        pytestconfig.option.otelcol
+        and getattr(request, "param", "") != "skip_otelcol"
+    )
+    listen_addr = f"127.0.0.1:{24317 if trace_service.use_otelcol else 14317}"
+    server.add_insecure_port(listen_addr)
+    if not trace_service.use_otelcol:
+        creds = grpc.ssl_server_credentials([cert])
+        server.add_secure_port("127.0.0.1:14318", creds)
+        listen_addr += " and 127.0.0.1:14318"
+    logger.info(f"Starting trace service at {listen_addr}...")
+    server.start()
+    yield trace_service
+    logger.info("Stopping trace service...")
+    server.stop(grace=None)
+
+
+@pytest.fixture(scope="module")
+def otelcol(pytestconfig, testdir, logger, trace_service, cert):
+    if not trace_service.use_otelcol:
+        yield
+        return
+
+    (testdir / "otel-config.yaml").write_text(
+        f"""receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: 127.0.0.1:14317
+
+  otlp/tls:
+    protocols:
+      grpc:
+        endpoint: 127.0.0.1:14318
+        tls:
+          cert_file: {testdir}/localhost.crt
+          key_file: {testdir}/localhost.key
+
+exporters:
+  otlp:
+    endpoint: 127.0.0.1:24317
+    tls:
+      insecure: true
+
+service:
+  pipelines:
+    traces:
+      receivers: [otlp, otlp/tls]
+      exporters: [otlp]
+  telemetry:
+    metrics:
+      # prevent otelcol from opening 8888 port
+      level: none"""
+    )
+    logger.info("Starting otelcol at 127.0.0.1:14317...")
+    proc = subprocess.Popen(
+        [pytestconfig.option.otelcol, "--config", testdir / "otel-config.yaml"]
+    )
+    time.sleep(1)  # give some time to get ready
+    assert proc.poll() is None, "Can't start otelcol"
+    yield
+    logger.info("Stopping otelcol...")
+    proc.terminate()
+    try:
+        proc.wait(timeout=5)
+    except subprocess.TimeoutExpired:
+        proc.kill()