From f6f2525dd71658effa09018c7d0e056bfd0ccd32 Mon Sep 17 00:00:00 2001 From: Evgeny <54681898+jimf5@users.noreply.github.com> Date: Wed, 14 Jun 2023 09:58:46 -0700 Subject: [PATCH 01/44] fix: building with nginx with HTTP/3 (#4) --- .github/workflows/nginx-otel-module-check.yml | 31 +++++++++++++++++++ CMakeLists.txt | 3 ++ 2 files changed, 34 insertions(+) create mode 100644 .github/workflows/nginx-otel-module-check.yml diff --git a/.github/workflows/nginx-otel-module-check.yml b/.github/workflows/nginx-otel-module-check.yml new file mode 100644 index 0000000..9493af5 --- /dev/null +++ b/.github/workflows/nginx-otel-module-check.yml @@ -0,0 +1,31 @@ +name: nginx-otel-module-check +run-name: ${{ github.actor }} is triggering pipeline +on: [push] +jobs: + build-module: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v3 + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y cmake libc-ares-dev libre2-dev + - name: Checkout nginx + run: hg clone http://hg.nginx.org/nginx/ + - name: Configure nginx + working-directory: nginx + run: auto/configure --with-compat + - name: Create build directory + run: mkdir build + - name: Build module + working-directory: build + run: | + cmake -DNGX_OTEL_NGINX_BUILD_DIR=${PWD}/../nginx/objs .. + make -j 4 + strip ngx_otel_module.so + - name: Archive module + uses: actions/upload-artifact@v3 + with: + name: nginx-otel-module + path: build/ngx_otel_module.so diff --git a/CMakeLists.txt b/CMakeLists.txt index 2f204d7..96ac4cd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -117,10 +117,13 @@ target_include_directories(ngx_otel_module PRIVATE ${NGX_OTEL_NGINX_BUILD_DIR} ${NGX_OTEL_NGINX_DIR}/src/core ${NGX_OTEL_NGINX_DIR}/src/event + ${NGX_OTEL_NGINX_DIR}/src/event/modules + ${NGX_OTEL_NGINX_DIR}/src/event/quic ${NGX_OTEL_NGINX_DIR}/src/os/unix ${NGX_OTEL_NGINX_DIR}/src/http ${NGX_OTEL_NGINX_DIR}/src/http/modules ${NGX_OTEL_NGINX_DIR}/src/http/v2 + ${NGX_OTEL_NGINX_DIR}/src/http/v3 ${PROTO_OUT_DIR}) target_link_libraries(ngx_otel_module From 3d528bb093a1334ec72c5e37a0e7e8750b296bcb Mon Sep 17 00:00:00 2001 From: Evgeny <54681898+jimf5@users.noreply.github.com> Date: Wed, 16 Aug 2023 17:20:25 -0700 Subject: [PATCH 02/44] add: functional tests (#5) * add functional tests --- .github/workflows/nginx-otel-module-check.yml | 51 ++ tests/h2_otel.t | 553 ++++++++++++++++++ tests/h3_otel.t | 508 ++++++++++++++++ tests/otel.t | 514 ++++++++++++++++ tests/otel_collector.t | 403 +++++++++++++ 5 files changed, 2029 insertions(+) create mode 100644 tests/h2_otel.t create mode 100644 tests/h3_otel.t create mode 100644 tests/otel.t create mode 100644 tests/otel_collector.t diff --git a/.github/workflows/nginx-otel-module-check.yml b/.github/workflows/nginx-otel-module-check.yml index 9493af5..ba81a95 100644 --- a/.github/workflows/nginx-otel-module-check.yml +++ b/.github/workflows/nginx-otel-module-check.yml @@ -29,3 +29,54 @@ jobs: with: name: nginx-otel-module path: build/ngx_otel_module.so + - name: Archive protoc and opentelemetry-proto + uses: actions/upload-artifact@v3 + with: + name: protoc-opentelemetry-proto + path: | + build/_deps/grpc-build/third_party/protobuf/protoc + build/_deps/otelcpp-src/third_party/opentelemetry-proto + test-module: + needs: build-module + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v3 + - name: Download module + uses: actions/download-artifact@v3 + with: + name: nginx-otel-module + path: build + - name: Download protoc and opentelemetry-proto + uses: actions/download-artifact@v3 + with: + name: protoc-opentelemetry-proto + path: build/_deps + - name: List files + run: ls -laR . + - name: Fix protoc file permissions + run: chmod +x build/_deps/grpc-build/third_party/protobuf/protoc + - name: Install perl modules + run: sudo cpan IO::Socket::SSL Crypt::Misc + - name: Download otelcol + run: | + curl -LO https://github.com/\ + open-telemetry/opentelemetry-collector-releases/releases/download/\ + v0.76.1/otelcol_0.76.1_linux_amd64.tar.gz + tar -xzf otelcol_0.76.1_linux_amd64.tar.gz + - name: Checkout nginx and nginx-test + run: | + hg clone http://hg.nginx.org/nginx/ + hg clone http://hg.nginx.org/nginx-tests/ + - name: Build nginx + working-directory: nginx + run: | + auto/configure --with-compat --with-debug --with-http_ssl_module \ + --with-http_v2_module --with-http_v3_module + make -j 4 + - name: Run tests + working-directory: tests + run: | + PERL5LIB=../nginx-tests/lib TEST_NGINX_UNSAFE=1 \ + TEST_NGINX_VERBOSE=1 TEST_NGINX_GLOBALS="load_module \ + ${PWD}/../build/ngx_otel_module.so;" prove -v . diff --git a/tests/h2_otel.t b/tests/h2_otel.t new file mode 100644 index 0000000..24987d7 --- /dev/null +++ b/tests/h2_otel.t @@ -0,0 +1,553 @@ +#!/usr/bin/perl + +# (C) Nginx, Inc. + +# Tests for OTel exporter in case HTTP/2. + +############################################################################### + +use warnings; +use strict; + +use Test::More; + +BEGIN { use FindBin; chdir($FindBin::Bin); } + +use Test::Nginx; +use Test::Nginx::HTTP2; +use MIME::Base64; + +############################################################################### + +select STDERR; $| = 1; +select STDOUT; $| = 1; + +my $t = Test::Nginx->new() + ->has(qw/http_v2 http_ssl rewrite mirror grpc socket_ssl_alpn/) + ->has_daemon(qw/openssl base64/) + ->write_file_expand('nginx.conf', <<'EOF'); + +%%TEST_GLOBALS%% + +daemon off; + +events { +} + +http { + %%TEST_GLOBALS_HTTP%% + + ssl_certificate_key localhost.key; + ssl_certificate localhost.crt; + + otel_exporter { + endpoint 127.0.0.1:8083; + interval 1s; + batch_size 10; + batch_count 1; + } + + otel_service_name test_server; + otel_trace on; + + server { + listen 127.0.0.1:8080 http2; + listen 127.0.0.1:8081; + listen 127.0.0.1:8082 http2 ssl; + server_name localhost; + + location /trace-on { + otel_trace_context extract; + otel_span_name default_location; + otel_span_attr http.request.header.completion + $request_completion; + otel_span_attr http.response.header.content.type + $sent_http_content_type; + otel_span_attr http.request $request; + add_header "X-Otel-Trace-Id" $otel_trace_id; + add_header "X-Otel-Span-Id" $otel_span_id; + add_header "X-Otel-Parent-Id" $otel_parent_id; + add_header "X-Otel-Parent-Sampled" $otel_parent_sampled; + return 200 "TRACE-ON"; + } + + location /context-ignore { + otel_trace_context ignore; + otel_span_name context_ignore; + add_header "X-Otel-Parent-Id" $otel_parent_id; + proxy_pass http://127.0.0.1:8081/trace-off; + } + + location /context-extract { + otel_trace_context extract; + otel_span_name context_extract; + add_header "X-Otel-Parent-Id" $otel_parent_id; + proxy_pass http://127.0.0.1:8081/trace-off; + } + + location /context-inject { + otel_trace_context inject; + otel_span_name context_inject; + add_header "X-Otel-Parent-Id" $otel_parent_id; + proxy_pass http://127.0.0.1:8081/trace-off; + } + + location /context-propagate { + otel_trace_context propagate; + otel_span_name context_propogate; + add_header "X-Otel-Parent-Id" $otel_parent_id; + proxy_pass http://127.0.0.1:8081/trace-off; + } + + location /trace-off { + otel_trace off; + add_header "X-Otel-Traceparent" $http_traceparent; + add_header "X-Otel-Tracestate" $http_tracestate; + return 200 "TRACE-OFF"; + } + } + + server { + listen 127.0.0.1:8083 http2; + server_name localhost; + otel_trace off; + + location / { + mirror /mirror; + grpc_pass 127.0.0.1:8084; + } + + location /mirror { + internal; + grpc_pass 127.0.0.1:%%PORT_4317%%; + } + } + + server { + listen 127.0.0.1:8084 http2; + server_name localhost; + otel_trace off; + + location / { + add_header content-type application/grpc; + add_header grpc-status 0; + add_header grpc-message ""; + return 200; + } + } + +} + +EOF + +$t->write_file('openssl.conf', <<'EOF'); +[ req ] +default_bits = 2048 +encrypt_key = no +distinguished_name = req_distinguished_name +[ req_distinguished_name ] + +EOF + +my $d = $t->testdir(); + +foreach my $name ('localhost') { + system('openssl req -x509 -new ' + . "-config $d/openssl.conf -subj /CN=$name/ " + . "-out $d/$name.crt -keyout $d/$name.key " + . ">>$d/openssl.out 2>&1") == 0 + or die "Can't create certificate for $name: $!\n"; +} + +$t->try_run('no OTel module')->plan(69); + +############################################################################### + +my $p = port(4317); +my $f = grpc(); + +#do requests +(undef, my $t_off_resp) = http2_get('/trace-off'); + +#batch0 (10 requests) +my ($tp_headers, $tp_resp) = http2_get('/trace-on', trace_headers => 1); +my ($t_headers, $t_resp) = http2_get('/trace-on', port => 8082, ssl => 1); + +(my $t_headers_ignore, undef) = http2_get('/context-ignore'); +(my $tp_headers_ignore, undef) = http2_get('/context-ignore', + trace_headers => 1); +(my $t_headers_extract, undef) = http2_get('/context-extract'); +(my $tp_headers_extract, undef) = http2_get('/context-extract', + trace_headers => 1); +(my $t_headers_inject, undef) = http2_get('/context-inject'); +(my $tp_headers_inject, undef) = http2_get('/context-inject', + trace_headers => 1); +(my $t_headers_propagate, undef) = http2_get('/context-propagate'); +(my $tp_headers_propagate, undef) = + http2_get('/context-propagate', trace_headers => 1); + +my ($frame) = grep { $_->{type} eq "DATA" } @{$f->{http_start}()}; +my $batch0 = to_hash(decode_protobuf(substr $frame->{data}, 8)); +my $spans = $$batch0{scope_spans}; + +#batch1 (5 reqeusts) +http2_get('/trace-on') for (1..5); + +($frame) = grep { $_->{type} eq "DATA" } @{$f->{http_start}()}; +my $batch1 = to_hash(decode_protobuf(substr $frame->{data}, 8)); + +#validate responses +like($tp_resp, qr/TRACE-ON/, 'http request1 - trace on'); +like($t_resp, qr/TRACE-ON/, 'http request2 - trace on'); +like($t_off_resp, qr/TRACE-OFF/, 'http request - trace off'); + +#validate batch size +delete $$spans{scope}; #remove 'scope' entry +is(scalar keys %{$spans}, 10, 'batch0 size - trace on'); +delete $$batch1{scope_spans}{scope}; #remove 'scope' entry +is(scalar keys %{$$batch1{scope_spans}}, 5, 'batch1 size - trace on'); + +#validate general attributes +is(get_attr("service.name", "string_value", + $$batch0{resource}), + 'test_server', 'service.name - trace on'); +is($$spans{span0}{name}, '"default_location"', 'span.name - trace on'); + +#validate http metrics +is(get_attr("http.method", "string_value", $$spans{span0}), 'GET', + 'http.method metric - trace on'); +is(get_attr("http.target", "string_value", $$spans{span0}), '/trace-on', + 'http.target metric - trace on'); +is(get_attr("http.route", "string_value", $$spans{span0}), '/trace-on', + 'http.route metric - trace on'); +is(get_attr("http.scheme", "string_value", $$spans{span0}), 'http', + 'http.scheme metric - trace on'); +is(get_attr("http.flavor", "string_value", $$spans{span0}), '2.0', + 'http.flavor metric - trace on'); +is(get_attr("http.user_agent", "string_value", $$spans{span0}), 'nginx-tests', + 'http.user_agent metric - trace on'); +is(get_attr("http.request_content_length", "int_value", $$spans{span0}), 0, + 'http.request_content_length metric - trace on'); +is(get_attr("http.response_content_length", "int_value", $$spans{span0}), 8, + 'http.response_content_length metric - trace on'); +is(get_attr("http.status_code", "int_value", $$spans{span0}), 200, + 'http.status_code metric - trace on'); +is(get_attr("net.host.name", "string_value", $$spans{span0}), 'localhost', + 'net.host.name metric - trace on'); +is(get_attr("net.host.port", "int_value", $$spans{span0}), 8080, + 'net.host.port metric - trace on'); +is(get_attr("net.sock.peer.addr", "string_value", $$spans{span0}), '127.0.0.1', + 'net.sock.peer.addr metric - trace on'); +like(get_attr("net.sock.peer.port", "int_value", $$spans{span0}), qr/\d+/, + 'net.sock.peer.port metric - trace on'); + +#validate https metrics +is(get_attr("http.method", "string_value", $$spans{span1}), 'GET', + 'http.method metric - trace on (https)'); +is(get_attr("http.target", "string_value", $$spans{span1}), '/trace-on', + 'http.target metric - trace on (https)'); +is(get_attr("http.route", "string_value", $$spans{span1}), '/trace-on', + 'http.route metric - trace on (https)'); +is(get_attr("http.scheme", "string_value", $$spans{span1}), 'https', + 'http.scheme metric - trace on (https)'); +is(get_attr("http.flavor", "string_value", $$spans{span1}), '2.0', + 'http.flavor metric - trace on (https)'); +isnt(get_attr("http.user_agent", "string_value", $$spans{span1}), + 'nginx-tests', 'http.user_agent metric - trace on (https)'); +is(get_attr("http.request_content_length", "int_value", $$spans{span1}), 0, + 'http.request_content_length metric - trace on (https)'); +is(get_attr("http.response_content_length", "int_value", $$spans{span1}), 8, + 'http.response_content_length metric - trace on (https)'); +is(get_attr("http.status_code", "int_value", $$spans{span1}), 200, + 'http.status_code metric - trace on (https)'); +is(get_attr("net.host.name", "string_value", $$spans{span1}), 'localhost', + 'net.host.name metric - trace on (https)'); +is(get_attr("net.host.port", "int_value", $$spans{span1}), 8082, + 'net.host.port metric - trace on (https)'); +is(get_attr("net.sock.peer.addr", "string_value", $$spans{span1}), '127.0.0.1', + 'net.sock.peer.addr metric - trace on (https)'); +like(get_attr("net.sock.peer.port", "int_value", $$spans{span1}), qr/\d+/, + 'net.sock.peer.port metric - trace on (https)'); + +#validate custom http metrics +is(${get_attr("http.request.header.completion", "array_value", $$spans{span0})} + {values}{string_value}, '"OK"', + 'http.request.header.completion metric - trace on'); +is(${get_attr( + "http.response.header.content.type", "array_value", $$spans{span0} + )}{values}{string_value}, '"text/plain"', + 'http.response.header.content.type metric - trace on'); +is(get_attr("http.request", "string_value", $$spans{span0}), + 'GET /trace-on HTTP/2.0', 'http.request metric - trace on'); + +#extract trace info +is($$spans{span0}{parent_span_id}, 'b9c7c989f97918e1', + 'traceparent - trace on'); +is($$spans{span0}{trace_state}, '"congo=ucfJifl5GOE,rojo=00f067aa0ba902b7"', + 'tracestate - trace on'); +is($$spans{span1}{parent_span_id}, undef, 'no traceparent - trace on'); +is($$spans{span1}{trace_state}, undef, 'no tracestate - trace on'); + +#variables +is($tp_headers->{'x-otel-trace-id'}, $$spans{span0}{trace_id}, + '$otel_trace_id variable - trace on'); +is($tp_headers->{'x-otel-span-id'}, $$spans{span0}{span_id}, + '$otel_span_id variable - trace on'); +is($tp_headers->{'x-otel-parent-id'}, $$spans{span0}{parent_span_id}, + '$otel_parent_id variable - trace on'); +is($tp_headers->{'x-otel-parent-sampled'}, 1, + '$otel_parent_sampled variable - trace on'); +is($t_headers->{'x-otel-parent-sampled'}, 0, + '$otel_parent_sampled variable - trace on (no traceparent header)'); + +#trace off +is((scalar grep { + get_attr("http.target", "string_value", $$spans{$_}) eq '/trace-off' + } keys %{$spans}), 0, 'no metric in batch0 - trace off'); +is((scalar grep { + get_attr("http.target", "string_value", $$spans{$_}) eq '/trace-off' + } keys %{$$batch1{scope_spans}}), 0, 'no metric in batch1 - trace off'); + +#trace context: ignore +is($t_headers_ignore->{'x-otel-traceparent'}, undef, + 'no traceparent - trace context ignore (no trace headers)'); +is($t_headers_ignore->{'x-otel-tracestate'}, undef, + 'no tracestate - trace context ignore (no trace headers)'); + +is($tp_headers_ignore->{'x-otel-parent-id'}, undef, + 'no parent span id - trace context ignore (trace headers)'); +is($tp_headers_ignore->{'x-otel-traceparent'}, + '00-0af7651916cd43dd8448eb211c80319c-b9c7c989f97918e1-01', + 'traceparent - trace context ignore (trace headers)'); +is($tp_headers_ignore->{'x-otel-tracestate'}, + 'congo=ucfJifl5GOE,rojo=00f067aa0ba902b7', + 'tracestate - trace context ignore (trace headers)'); + +#trace context: extract +is($t_headers_extract->{'x-otel-traceparent'}, undef, + 'no traceparent - trace context extract (no trace headers)'); +is($t_headers_extract->{'x-otel-tracestate'}, undef, + 'no tracestate - trace context extract (no trace headers)'); + +is($tp_headers_extract->{'x-otel-parent-id'}, 'b9c7c989f97918e1', + 'parent span id - trace context extract (trace headers)'); +is($tp_headers_extract->{'x-otel-traceparent'}, + '00-0af7651916cd43dd8448eb211c80319c-b9c7c989f97918e1-01', + 'traceparent - trace context extract (trace headers)'); +is($tp_headers_extract->{'x-otel-tracestate'}, + 'congo=ucfJifl5GOE,rojo=00f067aa0ba902b7', + 'tracestate - trace context extract (trace headers)'); + +#trace context: inject +is($t_headers_inject->{'x-otel-traceparent'}, + "00-$$spans{span6}{trace_id}-$$spans{span6}{span_id}-01", + 'traceparent - trace context inject (no trace headers)'); +is($t_headers_inject->{'x-otel-tracestate'}, undef, + 'no tracestate - trace context inject (no trace headers)'); + +is($tp_headers_inject->{'x-otel-parent-id'}, undef, + 'no parent span id - trace context inject (trace headers)'); +is($tp_headers_inject->{'x-otel-traceparent'}, + "00-$$spans{span7}{trace_id}-$$spans{span7}{span_id}-01", + 'traceparent - trace context inject (trace headers)'); +is($tp_headers_inject->{'x-otel-tracestate'}, undef, + 'no tracestate - trace context inject (trace headers)'); + +#trace context: propagate +is($t_headers_propagate->{'x-otel-traceparent'}, + "00-$$spans{span8}{trace_id}-$$spans{span8}{span_id}-01", + 'traceparent - trace context propagate (no trace headers)'); +is($t_headers_propagate->{'x-otel-tracestate'}, undef, + 'no tracestate - trace context propagate (no trace headers)'); + +is($tp_headers_propagate->{'x-otel-parent-id'}, 'b9c7c989f97918e1', + 'parent id - trace context propagate (trace headers)'); +is($tp_headers_propagate->{'x-otel-traceparent'}, + "00-0af7651916cd43dd8448eb211c80319c-$$spans{span9}{span_id}-01", + 'traceparent - trace context propagate (trace headers)'); +is($tp_headers_propagate->{'x-otel-tracestate'}, + 'congo=ucfJifl5GOE,rojo=00f067aa0ba902b7', + 'tracestate - trace context propagate (trace headers)'); + +SKIP: { +skip "depends on error log content", 2 unless $ENV{TEST_NGINX_UNSAFE}; + +$t->stop(); +my $log = $t->read_file("error.log"); + +like($log, qr/OTel\/grpc: Error parsing metadata: error=invalid value/, + 'log: error parsing metadata - no protobuf in response'); +unlike($log, qr/OTel export failure: No status received/, + 'log: no export failure'); + +} + +############################################################################### + +sub http2_get { + my ($path, %extra) = @_; + my ($frames, $frame); + + my $port = $extra{port} || 8080; + + my $s = $extra{ssl} + ? Test::Nginx::HTTP2->new( + undef, socket => get_ssl_socket($port, ['h2'])) + : Test::Nginx::HTTP2->new(); + + my $sid = $extra{trace_headers} + ? $s->new_stream({ headers => [ + { name => ':method', value => 'GET' }, + { name => ':scheme', value => 'http' }, + { name => ':path', value => $path }, + { name => ':authority', value => 'localhost' }, + { name => 'user-agent', value => 'nginx-tests', mode => 2 }, + { name => 'traceparent', + value => '00-0af7651916cd43dd8448eb211c80319c-' . + 'b9c7c989f97918e1-01', + mode => 2 + }, + { name => 'tracestate', + value => 'congo=ucfJifl5GOE,rojo=00f067aa0ba902b7', + mode => 2 + }]}) + : $s->new_stream({ path => $path }); + $frames = $s->read(all => [{ sid => $sid, fin => 1 }]); + + ($frame) = grep { $_->{type} eq "HEADERS" } @$frames; + my $headers = $frame->{headers}; + + ($frame) = grep { $_->{type} eq "DATA" } @$frames; + my $data = $frame->{data}; + + return $headers, $data; +} + +sub get_ssl_socket { + my ($port, $alpn) = @_; + + return http( + '', PeerAddr => '127.0.0.1:' . port($port), start => 1, + SSL => 1, + SSL_alpn_protocols => $alpn, + SSL_error_trap => sub { die $_[1] } + ); +} + +sub get_attr { + my($attr, $type, $obj) = @_; + + my ($res) = grep { + $_ =~ /^attribute\d+/ && $$obj{$_}{key} eq '"' . $attr . '"' + } keys %{$obj}; + + if (defined $res) { + $$obj{$res}{value}{$type} =~ s/(^\")|(\"$)//g + if $type eq 'string_value'; + + return $$obj{$res}{value}{$type}; + } + + return undef; +} + +sub decode_protobuf { + my ($protobuf) = @_; + + local $/; + open CMD, "echo '" . encode_base64($protobuf) . "' | base64 -d | " . + '$PWD/../build/_deps/grpc-build/third_party/protobuf/protoc '. + '--decode opentelemetry.proto.trace.v1.ResourceSpans -I ' . + '$PWD/../build/_deps/otelcpp-src/third_party/opentelemetry-proto ' . + 'opentelemetry/proto/collector/trace/v1/trace_service.proto |' + or die "Can't decode protobuf: $!\n"; + my $out = ; + close CMD; + + return $out; +} + +sub decode_bytes { + my ($bytes) = @_; + + my $c = sub { return chr oct(shift) }; + + $bytes =~ s/\\(\d{3})/$c->($1)/eg; + $bytes =~ s/(^\")|(\"$)//g; + $bytes =~ s/\\\\/\\/g; + $bytes =~ s/\\r/\r/g; + $bytes =~ s/\\n/\n/g; + $bytes =~ s/\\t/\t/g; + $bytes =~ s/\\"/\"/g; + $bytes =~ s/\\'/\'/g; + + return unpack("H*", unpack("a*", $bytes)); +} + +sub to_hash { + my ($textdata) = @_; + + my %out = (); + push my @stack, \%out; + my ($attr_count, $span_count) = (0, 0); + for my $line (split /\n/, $textdata) { + $line =~ s/(^\s+)|(\s+$)//g; + if ($line =~ /\:/) { + my ($k, $v) = split /\: /, $line; + $v = decode_bytes($v) if ($k =~ /trace_id|span_id|parent_span_id/); + $stack[$#stack]{$k} = $v; + } elsif ($line =~ /\{/) { + $line =~ s/\s\{//; + $line = 'attribute' . $attr_count++ if ($line eq 'attributes'); + if ($line eq 'spans') { + $line = 'span' . $span_count++; + $attr_count = 0; + } + my %new = (); + $stack[$#stack]{$line} = \%new; + push @stack, \%new; + } elsif ($line =~ /\}/) { + pop @stack; + } + } + + return \%out; +} + +sub grpc { + my ($server, $client, $f, $s, $c, $sid, $csid, $uri); + + $server = IO::Socket::INET->new( + Proto => 'tcp', + LocalHost => '127.0.0.1', + LocalPort => $p, + Listen => 5, + Reuse => 1 + ) or die "Can't create listening socket: $!\n"; + + $f->{http_start} = sub { + if (IO::Select->new($server)->can_read(5)) { + $client = $server->accept(); + } else { + # connection could be unexpectedly reused + goto reused if $client; + return undef; + } + + $client->sysread($_, 24) == 24 or return; # preface + + $c = Test::Nginx::HTTP2->new(1, socket => $client, + pure => 1, preface => "") or return; + +reused: + my $frames = $c->read(all => [{ fin => 1 }]); + + $client->close(); + + return $frames; + }; + + return $f; +} + +############################################################################### diff --git a/tests/h3_otel.t b/tests/h3_otel.t new file mode 100644 index 0000000..ef848cb --- /dev/null +++ b/tests/h3_otel.t @@ -0,0 +1,508 @@ +#!/usr/bin/perl + +# (C) Nginx, Inc. + +# Tests for OTel exporter in case HTTP/3. + +############################################################################### + +use warnings; +use strict; + +use Test::More; + +BEGIN { use FindBin; chdir($FindBin::Bin); } + +use Test::Nginx; +use Test::Nginx::HTTP2; +use Test::Nginx::HTTP3; +use MIME::Base64; + +############################################################################### + +select STDERR; $| = 1; +select STDOUT; $| = 1; + +my $t = Test::Nginx->new()->has(qw/http_v2 http_v3 rewrite mirror grpc cryptx/) + ->has_daemon(qw/openssl base64/) + ->write_file_expand('nginx.conf', <<'EOF'); + +%%TEST_GLOBALS%% + +daemon off; + +events { +} + +http { + %%TEST_GLOBALS_HTTP%% + + ssl_certificate_key localhost.key; + ssl_certificate localhost.crt; + ssl_protocols TLSv1.3; + + otel_exporter { + endpoint 127.0.0.1:8082; + interval 1s; + batch_size 10; + batch_count 2; + } + + otel_service_name test_server; + otel_trace on; + + server { + listen 127.0.0.1:%%PORT_8980_UDP%% quic; + listen 127.0.0.1:8081; + server_name localhost; + + location /trace-on { + otel_trace_context extract; + otel_span_name default_location; + otel_span_attr http.request.header.completion + $request_completion; + otel_span_attr http.response.header.content.type + $sent_http_content_type; + otel_span_attr http.request $request; + add_header "X-Otel-Trace-Id" $otel_trace_id; + add_header "X-Otel-Span-Id" $otel_span_id; + add_header "X-Otel-Parent-Id" $otel_parent_id; + add_header "X-Otel-Parent-Sampled" $otel_parent_sampled; + return 200 "TRACE-ON"; + } + + location /context-ignore { + otel_trace_context ignore; + otel_span_name context_ignore; + add_header "X-Otel-Parent-Id" $otel_parent_id; + proxy_pass http://127.0.0.1:8081/trace-off; + } + + location /context-extract { + otel_trace_context extract; + otel_span_name context_extract; + add_header "X-Otel-Parent-Id" $otel_parent_id; + proxy_pass http://127.0.0.1:8081/trace-off; + } + + location /context-inject { + otel_trace_context inject; + otel_span_name context_inject; + add_header "X-Otel-Parent-Id" $otel_parent_id; + proxy_pass http://127.0.0.1:8081/trace-off; + } + + location /context-propagate { + otel_trace_context propagate; + otel_span_name context_propogate; + add_header "X-Otel-Parent-Id" $otel_parent_id; + proxy_pass http://127.0.0.1:8081/trace-off; + } + + location /trace-off { + otel_trace off; + add_header "X-Otel-Traceparent" $http_traceparent; + add_header "X-Otel-Tracestate" $http_tracestate; + return 200 "TRACE-OFF"; + } + } + + server { + listen 127.0.0.1:8082 http2; + server_name localhost; + otel_trace off; + + location / { + mirror /mirror; + grpc_pass 127.0.0.1:8083; + } + + location /mirror { + internal; + grpc_pass 127.0.0.1:%%PORT_4317%%; + } + } + + server { + listen 127.0.0.1:8083 http2; + server_name localhost; + otel_trace off; + + location / { + add_header content-type application/grpc; + add_header grpc-status 0; + add_header grpc-message ""; + return 200; + } + } + +} + +EOF + +$t->write_file('openssl.conf', <<'EOF'); +[ req ] +default_bits = 2048 +encrypt_key = no +distinguished_name = req_distinguished_name +[ req_distinguished_name ] + +EOF + +my $d = $t->testdir(); + +foreach my $name ('localhost') { + system('openssl req -x509 -new ' + . "-config $d/openssl.conf -subj /CN=$name/ " + . "-out $d/$name.crt -keyout $d/$name.key " + . ">>$d/openssl.out 2>&1") == 0 + or die "Can't create certificate for $name: $!\n"; +} + +$t->try_run('no OTel module')->plan(56); + +############################################################################### + +my $p = port(4317); +my $f = grpc(); + +#do requests +(undef, my $t_off_resp) = http3_get('/trace-off'); + +#batch0 (10 requests) +my ($tp_headers, $tp_resp) = http3_get('/trace-on', trace_headers => 1); +my ($t_headers, $t_resp) = http3_get('/trace-on'); + +(my $t_headers_ignore, undef) = http3_get('/context-ignore'); +(my $tp_headers_ignore, undef) = http3_get('/context-ignore', + trace_headers => 1); +(my $t_headers_extract, undef) = http3_get('/context-extract'); +(my $tp_headers_extract, undef) = http3_get('/context-extract', + trace_headers => 1); +(my $t_headers_inject, undef) = http3_get('/context-inject'); +(my $tp_headers_inject, undef) = http3_get('/context-inject', + trace_headers => 1); +(my $t_headers_propagate, undef) = http3_get('/context-propagate'); +(my $tp_headers_propagate, undef) = + http3_get('/context-propagate', trace_headers => 1); + +my ($frame) = grep { $_->{type} eq "DATA" } @{$f->{http_start}()}; +my $batch0 = to_hash(decode_protobuf(substr $frame->{data}, 8)); +my $spans = $$batch0{scope_spans}; + +#batch1 (5 reqeusts) +http3_get('/trace-on') for (1..5); + +($frame) = grep { $_->{type} eq "DATA" } @{$f->{http_start}()}; +my $batch1 = to_hash(decode_protobuf(substr $frame->{data}, 8)); + +#validate responses +like($tp_resp, qr/TRACE-ON/, 'http request1 - trace on'); +like($t_resp, qr/TRACE-ON/, 'http request2 - trace on'); +like($t_off_resp, qr/TRACE-OFF/, 'http request - trace off'); + +#validate batch size +delete $$spans{scope}; #remove 'scope' entry +is(scalar keys %{$spans}, 10, 'batch0 size - trace on'); +delete $$batch1{scope_spans}{scope}; #remove 'scope' entry +is(scalar keys %{$$batch1{scope_spans}}, 5, 'batch1 size - trace on'); + +#validate general attributes +is(get_attr("service.name", "string_value", + $$batch0{resource}), + 'test_server', 'service.name - trace on'); +is($$spans{span0}{name}, '"default_location"', 'span.name - trace on'); + +#validate metrics +is(get_attr("http.method", "string_value", $$spans{span0}), 'GET', + 'http.method metric - trace on'); +is(get_attr("http.target", "string_value", $$spans{span0}), '/trace-on', + 'http.target metric - trace on'); +is(get_attr("http.route", "string_value", $$spans{span0}), '/trace-on', + 'http.route metric - trace on'); +is(get_attr("http.scheme", "string_value", $$spans{span0}), 'https', + 'http.scheme metric - trace on'); +is(get_attr("http.flavor", "string_value", $$spans{span0}), '3.0', + 'http.flavor metric - trace on'); +is(get_attr("http.user_agent", "string_value", $$spans{span0}), 'nginx-tests', + 'http.user_agent metric - trace on'); +is(get_attr("http.request_content_length", "int_value", $$spans{span0}), 0, + 'http.request_content_length metric - trace on'); +is(get_attr("http.response_content_length", "int_value", $$spans{span0}), 8, + 'http.response_content_length metric - trace on'); +is(get_attr("http.status_code", "int_value", $$spans{span0}), 200, + 'http.status_code metric - trace on'); +is(get_attr("net.host.name", "string_value", $$spans{span0}), 'localhost', + 'net.host.name metric - trace on'); +is(get_attr("net.host.port", "int_value", $$spans{span0}), 8980, + 'net.host.port metric - trace on'); +is(get_attr("net.sock.peer.addr", "string_value", $$spans{span0}), '127.0.0.1', + 'net.sock.peer.addr metric - trace on'); +like(get_attr("net.sock.peer.port", "int_value", $$spans{span0}), qr/\d+/, + 'net.sock.peer.port metric - trace on'); + +#validate custom http metrics +is(${get_attr("http.request.header.completion", "array_value", $$spans{span0})} + {values}{string_value}, '"OK"', + 'http.request.header.completion metric - trace on'); +is(${get_attr( + "http.response.header.content.type", "array_value", $$spans{span0} + )}{values}{string_value}, '"text/plain"', + 'http.response.header.content.type metric - trace on'); +is(get_attr("http.request", "string_value", $$spans{span0}), + 'GET /trace-on HTTP/3.0', 'http.request metric - trace on'); + +#extract trace info +is($$spans{span0}{parent_span_id}, 'b9c7c989f97918e1', + 'traceparent - trace on'); +is($$spans{span0}{trace_state}, '"congo=ucfJifl5GOE,rojo=00f067aa0ba902b7"', + 'tracestate - trace on'); +is($$spans{span1}{parent_span_id}, undef, 'no traceparent - trace on'); +is($$spans{span1}{trace_state}, undef, 'no tracestate - trace on'); + +#variables +is($tp_headers->{'x-otel-trace-id'}, $$spans{span0}{trace_id}, + '$otel_trace_id variable - trace on'); +is($tp_headers->{'x-otel-span-id'}, $$spans{span0}{span_id}, + '$otel_span_id variable - trace on'); +is($tp_headers->{'x-otel-parent-id'}, $$spans{span0}{parent_span_id}, + '$otel_parent_id variable - trace on'); +is($tp_headers->{'x-otel-parent-sampled'}, 1, + '$otel_parent_sampled variable - trace on'); +is($t_headers->{'x-otel-parent-sampled'}, 0, + '$otel_parent_sampled variable - trace on (no traceparent header)'); + +#trace off +is((scalar grep { + get_attr("http.target", "string_value", $$spans{$_}) eq '/trace-off' + } keys %{$spans}), 0, 'no metric in batch0 - trace off'); +is((scalar grep { + get_attr("http.target", "string_value", $$spans{$_}) eq '/trace-off' + } keys %{$$batch1{scope_spans}}), 0, 'no metric in batch1 - trace off'); + +#trace context: ignore +is($t_headers_ignore->{'x-otel-traceparent'}, undef, + 'no traceparent - trace context ignore (no trace headers)'); +is($t_headers_ignore->{'x-otel-tracestate'}, undef, + 'no tracestate - trace context ignore (no trace headers)'); + +is($tp_headers_ignore->{'x-otel-parent-id'}, undef, + 'no parent span id - trace context ignore (trace headers)'); +is($tp_headers_ignore->{'x-otel-traceparent'}, + '00-0af7651916cd43dd8448eb211c80319c-b9c7c989f97918e1-01', + 'traceparent - trace context ignore (trace headers)'); +is($tp_headers_ignore->{'x-otel-tracestate'}, + 'congo=ucfJifl5GOE,rojo=00f067aa0ba902b7', + 'tracestate - trace context ignore (trace headers)'); + +#trace context: extract +is($t_headers_extract->{'x-otel-traceparent'}, undef, + 'no traceparent - trace context extract (no trace headers)'); +is($t_headers_extract->{'x-otel-tracestate'}, undef, + 'no tracestate - trace context extract (no trace headers)'); + +is($tp_headers_extract->{'x-otel-parent-id'}, 'b9c7c989f97918e1', + 'parent span id - trace context extract (trace headers)'); +is($tp_headers_extract->{'x-otel-traceparent'}, + '00-0af7651916cd43dd8448eb211c80319c-b9c7c989f97918e1-01', + 'traceparent - trace context extract (trace headers)'); +is($tp_headers_extract->{'x-otel-tracestate'}, + 'congo=ucfJifl5GOE,rojo=00f067aa0ba902b7', + 'tracestate - trace context extract (trace headers)'); + +#trace context: inject +is($t_headers_inject->{'x-otel-traceparent'}, + "00-$$spans{span6}{trace_id}-$$spans{span6}{span_id}-01", + 'traceparent - trace context inject (no trace headers)'); +is($t_headers_inject->{'x-otel-tracestate'}, undef, + 'no tracestate - trace context inject (no trace headers)'); + +is($tp_headers_inject->{'x-otel-parent-id'}, undef, + 'no parent span id - trace context inject (trace headers)'); +is($tp_headers_inject->{'x-otel-traceparent'}, + "00-$$spans{span7}{trace_id}-$$spans{span7}{span_id}-01", + 'traceparent - trace context inject (trace headers)'); +is($tp_headers_inject->{'x-otel-tracestate'}, undef, + 'no tracestate - trace context inject (trace headers)'); + +#trace context: propagate +is($t_headers_propagate->{'x-otel-traceparent'}, + "00-$$spans{span8}{trace_id}-$$spans{span8}{span_id}-01", + 'traceparent - trace context propagate (no trace headers)'); +is($t_headers_propagate->{'x-otel-tracestate'}, undef, + 'no tracestate - trace context propagate (no trace headers)'); + +is($tp_headers_propagate->{'x-otel-parent-id'}, 'b9c7c989f97918e1', + 'parent id - trace context propagate (trace headers)'); +is($tp_headers_propagate->{'x-otel-traceparent'}, + "00-0af7651916cd43dd8448eb211c80319c-$$spans{span9}{span_id}-01", + 'traceparent - trace context propagate (trace headers)'); +is($tp_headers_propagate->{'x-otel-tracestate'}, + 'congo=ucfJifl5GOE,rojo=00f067aa0ba902b7', + 'tracestate - trace context propagate (trace headers)'); + +SKIP: { +skip "depends on error log content", 2 unless $ENV{TEST_NGINX_UNSAFE}; + +$t->stop(); +my $log = $t->read_file("error.log"); + +like($log, qr/OTel\/grpc: Error parsing metadata: error=invalid value/, + 'log: error parsing metadata - no protobuf in response'); +unlike($log, qr/OTel export failure: No status received/, + 'log: no export failure'); + +} + +############################################################################### + +sub http3_get { + my ($path, %extra) = @_; + my ($frames, $frame); + + my $s = Test::Nginx::HTTP3->new(); + + my $sid = $extra{trace_headers} + ? $s->new_stream({ headers => [ + { name => ':method', value => 'GET' }, + { name => ':scheme', value => 'http' }, + { name => ':path', value => $path }, + { name => ':authority', value => 'localhost' }, + { name => 'user-agent', value => 'nginx-tests' }, + { name => 'traceparent', + value => '00-0af7651916cd43dd8448eb211c80319c-' . + 'b9c7c989f97918e1-01' + }, + { name => 'tracestate', + value => 'congo=ucfJifl5GOE,rojo=00f067aa0ba902b7' + }]}) + : $s->new_stream({ path => $path }); + + $frames = $s->read(all => [{ sid => $sid, fin => 1 }]); + + ($frame) = grep { $_->{type} eq "HEADERS" } @$frames; + my $headers = $frame->{headers}; + + ($frame) = grep { $_->{type} eq "DATA" } @$frames; + my $data = $frame->{data}; + + return $headers, $data; +} + +sub get_attr { + my($attr, $type, $obj) = @_; + + my ($res) = grep { + $_ =~ /^attribute\d+/ && $$obj{$_}{key} eq '"' . $attr . '"' + } keys %{$obj}; + + if (defined $res) { + $$obj{$res}{value}{$type} =~ s/(^\")|(\"$)//g + if $type eq 'string_value'; + + return $$obj{$res}{value}{$type}; + } + + return undef; +} + +sub decode_protobuf { + my ($protobuf) = @_; + + local $/; + open CMD, "echo '" . encode_base64($protobuf) . "' | base64 -d | " . + '$PWD/../build/_deps/grpc-build/third_party/protobuf/protoc '. + '--decode opentelemetry.proto.trace.v1.ResourceSpans -I ' . + '$PWD/../build/_deps/otelcpp-src/third_party/opentelemetry-proto ' . + 'opentelemetry/proto/collector/trace/v1/trace_service.proto |' + or die "Can't decode protobuf: $!\n"; + my $out = ; + close CMD; + + return $out; +} + +sub decode_bytes { + my ($bytes) = @_; + + my $c = sub { return chr oct(shift) }; + + $bytes =~ s/\\(\d{3})/$c->($1)/eg; + $bytes =~ s/(^\")|(\"$)//g; + $bytes =~ s/\\\\/\\/g; + $bytes =~ s/\\r/\r/g; + $bytes =~ s/\\n/\n/g; + $bytes =~ s/\\t/\t/g; + $bytes =~ s/\\"/\"/g; + $bytes =~ s/\\'/\'/g; + + return unpack("H*", unpack("a*", $bytes)); +} + +sub to_hash { + my ($textdata) = @_; + + my %out = (); + push my @stack, \%out; + my ($attr_count, $span_count) = (0, 0); + for my $line (split /\n/, $textdata) { + $line =~ s/(^\s+)|(\s+$)//g; + if ($line =~ /\:/) { + my ($k, $v) = split /\: /, $line; + $v = decode_bytes($v) if ($k =~ /trace_id|span_id|parent_span_id/); + $stack[$#stack]{$k} = $v; + } elsif ($line =~ /\{/) { + $line =~ s/\s\{//; + $line = 'attribute' . $attr_count++ if ($line eq 'attributes'); + if ($line eq 'spans') { + $line = 'span' . $span_count++; + $attr_count = 0; + } + my %new = (); + $stack[$#stack]{$line} = \%new; + push @stack, \%new; + } elsif ($line =~ /\}/) { + pop @stack; + } + } + + return \%out; +} + +sub grpc { + my ($server, $client, $f, $s, $c, $sid, $csid, $uri); + + $server = IO::Socket::INET->new( + Proto => 'tcp', + LocalHost => '127.0.0.1', + LocalPort => $p, + Listen => 5, + Reuse => 1 + ) or die "Can't create listening socket: $!\n"; + + $f->{http_start} = sub { + if (IO::Select->new($server)->can_read(5)) { + $client = $server->accept(); + } else { + # connection could be unexpectedly reused + goto reused if $client; + return undef; + } + + $client->sysread($_, 24) == 24 or return; # preface + + $c = Test::Nginx::HTTP2->new(1, socket => $client, + pure => 1, preface => "") or return; + +reused: + my $frames = $c->read(all => [{ fin => 1 }]); + + $client->close(); + + return $frames; + }; + + return $f; +} + +############################################################################### diff --git a/tests/otel.t b/tests/otel.t new file mode 100644 index 0000000..9887b2f --- /dev/null +++ b/tests/otel.t @@ -0,0 +1,514 @@ +#!/usr/bin/perl + +# (C) Nginx, Inc. + +# Tests for OTel exporter in case HTTP. + +############################################################################### + +use warnings; +use strict; + +use Test::More; + +BEGIN { use FindBin; chdir($FindBin::Bin); } + +use Test::Nginx; +use Test::Nginx::HTTP2; +use MIME::Base64; + +############################################################################### + +select STDERR; $| = 1; +select STDOUT; $| = 1; + +my $t = Test::Nginx->new()->has(qw/http http_ssl http_v2 mirror rewrite/) + ->has_daemon(qw/openssl base64/) + ->write_file_expand('nginx.conf', <<'EOF'); + +%%TEST_GLOBALS%% + +daemon off; + +events { +} + +http { + %%TEST_GLOBALS_HTTP%% + + ssl_certificate_key localhost.key; + ssl_certificate localhost.crt; + + otel_exporter { + endpoint 127.0.0.1:8082; + interval 1s; + batch_size 10; + batch_count 2; + } + + otel_service_name test_server; + otel_trace on; + + server { + listen 127.0.0.1:8080; + listen 127.0.0.1:8081 ssl; + server_name localhost; + + location /trace-on { + otel_trace_context extract; + otel_span_name default_location; + otel_span_attr http.request.header.completion + $request_completion; + otel_span_attr http.response.header.content.type + $sent_http_content_type; + otel_span_attr http.request $request; + add_header "X-Otel-Trace-Id" $otel_trace_id; + add_header "X-Otel-Span-Id" $otel_span_id; + add_header "X-Otel-Parent-Id" $otel_parent_id; + add_header "X-Otel-Parent-Sampled" $otel_parent_sampled; + return 200 "TRACE-ON"; + } + + location /context-ignore { + otel_trace_context ignore; + otel_span_name context_ignore; + add_header "X-Otel-Parent-Id" $otel_parent_id; + proxy_pass http://localhost:8080/trace-off; + } + + location /context-extract { + otel_trace_context extract; + otel_span_name context_extract; + add_header "X-Otel-Parent-Id" $otel_parent_id; + proxy_pass http://localhost:8080/trace-off; + } + + location /context-inject { + otel_trace_context inject; + otel_span_name context_inject; + add_header "X-Otel-Parent-Id" $otel_parent_id; + proxy_pass http://localhost:8080/trace-off; + } + + location /context-propagate { + otel_trace_context propagate; + otel_span_name context_propogate; + add_header "X-Otel-Parent-Id" $otel_parent_id; + proxy_pass http://localhost:8080/trace-off; + } + + location /trace-off { + otel_trace off; + add_header "X-Otel-Traceparent" $http_traceparent; + add_header "X-Otel-Tracestate" $http_tracestate; + return 200 "TRACE-OFF"; + } + } + + server { + listen 127.0.0.1:8082 http2; + server_name localhost; + otel_trace off; + + location / { + mirror /mirror; + grpc_pass 127.0.0.1:8083; + } + + location /mirror { + internal; + grpc_pass 127.0.0.1:%%PORT_4317%%; + } + } + + server { + listen 127.0.0.1:8083 http2; + server_name localhost; + otel_trace off; + + location / { + add_header content-type application/grpc; + add_header grpc-status 0; + add_header grpc-message ""; + return 200; + } + } +} + +EOF + +$t->write_file('openssl.conf', <<'EOF'); +[ req ] +default_bits = 2048 +encrypt_key = no +distinguished_name = req_distinguished_name +[ req_distinguished_name ] + +EOF + +my $d = $t->testdir(); + +foreach my $name ('localhost') { + system('openssl req -x509 -new ' + . "-config $d/openssl.conf -subj /CN=$name/ " + . "-out $d/$name.crt -keyout $d/$name.key " + . ">>$d/openssl.out 2>&1") == 0 + or die "Can't create certificate for $name: $!\n"; +} + +$t->try_run('no OTel module')->plan(69); + +############################################################################### + +my $p = port(4317); +my $f = grpc(); + +#do requests +my $t_off_resp = http1_get('/trace-off'); + +#batch0 (10 requests) +my $tp_resp = http1_get('/trace-on', trace_headers => 1); +my $t_resp = http1_get('/trace-on', port => 8081, ssl => 1); + +my $t_resp_ignore = http1_get('/context-ignore'); +my $tp_resp_ignore = http1_get('/context-ignore', trace_headers => 1); +my $t_resp_extract = http1_get('/context-extract'); +my $tp_resp_extract = http1_get('/context-extract', trace_headers => 1); +my $t_resp_inject = http1_get('/context-inject'); +my $tp_resp_inject = http1_get('/context-inject', trace_headers => 1); +my $t_resp_propagate = http1_get('/context-propagate'); +my $tp_resp_propagate = http1_get('/context-propagate', trace_headers => 1); + +my ($frame) = grep { $_->{type} eq "DATA" } @{$f->{http_start}()}; +my $batch0 = to_hash(decode_protobuf(substr $frame->{data}, 8)); +my $spans = $$batch0{scope_spans}; + +#batch1 (5 reqeusts) +http1_get('/trace-on') for (1..5); + +($frame) = grep { $_->{type} eq "DATA" } @{$f->{http_start}()}; +my $batch1 = to_hash(decode_protobuf(substr $frame->{data}, 8)); + +#validate responses +like($tp_resp, qr/TRACE-ON/, 'http request1 - trace on'); +like($t_resp, qr/TRACE-ON/, 'http request2 - trace on'); +like($t_off_resp, qr/TRACE-OFF/, 'http request - trace off'); + +#validate batch size +delete $$spans{scope}; #remove 'scope' entry +is(scalar keys %{$spans}, 10, 'batch0 size - trace on'); +delete $$batch1{scope_spans}{scope}; #remove 'scope' entry +is(scalar keys %{$$batch1{scope_spans}}, 5, 'batch1 size - trace on'); + +#validate general attributes +is(get_attr("service.name", "string_value", + $$batch0{resource}), 'test_server', 'service.name - trace on'); +is($$spans{span0}{name}, '"default_location"', 'span.name - trace on'); + +#validate http metrics +is(get_attr("http.method", "string_value", $$spans{span0}), 'GET', + 'http.method metric - trace on'); +is(get_attr("http.target", "string_value", $$spans{span0}), '/trace-on', + 'http.target metric - trace on'); +is(get_attr("http.route", "string_value", $$spans{span0}), '/trace-on', + 'http.route metric - trace on'); +is(get_attr("http.scheme", "string_value", $$spans{span0}), 'http', + 'http.scheme metric - trace on'); +is(get_attr("http.flavor", "string_value", $$spans{span0}), '1.0', + 'http.flavor metric - trace on'); +is(get_attr("http.user_agent", "string_value", $$spans{span0}), 'nginx-tests', + 'http.user_agent metric - trace on'); +is(get_attr("http.request_content_length", "int_value", $$spans{span0}), 0, + 'http.request_content_length metric - trace on'); +is(get_attr("http.response_content_length", "int_value", $$spans{span0}), 8, + 'http.response_content_length metric - trace on'); +is(get_attr("http.status_code", "int_value", $$spans{span0}), 200, + 'http.status_code metric - trace on'); +is(get_attr("net.host.name", "string_value", $$spans{span0}), 'localhost', + 'net.host.name metric - trace on'); +is(get_attr("net.host.port", "int_value", $$spans{span0}), 8080, + 'net.host.port metric - trace on'); +is(get_attr("net.sock.peer.addr", "string_value", $$spans{span0}), '127.0.0.1', + 'net.sock.peer.addr metric - trace on'); +like(get_attr("net.sock.peer.port", "int_value", $$spans{span0}), qr/\d+/, + 'net.sock.peer.port metric - trace on'); + +#validate https metrics +is(get_attr("http.method", "string_value", $$spans{span1}), 'GET', + 'http.method metric - trace on (https)'); +is(get_attr("http.target", "string_value", $$spans{span1}), '/trace-on', + 'http.target metric - trace on (https)'); +is(get_attr("http.route", "string_value", $$spans{span1}), '/trace-on', + 'http.route metric - trace on (https)'); +is(get_attr("http.scheme", "string_value", $$spans{span1}), 'https', + 'http.scheme metric - trace on (https)'); +is(get_attr("http.flavor", "string_value", $$spans{span1}), '1.0', + 'http.flavor metric - trace on (https)'); +is(get_attr("http.user_agent", "string_value", $$spans{span1}), + 'nginx-tests', 'http.user_agent metric - trace on (https)'); +is(get_attr("http.request_content_length", "int_value", $$spans{span1}), 0, + 'http.request_content_length metric - trace on (https)'); +is(get_attr("http.response_content_length", "int_value", $$spans{span1}), 8, + 'http.response_content_length metric - trace on (https)'); +is(get_attr("http.status_code", "int_value", $$spans{span1}), 200, + 'http.status_code metric - trace on (https)'); +is(get_attr("net.host.name", "string_value", $$spans{span1}), 'localhost', + 'net.host.name metric - trace on (https)'); +is(get_attr("net.host.port", "int_value", $$spans{span1}), 8081, + 'net.host.port metric - trace on (https)'); +is(get_attr("net.sock.peer.addr", "string_value", $$spans{span1}), '127.0.0.1', + 'net.sock.peer.addr metric - trace on (https)'); +like(get_attr("net.sock.peer.port", "int_value", $$spans{span1}), qr/\d+/, + 'net.sock.peer.port metric - trace on (https)'); + +#validate custom http metrics +is(${get_attr("http.request.header.completion", "array_value", $$spans{span0})} + {values}{string_value}, '"OK"', + 'http.request.header.completion metric - trace on'); +is(${get_attr("http.response.header.content.type", + "array_value", $$spans{span0})}{values}{string_value}, '"text/plain"', + 'http.response.header.content.type metric - trace on'); +is(get_attr("http.request", "string_value", $$spans{span0}), + 'GET /trace-on HTTP/1.0', 'http.request metric - trace on'); + +#extract trace info +is($$spans{span0}{parent_span_id}, 'b9c7c989f97918e1', + 'traceparent - trace on'); +is($$spans{span0}{trace_state}, '"congo=ucfJifl5GOE,rojo=00f067aa0ba902b7"', + 'tracestate - trace on'); +is($$spans{span1}{parent_span_id}, undef, 'no traceparent - trace on'); +is($$spans{span1}{trace_state}, undef, 'no tracestate - trace on'); + +#variables +like($tp_resp, qr/X-Otel-Trace-Id: $$spans{span0}{trace_id}/, + '$otel_trace_id variable - trace on'); +like($tp_resp, qr/X-Otel-Span-Id: $$spans{span0}{span_id}/, + '$otel_span_id variable - trace on'); +like($tp_resp, qr/X-Otel-Parent-Id: $$spans{span0}{parent_span_id}/, + '$otel_parent_id variable - trace on'); +like($tp_resp, qr/X-Otel-Parent-Sampled: 1/, + '$otel_parent_sampled variable - trace on'); +like($t_resp, qr/X-Otel-Parent-Sampled: 0/, + '$otel_parent_sampled variable - trace on (no traceparent header)'); + +#trace off +is((scalar grep { + get_attr("http.target", "string_value", $$spans{$_}) eq '/trace-off' + } keys %{$spans}), 0, 'no metric in batch0 - trace off'); +is((scalar grep { + get_attr("http.target", "string_value", $$spans{$_}) eq '/trace-off' + } keys %{$$batch1{scope_spans}}), 0, 'no metric in batch1 - trace off'); + +#trace context: ignore +unlike($t_resp_ignore, qr/X-Otel-Traceparent/, + 'no traceparent - trace context ignore (no trace headers)'); +unlike($t_resp_ignore, qr/X-Otel-Tracestate/, + 'no tracestate - trace context ignore (no trace headers)'); + +unlike($tp_resp_ignore, qr/X-Otel-Parent-Id/, + 'no parent span id - trace context ignore (trace headers)'); +like($tp_resp_ignore, + qr/Traceparent: 00-0af7651916cd43dd8448eb211c80319c-b9c7c989f97918e1-01/, + 'traceparent - trace context ignore (trace headers)'); +like($tp_resp_ignore, + qr/Tracestate: congo=ucfJifl5GOE,rojo=00f067aa0ba902b7/, + 'tracestate - trace context ignore (trace headers)'); + +#trace context: extract +unlike($t_resp_extract, qr/X-Otel-Traceparent/, + 'no traceparent - trace context extract (no trace headers)'); +unlike($t_resp_extract, qr/X-Otel-Tracestate/, + 'no tracestate - trace context extract (no trace headers)'); + +like($tp_resp_extract, qr/X-Otel-Parent-Id: b9c7c989f97918e1/, + 'parent span id - trace context extract (trace headers)'); +like($tp_resp_extract, + qr/Traceparent: 00-0af7651916cd43dd8448eb211c80319c-b9c7c989f97918e1-01/, + 'traceparent - trace context extract (trace headers)'); +like($tp_resp_extract, + qr/Tracestate: congo=ucfJifl5GOE,rojo=00f067aa0ba902b7/, + 'tracestate - trace context extract (trace headers)'); + +#trace context: inject +like($t_resp_inject, + qr/Traceparent: 00-$$spans{span6}{trace_id}-$$spans{span6}{span_id}-01/, + 'traceparent - trace context inject (no trace headers)'); +unlike($t_resp_inject, qr/X-Otel-Tracestate/, + 'no tracestate - trace context inject (no trace headers)'); + +unlike($tp_resp_inject, qr/X-Otel-Parent-Id/, + 'no parent span id - trace context inject (trace headers)'); +like($tp_resp_inject, + qr/Traceparent: 00-$$spans{span7}{trace_id}-$$spans{span7}{span_id}-01/, + 'traceparent - trace context inject (trace headers)'); +unlike($tp_resp_inject, qr/Tracestate:/, + 'no tracestate - trace context inject (trace headers)'); + +#trace context: propagate +like($t_resp_propagate, + qr/Traceparent: 00-$$spans{span8}{trace_id}-$$spans{span8}{span_id}-01/, + 'traceparent - trace context propagate (no trace headers)'); +unlike($t_resp_propagate, qr/X-Otel-Tracestate/, + 'no tracestate - trace context propagate (no trace headers)'); + +like($tp_resp_propagate, qr/X-Otel-Parent-Id: b9c7c989f97918e1/, + 'parent id - trace context propagate (trace headers)'); +like($tp_resp_propagate, + qr/parent: 00-0af7651916cd43dd8448eb211c80319c-$$spans{span9}{span_id}-01/, + 'traceparent - trace context propagate (trace headers)'); +like($tp_resp_propagate, + qr/Tracestate: congo=ucfJifl5GOE,rojo=00f067aa0ba902b7/, + 'tracestate - trace context propagate (trace headers)'); + +SKIP: { +skip "depends on error log contents", 2 unless $ENV{TEST_NGINX_UNSAFE}; + +$t->stop(); +my $log = $t->read_file("error.log"); + +like($log, qr/OTel\/grpc: Error parsing metadata: error=invalid value/, + 'log: error parsing metadata - no protobuf in response'); +unlike($log, qr/OTel export failure: No status received/, + 'log: no export failure'); + +} + +############################################################################### + +sub http1_get { + my ($path, %extra) = @_; + + my $port = $extra{port} || 8080; + + my $r = < '127.0.0.1:' . port($port), + SSL => $extra{ssl}); +} + +sub get_attr { + my($attr, $type, $obj) = @_; + + my ($res) = grep { + $_ =~ /^attribute\d+/ && $$obj{$_}{key} eq '"' . $attr . '"' + } keys %{$obj}; + + if (defined $res) { + $$obj{$res}{value}{$type} =~ s/(^\")|(\"$)//g + if $type eq 'string_value'; + + return $$obj{$res}{value}{$type}; + } + + return undef; +} + +sub decode_protobuf { + my ($protobuf) = @_; + + local $/; + open CMD, "echo '" . encode_base64($protobuf) . "' | base64 -d | " . + '$PWD/../build/_deps/grpc-build/third_party/protobuf/protoc '. + '--decode opentelemetry.proto.trace.v1.ResourceSpans -I ' . + '$PWD/../build/_deps/otelcpp-src/third_party/opentelemetry-proto ' . + 'opentelemetry/proto/collector/trace/v1/trace_service.proto |' + or die "Can't decode protobuf: $!\n"; + my $out = ; + close CMD; + + return $out; +} + +sub decode_bytes { + my ($bytes) = @_; + + my $c = sub { return chr oct(shift) }; + + $bytes =~ s/\\(\d{3})/$c->($1)/eg; + $bytes =~ s/(^\")|(\"$)//g; + $bytes =~ s/\\\\/\\/g; + $bytes =~ s/\\r/\r/g; + $bytes =~ s/\\n/\n/g; + $bytes =~ s/\\t/\t/g; + $bytes =~ s/\\"/\"/g; + $bytes =~ s/\\'/\'/g; + + return unpack("H*", unpack("a*", $bytes)); +} + +sub to_hash { + my ($textdata) = @_; + + my %out = (); + push my @stack, \%out; + my ($attr_count, $span_count) = (0, 0); + for my $line (split /\n/, $textdata) { + $line =~ s/(^\s+)|(\s+$)//g; + if ($line =~ /\:/) { + my ($k, $v) = split /\: /, $line; + $v = decode_bytes($v) if ($k =~ /trace_id|span_id|parent_span_id/); + $stack[$#stack]{$k} = $v; + } elsif ($line =~ /\{/) { + $line =~ s/\s\{//; + $line = 'attribute' . $attr_count++ if ($line eq 'attributes'); + if ($line eq 'spans') { + $line = 'span' . $span_count++; + $attr_count = 0; + } + my %new = (); + $stack[$#stack]{$line} = \%new; + push @stack, \%new; + } elsif ($line =~ /\}/) { + pop @stack; + } + } + + return \%out; +} + +sub grpc { + my ($server, $client, $f, $s, $c, $sid, $csid, $uri); + + $server = IO::Socket::INET->new( + Proto => 'tcp', + LocalHost => '127.0.0.1', + LocalPort => $p, + Listen => 5, + Reuse => 1 + ) or die "Can't create listening socket: $!\n"; + + $f->{http_start} = sub { + if (IO::Select->new($server)->can_read(5)) { + $client = $server->accept(); + } else { + # connection could be unexpectedly reused + goto reused if $client; + return undef; + } + + $client->sysread($_, 24) == 24 or return; # preface + + $c = Test::Nginx::HTTP2->new(1, socket => $client, + pure => 1, preface => "") or return; + +reused: + my $frames = $c->read(all => [{ fin => 1 }]); + + $client->close(); + + return $frames; + }; + + return $f; +} + +############################################################################### diff --git a/tests/otel_collector.t b/tests/otel_collector.t new file mode 100644 index 0000000..158a2fd --- /dev/null +++ b/tests/otel_collector.t @@ -0,0 +1,403 @@ +#!/usr/bin/perl + +# (C) Nginx, Inc. + +# Tests for OTel exporter in case HTTP using otelcol. + +############################################################################### + +use warnings; +use strict; + +use Test::More; + +BEGIN { use FindBin; chdir($FindBin::Bin); } + +use Test::Nginx; + +############################################################################### + +select STDERR; $| = 1; +select STDOUT; $| = 1; + +plan(skip_all => "depends on logs content") unless $ENV{TEST_NGINX_UNSAFE}; + +eval { require JSON::PP; }; +plan(skip_all => "JSON::PP not installed") if $@; + +my $t = Test::Nginx->new()->has(qw/http http_ssl rewrite/) + ->write_file_expand('nginx.conf', <<'EOF'); + +%%TEST_GLOBALS%% + +daemon off; + +events { +} + +http { + %%TEST_GLOBALS_HTTP%% + + ssl_certificate_key localhost.key; + ssl_certificate localhost.crt; + + otel_exporter { + endpoint 127.0.0.1:%%PORT_4317%%; + interval 1s; + batch_size 10; + batch_count 2; + } + + otel_service_name test_server; + otel_trace on; + + server { + listen 127.0.0.1:8080; + listen 127.0.0.1:8081 ssl; + server_name localhost; + + location /trace-on { + otel_trace_context extract; + otel_span_name default_location; + otel_span_attr http.request.header.completion + $request_completion; + otel_span_attr http.response.header.content.type + $sent_http_content_type; + otel_span_attr http.request $request; + add_header "X-Otel-Trace-Id" $otel_trace_id; + add_header "X-Otel-Span-Id" $otel_span_id; + add_header "X-Otel-Parent-Id" $otel_parent_id; + add_header "X-Otel-Parent-Sampled" $otel_parent_sampled; + return 200 "TRACE-ON"; + } + + location /context-ignore { + otel_trace_context ignore; + otel_span_name context_ignore; + add_header "X-Otel-Parent-Id" $otel_parent_id; + proxy_pass http://localhost:8080/trace-off; + } + + location /context-extract { + otel_trace_context extract; + otel_span_name context_extract; + add_header "X-Otel-Parent-Id" $otel_parent_id; + proxy_pass http://localhost:8080/trace-off; + } + + location /context-inject { + otel_trace_context inject; + otel_span_name context_inject; + add_header "X-Otel-Parent-Id" $otel_parent_id; + proxy_pass http://localhost:8080/trace-off; + } + + location /context-propagate { + otel_trace_context propagate; + otel_span_name context_propogate; + add_header "X-Otel-Parent-Id" $otel_parent_id; + proxy_pass http://localhost:8080/trace-off; + } + + location /trace-off { + otel_trace off; + add_header "X-Otel-Traceparent" $http_traceparent; + add_header "X-Otel-Tracestate" $http_tracestate; + return 200 "TRACE-OFF"; + } + } +} + +EOF + +$t->write_file_expand('otel-config.yaml', <testdir() }/otel.json + +service: + pipelines: + traces: + receivers: [otlp] + exporters: [logging, file] + metrics: + receivers: [otlp] + exporters: [logging, file] + +EOF + +$t->write_file('openssl.conf', <<'EOF'); +[ req ] +default_bits = 2048 +encrypt_key = no +distinguished_name = req_distinguished_name +[ req_distinguished_name ] + +EOF + +my $d = $t->testdir(); + +foreach my $name ('localhost') { + system('openssl req -x509 -new ' + . "-config $d/openssl.conf -subj /CN=$name/ " + . "-out $d/$name.crt -keyout $d/$name.key " + . ">>$d/openssl.out 2>&1") == 0 + or die "Can't create certificate for $name: $!\n"; +} + +#suppress otel collector output +open OLDERR, ">&", \*STDERR; +open STDERR, ">>" , $^O eq 'MSWin32' ? 'nul' : '/dev/null'; +$t->run_daemon('../otelcol', '--config', $t->testdir().'/otel-config.yaml'); +open STDERR, ">&", \*OLDERR; +$t->waitforsocket('127.0.0.1:' . port(4317)) or + die 'No otel collector open socket'; + +$t->try_run('no OTel module')->plan(69); + +############################################################################### + +#do requests +my $t_off_resp = http1_get('/trace-off'); + +#batch0 (10 requests) +my $tp_resp = http1_get('/trace-on', trace_headers => 1); +my $t_resp = http1_get('/trace-on', port => 8081, ssl => 1); + +my $t_resp_ignore = http1_get('/context-ignore'); +my $tp_resp_ignore = http1_get('/context-ignore', trace_headers => 1); +my $t_resp_extract = http1_get('/context-extract'); +my $tp_resp_extract = http1_get('/context-extract', trace_headers => 1); +my $t_resp_inject = http1_get('/context-inject'); +my $tp_resp_inject = http1_get('/context-inject', trace_headers => 1); +my $t_resp_propagate = http1_get('/context-propagate'); +my $tp_resp_propagate = http1_get('/context-propagate', trace_headers => 1); + +#batch1 (5 reqeusts) +http1_get('/trace-on') for (1..5); + +#waiting batch1 is sent to collector for 1s +select undef, undef, undef, 1; + +my @batches = split /\n/, $t->read_file('otel.json'); +my $batch_json = JSON::PP::decode_json($batches[0]); +my $spans = $$batch_json{"resourceSpans"}[0]{"scopeSpans"}[0]{"spans"}; + +#validate responses +like($tp_resp, qr/TRACE-ON/, 'http request1 - trace on'); +like($t_resp, qr/TRACE-ON/, 'http request2 - trace on'); +like($t_off_resp, qr/TRACE-OFF/, 'http request - trace off'); + +#validate amount of batches +is(scalar @batches, 2, 'amount of batches - trace on'); + +#validate batch size +is(scalar @{$spans}, 10, 'batch0 size - trace on'); +is(scalar @{${JSON::PP::decode_json($batches[1])}{"resourceSpans"}[0] + {"scopeSpans"}[0]{"spans"}}, 5, 'batch1 size - trace on'); + +#validate general attributes +is(get_attr("service.name", "stringValue", + $$batch_json{resourceSpans}[0]{resource}), + 'test_server', 'service.name - trace on'); +is($$spans[0]{name}, 'default_location', 'span.name - trace on'); + +#validate http metrics +is(get_attr("http.method", "stringValue", $$spans[0]), 'GET', + 'http.method metric - trace on'); +is(get_attr("http.target", "stringValue", $$spans[0]), '/trace-on', + 'http.target metric - trace on'); +is(get_attr("http.route", "stringValue", $$spans[0]), '/trace-on', + 'http.route metric - trace on'); +is(get_attr("http.scheme", "stringValue", $$spans[0]), 'http', + 'http.scheme metric - trace on'); +is(get_attr("http.flavor", "stringValue", $$spans[0]), '1.0', + 'http.flavor metric - trace on'); +is(get_attr("http.user_agent", "stringValue", $$spans[0]), 'nginx-tests', + 'http.user_agent metric - trace on'); +is(get_attr("http.request_content_length", "intValue", $$spans[0]), 0, + 'http.request_content_length metric - trace on'); +is(get_attr("http.response_content_length", "intValue", $$spans[0]), 8, + 'http.response_content_length metric - trace on'); +is(get_attr("http.status_code", "intValue", $$spans[0]), 200, + 'http.status_code metric - trace on'); +is(get_attr("net.host.name", "stringValue", $$spans[0]), 'localhost', + 'net.host.name metric - trace on'); +is(get_attr("net.host.port", "intValue", $$spans[0]), 8080, + 'net.host.port metric - trace on'); +is(get_attr("net.sock.peer.addr", "stringValue", $$spans[0]), '127.0.0.1', + 'net.sock.peer.addr metric - trace on'); +like(get_attr("net.sock.peer.port", "intValue", $$spans[0]), qr/\d+/, + 'net.sock.peer.port metric - trace on'); + +#validate custom http metrics +is(${get_attr("http.request.header.completion", "arrayValue", $$spans[0])} + {values}[0]{stringValue}, 'OK', + 'http.request.header.completion metric - trace on'); +is(${get_attr("http.response.header.content.type", "arrayValue",$$spans[0])} + {values}[0]{stringValue}, 'text/plain', + 'http.response.header.content.type metric - trace on'); +is(get_attr("http.request", "stringValue", $$spans[0]), + 'GET /trace-on HTTP/1.0', 'http.request metric - trace on'); + +#validate https metrics +is(get_attr("http.method", "stringValue", $$spans[1]), 'GET', + 'http.method metric - trace on (https)'); +is(get_attr("http.target", "stringValue", $$spans[1]), '/trace-on', + 'http.target metric - trace on (https)'); +is(get_attr("http.route", "stringValue", $$spans[1]), '/trace-on', + 'http.route metric - trace on (https)'); +is(get_attr("http.scheme", "stringValue", $$spans[1]), 'https', + 'http.scheme metric - trace on (https)'); +is(get_attr("http.flavor", "stringValue", $$spans[1]), '1.0', + 'http.flavor metric - trace on (https)'); +is(get_attr("http.user_agent", "stringValue", $$spans[1]), 'nginx-tests', + 'http.user_agent metric - trace on (https)'); +is(get_attr("http.request_content_length", "intValue", $$spans[1]), 0, + 'http.request_content_length metric - trace on (https)'); +is(get_attr("http.response_content_length", "intValue", $$spans[1]), 8, + 'http.response_content_length metric - trace on (https)'); +is(get_attr("http.status_code", "intValue", $$spans[1]), 200, + 'http.status_code metric - trace on (https)'); +is(get_attr("net.host.name", "stringValue", $$spans[1]), 'localhost', + 'net.host.name metric - trace on (https)'); +is(get_attr("net.host.port", "intValue", $$spans[1]), 8081, + 'net.host.port metric - trace on (https)'); +is(get_attr("net.sock.peer.addr", "stringValue", $$spans[1]), '127.0.0.1', + 'net.sock.peer.addr metric - trace on (https)'); +like(get_attr("net.sock.peer.port", "intValue", $$spans[1]), qr/\d+/, + 'net.sock.peer.port metric - trace on (https)'); + +#extract trace info +is($$spans[0]{parentSpanId}, 'b9c7c989f97918e1', 'traceparent - trace on'); +is($$spans[0]{traceState}, 'congo=ucfJifl5GOE,rojo=00f067aa0ba902b7', + 'tracestate - trace on'); +is($$spans[1]{parentSpanId}, '', 'no traceparent - trace on'); +is($$spans[1]{traceState}, undef, 'no tracestate - trace on'); + +#variables +like($tp_resp, qr/X-Otel-Trace-Id: $$spans[0]{traceId}/, + '$otel_trace_id variable - trace on'); +like($tp_resp, qr/X-Otel-Span-Id: $$spans[0]{spanId}/, + '$otel_span_id variable - trace on'); +like($tp_resp, qr/X-Otel-Parent-Id: $$spans[0]{parentSpanId}/, + '$otel_parent_id variable - trace on'); +like($tp_resp, qr/X-Otel-Parent-Sampled: 1/, + '$otel_parent_sampled variable - trace on'); +like($t_resp, qr/X-Otel-Parent-Sampled: 0/, + '$otel_parent_sampled variable - trace on (no traceparent header)'); + +#trace off +unlike($batches[0].$batches[1], + qr/\Q{"key":"http.target","value":{"stringValue":"\/trace-off"}}\E/, + 'no metrics - trace off'); + +#trace context: ignore +unlike($t_resp_ignore, qr/X-Otel-Traceparent/, + 'no traceparent - trace context ignore (no trace headers)'); +unlike($t_resp_ignore, qr/X-Otel-Tracestate/, + 'no tracestate - trace context ignore (no trace headers)'); + +unlike($tp_resp_ignore, qr/X-Otel-Parent-Id/, + 'no parent span id - trace context ignore (trace headers)'); +like($tp_resp_ignore, + qr/Traceparent: 00-0af7651916cd43dd8448eb211c80319c-b9c7c989f97918e1-01/, + 'traceparent - trace context ignore (trace headers)'); +like($tp_resp_ignore, + qr/Tracestate: congo=ucfJifl5GOE,rojo=00f067aa0ba902b7/, + 'tracestate - trace context ignore (trace headers)'); + +#trace context: extract +unlike($t_resp_extract, qr/X-Otel-Traceparent/, + 'no traceparent - trace context extract (no trace headers)'); +unlike($t_resp_extract, qr/X-Otel-Tracestate/, + 'no tracestate - trace context extract (no trace headers)'); + +like($tp_resp_extract, qr/X-Otel-Parent-Id: b9c7c989f97918e1/, + 'parent span id - trace context extract (trace headers)'); +like($tp_resp_extract, + qr/Traceparent: 00-0af7651916cd43dd8448eb211c80319c-b9c7c989f97918e1-01/, + 'traceparent - trace context extract (trace headers)'); +like($tp_resp_extract, + qr/Tracestate: congo=ucfJifl5GOE,rojo=00f067aa0ba902b7/, + 'tracestate - trace context extract (trace headers)'); + +#trace context: inject +like($t_resp_inject, + qr/X-Otel-Traceparent: 00-$$spans[6]{traceId}-$$spans[6]{spanId}-01/, + 'traceparent - trace context inject (no trace headers)'); +unlike($t_resp_inject, qr/X-Otel-Tracestate/, + 'no tracestate - trace context inject (no trace headers)'); + +unlike($tp_resp_inject, qr/X-Otel-Parent-Id/, + 'no parent span id - trace context inject (trace headers)'); +like($tp_resp_inject, + qr/Traceparent: 00-$$spans[7]{traceId}-$$spans[7]{spanId}-01/, + 'traceparent - trace context inject (trace headers)'); +unlike($tp_resp_inject, qr/Tracestate:/, + 'no tracestate - trace context inject (trace headers)'); + +#trace context: propagate +like($t_resp_propagate, + qr/X-Otel-Traceparent: 00-$$spans[8]{traceId}-$$spans[8]{spanId}-01/, + 'traceparent - trace context propagate (no trace headers)'); +unlike($t_resp_propagate, qr/X-Otel-Tracestate/, + 'no tracestate - trace context propagate (no trace headers)'); + +like($tp_resp_propagate, qr/X-Otel-Parent-Id: b9c7c989f97918e1/, + 'parent id - trace context propagate (trace headers)'); +like($tp_resp_propagate, + qr/Traceparent: 00-0af7651916cd43dd8448eb211c80319c-$$spans[9]{spanId}-01/, + 'traceparent - trace context propagate (trace headers)'); +like($tp_resp_propagate, + qr/Tracestate: congo=ucfJifl5GOE,rojo=00f067aa0ba902b7/, + 'tracestate - trace context propagate (trace headers)'); + +$t->stop(); +my $log = $t->read_file("error.log"); + +unlike($log, qr/OTel\/grpc: Error parsing metadata: error=invalid value/, + 'log: no error parsing metadata'); +unlike($log, qr/OTel export failure: No status received/, + 'log: no export failure'); + +############################################################################### + +sub http1_get { + my ($path, %extra) = @_; + + my $port = $extra{port} || 8080; + + my $r = < '127.0.0.1:' . port($port), + SSL => $extra{ssl}); +} + +sub get_attr { + my($attr, $type, $obj) = @_; + + my ($res) = grep { $$_{"key"} eq $attr } @{$$obj{"attributes"}}; + + return defined $res ? $res->{"value"}{$type} : undef; +} + +############################################################################### From c732ff5fd11c044fc0bf2792d59c7917af57271f Mon Sep 17 00:00:00 2001 From: dplotnikov-f5 <67346902+dplotnikov-f5@users.noreply.github.com> Date: Mon, 21 Aug 2023 13:51:06 -0700 Subject: [PATCH 03/44] Added docs. (#6) Added docs. --- CODE_OF_CONDUCT.md | 74 ++++++++++++++++++++++++++++++++++++++++++++++ CONTRIBUTING.md | 53 +++++++++++++++++++++++++++++++++ SECURITY.md | 14 +++++++++ 3 files changed, 141 insertions(+) create mode 100644 CODE_OF_CONDUCT.md create mode 100644 CONTRIBUTING.md create mode 100644 SECURITY.md diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..3f2f0d0 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,74 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to make participation in our project and +our community a harassment-free experience for everyone, regardless of age, body +size, disability, ethnicity, sex characteristics, gender identity and expression, +level of experience, education, socio-economic status, nationality, personal +appearance, race, religion, or sexual identity and orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment +include: + +- Using welcoming and inclusive language +- Being respectful of differing viewpoints and experiences +- Gracefully accepting constructive criticism +- Focusing on what is best for the community +- Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +- The use of sexualized language or imagery and unwelcome sexual attention or + advances +- Trolling, insulting/derogatory comments, and personal or political attacks +- Public or private harassment +- Publishing others' private information, such as a physical or electronic + address, without explicit permission +- Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable +behavior and are expected to take appropriate and fair corrective action in +response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or +reject comments, commits, code, wiki edits, issues, and other contributions +that are not aligned to this Code of Conduct, or to ban temporarily or +permanently any contributor for other behaviors that they deem inappropriate, +threatening, offensive, or harmful. + +## Scope + +This Code of Conduct applies both within project spaces and in public spaces +when an individual is representing the project or its community. Examples of +representing a project or community include using an official project e-mail +address, posting via an official social media account, or acting as an appointed +representative at an online or offline event. Representation of a project may be +further defined and clarified by project maintainers. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported by contacting the moderation team at nginx-oss-community@f5.com. All +complaints will be reviewed and investigated and will result in a response that +is deemed necessary and appropriate to the circumstances. The project team is +obligated to maintain confidentiality with regard to the reporter of an incident. +Further details of specific enforcement policies may be posted separately. + +Project maintainers who do not follow or enforce the Code of Conduct in good +faith may face temporary or permanent repercussions as determined by other +members of the project's leadership. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant](https://www.contributor-covenant.org), version 1.4, +available at + +For answers to common questions about this code of conduct, see + diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..9d84fc3 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,53 @@ +# Contributing Guidelines + +The following is a set of guidelines for contributing to this project. We really appreciate that you are considering contributing! + +#### Table Of Contents + +[Getting Started](#getting-started) + +[Contributing](#contributing) + +[Code Guidelines](#code-guidelines) + +[Code of Conduct](https://github.com/nginxinc/nginx-otel/blob/main/CODE_OF_CONDUCT.md) + +## Getting Started + +Follow our [Getting Started Guide](https://github.com/nginxinc/nginx-otel/blob/main/README.md) to get this project up and running. + + + +## Contributing + +### Report a Bug + +To report a bug, open an issue on GitHub with the label `bug` using the available bug report issue template. Please ensure the bug has not already been reported. **If the bug is a potential security vulnerability, please report it using our [security policy](https://github.com/nginxinc/nginx-otel/blob/main/SECURITY.md).** + +### Suggest a Feature or Enhancement + +To suggest a new feature or other improvement, create an issue on GitHub and choose the type 'Feature request'. Please fill in the template as provided. + +### Open a Pull Request + +- Fork the repo, create a branch, implement your changes, add any relevant tests, submit a PR when your changes are **tested** and ready for review. +- Fill in [our pull request template](https://github.com/nginxinc/nginx-otel/blob/main/.github/pull_request_template.md). + +## Code Guidelines + + + +### NGINX Code Guidelines + +Before diving into the NGINX codebase or contributing, it's important to understand the fundamental principles and techniques outlined in the [NGINX Development Guide] (http://nginx.org/en/docs/dev/development_guide.html). + +### Git Guidelines + +- Keep a clean, concise and meaningful git commit history on your branch (within reason), rebasing locally and squashing before submitting a PR. +- If possible and/or relevant, use the [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/) format when writing a commit message, so that changelogs can be automatically generated +- Follow the guidelines of writing a good commit message as described here and summarised in the next few points: + - In the subject line, use the present tense ("Add feature" not "Added feature"). + - In the subject line, use the imperative mood ("Move cursor to..." not "Moves cursor to..."). + - Limit the subject line to 72 characters or less. + - Reference issues and pull requests liberally after the subject line. + - Add more detailed description in the body of the git message (`git commit -a` to give you more space and time in your text editor to write a good message instead of `git commit -am`). diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..fe15251 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,14 @@ +# Security Policy + +## Latest Versions + +We advise users to run or update to the most recent release of this project. Older versions of this project may not have all enhancements and/or bug fixes applied to them. + +## Reporting a Vulnerability + +The F5 Security Incident Response Team (F5 SIRT) has an email alias that makes it easy to report potential security vulnerabilities. + +- If you’re an F5 customer with an active support contract, please contact [F5 Technical Support](https://www.f5.com/services/support). +- If you aren’t an F5 customer, please report any potential or current instances of security vulnerabilities with any F5 product to the F5 Security Incident Response Team at F5SIRT@f5.com + +For more information visit [https://www.f5.com/services/support/report-a-vulnerability](https://www.f5.com/services/support/report-a-vulnerability) From 98fb4d1d21a58334f81eadc1e4ef8fc70ddc2225 Mon Sep 17 00:00:00 2001 From: Michael Vernik Date: Thu, 31 Aug 2023 22:32:56 -0700 Subject: [PATCH 04/44] =?UTF-8?q?Added=20to=20intro,=20more=20context/deta?= =?UTF-8?q?ils=20to=20building/installation.=20Remove=E2=80=A6=20(#7)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Updated README with more detailed instructions on building, installing, running and testing --- README.md | 209 +++++++++++++++++++++++------------------------------- 1 file changed, 88 insertions(+), 121 deletions(-) diff --git a/README.md b/README.md index 38d1448..bdf9ce7 100644 --- a/README.md +++ b/README.md @@ -1,51 +1,101 @@ -# nginx_otel +# NGINX Native OpenTelemetry (OTel) Module -This project provides support for OpenTelemetry distributed tracing in Nginx, offering: +## What is OpenTelemetry +OpenTelemetry (OTel) is an observability framework for monitoring, tracing, troubleshooting, and optimizing applications. OTel enables the collection of telemetry data from a deployed application stack. -- Lightweight and high-performance incoming HTTP request tracing -- [W3C trace context](https://www.w3.org/TR/trace-context/) propagation -- OTLP/gRPC trace export -- Fully Dynamic Variable-Based Sampling +## What is the NGINX Native OTel Module +The `ngx_otel_module` dynamic module enables NGINX Open-Source or NGINX Plus to send telemetry data to an OTel collector. It provides support for [W3C trace context](https://www.w3.org/TR/trace-context/) propagation, OTLP/gRPC trace exports and offers several benefits over exiting OTel modules, including: + +### Better Performance ### +3rd-party OTel implementations reduce performance of request processing by as much as 50% when tracing is enabled. The NGINX Native module limits this impact to approximately 10-15%. + +### Easy Provisioning ### +Setup and configuration can be done right in NGINX configuration files. + +### Fully Dynamic Variable-Based Sampling ### +The module provides the ability to trace a particular session by cookie/token. Additionally, [NGINX Plus](https://www.nginx.com/products/nginx/), available as part of a [commercial subscription](https://www.nginx.com/products/), enables dynamic module control via the [NGINX Plus API](http://nginx.org/en/docs/http/ngx_http_api_module.html) and [key-value store](http://nginx.org/en/docs/http/ngx_http_keyval_module.html) modules. ## Building +Follow these steps to build the `ngx_otel_module` dynamic module on Ubuntu or Debian based systems: -Install build tools and dependencies: +Install build tools and dependencies. ```bash - $ sudo apt install cmake build-essential libssl-dev zlib1g-dev libpcre3-dev - $ sudo apt install pkg-config libc-ares-dev libre2-dev # for gRPC +sudo apt install cmake build-essential libssl-dev zlib1g-dev libpcre3-dev +sudo apt install pkg-config libc-ares-dev libre2-dev # for gRPC ``` -Configure Nginx: +For the next step, you will need the `configure` script that is packaged with the NGINX source code. There are several methods for obtaining NGINX sources. You may choose to [download](http://hg.nginx.org/nginx/archive/tip.tar.gz) them or clone them directly from the NGINX Github repository. + +**Important:** To ensure compatibility, the `ngx_otel_module` and the NGINX binary that it will be used with, will need to be built using the same NGINX source code and operating system. We will build and install NGINX from obtained sources in a later step. When obtaining NGINX sources from Github, please ensure that you switch to the branch that you intend to use with the module binary. For simplicity, we will assume that the `main` branch will be used for the remainder of this tutorial. + ```bash - $ ./configure --with-compat +git clone https://github.com/nginx/nginx.git ``` -Configure and build Nginx OTel module: +Configure NGINX to generate files necessary for dynamic module compilation. These files will be placed into the `nginx/objs` directory. + +**Important:** If you did not obtain NGINX source code via the clone method in the previous step, you will need to adjust paths in the following commands to conform to your specific directory structure. ```bash - $ mkdir build - $ cd build - $ cmake -DNGX_OTEL_NGINX_BUILD_DIR=/path/to/configured/nginx/objs .. - $ make +cd nginx +auto/configure --with-compat ``` -## Getting Started +Exit the NGINX directory and clone the `ngx_otel_module` repository. +```bash +cd .. +git clone https://github.com/nginxinc/nginx-otel.git +``` -### Simple Tracing +Configure and build the NGINX OTel module. -Dumping all the requests could be useful even in non-distributed environment. +**Important**: replace the path in the `cmake` command with the path to the `nginx/objs` directory from above. +```bash +cd nginx-otel +mkdir build +cd build +cmake -DNGX_OTEL_NGINX_BUILD_DIR=/path/to/configured/nginx/objs .. +make +``` + +Compilation will produce a binary named `ngx_otel_module.so`. + +## Installation +***Important:*** The built `ngx_otel_module.so` dynamic module binary will ONLY be compatible with the same version of NGINX source code that was used to build it. To guarantee proper operation, you will need to build and install NGINX from sources obtained in previous steps on the same operating system. + +Follow [instructions](https://docs.nginx.com/nginx/admin-guide/installing-nginx/installing-nginx-open-source/#compiling-and-installing-from-source) related to compiling and installing NGINX. Skip procedures for downloading source code. + +By default, this will install NGINX into `/usr/local/nginx`. The following steps assume this directory structure. + +Copy the `ngx_otel_module.so` dynamic module binary to `/usr/local/nginx/modules`. + +Load the module by adding the following line to the top of the main NGINX configuration file, located at: `/usr/local/nginx/conf/nginx.conf`. ```nginx - http { - otel_trace on; - server { - location / { - proxy_pass http://backend; - } - } - } +load_module modules/ngx_otel_module.so; +``` + +## Configuring the Module +For a complete list of directives, embedded variables, default span attributes and sample configurations, please refer to the [`ngx_otel_module` documentation](https://nginx.org/en/docs/ngx_otel_module.html). + +## Examples +Use these examples to configure some common use-cases for OTel tracing. + +### Simple Tracing +This example sends telemetry data for all http requests. + +```nginx +http { + otel_trace on; + server { + location / { + proxy_pass http://backend; + } + } +} ``` ### Parent-based Tracing +In this example, we inherit trace contexts from incoming requests and record spans only if a parent span is sampled. We also propagate trace contexts and sampling decisions to upstream servers. ```nginx http { @@ -61,6 +111,7 @@ http { ``` ### Ratio-based Tracing +In this ratio-based example, tracing is configured for a percentage of traffic (in this case 10%): ```nginx http { @@ -87,105 +138,21 @@ http { } ``` -## How to Use +## Collecting and Viewing Traces +There are several methods and available software packages for viewing traces. For a quick start, [Jaeger](https://www.jaegertracing.io/) provides an all-in-one container to collect, process and view OTel trace data. Follow [these steps](https://www.jaegertracing.io/docs/next-release/deployment/#all-in-one) to download, install, launch and use Jaeger's OTel services. -### Directives +# Community +- Our Slack channel [#nginx-opentelemetry-module](https://nginxcommunity.slack.com/archives/C05NMNAQDU6), is the go-to place to start asking questions and sharing your thoughts. -#### Available in `http/server/location` contexts +- Our [GitHub issues page](https://github.com/nginxinc/nginx-otel/issues) offers space for a more technical discussion at your own pace. -**`otel_trace`** `on | off | “$var“;` +# Contributing +Get involved with the project by contributing! Please see our [contributing guide](CONTRIBUTING.md) for details. -The argument is a “complex value”, which should result in `on`/`off` or `1`/`0`. Default is `off`. - -**`otel_trace_context`** `ignore | extract | inject | propagate;` - -Defines how to propagate traceparent/tracestate headers. `extract` uses existing trace context from request. `inject` adds new context to request, rewriting existing headers if any. `propagate` updates existing context (i.e. combines `extract` and `inject`). `ignore` skips context headers processing. Default is `ignore`. - -**`otel_span_name`** `name;` - -Default is request’s location name. - -**`otel_span_attr`** `name “$var”;` - -If name starts with `http.(request|response).header.` the type of added attribute will be `string[]` to match semantic conventions (i.e. header value will be represented as a single element array). Otherwise, the attribute type will be `string`. - -#### Available in `http` context - -**`otel_exporter`**`;` - -Defines how to export tracing data. There can only be one `otel_exporter` directive in a given `http` context. - -```nginx -otel_exporter { - endpoint “host:port“; - interval 5s; # max interval between two exports - batch_size 512; # max number of spans to be sent in one batch per worker - batch_count 4; # max number of pending batches per worker, over the limit spans are dropped -} -``` - -**`otel_service_name`** `name;` - -Sets `service.name` attribute of OTel resource. By default, it is set to `unknown_service:nginx`. - -### Available in `otel_exporter` context - -**`endpoint`** `"host:post";` - -Defines exporter endpoint `host` and `port`. Only one endpoint per `otel_exporter` can be specified. - -**`interval`** `5s;` - -Maximum interval between two exports. Default is `5s`. - -**`batch_size`** `512;` - -Maximum number of spans to be sent in one batch per worker. Detault is 512. - -**`batch_count`** `4;` - -Maximum number of pending batches per worker, over the limit spans are dropped. Default is 4. - -### Variables - -`$otel_trace_id` - trace id. - -`$otel_span_id` - current span id. - -`$otel_parent_id` - parent span id. - -`$otel_parent_sampled` - `sampled` flag of parent span, `1`/`0`. - -### Default span [attributes](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/semantic_conventions/http.md) - -`http.method` - -`http.target` - -`http.route` - -`http.scheme` - -`http.flavor` - -`http.user_agent` - -`http.request_content_length` - -`http.response_content_length` - -`http.status_code` - -`net.host.name` - -`net.host.port` - -`net.sock.peer.addr` - -`net.sock.peer.port` - -## License +# Change Log +See our [release page](https://github.com/nginxinc/nginx-otel/releases) to keep track of updates. +# License [Apache License, Version 2.0](https://github.com/nginxinc/nginx-otel/blob/main/LICENSE) © [F5, Inc.](https://www.f5.com/) 2023 From 0da0f1537ec73f5683f6ca0d08b1e3a3817577da Mon Sep 17 00:00:00 2001 From: Michael Vernik Date: Fri, 1 Sep 2023 13:00:56 -0700 Subject: [PATCH 05/44] Updated README: Clarified dynamic control. Added link to Github repo. --- README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index bdf9ce7..f19894b 100644 --- a/README.md +++ b/README.md @@ -12,8 +12,10 @@ The `ngx_otel_module` dynamic module enables NGINX Open-Source or NGINX Plus to ### Easy Provisioning ### Setup and configuration can be done right in NGINX configuration files. -### Fully Dynamic Variable-Based Sampling ### -The module provides the ability to trace a particular session by cookie/token. Additionally, [NGINX Plus](https://www.nginx.com/products/nginx/), available as part of a [commercial subscription](https://www.nginx.com/products/), enables dynamic module control via the [NGINX Plus API](http://nginx.org/en/docs/http/ngx_http_api_module.html) and [key-value store](http://nginx.org/en/docs/http/ngx_http_keyval_module.html) modules. +### Dynamic, Variable-Based Control ### +The ability to control trace parameters dynamically using cookies, tokens, and variables. Please see our [Ratio-based Tracing](#ratio-based-tracing) example for more details. + +Additionally, [NGINX Plus](https://www.nginx.com/products/nginx/), available as part of a [commercial subscription](https://www.nginx.com/products/), enables dynamic control of sampling parameters via the [NGINX Plus API](http://nginx.org/en/docs/http/ngx_http_api_module.html) and [key-value store](http://nginx.org/en/docs/http/ngx_http_keyval_module.html) modules. ## Building Follow these steps to build the `ngx_otel_module` dynamic module on Ubuntu or Debian based systems: @@ -24,7 +26,7 @@ sudo apt install cmake build-essential libssl-dev zlib1g-dev libpcre3-dev sudo apt install pkg-config libc-ares-dev libre2-dev # for gRPC ``` -For the next step, you will need the `configure` script that is packaged with the NGINX source code. There are several methods for obtaining NGINX sources. You may choose to [download](http://hg.nginx.org/nginx/archive/tip.tar.gz) them or clone them directly from the NGINX Github repository. +For the next step, you will need the `configure` script that is packaged with the NGINX source code. There are several methods for obtaining NGINX sources. You may choose to [download](http://hg.nginx.org/nginx/archive/tip.tar.gz) them or clone them directly from the [NGINX Github repository](https://github.com/nginx/nginx). **Important:** To ensure compatibility, the `ngx_otel_module` and the NGINX binary that it will be used with, will need to be built using the same NGINX source code and operating system. We will build and install NGINX from obtained sources in a later step. When obtaining NGINX sources from Github, please ensure that you switch to the branch that you intend to use with the module binary. For simplicity, we will assume that the `main` branch will be used for the remainder of this tutorial. From f6d041fbf2fbd495e1e731899c317ba02db23708 Mon Sep 17 00:00:00 2001 From: Michael Vernik Date: Tue, 24 Oct 2023 13:04:59 -0700 Subject: [PATCH 06/44] add otel_exporter directive to simple tracing example in readme (#15) --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index f19894b..98847c8 100644 --- a/README.md +++ b/README.md @@ -87,7 +87,12 @@ This example sends telemetry data for all http requests. ```nginx http { + otel_exporter { + endpoint localhost:4317; + } + otel_trace on; + server { location / { proxy_pass http://backend; From 9f359ff0ebb7b1550a3cf3b191f396210532a7c1 Mon Sep 17 00:00:00 2001 From: Nina Forsyth Date: Tue, 24 Oct 2023 13:08:59 -0700 Subject: [PATCH 07/44] Update README.md (#14) Updating two items --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 98847c8..3a12e5d 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ OpenTelemetry (OTel) is an observability framework for monitoring, tracing, troubleshooting, and optimizing applications. OTel enables the collection of telemetry data from a deployed application stack. ## What is the NGINX Native OTel Module -The `ngx_otel_module` dynamic module enables NGINX Open-Source or NGINX Plus to send telemetry data to an OTel collector. It provides support for [W3C trace context](https://www.w3.org/TR/trace-context/) propagation, OTLP/gRPC trace exports and offers several benefits over exiting OTel modules, including: +The `ngx_otel_module` dynamic module enables NGINX Open Source or NGINX Plus to send telemetry data to an OTel collector. It provides support for [W3C trace context](https://www.w3.org/TR/trace-context/) propagation, OpenTelemetry Protocol (OTLP)/gRPC trace exports and offers several benefits over exiting OTel modules, including: ### Better Performance ### 3rd-party OTel implementations reduce performance of request processing by as much as 50% when tracing is enabled. The NGINX Native module limits this impact to approximately 10-15%. From fc7e69a0a7a9ffad74102dc356667ee2e8dbf64d Mon Sep 17 00:00:00 2001 From: Michael Vernik Date: Tue, 7 Nov 2023 15:37:30 -0800 Subject: [PATCH 08/44] Updated README: Added steps for installation from packages. (#19) --- README.md | 122 +++++++++++++++++++++++++++++++++--------------------- 1 file changed, 75 insertions(+), 47 deletions(-) diff --git a/README.md b/README.md index 3a12e5d..1f84fe0 100644 --- a/README.md +++ b/README.md @@ -17,60 +17,29 @@ The ability to control trace parameters dynamically using cookies, tokens, and v Additionally, [NGINX Plus](https://www.nginx.com/products/nginx/), available as part of a [commercial subscription](https://www.nginx.com/products/), enables dynamic control of sampling parameters via the [NGINX Plus API](http://nginx.org/en/docs/http/ngx_http_api_module.html) and [key-value store](http://nginx.org/en/docs/http/ngx_http_keyval_module.html) modules. -## Building -Follow these steps to build the `ngx_otel_module` dynamic module on Ubuntu or Debian based systems: +## Installing +Prebuilt packages of the module are available for easy installation. Follow these steps to install NGINX Open Source with the OTel module. See list of [compatible operating systems](https://nginx.org/en/linux_packages.html#distributions). -Install build tools and dependencies. +### Adding Package Repositories and Installing NGINX Open Source +Follow the official NGINX Open Source [installation steps](https://nginx.org/en/linux_packages.html#instructions) to set up package repositories for your specific operating system and install NGINX. + +**Important:** To ensure module compatibility, you must use officially distributed NGINX binaries. Compatibility with community distributed binaries, commonly available through various operating system vendors, is not guaranteed. + +### Installing the OTel Module from Packages +Once remote package repositories have been added and local package records have been updated, you may install the OTel module (`nginx-module-otel`) for your specific operating system. As an example, run the following commands to install on: + +#### RedHat, RHEL and Derivatives ```bash -sudo apt install cmake build-essential libssl-dev zlib1g-dev libpcre3-dev -sudo apt install pkg-config libc-ares-dev libre2-dev # for gRPC +sudo yum install nginx-module-otel ``` -For the next step, you will need the `configure` script that is packaged with the NGINX source code. There are several methods for obtaining NGINX sources. You may choose to [download](http://hg.nginx.org/nginx/archive/tip.tar.gz) them or clone them directly from the [NGINX Github repository](https://github.com/nginx/nginx). - -**Important:** To ensure compatibility, the `ngx_otel_module` and the NGINX binary that it will be used with, will need to be built using the same NGINX source code and operating system. We will build and install NGINX from obtained sources in a later step. When obtaining NGINX sources from Github, please ensure that you switch to the branch that you intend to use with the module binary. For simplicity, we will assume that the `main` branch will be used for the remainder of this tutorial. - +#### Debian, Ubuntu and derivatives ```bash -git clone https://github.com/nginx/nginx.git +sudo apt install nginx-module-otel ``` -Configure NGINX to generate files necessary for dynamic module compilation. These files will be placed into the `nginx/objs` directory. - -**Important:** If you did not obtain NGINX source code via the clone method in the previous step, you will need to adjust paths in the following commands to conform to your specific directory structure. -```bash -cd nginx -auto/configure --with-compat -``` - -Exit the NGINX directory and clone the `ngx_otel_module` repository. -```bash -cd .. -git clone https://github.com/nginxinc/nginx-otel.git -``` - -Configure and build the NGINX OTel module. - -**Important**: replace the path in the `cmake` command with the path to the `nginx/objs` directory from above. -```bash -cd nginx-otel -mkdir build -cd build -cmake -DNGX_OTEL_NGINX_BUILD_DIR=/path/to/configured/nginx/objs .. -make -``` - -Compilation will produce a binary named `ngx_otel_module.so`. - -## Installation -***Important:*** The built `ngx_otel_module.so` dynamic module binary will ONLY be compatible with the same version of NGINX source code that was used to build it. To guarantee proper operation, you will need to build and install NGINX from sources obtained in previous steps on the same operating system. - -Follow [instructions](https://docs.nginx.com/nginx/admin-guide/installing-nginx/installing-nginx-open-source/#compiling-and-installing-from-source) related to compiling and installing NGINX. Skip procedures for downloading source code. - -By default, this will install NGINX into `/usr/local/nginx`. The following steps assume this directory structure. - -Copy the `ngx_otel_module.so` dynamic module binary to `/usr/local/nginx/modules`. - -Load the module by adding the following line to the top of the main NGINX configuration file, located at: `/usr/local/nginx/conf/nginx.conf`. +### Enabling the OTel Module +Following the installation steps above will install the module into `/etc/nginx/modules` by default. Load the module by adding the following line to the top of the main NGINX configuration file, located at `/etc/nginx/nginx.conf`. ```nginx load_module modules/ngx_otel_module.so; @@ -148,6 +117,65 @@ http { ## Collecting and Viewing Traces There are several methods and available software packages for viewing traces. For a quick start, [Jaeger](https://www.jaegertracing.io/) provides an all-in-one container to collect, process and view OTel trace data. Follow [these steps](https://www.jaegertracing.io/docs/next-release/deployment/#all-in-one) to download, install, launch and use Jaeger's OTel services. +## Building +Follow these steps to build the `ngx_otel_module` dynamic module on Ubuntu or Debian based systems: + +Install build tools and dependencies. +```bash +sudo apt install cmake build-essential libssl-dev zlib1g-dev libpcre3-dev +sudo apt install pkg-config libc-ares-dev libre2-dev # for gRPC +``` + +For the next step, you will need the `configure` script that is packaged with the NGINX source code. There are several methods for obtaining NGINX sources. You may choose to [download](http://hg.nginx.org/nginx/archive/tip.tar.gz) them or clone them directly from the [NGINX Github repository](https://github.com/nginx/nginx). + +**Important:** To ensure compatibility, the `ngx_otel_module` and the NGINX binary that it will be used with, will need to be built using the same NGINX source code and operating system. We will build and install NGINX from obtained sources in a later step. When obtaining NGINX sources from Github, please ensure that you switch to the branch that you intend to use with the module binary. For simplicity, we will assume that the `main` branch will be used for the remainder of this tutorial. + +```bash +git clone https://github.com/nginx/nginx.git +``` + +Configure NGINX to generate files necessary for dynamic module compilation. These files will be placed into the `nginx/objs` directory. + +**Important:** If you did not obtain NGINX source code via the clone method in the previous step, you will need to adjust paths in the following commands to conform to your specific directory structure. +```bash +cd nginx +auto/configure --with-compat +``` + +Exit the NGINX directory and clone the `ngx_otel_module` repository. +```bash +cd .. +git clone https://github.com/nginxinc/nginx-otel.git +``` + +Configure and build the NGINX OTel module. + +**Important**: replace the path in the `cmake` command with the path to the `nginx/objs` directory from above. +```bash +cd nginx-otel +mkdir build +cd build +cmake -DNGX_OTEL_NGINX_BUILD_DIR=/path/to/configured/nginx/objs .. +make +``` + +Compilation will produce a binary named `ngx_otel_module.so`. + +## Installing from Built Binaries +***Important:*** The built `ngx_otel_module.so` dynamic module binary will ONLY be compatible with the same version of NGINX source code that was used to build it. To guarantee proper operation, you will need to build and install NGINX from sources obtained in previous steps on the same operating system. + +Follow [instructions](https://docs.nginx.com/nginx/admin-guide/installing-nginx/installing-nginx-open-source/#compiling-and-installing-from-source) related to compiling and installing NGINX. Skip procedures for downloading source code. + +By default, this will install NGINX into `/usr/local/nginx`. The following steps assume this directory structure. + +Copy the `ngx_otel_module.so` dynamic module binary to `/usr/local/nginx/modules`. + +Load the module by adding the following line to the top of the main NGINX configuration file, located at `/usr/local/nginx/conf/nginx.conf`. + +```nginx +load_module modules/ngx_otel_module.so; +``` + # Community - Our Slack channel [#nginx-opentelemetry-module](https://nginxcommunity.slack.com/archives/C05NMNAQDU6), is the go-to place to start asking questions and sharing your thoughts. From 958a4b696275c24b7cc5beda215bbc243dfcc18e Mon Sep 17 00:00:00 2001 From: Pavel Pautov Date: Thu, 7 Dec 2023 17:41:05 -0800 Subject: [PATCH 09/44] Prevent crash for HTTP/0.9 requests (fix #22). --- src/http_module.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/http_module.cpp b/src/http_module.cpp index a8a6ad2..5ebe0a0 100644 --- a/src/http_module.cpp +++ b/src/http_module.cpp @@ -243,7 +243,13 @@ ngx_int_t setHeader(ngx_http_request_t* r, StrView name, StrView value) return NGX_OK; } - header = (ngx_table_elt_t*)ngx_list_push(&r->headers_in.headers); + auto headers = &r->headers_in.headers; + if (!headers->pool && ngx_list_init(headers, r->pool, 2, + sizeof(ngx_table_elt_t)) != NGX_OK) { + return NGX_ERROR; + } + + header = (ngx_table_elt_t*)ngx_list_push(headers); if (header == NULL) { return NGX_ERROR; } From 1a43ddc6f198af8d5a1bccc4ddc9f9472ccdafc9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Sep 2023 00:27:50 +0000 Subject: [PATCH 10/44] Bump actions/checkout from 3 to 4 Bumps [actions/checkout](https://github.com/actions/checkout) from 3 to 4. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/nginx-otel-module-check.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/nginx-otel-module-check.yml b/.github/workflows/nginx-otel-module-check.yml index ba81a95..698f678 100644 --- a/.github/workflows/nginx-otel-module-check.yml +++ b/.github/workflows/nginx-otel-module-check.yml @@ -6,7 +6,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Install dependencies run: | sudo apt-get update @@ -41,7 +41,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Download module uses: actions/download-artifact@v3 with: From 1971b4f17f6ba323fa0382a608ec8a7387fb6450 Mon Sep 17 00:00:00 2001 From: Evgeny <54681898+jimf5@users.noreply.github.com> Date: Tue, 26 Dec 2023 12:14:30 -0800 Subject: [PATCH 11/44] Bumps actions/download-artifact and actions/upload-artifact from 3 to 4. --- .github/workflows/nginx-otel-module-check.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/nginx-otel-module-check.yml b/.github/workflows/nginx-otel-module-check.yml index 698f678..03fe48e 100644 --- a/.github/workflows/nginx-otel-module-check.yml +++ b/.github/workflows/nginx-otel-module-check.yml @@ -25,12 +25,12 @@ jobs: make -j 4 strip ngx_otel_module.so - name: Archive module - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: nginx-otel-module path: build/ngx_otel_module.so - name: Archive protoc and opentelemetry-proto - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: protoc-opentelemetry-proto path: | @@ -43,12 +43,12 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 - name: Download module - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: nginx-otel-module path: build - name: Download protoc and opentelemetry-proto - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: protoc-opentelemetry-proto path: build/_deps From 0491cc05ee6a9277c55caeee8563fb639db7a9b0 Mon Sep 17 00:00:00 2001 From: Eugene Grebenschikov Date: Tue, 6 Feb 2024 09:18:30 -0800 Subject: [PATCH 12/44] Add manual run of actions. --- .github/workflows/nginx-otel-module-check.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/nginx-otel-module-check.yml b/.github/workflows/nginx-otel-module-check.yml index 03fe48e..2d8f5ee 100644 --- a/.github/workflows/nginx-otel-module-check.yml +++ b/.github/workflows/nginx-otel-module-check.yml @@ -1,6 +1,9 @@ name: nginx-otel-module-check run-name: ${{ github.actor }} is triggering pipeline -on: [push] +on: + push: + workflow_dispatch: + jobs: build-module: runs-on: ubuntu-latest From fa280e1ffe840ba091d8611962087d4b68fa7c64 Mon Sep 17 00:00:00 2001 From: Dmitry Plotnikov Date: Mon, 5 Feb 2024 11:51:34 -0800 Subject: [PATCH 13/44] Bump gRPC and opentelemetry-cpp versions. Move on to gRPC v1.46.7 and opetelemtry-cpp v1.11.0, last versions that support C++11. --- CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 96ac4cd..baca6f8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -21,7 +21,7 @@ if(NGX_OTEL_FETCH_DEPS) FetchContent_Declare( grpc GIT_REPOSITORY https://github.com/grpc/grpc - GIT_TAG 18dda3c586b2607d8daead6b97922e59d867bb7d # v1.46.6 + GIT_TAG 02384e39185f109bd299eb8482306229967dc970 # v1.46.7 GIT_SUBMODULES third_party/protobuf third_party/abseil-cpp GIT_SHALLOW ON) @@ -35,7 +35,7 @@ if(NGX_OTEL_FETCH_DEPS) FetchContent_Declare( otelcpp GIT_REPOSITORY https://github.com/open-telemetry/opentelemetry-cpp - GIT_TAG 57bf8c2b0e85215a61602f559522d08caa4d2fb8 # v1.8.1 + GIT_TAG 11d5d9e0d8fd8ba876c8994714cc2647479b6574 # v1.11.0 GIT_SUBMODULES third_party/opentelemetry-proto GIT_SHALLOW ON) From 7edca7a3ec5723235693e051074a6393d0332173 Mon Sep 17 00:00:00 2001 From: Pavel Pautov Date: Tue, 6 Feb 2024 12:59:06 -0800 Subject: [PATCH 14/44] Removed NOTICE, it's covered by README.md and LICENSE already. --- NOTICE | 16 ---------------- 1 file changed, 16 deletions(-) delete mode 100644 NOTICE diff --git a/NOTICE b/NOTICE deleted file mode 100644 index 5c9f3ea..0000000 --- a/NOTICE +++ /dev/null @@ -1,16 +0,0 @@ - - NGINX OTel. - - Copyright 2017-2023 NGINX, Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. From c8c71688cf9d3d1611e19be8c8a110af02b805ef Mon Sep 17 00:00:00 2001 From: Pavel Pautov Date: Tue, 6 Feb 2024 15:42:58 -0800 Subject: [PATCH 15/44] Update commit message guidelines. --- CONTRIBUTING.md | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9d84fc3..84f3267 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -35,8 +35,6 @@ To suggest a new feature or other improvement, create an issue on GitHub and cho ## Code Guidelines - - ### NGINX Code Guidelines Before diving into the NGINX codebase or contributing, it's important to understand the fundamental principles and techniques outlined in the [NGINX Development Guide] (http://nginx.org/en/docs/dev/development_guide.html). @@ -44,10 +42,10 @@ Before diving into the NGINX codebase or contributing, it's important to underst ### Git Guidelines - Keep a clean, concise and meaningful git commit history on your branch (within reason), rebasing locally and squashing before submitting a PR. -- If possible and/or relevant, use the [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/) format when writing a commit message, so that changelogs can be automatically generated -- Follow the guidelines of writing a good commit message as described here and summarised in the next few points: +- Follow below guidelines for writing commit messages: - In the subject line, use the present tense ("Add feature" not "Added feature"). - In the subject line, use the imperative mood ("Move cursor to..." not "Moves cursor to..."). + - End subject line with a period. - Limit the subject line to 72 characters or less. - - Reference issues and pull requests liberally after the subject line. + - Reference issues in the subject line and/or body. - Add more detailed description in the body of the git message (`git commit -a` to give you more space and time in your text editor to write a good message instead of `git commit -am`). From aac5678defd4a704d7fff216244755a8c9d33d8d Mon Sep 17 00:00:00 2001 From: Pavel Pautov Date: Tue, 6 Feb 2024 15:59:27 -0800 Subject: [PATCH 16/44] Update PR template. Removed documentation task, as most of it is hosted externally for now. --- .github/pull_request_template.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index fa49d47..de75aea 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,6 +1,6 @@ ### Proposed changes -Describe the use case and detail of the change. If this PR addresses an issue on GitHub, make sure to include a link to that issue using one of the [supported keywords](https://docs.github.com/en/github/managing-your-work-on-github/linking-a-pull-request-to-an-issue) here in this description (not in the title of the PR). +Describe the use case and detail of the change. If this PR addresses an issue on GitHub, make sure to include a link to that issue using `fix` [keyword](https://docs.github.com/en/github/managing-your-work-on-github/linking-a-pull-request-to-an-issue) here in this description and in corresponding commit message. ### Checklist @@ -9,4 +9,3 @@ Before creating a PR, run through this checklist and mark each as complete. - [ ] I have read the [`CONTRIBUTING`](https://github.com/nginxinc/nginx-otel/blob/main/CONTRIBUTING.md) document - [ ] If applicable, I have added tests that prove my fix is effective or that my feature works - [ ] If applicable, I have checked that any relevant tests pass after adding my changes -- [ ] I have updated any relevant documentation ([`README.md`](https://github.com/nginxinc/nginx-otel/blob/main/README.md) and [`CHANGELOG.md`](https://github.com/nginxinc/nginx-otel/blob/main/CHANGELOG.md)) From b54c65005af95bdd04f5c49baa3f16e08a9301cf Mon Sep 17 00:00:00 2001 From: Dmitry Plotnikov Date: Thu, 29 Feb 2024 10:06:37 -0800 Subject: [PATCH 17/44] Move gRPC/Protobuf logs handling to a dedicated file. --- CMakeLists.txt | 1 + src/grpc_log.cpp | 52 +++++++++++++++++++++++++++++++++++++++++++++ src/grpc_log.hpp | 3 +++ src/http_module.cpp | 34 +++-------------------------- src/ngx.hpp | 7 ++++++ 5 files changed, 66 insertions(+), 31 deletions(-) create mode 100644 src/grpc_log.cpp create mode 100644 src/grpc_log.hpp create mode 100644 src/ngx.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index baca6f8..910300c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -104,6 +104,7 @@ add_compile_options(-Wall -Wtype-limits -Werror) add_library(ngx_otel_module MODULE src/http_module.cpp + src/grpc_log.cpp src/modules.c ${PROTO_SOURCES}) diff --git a/src/grpc_log.cpp b/src/grpc_log.cpp new file mode 100644 index 0000000..d6e1150 --- /dev/null +++ b/src/grpc_log.cpp @@ -0,0 +1,52 @@ +#include "ngx.hpp" + +#include "grpc_log.hpp" + +#include +#include + +class ProtobufLog { +public: + ProtobufLog() { google::protobuf::SetLogHandler(protobufLogHandler); } + ~ProtobufLog() { google::protobuf::SetLogHandler(NULL); } + +private: + static void protobufLogHandler(google::protobuf::LogLevel logLevel, + const char* filename, int line, const std::string& msg) + { + using namespace google::protobuf; + + ngx_uint_t level = logLevel == LOGLEVEL_FATAL ? NGX_LOG_EMERG : + logLevel == LOGLEVEL_ERROR ? NGX_LOG_ERR : + logLevel == LOGLEVEL_WARNING ? NGX_LOG_WARN : + /*LOGLEVEL_INFO*/ NGX_LOG_INFO; + + ngx_log_error(level, ngx_cycle->log, 0, "OTel/protobuf: %s", + msg.c_str()); + } +}; + +class GrpcLog { +public: + GrpcLog() { gpr_set_log_function(grpcLogHandler); } + ~GrpcLog() { gpr_set_log_function(NULL); } + +private: + static void grpcLogHandler(gpr_log_func_args* args) + { + ngx_uint_t level = + args->severity == GPR_LOG_SEVERITY_ERROR ? NGX_LOG_ERR : + args->severity == GPR_LOG_SEVERITY_INFO ? NGX_LOG_INFO : + /*GPR_LOG_SEVERITY_DEBUG*/ NGX_LOG_DEBUG; + + ngx_log_error(level, ngx_cycle->log, 0, "OTel/grpc: %s", + args->message); + } + + ProtobufLog protoLog; +}; + +void initGrpcLog() +{ + static GrpcLog init; +} diff --git a/src/grpc_log.hpp b/src/grpc_log.hpp new file mode 100644 index 0000000..e6da5c9 --- /dev/null +++ b/src/grpc_log.hpp @@ -0,0 +1,3 @@ +#pragma once + +void initGrpcLog(); diff --git a/src/http_module.cpp b/src/http_module.cpp index 5ebe0a0..1ddffc3 100644 --- a/src/http_module.cpp +++ b/src/http_module.cpp @@ -1,11 +1,6 @@ -extern "C" { -#include -#include -#include -} +#include "ngx.hpp" -#include -#include +#include "grpc_log.hpp" #include "str_view.hpp" #include "trace_context.hpp" @@ -504,28 +499,6 @@ ngx_int_t onRequestEnd(ngx_http_request_t* r) return NGX_DECLINED; } -void grpcLogHandler(gpr_log_func_args* args) -{ - ngx_uint_t level = args->severity == GPR_LOG_SEVERITY_ERROR ? NGX_LOG_ERR : - args->severity == GPR_LOG_SEVERITY_INFO ? NGX_LOG_INFO : - /*GPR_LOG_SEVERITY_DEBUG*/ NGX_LOG_DEBUG; - - ngx_log_error(level, ngx_cycle->log, 0, "OTel/grpc: %s", args->message); -} - -void protobufLogHandler(google::protobuf::LogLevel logLevel, - const char* filename, int line, const std::string& msg) -{ - using namespace google::protobuf; - - ngx_uint_t level = logLevel == LOGLEVEL_FATAL ? NGX_LOG_EMERG : - logLevel == LOGLEVEL_ERROR ? NGX_LOG_ERR : - logLevel == LOGLEVEL_WARNING ? NGX_LOG_WARN : - /*LOGLEVEL_INFO*/ NGX_LOG_INFO; - - ngx_log_error(level, ngx_cycle->log, 0, "OTel/protobuf: %s", msg.c_str()); -} - ngx_int_t initModule(ngx_conf_t* cf) { auto cmcf = (ngx_http_core_main_conf_t*)ngx_http_conf_get_module_main_conf( @@ -547,8 +520,7 @@ ngx_int_t initModule(ngx_conf_t* cf) *h = onRequestEnd; - gpr_set_log_function(grpcLogHandler); - google::protobuf::SetLogHandler(protobufLogHandler); + initGrpcLog(); return NGX_OK; } diff --git a/src/ngx.hpp b/src/ngx.hpp new file mode 100644 index 0000000..63351fa --- /dev/null +++ b/src/ngx.hpp @@ -0,0 +1,7 @@ +#pragma once + +extern "C" { +#include +#include +#include +} From 93dc2b1878aa72d7c92dcb8fc8a08a198067b22d Mon Sep 17 00:00:00 2001 From: Dmitry Plotnikov Date: Thu, 29 Feb 2024 10:06:53 -0800 Subject: [PATCH 18/44] Use Abseil logging for Protobuf v22 and above (fix #16). --- src/grpc_log.cpp | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/src/grpc_log.cpp b/src/grpc_log.cpp index d6e1150..84d796c 100644 --- a/src/grpc_log.cpp +++ b/src/grpc_log.cpp @@ -3,6 +3,10 @@ #include "grpc_log.hpp" #include +#include + +#if GOOGLE_PROTOBUF_VERSION < 4022000 + #include class ProtobufLog { @@ -26,6 +30,43 @@ private: } }; +#else + +#include +#include +#include + +class ProtobufLog : absl::LogSink { +public: + ProtobufLog() + { + absl::InitializeLog(); + absl::AddLogSink(this); + // Disable logging to stderr + absl::SetStderrThreshold(static_cast(100)); + } + + ~ProtobufLog() override { absl::RemoveLogSink(this); } + + void Send(const absl::LogEntry& entry) override + { + auto severity = entry.log_severity(); + + ngx_uint_t level = + severity == absl::LogSeverity::kFatal ? NGX_LOG_EMERG : + severity == absl::LogSeverity::kError ? NGX_LOG_ERR : + severity == absl::LogSeverity::kWarning ? NGX_LOG_WARN : + /*absl::LogSeverity::kInfo*/ NGX_LOG_INFO; + + ngx_str_t message { entry.text_message().size(), + (u_char*)entry.text_message().data() }; + + ngx_log_error(level, ngx_cycle->log, 0, "OTel/protobuf: %V", &message); + } +}; + +#endif + class GrpcLog { public: GrpcLog() { gpr_set_log_function(grpcLogHandler); } From fa28f37dab4e7836fe92407940e407b843e41f2e Mon Sep 17 00:00:00 2001 From: Dmitry Plotnikov Date: Fri, 1 Mar 2024 18:48:39 +0000 Subject: [PATCH 19/44] Stop using system provided RE2 for static build. Starting with 2023-06-01 RE2 publicly depends on Abseil, so we can't use system provided RE2 together with gRPC-bundled Abseil. --- CMakeLists.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 910300c..c04c400 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -22,13 +22,12 @@ if(NGX_OTEL_FETCH_DEPS) grpc GIT_REPOSITORY https://github.com/grpc/grpc GIT_TAG 02384e39185f109bd299eb8482306229967dc970 # v1.46.7 - GIT_SUBMODULES third_party/protobuf third_party/abseil-cpp + GIT_SUBMODULES third_party/protobuf third_party/abseil-cpp third_party/re2 GIT_SHALLOW ON) set(gRPC_USE_PROTO_LITE ON CACHE INTERNAL "") set(gRPC_INSTALL OFF CACHE INTERNAL "") set(gRPC_CARES_PROVIDER package CACHE INTERNAL "") - set(gRPC_RE2_PROVIDER package CACHE INTERNAL "") set(gRPC_SSL_PROVIDER package CACHE INTERNAL "") set(gRPC_ZLIB_PROVIDER package CACHE INTERNAL "") From d6d7ce10151429f89b0fc0409d4340566297752d Mon Sep 17 00:00:00 2001 From: Dmitry Plotnikov Date: Tue, 30 Jan 2024 09:51:41 -0800 Subject: [PATCH 20/44] Bump gRPC to v1.49.4 to support building with GCC 13 (fix #13). --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c04c400..ea972ad 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -21,7 +21,7 @@ if(NGX_OTEL_FETCH_DEPS) FetchContent_Declare( grpc GIT_REPOSITORY https://github.com/grpc/grpc - GIT_TAG 02384e39185f109bd299eb8482306229967dc970 # v1.46.7 + GIT_TAG e241f37befe7ba4688effd84bfbf99b0f681a2f7 # v1.49.4 GIT_SUBMODULES third_party/protobuf third_party/abseil-cpp third_party/re2 GIT_SHALLOW ON) From 5a0071f497211ac12c018beeb3f8906981fec7fa Mon Sep 17 00:00:00 2001 From: Dmitry Plotnikov Date: Fri, 1 Mar 2024 06:53:55 +0000 Subject: [PATCH 21/44] Enforce compiler warnings in CI build only. --- .github/workflows/nginx-otel-module-check.yml | 9 +++++---- CMakeLists.txt | 9 ++++++--- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/.github/workflows/nginx-otel-module-check.yml b/.github/workflows/nginx-otel-module-check.yml index 2d8f5ee..bad1a32 100644 --- a/.github/workflows/nginx-otel-module-check.yml +++ b/.github/workflows/nginx-otel-module-check.yml @@ -24,7 +24,8 @@ jobs: - name: Build module working-directory: build run: | - cmake -DNGX_OTEL_NGINX_BUILD_DIR=${PWD}/../nginx/objs .. + cmake -DNGX_OTEL_NGINX_BUILD_DIR=${PWD}/../nginx/objs \ + -DNGX_OTEL_DEV=ON .. make -j 4 strip ngx_otel_module.so - name: Archive module @@ -75,11 +76,11 @@ jobs: working-directory: nginx run: | auto/configure --with-compat --with-debug --with-http_ssl_module \ - --with-http_v2_module --with-http_v3_module + --with-http_v2_module --with-http_v3_module make -j 4 - name: Run tests working-directory: tests run: | PERL5LIB=../nginx-tests/lib TEST_NGINX_UNSAFE=1 \ - TEST_NGINX_VERBOSE=1 TEST_NGINX_GLOBALS="load_module \ - ${PWD}/../build/ngx_otel_module.so;" prove -v . + TEST_NGINX_VERBOSE=1 TEST_NGINX_GLOBALS="load_module \ + ${PWD}/../build/ngx_otel_module.so;" prove -v . diff --git a/CMakeLists.txt b/CMakeLists.txt index ea972ad..50c0c55 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,8 +6,9 @@ set(NGX_OTEL_NGINX_BUILD_DIR "" set(NGX_OTEL_NGINX_DIR "${NGX_OTEL_NGINX_BUILD_DIR}/.." CACHE PATH "Nginx source dir") -set(NGX_OTEL_FETCH_DEPS ON CACHE BOOL "Download dependencies") -set(NGX_OTEL_PROTO_DIR "" CACHE PATH "OTel proto files root") +set(NGX_OTEL_FETCH_DEPS ON CACHE BOOL "Download dependencies") +set(NGX_OTEL_PROTO_DIR "" CACHE PATH "OTel proto files root") +set(NGX_OTEL_DEV OFF CACHE BOOL "Enforce compiler warnings") if(NOT CMAKE_BUILD_TYPE) set(CMAKE_BUILD_TYPE RelWithDebInfo) @@ -99,7 +100,9 @@ add_custom_command( set(CMAKE_CXX_STANDARD 11) set(CMAKE_CXX_EXTENSIONS OFF) -add_compile_options(-Wall -Wtype-limits -Werror) +if (NGX_OTEL_DEV) + add_compile_options(-Wall -Wtype-limits -Werror) +endif() add_library(ngx_otel_module MODULE src/http_module.cpp From 30b9b73546935a72cf3daab44e081e1364a90d2e Mon Sep 17 00:00:00 2001 From: Pavel Pautov Date: Mon, 11 Mar 2024 22:36:34 -0700 Subject: [PATCH 22/44] Disable unnecessary gRPC features to reduce binary size. Incidentally, this also removes RE2 code from the binary. --- CMakeLists.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 50c0c55..4b85113 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -47,6 +47,9 @@ if(NGX_OTEL_FETCH_DEPS) FetchContent_MakeAvailable(grpc otelcpp) + # reconsider once https://github.com/grpc/grpc/issues/36023 is done + target_compile_definitions(grpc PRIVATE GRPC_NO_XDS GRPC_NO_RLS) + set_property(DIRECTORY ${grpc_SOURCE_DIR} PROPERTY EXCLUDE_FROM_ALL YES) set_property(DIRECTORY ${otelcpp_SOURCE_DIR} From d3817136395a326dd9626580d81e0c4703fc1463 Mon Sep 17 00:00:00 2001 From: Dmitry Plotnikov Date: Tue, 12 Mar 2024 12:04:45 -0700 Subject: [PATCH 23/44] Fix undefined symbols error on Mac (fix #38). --- CMakeLists.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4b85113..8588f52 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -119,6 +119,10 @@ set_target_properties(ngx_otel_module PROPERTIES PREFIX "") # can't use OTel's WITH_ABSEIL until cmake 3.24, as it triggers find_package() target_compile_definitions(ngx_otel_module PRIVATE HAVE_ABSEIL) +if (APPLE) + target_link_options(ngx_otel_module PRIVATE -undefined dynamic_lookup) +endif() + target_include_directories(ngx_otel_module PRIVATE ${NGX_OTEL_NGINX_BUILD_DIR} ${NGX_OTEL_NGINX_DIR}/src/core From 01a40c271e4cf5bb4630229db4f228060e2af8a1 Mon Sep 17 00:00:00 2001 From: Pavel Pautov Date: Tue, 12 Mar 2024 23:43:09 -0700 Subject: [PATCH 24/44] Unify CMake and Nginx build system defaults. Provide generic environment variable to adjust CMake settings from Nginx build system. --- config | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/config b/config index 11d3317..75378e2 100644 --- a/config +++ b/config @@ -1,10 +1,9 @@ ngx_addon_name=ngx_otel_module cmake -D NGX_OTEL_NGINX_BUILD_DIR=$NGX_OBJS \ - -D NGX_OTEL_FETCH_DEPS=OFF \ - -D NGX_OTEL_PROTO_DIR=$NGX_OTEL_PROTO_DIR \ -D CMAKE_LIBRARY_OUTPUT_DIRECTORY=$PWD/$NGX_OBJS \ -D "CMAKE_C_FLAGS=$NGX_CC_OPT" \ -D "CMAKE_CXX_FLAGS=$NGX_CC_OPT" \ -D "CMAKE_MODULE_LINKER_FLAGS=$NGX_LD_OPT" \ + $NGX_OTEL_CMAKE_OPTS \ -S $ngx_addon_dir -B $NGX_OBJS/otel || exit 1 From 6ed3910afbfcdec9123238c5a625be60c53d6479 Mon Sep 17 00:00:00 2001 From: Pavel Pautov Date: Mon, 15 Jul 2024 16:12:19 -0700 Subject: [PATCH 25/44] Support custom versions of auto-fetched build dependencies. --- CMakeLists.txt | 46 ++++++++++++++++++++++++++++++---------------- 1 file changed, 30 insertions(+), 16 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8588f52..645c5d5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,7 +6,10 @@ set(NGX_OTEL_NGINX_BUILD_DIR "" set(NGX_OTEL_NGINX_DIR "${NGX_OTEL_NGINX_BUILD_DIR}/.." CACHE PATH "Nginx source dir") -set(NGX_OTEL_FETCH_DEPS ON CACHE BOOL "Download dependencies") +set(NGX_OTEL_GRPC e241f37befe7ba4688effd84bfbf99b0f681a2f7 # v1.49.4 + CACHE STRING "gRPC tag to download or 'package' to use preinstalled") +set(NGX_OTEL_SDK 11d5d9e0d8fd8ba876c8994714cc2647479b6574 # v1.11.0 + CACHE STRING "OTel SDK tag to download or 'package' to use preinstalled") set(NGX_OTEL_PROTO_DIR "" CACHE PATH "OTel proto files root") set(NGX_OTEL_DEV OFF CACHE BOOL "Enforce compiler warnings") @@ -16,13 +19,16 @@ endif() set(CMAKE_CXX_VISIBILITY_PRESET hidden) -if(NGX_OTEL_FETCH_DEPS) +if(NGX_OTEL_GRPC STREQUAL "package") + find_package(protobuf REQUIRED) + find_package(gRPC REQUIRED) +else() include(FetchContent) FetchContent_Declare( grpc GIT_REPOSITORY https://github.com/grpc/grpc - GIT_TAG e241f37befe7ba4688effd84bfbf99b0f681a2f7 # v1.49.4 + GIT_TAG ${NGX_OTEL_GRPC} GIT_SUBMODULES third_party/protobuf third_party/abseil-cpp third_party/re2 GIT_SHALLOW ON) @@ -32,10 +38,29 @@ if(NGX_OTEL_FETCH_DEPS) set(gRPC_SSL_PROVIDER package CACHE INTERNAL "") set(gRPC_ZLIB_PROVIDER package CACHE INTERNAL "") + set(CMAKE_POSITION_INDEPENDENT_CODE ON) + + FetchContent_MakeAvailable(grpc) + + # reconsider once https://github.com/grpc/grpc/issues/36023 is done + target_compile_definitions(grpc PRIVATE GRPC_NO_XDS GRPC_NO_RLS) + + set_property(DIRECTORY ${grpc_SOURCE_DIR} + PROPERTY EXCLUDE_FROM_ALL YES) + + add_library(gRPC::grpc++ ALIAS grpc++) + add_executable(gRPC::grpc_cpp_plugin ALIAS grpc_cpp_plugin) +endif() + +if(NGX_OTEL_SDK STREQUAL "package") + find_package(opentelemetry-cpp REQUIRED) +else() + include(FetchContent) + FetchContent_Declare( otelcpp GIT_REPOSITORY https://github.com/open-telemetry/opentelemetry-cpp - GIT_TAG 11d5d9e0d8fd8ba876c8994714cc2647479b6574 # v1.11.0 + GIT_TAG ${NGX_OTEL_SDK} GIT_SUBMODULES third_party/opentelemetry-proto GIT_SHALLOW ON) @@ -45,13 +70,8 @@ if(NGX_OTEL_FETCH_DEPS) set(CMAKE_POSITION_INDEPENDENT_CODE ON) set(CMAKE_POLICY_DEFAULT_CMP0063 NEW) - FetchContent_MakeAvailable(grpc otelcpp) + FetchContent_MakeAvailable(otelcpp) - # reconsider once https://github.com/grpc/grpc/issues/36023 is done - target_compile_definitions(grpc PRIVATE GRPC_NO_XDS GRPC_NO_RLS) - - set_property(DIRECTORY ${grpc_SOURCE_DIR} - PROPERTY EXCLUDE_FROM_ALL YES) set_property(DIRECTORY ${otelcpp_SOURCE_DIR} PROPERTY EXCLUDE_FROM_ALL YES) @@ -61,12 +81,6 @@ if(NGX_OTEL_FETCH_DEPS) endif() add_library(opentelemetry-cpp::trace ALIAS opentelemetry_trace) - add_library(gRPC::grpc++ ALIAS grpc++) - add_executable(gRPC::grpc_cpp_plugin ALIAS grpc_cpp_plugin) -else() - find_package(opentelemetry-cpp REQUIRED) - find_package(protobuf REQUIRED) - find_package(gRPC REQUIRED) endif() set(PROTO_DIR ${NGX_OTEL_PROTO_DIR}) From 10215eee1b1fa15d701aa3f30a06a71b09f77720 Mon Sep 17 00:00:00 2001 From: Pavel Pautov Date: Tue, 16 Jul 2024 14:13:31 -0700 Subject: [PATCH 26/44] Support building with latest gRPC versions (up to v1.65.0). --- CMakeLists.txt | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 645c5d5..b745342 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -34,10 +34,14 @@ else() set(gRPC_USE_PROTO_LITE ON CACHE INTERNAL "") set(gRPC_INSTALL OFF CACHE INTERNAL "") + set(gRPC_USE_SYSTEMD OFF CACHE INTERNAL "") + set(gRPC_DOWNLOAD_ARCHIVES OFF CACHE INTERNAL "") set(gRPC_CARES_PROVIDER package CACHE INTERNAL "") set(gRPC_SSL_PROVIDER package CACHE INTERNAL "") set(gRPC_ZLIB_PROVIDER package CACHE INTERNAL "") + set(protobuf_INSTALL OFF CACHE INTERNAL "") + set(CMAKE_POSITION_INDEPENDENT_CODE ON) FetchContent_MakeAvailable(grpc) @@ -108,8 +112,8 @@ add_custom_command( --plugin protoc-gen-grpc=$ ${PROTOS} # remove inconsequential UTF8 check during serialization to aid performance - COMMAND sed -i.bak - -e [[/ ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String(/,/);/d]] + COMMAND sed -i.bak -E + -e [[/ ::(PROTOBUF_NAMESPACE_ID|google::protobuf)::internal::WireFormatLite::VerifyUtf8String\(/,/\);/d]] ${PROTO_SOURCES} DEPENDS ${PROTOS} protobuf::protoc gRPC::grpc_cpp_plugin VERBATIM) From 4c24716eef5709489cf6fd61736c0e1beae80ce5 Mon Sep 17 00:00:00 2001 From: Pavel Pautov Date: Tue, 16 Jul 2024 22:45:46 -0700 Subject: [PATCH 27/44] Don't force C++ standard for user builds. This fixes build against C++17 enabled prebuilt dependencies. --- CMakeLists.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b745342..7a74f0a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -118,10 +118,10 @@ add_custom_command( DEPENDS ${PROTOS} protobuf::protoc gRPC::grpc_cpp_plugin VERBATIM) -set(CMAKE_CXX_STANDARD 11) -set(CMAKE_CXX_EXTENSIONS OFF) - if (NGX_OTEL_DEV) + set(CMAKE_CXX_STANDARD 11) + set(CMAKE_CXX_EXTENSIONS OFF) + add_compile_options(-Wall -Wtype-limits -Werror) endif() From 4c841c1c55ea98f22806b19cdb9a7efbab43dc0f Mon Sep 17 00:00:00 2001 From: Pavel Pautov Date: Wed, 17 Jul 2024 16:34:06 -0700 Subject: [PATCH 28/44] Use Abseil logging for gRPC v1.65.0 and above. Original logging method is now deprecated and results in error message on Nginx startup. --- src/grpc_log.cpp | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/src/grpc_log.cpp b/src/grpc_log.cpp index 84d796c..bb83364 100644 --- a/src/grpc_log.cpp +++ b/src/grpc_log.cpp @@ -2,8 +2,8 @@ #include "grpc_log.hpp" -#include #include +#include #if GOOGLE_PROTOBUF_VERSION < 4022000 @@ -36,9 +36,9 @@ private: #include #include -class ProtobufLog : absl::LogSink { +class NgxLogSink : absl::LogSink { public: - ProtobufLog() + NgxLogSink() { absl::InitializeLog(); absl::AddLogSink(this); @@ -46,7 +46,7 @@ public: absl::SetStderrThreshold(static_cast(100)); } - ~ProtobufLog() override { absl::RemoveLogSink(this); } + ~NgxLogSink() override { absl::RemoveLogSink(this); } void Send(const absl::LogEntry& entry) override { @@ -61,12 +61,19 @@ public: ngx_str_t message { entry.text_message().size(), (u_char*)entry.text_message().data() }; - ngx_log_error(level, ngx_cycle->log, 0, "OTel/protobuf: %V", &message); + ngx_log_error(level, ngx_cycle->log, 0, "OTel/grpc: %V", &message); } }; +typedef NgxLogSink ProtobufLog; + #endif +#if (GRPC_CPP_VERSION_MAJOR < 1) || \ + (GRPC_CPP_VERSION_MAJOR == 1 && GRPC_CPP_VERSION_MINOR < 65) + +#include + class GrpcLog { public: GrpcLog() { gpr_set_log_function(grpcLogHandler); } @@ -87,6 +94,13 @@ private: ProtobufLog protoLog; }; +#else + +// newer gRPC implies newer protobuf, and both use Abseil for logging +typedef NgxLogSink GrpcLog; + +#endif + void initGrpcLog() { static GrpcLog init; From 1074d02758156a8c2e5a9e32f16293353226e830 Mon Sep 17 00:00:00 2001 From: Eugene Grebenschikov Date: Fri, 4 Oct 2024 11:21:01 -0700 Subject: [PATCH 29/44] Use github repos for nginx and nginx-tests. --- .github/workflows/nginx-otel-module-check.yml | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/.github/workflows/nginx-otel-module-check.yml b/.github/workflows/nginx-otel-module-check.yml index bad1a32..77a4332 100644 --- a/.github/workflows/nginx-otel-module-check.yml +++ b/.github/workflows/nginx-otel-module-check.yml @@ -15,7 +15,10 @@ jobs: sudo apt-get update sudo apt-get install -y cmake libc-ares-dev libre2-dev - name: Checkout nginx - run: hg clone http://hg.nginx.org/nginx/ + uses: actions/checkout@v4 + with: + repository: nginx/nginx + path: nginx - name: Configure nginx working-directory: nginx run: auto/configure --with-compat @@ -68,16 +71,24 @@ jobs: open-telemetry/opentelemetry-collector-releases/releases/download/\ v0.76.1/otelcol_0.76.1_linux_amd64.tar.gz tar -xzf otelcol_0.76.1_linux_amd64.tar.gz - - name: Checkout nginx and nginx-test - run: | - hg clone http://hg.nginx.org/nginx/ - hg clone http://hg.nginx.org/nginx-tests/ + - name: Checkout nginx + uses: actions/checkout@v4 + with: + repository: nginx/nginx + path: nginx - name: Build nginx working-directory: nginx run: | auto/configure --with-compat --with-debug --with-http_ssl_module \ --with-http_v2_module --with-http_v3_module make -j 4 + - name: Checkout lib from nginx-tests + uses: actions/checkout@v4 + with: + repository: nginx/nginx-tests + sparse-checkout: | + lib + path: nginx-tests - name: Run tests working-directory: tests run: | From b5c8cd8de3e5ea131681d425d0f82e378ed928c5 Mon Sep 17 00:00:00 2001 From: Pavel Pautov Date: Sat, 9 Nov 2024 15:51:33 -0800 Subject: [PATCH 30/44] Support custom resource attributes (fix #32). Now attributes can be set with "otel_resource_attr" directive, e.g. otel_resource_attr my.name "my value"; --- src/batch_exporter.hpp | 11 +++-- src/http_module.cpp | 93 ++++++++++++++++++++++++++++++++---------- 2 files changed, 79 insertions(+), 25 deletions(-) diff --git a/src/batch_exporter.hpp b/src/batch_exporter.hpp index d160d2c..2432fe3 100644 --- a/src/batch_exporter.hpp +++ b/src/batch_exporter.hpp @@ -112,7 +112,8 @@ public: }; BatchExporter(StrView target, - size_t batchSize, size_t batchCount, StrView serviceName) : + size_t batchSize, size_t batchCount, + const std::map& resourceAttrs) : batchSize(batchSize), client(std::string(target)) { free.reserve(batchCount); @@ -120,9 +121,11 @@ public: free.emplace_back(); auto resourceSpans = free.back().add_resource_spans(); - auto attr = resourceSpans->mutable_resource()->add_attributes(); - attr->set_key("service.name"); - attr->mutable_value()->set_string_value(std::string(serviceName)); + for (auto& attr : resourceAttrs) { + auto kv = resourceSpans->mutable_resource()->add_attributes(); + kv->set_key(std::string(attr.first)); + kv->mutable_value()->set_string_value(std::string(attr.second)); + } auto scopeSpans = resourceSpans->add_scope_spans(); scopeSpans->mutable_scope()->set_name("nginx"); diff --git a/src/http_module.cpp b/src/http_module.cpp index 1ddffc3..5c73ef0 100644 --- a/src/http_module.cpp +++ b/src/http_module.cpp @@ -15,7 +15,7 @@ struct OtelCtx { TraceContext current; }; -struct MainConf { +struct MainConfBase { ngx_str_t endpoint; ngx_msec_t interval; size_t batchSize; @@ -24,6 +24,10 @@ struct MainConf { ngx_str_t serviceName; }; +struct MainConf : MainConfBase { + std::map resourceAttrs; +}; + struct SpanAttr { ngx_str_t name; ngx_http_complex_value_t value; @@ -38,6 +42,7 @@ struct LocationConf { }; char* setExporter(ngx_conf_t* cf, ngx_command_t* cmd, void* conf); +char* addResourceAttr(ngx_conf_t* cf, ngx_command_t* cmd, void* conf); char* addSpanAttr(ngx_conf_t* cf, ngx_command_t* cmd, void* conf); namespace Propagation { @@ -59,14 +64,17 @@ ngx_command_t gCommands[] = { { ngx_string("otel_exporter"), NGX_HTTP_MAIN_CONF|NGX_CONF_BLOCK|NGX_CONF_NOARGS, - setExporter, - NGX_HTTP_MAIN_CONF_OFFSET }, + setExporter }, + + { ngx_string("otel_resource_attr"), + NGX_HTTP_MAIN_CONF|NGX_CONF_TAKE2, + addResourceAttr }, { ngx_string("otel_service_name"), NGX_HTTP_MAIN_CONF|NGX_CONF_TAKE1, ngx_conf_set_str_slot, NGX_HTTP_MAIN_CONF_OFFSET, - offsetof(MainConf, serviceName) }, + offsetof(MainConfBase, serviceName) }, { ngx_string("otel_trace"), NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1, @@ -101,25 +109,25 @@ ngx_command_t gExporterCommands[] = { NGX_CONF_TAKE1, ngx_conf_set_str_slot, 0, - offsetof(MainConf, endpoint) }, + offsetof(MainConfBase, endpoint) }, { ngx_string("interval"), NGX_CONF_TAKE1, ngx_conf_set_msec_slot, 0, - offsetof(MainConf, interval) }, + offsetof(MainConfBase, interval) }, { ngx_string("batch_size"), NGX_CONF_TAKE1, ngx_conf_set_size_slot, 0, - offsetof(MainConf, batchSize) }, + offsetof(MainConfBase, batchSize) }, { ngx_string("batch_count"), NGX_CONF_TAKE1, ngx_conf_set_size_slot, 0, - offsetof(MainConf, batchCount) }, + offsetof(MainConfBase, batchCount) }, ngx_null_command }; @@ -136,6 +144,18 @@ ngx_str_t toNgxStr(StrView str) return ngx_str_t{str.size(), (u_char*)str.data()}; } +MainConf* getMainConf(ngx_conf_t* cf) +{ + return static_cast( + (MainConfBase*)ngx_http_conf_get_module_main_conf(cf, gHttpModule)); +} + +MainConf* getMainConf(ngx_cycle_t* cycle) +{ + return static_cast( + (MainConfBase*)ngx_http_cycle_get_module_main_conf(cycle, gHttpModule)); +} + LocationConf* getLocationConf(ngx_http_request_t* r) { return (LocationConf*)ngx_http_get_module_loc_conf(r, gHttpModule); @@ -527,8 +547,7 @@ ngx_int_t initModule(ngx_conf_t* cf) ngx_int_t initWorkerProcess(ngx_cycle_t* cycle) { - auto mcf = (MainConf*)ngx_http_cycle_get_module_main_conf( - cycle, gHttpModule); + auto mcf = getMainConf(cycle); // no 'http' or 'otel_exporter' blocks if (mcf == NULL || mcf->endpoint.len == 0) { @@ -540,7 +559,7 @@ ngx_int_t initWorkerProcess(ngx_cycle_t* cycle) toStrView(mcf->endpoint), mcf->batchSize, mcf->batchCount, - toStrView(mcf->serviceName))); + mcf->resourceAttrs)); } catch (const std::exception& e) { ngx_log_error(NGX_LOG_CRIT, cycle->log, 0, "OTel worker init error: %s", e.what()); @@ -561,8 +580,7 @@ ngx_int_t initWorkerProcess(ngx_cycle_t* cycle) "OTel flush error: %s", e.what()); } - auto mcf = (MainConf*)ngx_http_cycle_get_module_main_conf( - ngx_cycle, gHttpModule); + auto mcf = getMainConf((ngx_cycle_t*)ngx_cycle); ngx_add_timer(ev, mcf->interval); }; @@ -590,7 +608,7 @@ void exitWorkerProcess(ngx_cycle_t* cycle) char* setExporter(ngx_conf_t* cf, ngx_command_t* cmd, void* conf) { - auto mcf = (MainConf*)conf; + auto mcf = getMainConf(cf); if (mcf->endpoint.len) { return (char*)"is duplicate"; @@ -649,31 +667,64 @@ char* setExporter(ngx_conf_t* cf, ngx_command_t* cmd, void* conf) return NGX_CONF_OK; } +char* addResourceAttr(ngx_conf_t* cf, ngx_command_t* cmd, void* conf) +{ + auto mcf = getMainConf(cf); + + try { + auto args = (ngx_str_t*)cf->args->elts; + mcf->resourceAttrs[toStrView(args[1])] = toStrView(args[2]); + } catch (const std::exception& e) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "OTel: %s", e.what()); + return (char*)NGX_CONF_ERROR; + } + + return NGX_CONF_OK; +} + void* createMainConf(ngx_conf_t* cf) { - auto mcf = (MainConf*)ngx_pcalloc(cf->pool, sizeof(MainConf)); - if (mcf == NULL) { + auto cln = ngx_pool_cleanup_add(cf->pool, sizeof(MainConf)); + if (cln == NULL) { return NULL; } + MainConf* mcf; + try { + mcf = new (cln->data) MainConf{}; + } catch (const std::exception& e) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "OTel: %s", e.what()); + return NULL; + } + + cln->handler = [](void* data) { + ((MainConf*)data)->~MainConf(); + }; + mcf->interval = NGX_CONF_UNSET_MSEC; mcf->batchSize = NGX_CONF_UNSET_SIZE; mcf->batchCount = NGX_CONF_UNSET_SIZE; - return mcf; + return static_cast(mcf); } char* initMainConf(ngx_conf_t* cf, void* conf) { auto mcf = (MainConf*)conf; - ngx_conf_init_msec_value(mcf->interval, 5000); ngx_conf_init_size_value(mcf->batchSize, 512); ngx_conf_init_size_value(mcf->batchCount, 4); - if (mcf->serviceName.data == NULL) { - mcf->serviceName = ngx_string("unknown_service:nginx"); + try { + if (mcf->serviceName.data == NULL) { + mcf->resourceAttrs.emplace("service.name", "unknown_service:nginx"); + } else { + mcf->resourceAttrs["service.name"] = toStrView(mcf->serviceName); + } + } catch (const std::exception& e) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "OTel: %s", e.what()); + return (char*)NGX_CONF_ERROR; } return NGX_CONF_OK; @@ -811,7 +862,7 @@ char* mergeLocationConf(ngx_conf_t* cf, void* parent, void* child) conf->spanAttrs = prev->spanAttrs; } - auto mcf = (MainConf*)ngx_http_conf_get_module_main_conf(cf, gHttpModule); + auto mcf = getMainConf(cf); if (mcf->endpoint.len == 0 && conf->trace) { ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, From 668077dbf7cd0e5383c1265c9cecf11f310c7ebf Mon Sep 17 00:00:00 2001 From: Pavel Pautov Date: Tue, 12 Nov 2024 11:49:02 -0800 Subject: [PATCH 31/44] Allow HTTP scheme in endpoint (fix #60). --- src/http_module.cpp | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/http_module.cpp b/src/http_module.cpp index 5c73ef0..d8ae859 100644 --- a/src/http_module.cpp +++ b/src/http_module.cpp @@ -144,6 +144,18 @@ ngx_str_t toNgxStr(StrView str) return ngx_str_t{str.size(), (u_char*)str.data()}; } +bool iremovePrefix(ngx_str_t* str, StrView p) +{ + if (str->len >= p.size() && + ngx_strncasecmp(str->data, (u_char*)p.data(), p.size()) == 0) { + str->data += p.size(); + str->len -= p.size(); + return true; + } + + return false; +} + MainConf* getMainConf(ngx_conf_t* cf) { return static_cast( @@ -658,6 +670,14 @@ char* setExporter(ngx_conf_t* cf, ngx_command_t* cmd, void* conf) return rv; } + if (iremovePrefix(&mcf->endpoint, "https://")) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "\"otel_exporter\" doesn't support \"https\" endpoints"); + return (char*)NGX_CONF_ERROR; + } else { + iremovePrefix(&mcf->endpoint, "http://"); + } + if (mcf->endpoint.len == 0) { ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "\"otel_exporter\" requires \"endpoint\""); From f45b618931e4823bfb8735b3b699b50d65c2e522 Mon Sep 17 00:00:00 2001 From: Pavel Pautov Date: Tue, 19 Nov 2024 14:25:25 -0800 Subject: [PATCH 32/44] Rename CI workflow file to 'ubuntu.yml'. --- .github/workflows/{nginx-otel-module-check.yml => ubuntu.yml} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename .github/workflows/{nginx-otel-module-check.yml => ubuntu.yml} (99%) diff --git a/.github/workflows/nginx-otel-module-check.yml b/.github/workflows/ubuntu.yml similarity index 99% rename from .github/workflows/nginx-otel-module-check.yml rename to .github/workflows/ubuntu.yml index 77a4332..e3bd13e 100644 --- a/.github/workflows/nginx-otel-module-check.yml +++ b/.github/workflows/ubuntu.yml @@ -1,4 +1,4 @@ -name: nginx-otel-module-check +name: Ubuntu build run-name: ${{ github.actor }} is triggering pipeline on: push: From da2e4eb11b0ea70c67aa63db8f462f7923393a55 Mon Sep 17 00:00:00 2001 From: Pavel Pautov Date: Tue, 19 Nov 2024 14:32:15 -0800 Subject: [PATCH 33/44] Trigger CI build on pull requests. Also, use default 'run-name' for more informative message. --- .github/workflows/ubuntu.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index e3bd13e..55c2485 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -1,8 +1,10 @@ name: Ubuntu build -run-name: ${{ github.actor }} is triggering pipeline + on: push: - workflow_dispatch: + branches: + - main + pull_request: jobs: build-module: From 6c1659a20ba946cdde21e9dbc52e7c740b06d968 Mon Sep 17 00:00:00 2001 From: Nikita Vakula Date: Fri, 15 Nov 2024 11:39:30 +0100 Subject: [PATCH 34/44] Support export via TLS (fix #12). --- src/batch_exporter.hpp | 4 ++-- src/http_module.cpp | 45 +++++++++++++++++++++++++++++++++--- src/trace_service_client.hpp | 14 ++++++++--- 3 files changed, 55 insertions(+), 8 deletions(-) diff --git a/src/batch_exporter.hpp b/src/batch_exporter.hpp index 2432fe3..cb3e075 100644 --- a/src/batch_exporter.hpp +++ b/src/batch_exporter.hpp @@ -111,10 +111,10 @@ public: int attrSize{0}; }; - BatchExporter(StrView target, + BatchExporter(StrView target, bool ssl, const std::string& trustedCert, size_t batchSize, size_t batchCount, const std::map& resourceAttrs) : - batchSize(batchSize), client(std::string(target)) + batchSize(batchSize), client(std::string(target), ssl, trustedCert) { free.reserve(batchCount); while (batchCount-- > 0) { diff --git a/src/http_module.cpp b/src/http_module.cpp index d8ae859..df5702e 100644 --- a/src/http_module.cpp +++ b/src/http_module.cpp @@ -6,6 +6,8 @@ #include "trace_context.hpp" #include "batch_exporter.hpp" +#include + extern ngx_module_t gHttpModule; namespace { @@ -26,6 +28,8 @@ struct MainConfBase { struct MainConf : MainConfBase { std::map resourceAttrs; + bool ssl; + std::string trustedCert; }; struct SpanAttr { @@ -44,6 +48,7 @@ struct LocationConf { char* setExporter(ngx_conf_t* cf, ngx_command_t* cmd, void* conf); char* addResourceAttr(ngx_conf_t* cf, ngx_command_t* cmd, void* conf); char* addSpanAttr(ngx_conf_t* cf, ngx_command_t* cmd, void* conf); +char* setTrustedCertificate(ngx_conf_t* cf, ngx_command_t* cmd, void* conf); namespace Propagation { @@ -111,6 +116,10 @@ ngx_command_t gExporterCommands[] = { 0, offsetof(MainConfBase, endpoint) }, + { ngx_string("trusted_certificate"), + NGX_CONF_TAKE1, + setTrustedCertificate }, + { ngx_string("interval"), NGX_CONF_TAKE1, ngx_conf_set_msec_slot, @@ -569,6 +578,8 @@ ngx_int_t initWorkerProcess(ngx_cycle_t* cycle) try { gExporter.reset(new BatchExporter( toStrView(mcf->endpoint), + mcf->ssl, + mcf->trustedCert, mcf->batchSize, mcf->batchCount, mcf->resourceAttrs)); @@ -671,9 +682,7 @@ char* setExporter(ngx_conf_t* cf, ngx_command_t* cmd, void* conf) } if (iremovePrefix(&mcf->endpoint, "https://")) { - ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, - "\"otel_exporter\" doesn't support \"https\" endpoints"); - return (char*)NGX_CONF_ERROR; + mcf->ssl = true; } else { iremovePrefix(&mcf->endpoint, "http://"); } @@ -702,6 +711,36 @@ char* addResourceAttr(ngx_conf_t* cf, ngx_command_t* cmd, void* conf) return NGX_CONF_OK; } +char* setTrustedCertificate(ngx_conf_t* cf, ngx_command_t* cmd, void* conf) { + auto path = ((ngx_str_t*)cf->args->elts)[1]; + auto mcf = getMainConf(cf); + + if (ngx_get_full_name(cf->pool, &cf->cycle->conf_prefix, &path) != NGX_OK) { + return (char*)NGX_CONF_ERROR; + } + + try { + std::ifstream file{(const char*)path.data, std::ios::binary}; + if (!file.is_open()) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, ngx_errno, + "failed to open \"%V\"", &path); + return (char*)NGX_CONF_ERROR; + } + file.exceptions(std::ios::failbit | std::ios::badbit); + file.seekg(0, std::ios::end); + size_t size = file.tellg(); + mcf->trustedCert.resize(size); + file.seekg(0); + file.read(&mcf->trustedCert[0], mcf->trustedCert.size()); + } catch (const std::exception& e) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "failed to read \"%V\": %s", &path, e.what()); + return (char*)NGX_CONF_ERROR; + } + + return NGX_CONF_OK; +} + void* createMainConf(ngx_conf_t* cf) { auto cln = ngx_pool_cleanup_add(cf->pool, sizeof(MainConf)); diff --git a/src/trace_service_client.hpp b/src/trace_service_client.hpp index 6871019..d248f00 100644 --- a/src/trace_service_client.hpp +++ b/src/trace_service_client.hpp @@ -17,10 +17,18 @@ public: typedef std::function ResponseCb; - TraceServiceClient(const std::string& target) + TraceServiceClient(const std::string& target, bool ssl, + const std::string& trustedCert) { - auto channel = grpc::CreateChannel( - target, grpc::InsecureChannelCredentials()); + std::shared_ptr creds; + if (ssl) { + grpc::SslCredentialsOptions options; + options.pem_root_certs = trustedCert; + creds = grpc::SslCredentials(options); + } else { + creds = grpc::InsecureChannelCredentials(); + } + auto channel = grpc::CreateChannel(target, creds); channel->GetState(true); // trigger 'connecting' state stub = TraceService::NewStub(channel); From 1e183a3fa92d1b50bd94e902bce1fd05ccc4cb2b Mon Sep 17 00:00:00 2001 From: Eugene Grebenschikov Date: Tue, 6 Feb 2024 10:43:01 -0800 Subject: [PATCH 35/44] Use pytest for tests. --- .github/workflows/ubuntu.yml | 92 ++---- tests/conftest.py | 100 +++++++ tests/h2_otel.t | 553 ----------------------------------- tests/h3_otel.t | 508 -------------------------------- tests/otel.t | 514 -------------------------------- tests/otel_collector.t | 403 ------------------------- tests/requirements.txt | 6 + tests/test_otel.py | 262 +++++++++++++++++ tests/trace_service.py | 86 ++++++ 9 files changed, 474 insertions(+), 2050 deletions(-) create mode 100644 tests/conftest.py delete mode 100644 tests/h2_otel.t delete mode 100644 tests/h3_otel.t delete mode 100644 tests/otel.t delete mode 100644 tests/otel_collector.t create mode 100644 tests/requirements.txt create mode 100644 tests/test_otel.py create mode 100644 tests/trace_service.py diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 55c2485..1a8edac 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -8,71 +8,14 @@ on: jobs: build-module: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - name: Checkout repository uses: actions/checkout@v4 - name: Install dependencies run: | sudo apt-get update - sudo apt-get install -y cmake libc-ares-dev libre2-dev - - name: Checkout nginx - uses: actions/checkout@v4 - with: - repository: nginx/nginx - path: nginx - - name: Configure nginx - working-directory: nginx - run: auto/configure --with-compat - - name: Create build directory - run: mkdir build - - name: Build module - working-directory: build - run: | - cmake -DNGX_OTEL_NGINX_BUILD_DIR=${PWD}/../nginx/objs \ - -DNGX_OTEL_DEV=ON .. - make -j 4 - strip ngx_otel_module.so - - name: Archive module - uses: actions/upload-artifact@v4 - with: - name: nginx-otel-module - path: build/ngx_otel_module.so - - name: Archive protoc and opentelemetry-proto - uses: actions/upload-artifact@v4 - with: - name: protoc-opentelemetry-proto - path: | - build/_deps/grpc-build/third_party/protobuf/protoc - build/_deps/otelcpp-src/third_party/opentelemetry-proto - test-module: - needs: build-module - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - name: Download module - uses: actions/download-artifact@v4 - with: - name: nginx-otel-module - path: build - - name: Download protoc and opentelemetry-proto - uses: actions/download-artifact@v4 - with: - name: protoc-opentelemetry-proto - path: build/_deps - - name: List files - run: ls -laR . - - name: Fix protoc file permissions - run: chmod +x build/_deps/grpc-build/third_party/protobuf/protoc - - name: Install perl modules - run: sudo cpan IO::Socket::SSL Crypt::Misc - - name: Download otelcol - run: | - curl -LO https://github.com/\ - open-telemetry/opentelemetry-collector-releases/releases/download/\ - v0.76.1/otelcol_0.76.1_linux_amd64.tar.gz - tar -xzf otelcol_0.76.1_linux_amd64.tar.gz + sudo apt-get install -y cmake libc-ares-dev - name: Checkout nginx uses: actions/checkout@v4 with: @@ -83,17 +26,22 @@ jobs: run: | auto/configure --with-compat --with-debug --with-http_ssl_module \ --with-http_v2_module --with-http_v3_module - make -j 4 - - name: Checkout lib from nginx-tests - uses: actions/checkout@v4 - with: - repository: nginx/nginx-tests - sparse-checkout: | - lib - path: nginx-tests - - name: Run tests - working-directory: tests + make -j $(nproc) + - name: Build module run: | - PERL5LIB=../nginx-tests/lib TEST_NGINX_UNSAFE=1 \ - TEST_NGINX_VERBOSE=1 TEST_NGINX_GLOBALS="load_module \ - ${PWD}/../build/ngx_otel_module.so;" prove -v . + mkdir build + cd build + cmake -DNGX_OTEL_NGINX_BUILD_DIR=${PWD}/../nginx/objs \ + -DNGX_OTEL_DEV=ON .. + make -j $(nproc) + - name: Download otelcol + run: | + curl -sLo - https://github.com/\ + open-telemetry/opentelemetry-collector-releases/releases/download/\ + v0.115.1/otelcol_0.115.1_linux_amd64.tar.gz | tar -xzv + - name: Install test dependencies + run: pip install -r tests/requirements.txt + - name: Run tests + run: | + pytest tests --maxfail=10 --nginx=nginx/objs/nginx \ + --module=build/ngx_otel_module.so --otelcol=./otelcol diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..7978759 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,100 @@ +import jinja2 +import logging +from OpenSSL import crypto +import os +import pytest +import subprocess +import time + + +pytest_plugins = [ + "trace_service", +] + + +def pytest_addoption(parser): + parser.addoption("--nginx", required=True) + parser.addoption("--module", required=True) + parser.addoption("--otelcol") + parser.addoption("--globals", default="") + + +def self_signed_cert(test_dir, name): + k = crypto.PKey() + k.generate_key(crypto.TYPE_RSA, 2048) + cert = crypto.X509() + cert.get_subject().CN = name + cert.set_issuer(cert.get_subject()) + cert.gmtime_adj_notBefore(0) + cert.gmtime_adj_notAfter(365 * 86400) # 365 days + cert.set_pubkey(k) + cert.sign(k, "sha512") + (test_dir / f"{name}.key").write_text( + crypto.dump_privatekey(crypto.FILETYPE_PEM, k).decode("utf-8") + ) + (test_dir / f"{name}.crt").write_text( + crypto.dump_certificate(crypto.FILETYPE_PEM, cert).decode("utf-8") + ) + + +@pytest.fixture(scope="session") +def logger(): + logging.basicConfig(level=logging.INFO) + return logging.getLogger(__name__) + + +@pytest.fixture(scope="module") +def testdir(tmp_path_factory): + return tmp_path_factory.mktemp("nginx") + + +@pytest.fixture(scope="module") +def nginx_config(request, pytestconfig, testdir, logger): + tmpl = jinja2.Environment().from_string(request.module.NGINX_CONFIG) + params = getattr(request, "param", {}) + params["globals"] = ( + f"pid {testdir}/nginx.pid;\n" + + "error_log stderr info;\n" + + f"error_log {testdir}/error.log info;\n" + + f"load_module {os.path.abspath(pytestconfig.option.module)};\n" + + pytestconfig.option.globals + ) + params["http_globals"] = f"root {testdir};\n" + "access_log off;\n" + conf = tmpl.render(params) + logger.debug(conf) + return conf + + +@pytest.fixture(scope="module") +def nginx(testdir, pytestconfig, nginx_config, certs, logger, otelcol): + (testdir / "nginx.conf").write_text(nginx_config) + logger.info("Starting nginx...") + proc = subprocess.Popen( + [ + pytestconfig.option.nginx, + "-p", + str(testdir), + "-c", + "nginx.conf", + "-e", + "error.log", + ] + ) + logger.debug(f"args={' '.join(proc.args)}") + logger.debug(f"pid={proc.pid}") + while not (testdir / "nginx.pid").exists(): + time.sleep(0.1) + assert proc.poll() is None, "Can't start nginx" + yield proc + logger.info("Stopping nginx...") + proc.terminate() + try: + proc.wait(timeout=5) + except subprocess.TimeoutExpired: + proc.kill() + assert "[alert]" not in (testdir / "error.log").read_text() + + +@pytest.fixture(scope="module") +def certs(testdir): + self_signed_cert(testdir, "localhost") diff --git a/tests/h2_otel.t b/tests/h2_otel.t deleted file mode 100644 index 24987d7..0000000 --- a/tests/h2_otel.t +++ /dev/null @@ -1,553 +0,0 @@ -#!/usr/bin/perl - -# (C) Nginx, Inc. - -# Tests for OTel exporter in case HTTP/2. - -############################################################################### - -use warnings; -use strict; - -use Test::More; - -BEGIN { use FindBin; chdir($FindBin::Bin); } - -use Test::Nginx; -use Test::Nginx::HTTP2; -use MIME::Base64; - -############################################################################### - -select STDERR; $| = 1; -select STDOUT; $| = 1; - -my $t = Test::Nginx->new() - ->has(qw/http_v2 http_ssl rewrite mirror grpc socket_ssl_alpn/) - ->has_daemon(qw/openssl base64/) - ->write_file_expand('nginx.conf', <<'EOF'); - -%%TEST_GLOBALS%% - -daemon off; - -events { -} - -http { - %%TEST_GLOBALS_HTTP%% - - ssl_certificate_key localhost.key; - ssl_certificate localhost.crt; - - otel_exporter { - endpoint 127.0.0.1:8083; - interval 1s; - batch_size 10; - batch_count 1; - } - - otel_service_name test_server; - otel_trace on; - - server { - listen 127.0.0.1:8080 http2; - listen 127.0.0.1:8081; - listen 127.0.0.1:8082 http2 ssl; - server_name localhost; - - location /trace-on { - otel_trace_context extract; - otel_span_name default_location; - otel_span_attr http.request.header.completion - $request_completion; - otel_span_attr http.response.header.content.type - $sent_http_content_type; - otel_span_attr http.request $request; - add_header "X-Otel-Trace-Id" $otel_trace_id; - add_header "X-Otel-Span-Id" $otel_span_id; - add_header "X-Otel-Parent-Id" $otel_parent_id; - add_header "X-Otel-Parent-Sampled" $otel_parent_sampled; - return 200 "TRACE-ON"; - } - - location /context-ignore { - otel_trace_context ignore; - otel_span_name context_ignore; - add_header "X-Otel-Parent-Id" $otel_parent_id; - proxy_pass http://127.0.0.1:8081/trace-off; - } - - location /context-extract { - otel_trace_context extract; - otel_span_name context_extract; - add_header "X-Otel-Parent-Id" $otel_parent_id; - proxy_pass http://127.0.0.1:8081/trace-off; - } - - location /context-inject { - otel_trace_context inject; - otel_span_name context_inject; - add_header "X-Otel-Parent-Id" $otel_parent_id; - proxy_pass http://127.0.0.1:8081/trace-off; - } - - location /context-propagate { - otel_trace_context propagate; - otel_span_name context_propogate; - add_header "X-Otel-Parent-Id" $otel_parent_id; - proxy_pass http://127.0.0.1:8081/trace-off; - } - - location /trace-off { - otel_trace off; - add_header "X-Otel-Traceparent" $http_traceparent; - add_header "X-Otel-Tracestate" $http_tracestate; - return 200 "TRACE-OFF"; - } - } - - server { - listen 127.0.0.1:8083 http2; - server_name localhost; - otel_trace off; - - location / { - mirror /mirror; - grpc_pass 127.0.0.1:8084; - } - - location /mirror { - internal; - grpc_pass 127.0.0.1:%%PORT_4317%%; - } - } - - server { - listen 127.0.0.1:8084 http2; - server_name localhost; - otel_trace off; - - location / { - add_header content-type application/grpc; - add_header grpc-status 0; - add_header grpc-message ""; - return 200; - } - } - -} - -EOF - -$t->write_file('openssl.conf', <<'EOF'); -[ req ] -default_bits = 2048 -encrypt_key = no -distinguished_name = req_distinguished_name -[ req_distinguished_name ] - -EOF - -my $d = $t->testdir(); - -foreach my $name ('localhost') { - system('openssl req -x509 -new ' - . "-config $d/openssl.conf -subj /CN=$name/ " - . "-out $d/$name.crt -keyout $d/$name.key " - . ">>$d/openssl.out 2>&1") == 0 - or die "Can't create certificate for $name: $!\n"; -} - -$t->try_run('no OTel module')->plan(69); - -############################################################################### - -my $p = port(4317); -my $f = grpc(); - -#do requests -(undef, my $t_off_resp) = http2_get('/trace-off'); - -#batch0 (10 requests) -my ($tp_headers, $tp_resp) = http2_get('/trace-on', trace_headers => 1); -my ($t_headers, $t_resp) = http2_get('/trace-on', port => 8082, ssl => 1); - -(my $t_headers_ignore, undef) = http2_get('/context-ignore'); -(my $tp_headers_ignore, undef) = http2_get('/context-ignore', - trace_headers => 1); -(my $t_headers_extract, undef) = http2_get('/context-extract'); -(my $tp_headers_extract, undef) = http2_get('/context-extract', - trace_headers => 1); -(my $t_headers_inject, undef) = http2_get('/context-inject'); -(my $tp_headers_inject, undef) = http2_get('/context-inject', - trace_headers => 1); -(my $t_headers_propagate, undef) = http2_get('/context-propagate'); -(my $tp_headers_propagate, undef) = - http2_get('/context-propagate', trace_headers => 1); - -my ($frame) = grep { $_->{type} eq "DATA" } @{$f->{http_start}()}; -my $batch0 = to_hash(decode_protobuf(substr $frame->{data}, 8)); -my $spans = $$batch0{scope_spans}; - -#batch1 (5 reqeusts) -http2_get('/trace-on') for (1..5); - -($frame) = grep { $_->{type} eq "DATA" } @{$f->{http_start}()}; -my $batch1 = to_hash(decode_protobuf(substr $frame->{data}, 8)); - -#validate responses -like($tp_resp, qr/TRACE-ON/, 'http request1 - trace on'); -like($t_resp, qr/TRACE-ON/, 'http request2 - trace on'); -like($t_off_resp, qr/TRACE-OFF/, 'http request - trace off'); - -#validate batch size -delete $$spans{scope}; #remove 'scope' entry -is(scalar keys %{$spans}, 10, 'batch0 size - trace on'); -delete $$batch1{scope_spans}{scope}; #remove 'scope' entry -is(scalar keys %{$$batch1{scope_spans}}, 5, 'batch1 size - trace on'); - -#validate general attributes -is(get_attr("service.name", "string_value", - $$batch0{resource}), - 'test_server', 'service.name - trace on'); -is($$spans{span0}{name}, '"default_location"', 'span.name - trace on'); - -#validate http metrics -is(get_attr("http.method", "string_value", $$spans{span0}), 'GET', - 'http.method metric - trace on'); -is(get_attr("http.target", "string_value", $$spans{span0}), '/trace-on', - 'http.target metric - trace on'); -is(get_attr("http.route", "string_value", $$spans{span0}), '/trace-on', - 'http.route metric - trace on'); -is(get_attr("http.scheme", "string_value", $$spans{span0}), 'http', - 'http.scheme metric - trace on'); -is(get_attr("http.flavor", "string_value", $$spans{span0}), '2.0', - 'http.flavor metric - trace on'); -is(get_attr("http.user_agent", "string_value", $$spans{span0}), 'nginx-tests', - 'http.user_agent metric - trace on'); -is(get_attr("http.request_content_length", "int_value", $$spans{span0}), 0, - 'http.request_content_length metric - trace on'); -is(get_attr("http.response_content_length", "int_value", $$spans{span0}), 8, - 'http.response_content_length metric - trace on'); -is(get_attr("http.status_code", "int_value", $$spans{span0}), 200, - 'http.status_code metric - trace on'); -is(get_attr("net.host.name", "string_value", $$spans{span0}), 'localhost', - 'net.host.name metric - trace on'); -is(get_attr("net.host.port", "int_value", $$spans{span0}), 8080, - 'net.host.port metric - trace on'); -is(get_attr("net.sock.peer.addr", "string_value", $$spans{span0}), '127.0.0.1', - 'net.sock.peer.addr metric - trace on'); -like(get_attr("net.sock.peer.port", "int_value", $$spans{span0}), qr/\d+/, - 'net.sock.peer.port metric - trace on'); - -#validate https metrics -is(get_attr("http.method", "string_value", $$spans{span1}), 'GET', - 'http.method metric - trace on (https)'); -is(get_attr("http.target", "string_value", $$spans{span1}), '/trace-on', - 'http.target metric - trace on (https)'); -is(get_attr("http.route", "string_value", $$spans{span1}), '/trace-on', - 'http.route metric - trace on (https)'); -is(get_attr("http.scheme", "string_value", $$spans{span1}), 'https', - 'http.scheme metric - trace on (https)'); -is(get_attr("http.flavor", "string_value", $$spans{span1}), '2.0', - 'http.flavor metric - trace on (https)'); -isnt(get_attr("http.user_agent", "string_value", $$spans{span1}), - 'nginx-tests', 'http.user_agent metric - trace on (https)'); -is(get_attr("http.request_content_length", "int_value", $$spans{span1}), 0, - 'http.request_content_length metric - trace on (https)'); -is(get_attr("http.response_content_length", "int_value", $$spans{span1}), 8, - 'http.response_content_length metric - trace on (https)'); -is(get_attr("http.status_code", "int_value", $$spans{span1}), 200, - 'http.status_code metric - trace on (https)'); -is(get_attr("net.host.name", "string_value", $$spans{span1}), 'localhost', - 'net.host.name metric - trace on (https)'); -is(get_attr("net.host.port", "int_value", $$spans{span1}), 8082, - 'net.host.port metric - trace on (https)'); -is(get_attr("net.sock.peer.addr", "string_value", $$spans{span1}), '127.0.0.1', - 'net.sock.peer.addr metric - trace on (https)'); -like(get_attr("net.sock.peer.port", "int_value", $$spans{span1}), qr/\d+/, - 'net.sock.peer.port metric - trace on (https)'); - -#validate custom http metrics -is(${get_attr("http.request.header.completion", "array_value", $$spans{span0})} - {values}{string_value}, '"OK"', - 'http.request.header.completion metric - trace on'); -is(${get_attr( - "http.response.header.content.type", "array_value", $$spans{span0} - )}{values}{string_value}, '"text/plain"', - 'http.response.header.content.type metric - trace on'); -is(get_attr("http.request", "string_value", $$spans{span0}), - 'GET /trace-on HTTP/2.0', 'http.request metric - trace on'); - -#extract trace info -is($$spans{span0}{parent_span_id}, 'b9c7c989f97918e1', - 'traceparent - trace on'); -is($$spans{span0}{trace_state}, '"congo=ucfJifl5GOE,rojo=00f067aa0ba902b7"', - 'tracestate - trace on'); -is($$spans{span1}{parent_span_id}, undef, 'no traceparent - trace on'); -is($$spans{span1}{trace_state}, undef, 'no tracestate - trace on'); - -#variables -is($tp_headers->{'x-otel-trace-id'}, $$spans{span0}{trace_id}, - '$otel_trace_id variable - trace on'); -is($tp_headers->{'x-otel-span-id'}, $$spans{span0}{span_id}, - '$otel_span_id variable - trace on'); -is($tp_headers->{'x-otel-parent-id'}, $$spans{span0}{parent_span_id}, - '$otel_parent_id variable - trace on'); -is($tp_headers->{'x-otel-parent-sampled'}, 1, - '$otel_parent_sampled variable - trace on'); -is($t_headers->{'x-otel-parent-sampled'}, 0, - '$otel_parent_sampled variable - trace on (no traceparent header)'); - -#trace off -is((scalar grep { - get_attr("http.target", "string_value", $$spans{$_}) eq '/trace-off' - } keys %{$spans}), 0, 'no metric in batch0 - trace off'); -is((scalar grep { - get_attr("http.target", "string_value", $$spans{$_}) eq '/trace-off' - } keys %{$$batch1{scope_spans}}), 0, 'no metric in batch1 - trace off'); - -#trace context: ignore -is($t_headers_ignore->{'x-otel-traceparent'}, undef, - 'no traceparent - trace context ignore (no trace headers)'); -is($t_headers_ignore->{'x-otel-tracestate'}, undef, - 'no tracestate - trace context ignore (no trace headers)'); - -is($tp_headers_ignore->{'x-otel-parent-id'}, undef, - 'no parent span id - trace context ignore (trace headers)'); -is($tp_headers_ignore->{'x-otel-traceparent'}, - '00-0af7651916cd43dd8448eb211c80319c-b9c7c989f97918e1-01', - 'traceparent - trace context ignore (trace headers)'); -is($tp_headers_ignore->{'x-otel-tracestate'}, - 'congo=ucfJifl5GOE,rojo=00f067aa0ba902b7', - 'tracestate - trace context ignore (trace headers)'); - -#trace context: extract -is($t_headers_extract->{'x-otel-traceparent'}, undef, - 'no traceparent - trace context extract (no trace headers)'); -is($t_headers_extract->{'x-otel-tracestate'}, undef, - 'no tracestate - trace context extract (no trace headers)'); - -is($tp_headers_extract->{'x-otel-parent-id'}, 'b9c7c989f97918e1', - 'parent span id - trace context extract (trace headers)'); -is($tp_headers_extract->{'x-otel-traceparent'}, - '00-0af7651916cd43dd8448eb211c80319c-b9c7c989f97918e1-01', - 'traceparent - trace context extract (trace headers)'); -is($tp_headers_extract->{'x-otel-tracestate'}, - 'congo=ucfJifl5GOE,rojo=00f067aa0ba902b7', - 'tracestate - trace context extract (trace headers)'); - -#trace context: inject -is($t_headers_inject->{'x-otel-traceparent'}, - "00-$$spans{span6}{trace_id}-$$spans{span6}{span_id}-01", - 'traceparent - trace context inject (no trace headers)'); -is($t_headers_inject->{'x-otel-tracestate'}, undef, - 'no tracestate - trace context inject (no trace headers)'); - -is($tp_headers_inject->{'x-otel-parent-id'}, undef, - 'no parent span id - trace context inject (trace headers)'); -is($tp_headers_inject->{'x-otel-traceparent'}, - "00-$$spans{span7}{trace_id}-$$spans{span7}{span_id}-01", - 'traceparent - trace context inject (trace headers)'); -is($tp_headers_inject->{'x-otel-tracestate'}, undef, - 'no tracestate - trace context inject (trace headers)'); - -#trace context: propagate -is($t_headers_propagate->{'x-otel-traceparent'}, - "00-$$spans{span8}{trace_id}-$$spans{span8}{span_id}-01", - 'traceparent - trace context propagate (no trace headers)'); -is($t_headers_propagate->{'x-otel-tracestate'}, undef, - 'no tracestate - trace context propagate (no trace headers)'); - -is($tp_headers_propagate->{'x-otel-parent-id'}, 'b9c7c989f97918e1', - 'parent id - trace context propagate (trace headers)'); -is($tp_headers_propagate->{'x-otel-traceparent'}, - "00-0af7651916cd43dd8448eb211c80319c-$$spans{span9}{span_id}-01", - 'traceparent - trace context propagate (trace headers)'); -is($tp_headers_propagate->{'x-otel-tracestate'}, - 'congo=ucfJifl5GOE,rojo=00f067aa0ba902b7', - 'tracestate - trace context propagate (trace headers)'); - -SKIP: { -skip "depends on error log content", 2 unless $ENV{TEST_NGINX_UNSAFE}; - -$t->stop(); -my $log = $t->read_file("error.log"); - -like($log, qr/OTel\/grpc: Error parsing metadata: error=invalid value/, - 'log: error parsing metadata - no protobuf in response'); -unlike($log, qr/OTel export failure: No status received/, - 'log: no export failure'); - -} - -############################################################################### - -sub http2_get { - my ($path, %extra) = @_; - my ($frames, $frame); - - my $port = $extra{port} || 8080; - - my $s = $extra{ssl} - ? Test::Nginx::HTTP2->new( - undef, socket => get_ssl_socket($port, ['h2'])) - : Test::Nginx::HTTP2->new(); - - my $sid = $extra{trace_headers} - ? $s->new_stream({ headers => [ - { name => ':method', value => 'GET' }, - { name => ':scheme', value => 'http' }, - { name => ':path', value => $path }, - { name => ':authority', value => 'localhost' }, - { name => 'user-agent', value => 'nginx-tests', mode => 2 }, - { name => 'traceparent', - value => '00-0af7651916cd43dd8448eb211c80319c-' . - 'b9c7c989f97918e1-01', - mode => 2 - }, - { name => 'tracestate', - value => 'congo=ucfJifl5GOE,rojo=00f067aa0ba902b7', - mode => 2 - }]}) - : $s->new_stream({ path => $path }); - $frames = $s->read(all => [{ sid => $sid, fin => 1 }]); - - ($frame) = grep { $_->{type} eq "HEADERS" } @$frames; - my $headers = $frame->{headers}; - - ($frame) = grep { $_->{type} eq "DATA" } @$frames; - my $data = $frame->{data}; - - return $headers, $data; -} - -sub get_ssl_socket { - my ($port, $alpn) = @_; - - return http( - '', PeerAddr => '127.0.0.1:' . port($port), start => 1, - SSL => 1, - SSL_alpn_protocols => $alpn, - SSL_error_trap => sub { die $_[1] } - ); -} - -sub get_attr { - my($attr, $type, $obj) = @_; - - my ($res) = grep { - $_ =~ /^attribute\d+/ && $$obj{$_}{key} eq '"' . $attr . '"' - } keys %{$obj}; - - if (defined $res) { - $$obj{$res}{value}{$type} =~ s/(^\")|(\"$)//g - if $type eq 'string_value'; - - return $$obj{$res}{value}{$type}; - } - - return undef; -} - -sub decode_protobuf { - my ($protobuf) = @_; - - local $/; - open CMD, "echo '" . encode_base64($protobuf) . "' | base64 -d | " . - '$PWD/../build/_deps/grpc-build/third_party/protobuf/protoc '. - '--decode opentelemetry.proto.trace.v1.ResourceSpans -I ' . - '$PWD/../build/_deps/otelcpp-src/third_party/opentelemetry-proto ' . - 'opentelemetry/proto/collector/trace/v1/trace_service.proto |' - or die "Can't decode protobuf: $!\n"; - my $out = ; - close CMD; - - return $out; -} - -sub decode_bytes { - my ($bytes) = @_; - - my $c = sub { return chr oct(shift) }; - - $bytes =~ s/\\(\d{3})/$c->($1)/eg; - $bytes =~ s/(^\")|(\"$)//g; - $bytes =~ s/\\\\/\\/g; - $bytes =~ s/\\r/\r/g; - $bytes =~ s/\\n/\n/g; - $bytes =~ s/\\t/\t/g; - $bytes =~ s/\\"/\"/g; - $bytes =~ s/\\'/\'/g; - - return unpack("H*", unpack("a*", $bytes)); -} - -sub to_hash { - my ($textdata) = @_; - - my %out = (); - push my @stack, \%out; - my ($attr_count, $span_count) = (0, 0); - for my $line (split /\n/, $textdata) { - $line =~ s/(^\s+)|(\s+$)//g; - if ($line =~ /\:/) { - my ($k, $v) = split /\: /, $line; - $v = decode_bytes($v) if ($k =~ /trace_id|span_id|parent_span_id/); - $stack[$#stack]{$k} = $v; - } elsif ($line =~ /\{/) { - $line =~ s/\s\{//; - $line = 'attribute' . $attr_count++ if ($line eq 'attributes'); - if ($line eq 'spans') { - $line = 'span' . $span_count++; - $attr_count = 0; - } - my %new = (); - $stack[$#stack]{$line} = \%new; - push @stack, \%new; - } elsif ($line =~ /\}/) { - pop @stack; - } - } - - return \%out; -} - -sub grpc { - my ($server, $client, $f, $s, $c, $sid, $csid, $uri); - - $server = IO::Socket::INET->new( - Proto => 'tcp', - LocalHost => '127.0.0.1', - LocalPort => $p, - Listen => 5, - Reuse => 1 - ) or die "Can't create listening socket: $!\n"; - - $f->{http_start} = sub { - if (IO::Select->new($server)->can_read(5)) { - $client = $server->accept(); - } else { - # connection could be unexpectedly reused - goto reused if $client; - return undef; - } - - $client->sysread($_, 24) == 24 or return; # preface - - $c = Test::Nginx::HTTP2->new(1, socket => $client, - pure => 1, preface => "") or return; - -reused: - my $frames = $c->read(all => [{ fin => 1 }]); - - $client->close(); - - return $frames; - }; - - return $f; -} - -############################################################################### diff --git a/tests/h3_otel.t b/tests/h3_otel.t deleted file mode 100644 index ef848cb..0000000 --- a/tests/h3_otel.t +++ /dev/null @@ -1,508 +0,0 @@ -#!/usr/bin/perl - -# (C) Nginx, Inc. - -# Tests for OTel exporter in case HTTP/3. - -############################################################################### - -use warnings; -use strict; - -use Test::More; - -BEGIN { use FindBin; chdir($FindBin::Bin); } - -use Test::Nginx; -use Test::Nginx::HTTP2; -use Test::Nginx::HTTP3; -use MIME::Base64; - -############################################################################### - -select STDERR; $| = 1; -select STDOUT; $| = 1; - -my $t = Test::Nginx->new()->has(qw/http_v2 http_v3 rewrite mirror grpc cryptx/) - ->has_daemon(qw/openssl base64/) - ->write_file_expand('nginx.conf', <<'EOF'); - -%%TEST_GLOBALS%% - -daemon off; - -events { -} - -http { - %%TEST_GLOBALS_HTTP%% - - ssl_certificate_key localhost.key; - ssl_certificate localhost.crt; - ssl_protocols TLSv1.3; - - otel_exporter { - endpoint 127.0.0.1:8082; - interval 1s; - batch_size 10; - batch_count 2; - } - - otel_service_name test_server; - otel_trace on; - - server { - listen 127.0.0.1:%%PORT_8980_UDP%% quic; - listen 127.0.0.1:8081; - server_name localhost; - - location /trace-on { - otel_trace_context extract; - otel_span_name default_location; - otel_span_attr http.request.header.completion - $request_completion; - otel_span_attr http.response.header.content.type - $sent_http_content_type; - otel_span_attr http.request $request; - add_header "X-Otel-Trace-Id" $otel_trace_id; - add_header "X-Otel-Span-Id" $otel_span_id; - add_header "X-Otel-Parent-Id" $otel_parent_id; - add_header "X-Otel-Parent-Sampled" $otel_parent_sampled; - return 200 "TRACE-ON"; - } - - location /context-ignore { - otel_trace_context ignore; - otel_span_name context_ignore; - add_header "X-Otel-Parent-Id" $otel_parent_id; - proxy_pass http://127.0.0.1:8081/trace-off; - } - - location /context-extract { - otel_trace_context extract; - otel_span_name context_extract; - add_header "X-Otel-Parent-Id" $otel_parent_id; - proxy_pass http://127.0.0.1:8081/trace-off; - } - - location /context-inject { - otel_trace_context inject; - otel_span_name context_inject; - add_header "X-Otel-Parent-Id" $otel_parent_id; - proxy_pass http://127.0.0.1:8081/trace-off; - } - - location /context-propagate { - otel_trace_context propagate; - otel_span_name context_propogate; - add_header "X-Otel-Parent-Id" $otel_parent_id; - proxy_pass http://127.0.0.1:8081/trace-off; - } - - location /trace-off { - otel_trace off; - add_header "X-Otel-Traceparent" $http_traceparent; - add_header "X-Otel-Tracestate" $http_tracestate; - return 200 "TRACE-OFF"; - } - } - - server { - listen 127.0.0.1:8082 http2; - server_name localhost; - otel_trace off; - - location / { - mirror /mirror; - grpc_pass 127.0.0.1:8083; - } - - location /mirror { - internal; - grpc_pass 127.0.0.1:%%PORT_4317%%; - } - } - - server { - listen 127.0.0.1:8083 http2; - server_name localhost; - otel_trace off; - - location / { - add_header content-type application/grpc; - add_header grpc-status 0; - add_header grpc-message ""; - return 200; - } - } - -} - -EOF - -$t->write_file('openssl.conf', <<'EOF'); -[ req ] -default_bits = 2048 -encrypt_key = no -distinguished_name = req_distinguished_name -[ req_distinguished_name ] - -EOF - -my $d = $t->testdir(); - -foreach my $name ('localhost') { - system('openssl req -x509 -new ' - . "-config $d/openssl.conf -subj /CN=$name/ " - . "-out $d/$name.crt -keyout $d/$name.key " - . ">>$d/openssl.out 2>&1") == 0 - or die "Can't create certificate for $name: $!\n"; -} - -$t->try_run('no OTel module')->plan(56); - -############################################################################### - -my $p = port(4317); -my $f = grpc(); - -#do requests -(undef, my $t_off_resp) = http3_get('/trace-off'); - -#batch0 (10 requests) -my ($tp_headers, $tp_resp) = http3_get('/trace-on', trace_headers => 1); -my ($t_headers, $t_resp) = http3_get('/trace-on'); - -(my $t_headers_ignore, undef) = http3_get('/context-ignore'); -(my $tp_headers_ignore, undef) = http3_get('/context-ignore', - trace_headers => 1); -(my $t_headers_extract, undef) = http3_get('/context-extract'); -(my $tp_headers_extract, undef) = http3_get('/context-extract', - trace_headers => 1); -(my $t_headers_inject, undef) = http3_get('/context-inject'); -(my $tp_headers_inject, undef) = http3_get('/context-inject', - trace_headers => 1); -(my $t_headers_propagate, undef) = http3_get('/context-propagate'); -(my $tp_headers_propagate, undef) = - http3_get('/context-propagate', trace_headers => 1); - -my ($frame) = grep { $_->{type} eq "DATA" } @{$f->{http_start}()}; -my $batch0 = to_hash(decode_protobuf(substr $frame->{data}, 8)); -my $spans = $$batch0{scope_spans}; - -#batch1 (5 reqeusts) -http3_get('/trace-on') for (1..5); - -($frame) = grep { $_->{type} eq "DATA" } @{$f->{http_start}()}; -my $batch1 = to_hash(decode_protobuf(substr $frame->{data}, 8)); - -#validate responses -like($tp_resp, qr/TRACE-ON/, 'http request1 - trace on'); -like($t_resp, qr/TRACE-ON/, 'http request2 - trace on'); -like($t_off_resp, qr/TRACE-OFF/, 'http request - trace off'); - -#validate batch size -delete $$spans{scope}; #remove 'scope' entry -is(scalar keys %{$spans}, 10, 'batch0 size - trace on'); -delete $$batch1{scope_spans}{scope}; #remove 'scope' entry -is(scalar keys %{$$batch1{scope_spans}}, 5, 'batch1 size - trace on'); - -#validate general attributes -is(get_attr("service.name", "string_value", - $$batch0{resource}), - 'test_server', 'service.name - trace on'); -is($$spans{span0}{name}, '"default_location"', 'span.name - trace on'); - -#validate metrics -is(get_attr("http.method", "string_value", $$spans{span0}), 'GET', - 'http.method metric - trace on'); -is(get_attr("http.target", "string_value", $$spans{span0}), '/trace-on', - 'http.target metric - trace on'); -is(get_attr("http.route", "string_value", $$spans{span0}), '/trace-on', - 'http.route metric - trace on'); -is(get_attr("http.scheme", "string_value", $$spans{span0}), 'https', - 'http.scheme metric - trace on'); -is(get_attr("http.flavor", "string_value", $$spans{span0}), '3.0', - 'http.flavor metric - trace on'); -is(get_attr("http.user_agent", "string_value", $$spans{span0}), 'nginx-tests', - 'http.user_agent metric - trace on'); -is(get_attr("http.request_content_length", "int_value", $$spans{span0}), 0, - 'http.request_content_length metric - trace on'); -is(get_attr("http.response_content_length", "int_value", $$spans{span0}), 8, - 'http.response_content_length metric - trace on'); -is(get_attr("http.status_code", "int_value", $$spans{span0}), 200, - 'http.status_code metric - trace on'); -is(get_attr("net.host.name", "string_value", $$spans{span0}), 'localhost', - 'net.host.name metric - trace on'); -is(get_attr("net.host.port", "int_value", $$spans{span0}), 8980, - 'net.host.port metric - trace on'); -is(get_attr("net.sock.peer.addr", "string_value", $$spans{span0}), '127.0.0.1', - 'net.sock.peer.addr metric - trace on'); -like(get_attr("net.sock.peer.port", "int_value", $$spans{span0}), qr/\d+/, - 'net.sock.peer.port metric - trace on'); - -#validate custom http metrics -is(${get_attr("http.request.header.completion", "array_value", $$spans{span0})} - {values}{string_value}, '"OK"', - 'http.request.header.completion metric - trace on'); -is(${get_attr( - "http.response.header.content.type", "array_value", $$spans{span0} - )}{values}{string_value}, '"text/plain"', - 'http.response.header.content.type metric - trace on'); -is(get_attr("http.request", "string_value", $$spans{span0}), - 'GET /trace-on HTTP/3.0', 'http.request metric - trace on'); - -#extract trace info -is($$spans{span0}{parent_span_id}, 'b9c7c989f97918e1', - 'traceparent - trace on'); -is($$spans{span0}{trace_state}, '"congo=ucfJifl5GOE,rojo=00f067aa0ba902b7"', - 'tracestate - trace on'); -is($$spans{span1}{parent_span_id}, undef, 'no traceparent - trace on'); -is($$spans{span1}{trace_state}, undef, 'no tracestate - trace on'); - -#variables -is($tp_headers->{'x-otel-trace-id'}, $$spans{span0}{trace_id}, - '$otel_trace_id variable - trace on'); -is($tp_headers->{'x-otel-span-id'}, $$spans{span0}{span_id}, - '$otel_span_id variable - trace on'); -is($tp_headers->{'x-otel-parent-id'}, $$spans{span0}{parent_span_id}, - '$otel_parent_id variable - trace on'); -is($tp_headers->{'x-otel-parent-sampled'}, 1, - '$otel_parent_sampled variable - trace on'); -is($t_headers->{'x-otel-parent-sampled'}, 0, - '$otel_parent_sampled variable - trace on (no traceparent header)'); - -#trace off -is((scalar grep { - get_attr("http.target", "string_value", $$spans{$_}) eq '/trace-off' - } keys %{$spans}), 0, 'no metric in batch0 - trace off'); -is((scalar grep { - get_attr("http.target", "string_value", $$spans{$_}) eq '/trace-off' - } keys %{$$batch1{scope_spans}}), 0, 'no metric in batch1 - trace off'); - -#trace context: ignore -is($t_headers_ignore->{'x-otel-traceparent'}, undef, - 'no traceparent - trace context ignore (no trace headers)'); -is($t_headers_ignore->{'x-otel-tracestate'}, undef, - 'no tracestate - trace context ignore (no trace headers)'); - -is($tp_headers_ignore->{'x-otel-parent-id'}, undef, - 'no parent span id - trace context ignore (trace headers)'); -is($tp_headers_ignore->{'x-otel-traceparent'}, - '00-0af7651916cd43dd8448eb211c80319c-b9c7c989f97918e1-01', - 'traceparent - trace context ignore (trace headers)'); -is($tp_headers_ignore->{'x-otel-tracestate'}, - 'congo=ucfJifl5GOE,rojo=00f067aa0ba902b7', - 'tracestate - trace context ignore (trace headers)'); - -#trace context: extract -is($t_headers_extract->{'x-otel-traceparent'}, undef, - 'no traceparent - trace context extract (no trace headers)'); -is($t_headers_extract->{'x-otel-tracestate'}, undef, - 'no tracestate - trace context extract (no trace headers)'); - -is($tp_headers_extract->{'x-otel-parent-id'}, 'b9c7c989f97918e1', - 'parent span id - trace context extract (trace headers)'); -is($tp_headers_extract->{'x-otel-traceparent'}, - '00-0af7651916cd43dd8448eb211c80319c-b9c7c989f97918e1-01', - 'traceparent - trace context extract (trace headers)'); -is($tp_headers_extract->{'x-otel-tracestate'}, - 'congo=ucfJifl5GOE,rojo=00f067aa0ba902b7', - 'tracestate - trace context extract (trace headers)'); - -#trace context: inject -is($t_headers_inject->{'x-otel-traceparent'}, - "00-$$spans{span6}{trace_id}-$$spans{span6}{span_id}-01", - 'traceparent - trace context inject (no trace headers)'); -is($t_headers_inject->{'x-otel-tracestate'}, undef, - 'no tracestate - trace context inject (no trace headers)'); - -is($tp_headers_inject->{'x-otel-parent-id'}, undef, - 'no parent span id - trace context inject (trace headers)'); -is($tp_headers_inject->{'x-otel-traceparent'}, - "00-$$spans{span7}{trace_id}-$$spans{span7}{span_id}-01", - 'traceparent - trace context inject (trace headers)'); -is($tp_headers_inject->{'x-otel-tracestate'}, undef, - 'no tracestate - trace context inject (trace headers)'); - -#trace context: propagate -is($t_headers_propagate->{'x-otel-traceparent'}, - "00-$$spans{span8}{trace_id}-$$spans{span8}{span_id}-01", - 'traceparent - trace context propagate (no trace headers)'); -is($t_headers_propagate->{'x-otel-tracestate'}, undef, - 'no tracestate - trace context propagate (no trace headers)'); - -is($tp_headers_propagate->{'x-otel-parent-id'}, 'b9c7c989f97918e1', - 'parent id - trace context propagate (trace headers)'); -is($tp_headers_propagate->{'x-otel-traceparent'}, - "00-0af7651916cd43dd8448eb211c80319c-$$spans{span9}{span_id}-01", - 'traceparent - trace context propagate (trace headers)'); -is($tp_headers_propagate->{'x-otel-tracestate'}, - 'congo=ucfJifl5GOE,rojo=00f067aa0ba902b7', - 'tracestate - trace context propagate (trace headers)'); - -SKIP: { -skip "depends on error log content", 2 unless $ENV{TEST_NGINX_UNSAFE}; - -$t->stop(); -my $log = $t->read_file("error.log"); - -like($log, qr/OTel\/grpc: Error parsing metadata: error=invalid value/, - 'log: error parsing metadata - no protobuf in response'); -unlike($log, qr/OTel export failure: No status received/, - 'log: no export failure'); - -} - -############################################################################### - -sub http3_get { - my ($path, %extra) = @_; - my ($frames, $frame); - - my $s = Test::Nginx::HTTP3->new(); - - my $sid = $extra{trace_headers} - ? $s->new_stream({ headers => [ - { name => ':method', value => 'GET' }, - { name => ':scheme', value => 'http' }, - { name => ':path', value => $path }, - { name => ':authority', value => 'localhost' }, - { name => 'user-agent', value => 'nginx-tests' }, - { name => 'traceparent', - value => '00-0af7651916cd43dd8448eb211c80319c-' . - 'b9c7c989f97918e1-01' - }, - { name => 'tracestate', - value => 'congo=ucfJifl5GOE,rojo=00f067aa0ba902b7' - }]}) - : $s->new_stream({ path => $path }); - - $frames = $s->read(all => [{ sid => $sid, fin => 1 }]); - - ($frame) = grep { $_->{type} eq "HEADERS" } @$frames; - my $headers = $frame->{headers}; - - ($frame) = grep { $_->{type} eq "DATA" } @$frames; - my $data = $frame->{data}; - - return $headers, $data; -} - -sub get_attr { - my($attr, $type, $obj) = @_; - - my ($res) = grep { - $_ =~ /^attribute\d+/ && $$obj{$_}{key} eq '"' . $attr . '"' - } keys %{$obj}; - - if (defined $res) { - $$obj{$res}{value}{$type} =~ s/(^\")|(\"$)//g - if $type eq 'string_value'; - - return $$obj{$res}{value}{$type}; - } - - return undef; -} - -sub decode_protobuf { - my ($protobuf) = @_; - - local $/; - open CMD, "echo '" . encode_base64($protobuf) . "' | base64 -d | " . - '$PWD/../build/_deps/grpc-build/third_party/protobuf/protoc '. - '--decode opentelemetry.proto.trace.v1.ResourceSpans -I ' . - '$PWD/../build/_deps/otelcpp-src/third_party/opentelemetry-proto ' . - 'opentelemetry/proto/collector/trace/v1/trace_service.proto |' - or die "Can't decode protobuf: $!\n"; - my $out = ; - close CMD; - - return $out; -} - -sub decode_bytes { - my ($bytes) = @_; - - my $c = sub { return chr oct(shift) }; - - $bytes =~ s/\\(\d{3})/$c->($1)/eg; - $bytes =~ s/(^\")|(\"$)//g; - $bytes =~ s/\\\\/\\/g; - $bytes =~ s/\\r/\r/g; - $bytes =~ s/\\n/\n/g; - $bytes =~ s/\\t/\t/g; - $bytes =~ s/\\"/\"/g; - $bytes =~ s/\\'/\'/g; - - return unpack("H*", unpack("a*", $bytes)); -} - -sub to_hash { - my ($textdata) = @_; - - my %out = (); - push my @stack, \%out; - my ($attr_count, $span_count) = (0, 0); - for my $line (split /\n/, $textdata) { - $line =~ s/(^\s+)|(\s+$)//g; - if ($line =~ /\:/) { - my ($k, $v) = split /\: /, $line; - $v = decode_bytes($v) if ($k =~ /trace_id|span_id|parent_span_id/); - $stack[$#stack]{$k} = $v; - } elsif ($line =~ /\{/) { - $line =~ s/\s\{//; - $line = 'attribute' . $attr_count++ if ($line eq 'attributes'); - if ($line eq 'spans') { - $line = 'span' . $span_count++; - $attr_count = 0; - } - my %new = (); - $stack[$#stack]{$line} = \%new; - push @stack, \%new; - } elsif ($line =~ /\}/) { - pop @stack; - } - } - - return \%out; -} - -sub grpc { - my ($server, $client, $f, $s, $c, $sid, $csid, $uri); - - $server = IO::Socket::INET->new( - Proto => 'tcp', - LocalHost => '127.0.0.1', - LocalPort => $p, - Listen => 5, - Reuse => 1 - ) or die "Can't create listening socket: $!\n"; - - $f->{http_start} = sub { - if (IO::Select->new($server)->can_read(5)) { - $client = $server->accept(); - } else { - # connection could be unexpectedly reused - goto reused if $client; - return undef; - } - - $client->sysread($_, 24) == 24 or return; # preface - - $c = Test::Nginx::HTTP2->new(1, socket => $client, - pure => 1, preface => "") or return; - -reused: - my $frames = $c->read(all => [{ fin => 1 }]); - - $client->close(); - - return $frames; - }; - - return $f; -} - -############################################################################### diff --git a/tests/otel.t b/tests/otel.t deleted file mode 100644 index 9887b2f..0000000 --- a/tests/otel.t +++ /dev/null @@ -1,514 +0,0 @@ -#!/usr/bin/perl - -# (C) Nginx, Inc. - -# Tests for OTel exporter in case HTTP. - -############################################################################### - -use warnings; -use strict; - -use Test::More; - -BEGIN { use FindBin; chdir($FindBin::Bin); } - -use Test::Nginx; -use Test::Nginx::HTTP2; -use MIME::Base64; - -############################################################################### - -select STDERR; $| = 1; -select STDOUT; $| = 1; - -my $t = Test::Nginx->new()->has(qw/http http_ssl http_v2 mirror rewrite/) - ->has_daemon(qw/openssl base64/) - ->write_file_expand('nginx.conf', <<'EOF'); - -%%TEST_GLOBALS%% - -daemon off; - -events { -} - -http { - %%TEST_GLOBALS_HTTP%% - - ssl_certificate_key localhost.key; - ssl_certificate localhost.crt; - - otel_exporter { - endpoint 127.0.0.1:8082; - interval 1s; - batch_size 10; - batch_count 2; - } - - otel_service_name test_server; - otel_trace on; - - server { - listen 127.0.0.1:8080; - listen 127.0.0.1:8081 ssl; - server_name localhost; - - location /trace-on { - otel_trace_context extract; - otel_span_name default_location; - otel_span_attr http.request.header.completion - $request_completion; - otel_span_attr http.response.header.content.type - $sent_http_content_type; - otel_span_attr http.request $request; - add_header "X-Otel-Trace-Id" $otel_trace_id; - add_header "X-Otel-Span-Id" $otel_span_id; - add_header "X-Otel-Parent-Id" $otel_parent_id; - add_header "X-Otel-Parent-Sampled" $otel_parent_sampled; - return 200 "TRACE-ON"; - } - - location /context-ignore { - otel_trace_context ignore; - otel_span_name context_ignore; - add_header "X-Otel-Parent-Id" $otel_parent_id; - proxy_pass http://localhost:8080/trace-off; - } - - location /context-extract { - otel_trace_context extract; - otel_span_name context_extract; - add_header "X-Otel-Parent-Id" $otel_parent_id; - proxy_pass http://localhost:8080/trace-off; - } - - location /context-inject { - otel_trace_context inject; - otel_span_name context_inject; - add_header "X-Otel-Parent-Id" $otel_parent_id; - proxy_pass http://localhost:8080/trace-off; - } - - location /context-propagate { - otel_trace_context propagate; - otel_span_name context_propogate; - add_header "X-Otel-Parent-Id" $otel_parent_id; - proxy_pass http://localhost:8080/trace-off; - } - - location /trace-off { - otel_trace off; - add_header "X-Otel-Traceparent" $http_traceparent; - add_header "X-Otel-Tracestate" $http_tracestate; - return 200 "TRACE-OFF"; - } - } - - server { - listen 127.0.0.1:8082 http2; - server_name localhost; - otel_trace off; - - location / { - mirror /mirror; - grpc_pass 127.0.0.1:8083; - } - - location /mirror { - internal; - grpc_pass 127.0.0.1:%%PORT_4317%%; - } - } - - server { - listen 127.0.0.1:8083 http2; - server_name localhost; - otel_trace off; - - location / { - add_header content-type application/grpc; - add_header grpc-status 0; - add_header grpc-message ""; - return 200; - } - } -} - -EOF - -$t->write_file('openssl.conf', <<'EOF'); -[ req ] -default_bits = 2048 -encrypt_key = no -distinguished_name = req_distinguished_name -[ req_distinguished_name ] - -EOF - -my $d = $t->testdir(); - -foreach my $name ('localhost') { - system('openssl req -x509 -new ' - . "-config $d/openssl.conf -subj /CN=$name/ " - . "-out $d/$name.crt -keyout $d/$name.key " - . ">>$d/openssl.out 2>&1") == 0 - or die "Can't create certificate for $name: $!\n"; -} - -$t->try_run('no OTel module')->plan(69); - -############################################################################### - -my $p = port(4317); -my $f = grpc(); - -#do requests -my $t_off_resp = http1_get('/trace-off'); - -#batch0 (10 requests) -my $tp_resp = http1_get('/trace-on', trace_headers => 1); -my $t_resp = http1_get('/trace-on', port => 8081, ssl => 1); - -my $t_resp_ignore = http1_get('/context-ignore'); -my $tp_resp_ignore = http1_get('/context-ignore', trace_headers => 1); -my $t_resp_extract = http1_get('/context-extract'); -my $tp_resp_extract = http1_get('/context-extract', trace_headers => 1); -my $t_resp_inject = http1_get('/context-inject'); -my $tp_resp_inject = http1_get('/context-inject', trace_headers => 1); -my $t_resp_propagate = http1_get('/context-propagate'); -my $tp_resp_propagate = http1_get('/context-propagate', trace_headers => 1); - -my ($frame) = grep { $_->{type} eq "DATA" } @{$f->{http_start}()}; -my $batch0 = to_hash(decode_protobuf(substr $frame->{data}, 8)); -my $spans = $$batch0{scope_spans}; - -#batch1 (5 reqeusts) -http1_get('/trace-on') for (1..5); - -($frame) = grep { $_->{type} eq "DATA" } @{$f->{http_start}()}; -my $batch1 = to_hash(decode_protobuf(substr $frame->{data}, 8)); - -#validate responses -like($tp_resp, qr/TRACE-ON/, 'http request1 - trace on'); -like($t_resp, qr/TRACE-ON/, 'http request2 - trace on'); -like($t_off_resp, qr/TRACE-OFF/, 'http request - trace off'); - -#validate batch size -delete $$spans{scope}; #remove 'scope' entry -is(scalar keys %{$spans}, 10, 'batch0 size - trace on'); -delete $$batch1{scope_spans}{scope}; #remove 'scope' entry -is(scalar keys %{$$batch1{scope_spans}}, 5, 'batch1 size - trace on'); - -#validate general attributes -is(get_attr("service.name", "string_value", - $$batch0{resource}), 'test_server', 'service.name - trace on'); -is($$spans{span0}{name}, '"default_location"', 'span.name - trace on'); - -#validate http metrics -is(get_attr("http.method", "string_value", $$spans{span0}), 'GET', - 'http.method metric - trace on'); -is(get_attr("http.target", "string_value", $$spans{span0}), '/trace-on', - 'http.target metric - trace on'); -is(get_attr("http.route", "string_value", $$spans{span0}), '/trace-on', - 'http.route metric - trace on'); -is(get_attr("http.scheme", "string_value", $$spans{span0}), 'http', - 'http.scheme metric - trace on'); -is(get_attr("http.flavor", "string_value", $$spans{span0}), '1.0', - 'http.flavor metric - trace on'); -is(get_attr("http.user_agent", "string_value", $$spans{span0}), 'nginx-tests', - 'http.user_agent metric - trace on'); -is(get_attr("http.request_content_length", "int_value", $$spans{span0}), 0, - 'http.request_content_length metric - trace on'); -is(get_attr("http.response_content_length", "int_value", $$spans{span0}), 8, - 'http.response_content_length metric - trace on'); -is(get_attr("http.status_code", "int_value", $$spans{span0}), 200, - 'http.status_code metric - trace on'); -is(get_attr("net.host.name", "string_value", $$spans{span0}), 'localhost', - 'net.host.name metric - trace on'); -is(get_attr("net.host.port", "int_value", $$spans{span0}), 8080, - 'net.host.port metric - trace on'); -is(get_attr("net.sock.peer.addr", "string_value", $$spans{span0}), '127.0.0.1', - 'net.sock.peer.addr metric - trace on'); -like(get_attr("net.sock.peer.port", "int_value", $$spans{span0}), qr/\d+/, - 'net.sock.peer.port metric - trace on'); - -#validate https metrics -is(get_attr("http.method", "string_value", $$spans{span1}), 'GET', - 'http.method metric - trace on (https)'); -is(get_attr("http.target", "string_value", $$spans{span1}), '/trace-on', - 'http.target metric - trace on (https)'); -is(get_attr("http.route", "string_value", $$spans{span1}), '/trace-on', - 'http.route metric - trace on (https)'); -is(get_attr("http.scheme", "string_value", $$spans{span1}), 'https', - 'http.scheme metric - trace on (https)'); -is(get_attr("http.flavor", "string_value", $$spans{span1}), '1.0', - 'http.flavor metric - trace on (https)'); -is(get_attr("http.user_agent", "string_value", $$spans{span1}), - 'nginx-tests', 'http.user_agent metric - trace on (https)'); -is(get_attr("http.request_content_length", "int_value", $$spans{span1}), 0, - 'http.request_content_length metric - trace on (https)'); -is(get_attr("http.response_content_length", "int_value", $$spans{span1}), 8, - 'http.response_content_length metric - trace on (https)'); -is(get_attr("http.status_code", "int_value", $$spans{span1}), 200, - 'http.status_code metric - trace on (https)'); -is(get_attr("net.host.name", "string_value", $$spans{span1}), 'localhost', - 'net.host.name metric - trace on (https)'); -is(get_attr("net.host.port", "int_value", $$spans{span1}), 8081, - 'net.host.port metric - trace on (https)'); -is(get_attr("net.sock.peer.addr", "string_value", $$spans{span1}), '127.0.0.1', - 'net.sock.peer.addr metric - trace on (https)'); -like(get_attr("net.sock.peer.port", "int_value", $$spans{span1}), qr/\d+/, - 'net.sock.peer.port metric - trace on (https)'); - -#validate custom http metrics -is(${get_attr("http.request.header.completion", "array_value", $$spans{span0})} - {values}{string_value}, '"OK"', - 'http.request.header.completion metric - trace on'); -is(${get_attr("http.response.header.content.type", - "array_value", $$spans{span0})}{values}{string_value}, '"text/plain"', - 'http.response.header.content.type metric - trace on'); -is(get_attr("http.request", "string_value", $$spans{span0}), - 'GET /trace-on HTTP/1.0', 'http.request metric - trace on'); - -#extract trace info -is($$spans{span0}{parent_span_id}, 'b9c7c989f97918e1', - 'traceparent - trace on'); -is($$spans{span0}{trace_state}, '"congo=ucfJifl5GOE,rojo=00f067aa0ba902b7"', - 'tracestate - trace on'); -is($$spans{span1}{parent_span_id}, undef, 'no traceparent - trace on'); -is($$spans{span1}{trace_state}, undef, 'no tracestate - trace on'); - -#variables -like($tp_resp, qr/X-Otel-Trace-Id: $$spans{span0}{trace_id}/, - '$otel_trace_id variable - trace on'); -like($tp_resp, qr/X-Otel-Span-Id: $$spans{span0}{span_id}/, - '$otel_span_id variable - trace on'); -like($tp_resp, qr/X-Otel-Parent-Id: $$spans{span0}{parent_span_id}/, - '$otel_parent_id variable - trace on'); -like($tp_resp, qr/X-Otel-Parent-Sampled: 1/, - '$otel_parent_sampled variable - trace on'); -like($t_resp, qr/X-Otel-Parent-Sampled: 0/, - '$otel_parent_sampled variable - trace on (no traceparent header)'); - -#trace off -is((scalar grep { - get_attr("http.target", "string_value", $$spans{$_}) eq '/trace-off' - } keys %{$spans}), 0, 'no metric in batch0 - trace off'); -is((scalar grep { - get_attr("http.target", "string_value", $$spans{$_}) eq '/trace-off' - } keys %{$$batch1{scope_spans}}), 0, 'no metric in batch1 - trace off'); - -#trace context: ignore -unlike($t_resp_ignore, qr/X-Otel-Traceparent/, - 'no traceparent - trace context ignore (no trace headers)'); -unlike($t_resp_ignore, qr/X-Otel-Tracestate/, - 'no tracestate - trace context ignore (no trace headers)'); - -unlike($tp_resp_ignore, qr/X-Otel-Parent-Id/, - 'no parent span id - trace context ignore (trace headers)'); -like($tp_resp_ignore, - qr/Traceparent: 00-0af7651916cd43dd8448eb211c80319c-b9c7c989f97918e1-01/, - 'traceparent - trace context ignore (trace headers)'); -like($tp_resp_ignore, - qr/Tracestate: congo=ucfJifl5GOE,rojo=00f067aa0ba902b7/, - 'tracestate - trace context ignore (trace headers)'); - -#trace context: extract -unlike($t_resp_extract, qr/X-Otel-Traceparent/, - 'no traceparent - trace context extract (no trace headers)'); -unlike($t_resp_extract, qr/X-Otel-Tracestate/, - 'no tracestate - trace context extract (no trace headers)'); - -like($tp_resp_extract, qr/X-Otel-Parent-Id: b9c7c989f97918e1/, - 'parent span id - trace context extract (trace headers)'); -like($tp_resp_extract, - qr/Traceparent: 00-0af7651916cd43dd8448eb211c80319c-b9c7c989f97918e1-01/, - 'traceparent - trace context extract (trace headers)'); -like($tp_resp_extract, - qr/Tracestate: congo=ucfJifl5GOE,rojo=00f067aa0ba902b7/, - 'tracestate - trace context extract (trace headers)'); - -#trace context: inject -like($t_resp_inject, - qr/Traceparent: 00-$$spans{span6}{trace_id}-$$spans{span6}{span_id}-01/, - 'traceparent - trace context inject (no trace headers)'); -unlike($t_resp_inject, qr/X-Otel-Tracestate/, - 'no tracestate - trace context inject (no trace headers)'); - -unlike($tp_resp_inject, qr/X-Otel-Parent-Id/, - 'no parent span id - trace context inject (trace headers)'); -like($tp_resp_inject, - qr/Traceparent: 00-$$spans{span7}{trace_id}-$$spans{span7}{span_id}-01/, - 'traceparent - trace context inject (trace headers)'); -unlike($tp_resp_inject, qr/Tracestate:/, - 'no tracestate - trace context inject (trace headers)'); - -#trace context: propagate -like($t_resp_propagate, - qr/Traceparent: 00-$$spans{span8}{trace_id}-$$spans{span8}{span_id}-01/, - 'traceparent - trace context propagate (no trace headers)'); -unlike($t_resp_propagate, qr/X-Otel-Tracestate/, - 'no tracestate - trace context propagate (no trace headers)'); - -like($tp_resp_propagate, qr/X-Otel-Parent-Id: b9c7c989f97918e1/, - 'parent id - trace context propagate (trace headers)'); -like($tp_resp_propagate, - qr/parent: 00-0af7651916cd43dd8448eb211c80319c-$$spans{span9}{span_id}-01/, - 'traceparent - trace context propagate (trace headers)'); -like($tp_resp_propagate, - qr/Tracestate: congo=ucfJifl5GOE,rojo=00f067aa0ba902b7/, - 'tracestate - trace context propagate (trace headers)'); - -SKIP: { -skip "depends on error log contents", 2 unless $ENV{TEST_NGINX_UNSAFE}; - -$t->stop(); -my $log = $t->read_file("error.log"); - -like($log, qr/OTel\/grpc: Error parsing metadata: error=invalid value/, - 'log: error parsing metadata - no protobuf in response'); -unlike($log, qr/OTel export failure: No status received/, - 'log: no export failure'); - -} - -############################################################################### - -sub http1_get { - my ($path, %extra) = @_; - - my $port = $extra{port} || 8080; - - my $r = < '127.0.0.1:' . port($port), - SSL => $extra{ssl}); -} - -sub get_attr { - my($attr, $type, $obj) = @_; - - my ($res) = grep { - $_ =~ /^attribute\d+/ && $$obj{$_}{key} eq '"' . $attr . '"' - } keys %{$obj}; - - if (defined $res) { - $$obj{$res}{value}{$type} =~ s/(^\")|(\"$)//g - if $type eq 'string_value'; - - return $$obj{$res}{value}{$type}; - } - - return undef; -} - -sub decode_protobuf { - my ($protobuf) = @_; - - local $/; - open CMD, "echo '" . encode_base64($protobuf) . "' | base64 -d | " . - '$PWD/../build/_deps/grpc-build/third_party/protobuf/protoc '. - '--decode opentelemetry.proto.trace.v1.ResourceSpans -I ' . - '$PWD/../build/_deps/otelcpp-src/third_party/opentelemetry-proto ' . - 'opentelemetry/proto/collector/trace/v1/trace_service.proto |' - or die "Can't decode protobuf: $!\n"; - my $out = ; - close CMD; - - return $out; -} - -sub decode_bytes { - my ($bytes) = @_; - - my $c = sub { return chr oct(shift) }; - - $bytes =~ s/\\(\d{3})/$c->($1)/eg; - $bytes =~ s/(^\")|(\"$)//g; - $bytes =~ s/\\\\/\\/g; - $bytes =~ s/\\r/\r/g; - $bytes =~ s/\\n/\n/g; - $bytes =~ s/\\t/\t/g; - $bytes =~ s/\\"/\"/g; - $bytes =~ s/\\'/\'/g; - - return unpack("H*", unpack("a*", $bytes)); -} - -sub to_hash { - my ($textdata) = @_; - - my %out = (); - push my @stack, \%out; - my ($attr_count, $span_count) = (0, 0); - for my $line (split /\n/, $textdata) { - $line =~ s/(^\s+)|(\s+$)//g; - if ($line =~ /\:/) { - my ($k, $v) = split /\: /, $line; - $v = decode_bytes($v) if ($k =~ /trace_id|span_id|parent_span_id/); - $stack[$#stack]{$k} = $v; - } elsif ($line =~ /\{/) { - $line =~ s/\s\{//; - $line = 'attribute' . $attr_count++ if ($line eq 'attributes'); - if ($line eq 'spans') { - $line = 'span' . $span_count++; - $attr_count = 0; - } - my %new = (); - $stack[$#stack]{$line} = \%new; - push @stack, \%new; - } elsif ($line =~ /\}/) { - pop @stack; - } - } - - return \%out; -} - -sub grpc { - my ($server, $client, $f, $s, $c, $sid, $csid, $uri); - - $server = IO::Socket::INET->new( - Proto => 'tcp', - LocalHost => '127.0.0.1', - LocalPort => $p, - Listen => 5, - Reuse => 1 - ) or die "Can't create listening socket: $!\n"; - - $f->{http_start} = sub { - if (IO::Select->new($server)->can_read(5)) { - $client = $server->accept(); - } else { - # connection could be unexpectedly reused - goto reused if $client; - return undef; - } - - $client->sysread($_, 24) == 24 or return; # preface - - $c = Test::Nginx::HTTP2->new(1, socket => $client, - pure => 1, preface => "") or return; - -reused: - my $frames = $c->read(all => [{ fin => 1 }]); - - $client->close(); - - return $frames; - }; - - return $f; -} - -############################################################################### diff --git a/tests/otel_collector.t b/tests/otel_collector.t deleted file mode 100644 index 158a2fd..0000000 --- a/tests/otel_collector.t +++ /dev/null @@ -1,403 +0,0 @@ -#!/usr/bin/perl - -# (C) Nginx, Inc. - -# Tests for OTel exporter in case HTTP using otelcol. - -############################################################################### - -use warnings; -use strict; - -use Test::More; - -BEGIN { use FindBin; chdir($FindBin::Bin); } - -use Test::Nginx; - -############################################################################### - -select STDERR; $| = 1; -select STDOUT; $| = 1; - -plan(skip_all => "depends on logs content") unless $ENV{TEST_NGINX_UNSAFE}; - -eval { require JSON::PP; }; -plan(skip_all => "JSON::PP not installed") if $@; - -my $t = Test::Nginx->new()->has(qw/http http_ssl rewrite/) - ->write_file_expand('nginx.conf', <<'EOF'); - -%%TEST_GLOBALS%% - -daemon off; - -events { -} - -http { - %%TEST_GLOBALS_HTTP%% - - ssl_certificate_key localhost.key; - ssl_certificate localhost.crt; - - otel_exporter { - endpoint 127.0.0.1:%%PORT_4317%%; - interval 1s; - batch_size 10; - batch_count 2; - } - - otel_service_name test_server; - otel_trace on; - - server { - listen 127.0.0.1:8080; - listen 127.0.0.1:8081 ssl; - server_name localhost; - - location /trace-on { - otel_trace_context extract; - otel_span_name default_location; - otel_span_attr http.request.header.completion - $request_completion; - otel_span_attr http.response.header.content.type - $sent_http_content_type; - otel_span_attr http.request $request; - add_header "X-Otel-Trace-Id" $otel_trace_id; - add_header "X-Otel-Span-Id" $otel_span_id; - add_header "X-Otel-Parent-Id" $otel_parent_id; - add_header "X-Otel-Parent-Sampled" $otel_parent_sampled; - return 200 "TRACE-ON"; - } - - location /context-ignore { - otel_trace_context ignore; - otel_span_name context_ignore; - add_header "X-Otel-Parent-Id" $otel_parent_id; - proxy_pass http://localhost:8080/trace-off; - } - - location /context-extract { - otel_trace_context extract; - otel_span_name context_extract; - add_header "X-Otel-Parent-Id" $otel_parent_id; - proxy_pass http://localhost:8080/trace-off; - } - - location /context-inject { - otel_trace_context inject; - otel_span_name context_inject; - add_header "X-Otel-Parent-Id" $otel_parent_id; - proxy_pass http://localhost:8080/trace-off; - } - - location /context-propagate { - otel_trace_context propagate; - otel_span_name context_propogate; - add_header "X-Otel-Parent-Id" $otel_parent_id; - proxy_pass http://localhost:8080/trace-off; - } - - location /trace-off { - otel_trace off; - add_header "X-Otel-Traceparent" $http_traceparent; - add_header "X-Otel-Tracestate" $http_tracestate; - return 200 "TRACE-OFF"; - } - } -} - -EOF - -$t->write_file_expand('otel-config.yaml', <testdir() }/otel.json - -service: - pipelines: - traces: - receivers: [otlp] - exporters: [logging, file] - metrics: - receivers: [otlp] - exporters: [logging, file] - -EOF - -$t->write_file('openssl.conf', <<'EOF'); -[ req ] -default_bits = 2048 -encrypt_key = no -distinguished_name = req_distinguished_name -[ req_distinguished_name ] - -EOF - -my $d = $t->testdir(); - -foreach my $name ('localhost') { - system('openssl req -x509 -new ' - . "-config $d/openssl.conf -subj /CN=$name/ " - . "-out $d/$name.crt -keyout $d/$name.key " - . ">>$d/openssl.out 2>&1") == 0 - or die "Can't create certificate for $name: $!\n"; -} - -#suppress otel collector output -open OLDERR, ">&", \*STDERR; -open STDERR, ">>" , $^O eq 'MSWin32' ? 'nul' : '/dev/null'; -$t->run_daemon('../otelcol', '--config', $t->testdir().'/otel-config.yaml'); -open STDERR, ">&", \*OLDERR; -$t->waitforsocket('127.0.0.1:' . port(4317)) or - die 'No otel collector open socket'; - -$t->try_run('no OTel module')->plan(69); - -############################################################################### - -#do requests -my $t_off_resp = http1_get('/trace-off'); - -#batch0 (10 requests) -my $tp_resp = http1_get('/trace-on', trace_headers => 1); -my $t_resp = http1_get('/trace-on', port => 8081, ssl => 1); - -my $t_resp_ignore = http1_get('/context-ignore'); -my $tp_resp_ignore = http1_get('/context-ignore', trace_headers => 1); -my $t_resp_extract = http1_get('/context-extract'); -my $tp_resp_extract = http1_get('/context-extract', trace_headers => 1); -my $t_resp_inject = http1_get('/context-inject'); -my $tp_resp_inject = http1_get('/context-inject', trace_headers => 1); -my $t_resp_propagate = http1_get('/context-propagate'); -my $tp_resp_propagate = http1_get('/context-propagate', trace_headers => 1); - -#batch1 (5 reqeusts) -http1_get('/trace-on') for (1..5); - -#waiting batch1 is sent to collector for 1s -select undef, undef, undef, 1; - -my @batches = split /\n/, $t->read_file('otel.json'); -my $batch_json = JSON::PP::decode_json($batches[0]); -my $spans = $$batch_json{"resourceSpans"}[0]{"scopeSpans"}[0]{"spans"}; - -#validate responses -like($tp_resp, qr/TRACE-ON/, 'http request1 - trace on'); -like($t_resp, qr/TRACE-ON/, 'http request2 - trace on'); -like($t_off_resp, qr/TRACE-OFF/, 'http request - trace off'); - -#validate amount of batches -is(scalar @batches, 2, 'amount of batches - trace on'); - -#validate batch size -is(scalar @{$spans}, 10, 'batch0 size - trace on'); -is(scalar @{${JSON::PP::decode_json($batches[1])}{"resourceSpans"}[0] - {"scopeSpans"}[0]{"spans"}}, 5, 'batch1 size - trace on'); - -#validate general attributes -is(get_attr("service.name", "stringValue", - $$batch_json{resourceSpans}[0]{resource}), - 'test_server', 'service.name - trace on'); -is($$spans[0]{name}, 'default_location', 'span.name - trace on'); - -#validate http metrics -is(get_attr("http.method", "stringValue", $$spans[0]), 'GET', - 'http.method metric - trace on'); -is(get_attr("http.target", "stringValue", $$spans[0]), '/trace-on', - 'http.target metric - trace on'); -is(get_attr("http.route", "stringValue", $$spans[0]), '/trace-on', - 'http.route metric - trace on'); -is(get_attr("http.scheme", "stringValue", $$spans[0]), 'http', - 'http.scheme metric - trace on'); -is(get_attr("http.flavor", "stringValue", $$spans[0]), '1.0', - 'http.flavor metric - trace on'); -is(get_attr("http.user_agent", "stringValue", $$spans[0]), 'nginx-tests', - 'http.user_agent metric - trace on'); -is(get_attr("http.request_content_length", "intValue", $$spans[0]), 0, - 'http.request_content_length metric - trace on'); -is(get_attr("http.response_content_length", "intValue", $$spans[0]), 8, - 'http.response_content_length metric - trace on'); -is(get_attr("http.status_code", "intValue", $$spans[0]), 200, - 'http.status_code metric - trace on'); -is(get_attr("net.host.name", "stringValue", $$spans[0]), 'localhost', - 'net.host.name metric - trace on'); -is(get_attr("net.host.port", "intValue", $$spans[0]), 8080, - 'net.host.port metric - trace on'); -is(get_attr("net.sock.peer.addr", "stringValue", $$spans[0]), '127.0.0.1', - 'net.sock.peer.addr metric - trace on'); -like(get_attr("net.sock.peer.port", "intValue", $$spans[0]), qr/\d+/, - 'net.sock.peer.port metric - trace on'); - -#validate custom http metrics -is(${get_attr("http.request.header.completion", "arrayValue", $$spans[0])} - {values}[0]{stringValue}, 'OK', - 'http.request.header.completion metric - trace on'); -is(${get_attr("http.response.header.content.type", "arrayValue",$$spans[0])} - {values}[0]{stringValue}, 'text/plain', - 'http.response.header.content.type metric - trace on'); -is(get_attr("http.request", "stringValue", $$spans[0]), - 'GET /trace-on HTTP/1.0', 'http.request metric - trace on'); - -#validate https metrics -is(get_attr("http.method", "stringValue", $$spans[1]), 'GET', - 'http.method metric - trace on (https)'); -is(get_attr("http.target", "stringValue", $$spans[1]), '/trace-on', - 'http.target metric - trace on (https)'); -is(get_attr("http.route", "stringValue", $$spans[1]), '/trace-on', - 'http.route metric - trace on (https)'); -is(get_attr("http.scheme", "stringValue", $$spans[1]), 'https', - 'http.scheme metric - trace on (https)'); -is(get_attr("http.flavor", "stringValue", $$spans[1]), '1.0', - 'http.flavor metric - trace on (https)'); -is(get_attr("http.user_agent", "stringValue", $$spans[1]), 'nginx-tests', - 'http.user_agent metric - trace on (https)'); -is(get_attr("http.request_content_length", "intValue", $$spans[1]), 0, - 'http.request_content_length metric - trace on (https)'); -is(get_attr("http.response_content_length", "intValue", $$spans[1]), 8, - 'http.response_content_length metric - trace on (https)'); -is(get_attr("http.status_code", "intValue", $$spans[1]), 200, - 'http.status_code metric - trace on (https)'); -is(get_attr("net.host.name", "stringValue", $$spans[1]), 'localhost', - 'net.host.name metric - trace on (https)'); -is(get_attr("net.host.port", "intValue", $$spans[1]), 8081, - 'net.host.port metric - trace on (https)'); -is(get_attr("net.sock.peer.addr", "stringValue", $$spans[1]), '127.0.0.1', - 'net.sock.peer.addr metric - trace on (https)'); -like(get_attr("net.sock.peer.port", "intValue", $$spans[1]), qr/\d+/, - 'net.sock.peer.port metric - trace on (https)'); - -#extract trace info -is($$spans[0]{parentSpanId}, 'b9c7c989f97918e1', 'traceparent - trace on'); -is($$spans[0]{traceState}, 'congo=ucfJifl5GOE,rojo=00f067aa0ba902b7', - 'tracestate - trace on'); -is($$spans[1]{parentSpanId}, '', 'no traceparent - trace on'); -is($$spans[1]{traceState}, undef, 'no tracestate - trace on'); - -#variables -like($tp_resp, qr/X-Otel-Trace-Id: $$spans[0]{traceId}/, - '$otel_trace_id variable - trace on'); -like($tp_resp, qr/X-Otel-Span-Id: $$spans[0]{spanId}/, - '$otel_span_id variable - trace on'); -like($tp_resp, qr/X-Otel-Parent-Id: $$spans[0]{parentSpanId}/, - '$otel_parent_id variable - trace on'); -like($tp_resp, qr/X-Otel-Parent-Sampled: 1/, - '$otel_parent_sampled variable - trace on'); -like($t_resp, qr/X-Otel-Parent-Sampled: 0/, - '$otel_parent_sampled variable - trace on (no traceparent header)'); - -#trace off -unlike($batches[0].$batches[1], - qr/\Q{"key":"http.target","value":{"stringValue":"\/trace-off"}}\E/, - 'no metrics - trace off'); - -#trace context: ignore -unlike($t_resp_ignore, qr/X-Otel-Traceparent/, - 'no traceparent - trace context ignore (no trace headers)'); -unlike($t_resp_ignore, qr/X-Otel-Tracestate/, - 'no tracestate - trace context ignore (no trace headers)'); - -unlike($tp_resp_ignore, qr/X-Otel-Parent-Id/, - 'no parent span id - trace context ignore (trace headers)'); -like($tp_resp_ignore, - qr/Traceparent: 00-0af7651916cd43dd8448eb211c80319c-b9c7c989f97918e1-01/, - 'traceparent - trace context ignore (trace headers)'); -like($tp_resp_ignore, - qr/Tracestate: congo=ucfJifl5GOE,rojo=00f067aa0ba902b7/, - 'tracestate - trace context ignore (trace headers)'); - -#trace context: extract -unlike($t_resp_extract, qr/X-Otel-Traceparent/, - 'no traceparent - trace context extract (no trace headers)'); -unlike($t_resp_extract, qr/X-Otel-Tracestate/, - 'no tracestate - trace context extract (no trace headers)'); - -like($tp_resp_extract, qr/X-Otel-Parent-Id: b9c7c989f97918e1/, - 'parent span id - trace context extract (trace headers)'); -like($tp_resp_extract, - qr/Traceparent: 00-0af7651916cd43dd8448eb211c80319c-b9c7c989f97918e1-01/, - 'traceparent - trace context extract (trace headers)'); -like($tp_resp_extract, - qr/Tracestate: congo=ucfJifl5GOE,rojo=00f067aa0ba902b7/, - 'tracestate - trace context extract (trace headers)'); - -#trace context: inject -like($t_resp_inject, - qr/X-Otel-Traceparent: 00-$$spans[6]{traceId}-$$spans[6]{spanId}-01/, - 'traceparent - trace context inject (no trace headers)'); -unlike($t_resp_inject, qr/X-Otel-Tracestate/, - 'no tracestate - trace context inject (no trace headers)'); - -unlike($tp_resp_inject, qr/X-Otel-Parent-Id/, - 'no parent span id - trace context inject (trace headers)'); -like($tp_resp_inject, - qr/Traceparent: 00-$$spans[7]{traceId}-$$spans[7]{spanId}-01/, - 'traceparent - trace context inject (trace headers)'); -unlike($tp_resp_inject, qr/Tracestate:/, - 'no tracestate - trace context inject (trace headers)'); - -#trace context: propagate -like($t_resp_propagate, - qr/X-Otel-Traceparent: 00-$$spans[8]{traceId}-$$spans[8]{spanId}-01/, - 'traceparent - trace context propagate (no trace headers)'); -unlike($t_resp_propagate, qr/X-Otel-Tracestate/, - 'no tracestate - trace context propagate (no trace headers)'); - -like($tp_resp_propagate, qr/X-Otel-Parent-Id: b9c7c989f97918e1/, - 'parent id - trace context propagate (trace headers)'); -like($tp_resp_propagate, - qr/Traceparent: 00-0af7651916cd43dd8448eb211c80319c-$$spans[9]{spanId}-01/, - 'traceparent - trace context propagate (trace headers)'); -like($tp_resp_propagate, - qr/Tracestate: congo=ucfJifl5GOE,rojo=00f067aa0ba902b7/, - 'tracestate - trace context propagate (trace headers)'); - -$t->stop(); -my $log = $t->read_file("error.log"); - -unlike($log, qr/OTel\/grpc: Error parsing metadata: error=invalid value/, - 'log: no error parsing metadata'); -unlike($log, qr/OTel export failure: No status received/, - 'log: no export failure'); - -############################################################################### - -sub http1_get { - my ($path, %extra) = @_; - - my $port = $extra{port} || 8080; - - my $r = < '127.0.0.1:' . port($port), - SSL => $extra{ssl}); -} - -sub get_attr { - my($attr, $type, $obj) = @_; - - my ($res) = grep { $$_{"key"} eq $attr } @{$$obj{"attributes"}}; - - return defined $res ? $res->{"value"}{$type} : undef; -} - -############################################################################### diff --git a/tests/requirements.txt b/tests/requirements.txt new file mode 100644 index 0000000..d27c6b1 --- /dev/null +++ b/tests/requirements.txt @@ -0,0 +1,6 @@ +pytest~=8.3 +jinja2~=3.1 +pyopenssl~=24.3 +niquests~=3.11 +grpcio~=1.68 +opentelemetry-proto~=1.28 diff --git a/tests/test_otel.py b/tests/test_otel.py new file mode 100644 index 0000000..5fbac51 --- /dev/null +++ b/tests/test_otel.py @@ -0,0 +1,262 @@ +from collections import namedtuple +import niquests +import pytest +import socket +import time +import urllib3 + + +NGINX_CONFIG = """ +{{ globals }} + +daemon off; + +events { +} + +http { + {{ http_globals }} + + ssl_certificate localhost.crt; + ssl_certificate_key localhost.key; + + otel_exporter { + endpoint 127.0.0.1:14317; + interval {{ interval or "1ms" }}; + batch_size 3; + batch_count 3; + } + + otel_trace on; + otel_service_name test_service; + + server { + listen 127.0.0.1:18443 ssl; + listen 127.0.0.1:18443 quic; + listen 127.0.0.1:18080; + + http2 on; + + server_name localhost; + + location /ok { + return 200 "OK"; + } + + location /err { + return 500 "ERR"; + } + + location /custom { + otel_span_name custom_location; + otel_span_attr http.request.completion + $request_completion; + otel_span_attr http.response.header.content.type + $sent_http_content_type; + otel_span_attr http.request $request; + return 200 "OK"; + } + + location /vars { + otel_trace_context extract; + add_header "X-Otel-Trace-Id" $otel_trace_id; + add_header "X-Otel-Span-Id" $otel_span_id; + add_header "X-Otel-Parent-Id" $otel_parent_id; + add_header "X-Otel-Parent-Sampled" $otel_parent_sampled; + return 204; + } + + location /ignore { + proxy_pass http://127.0.0.1:18080/notrace; + } + + location /extract { + otel_trace_context extract; + proxy_pass http://127.0.0.1:18080/notrace; + } + + location /inject { + otel_trace_context inject; + proxy_pass http://127.0.0.1:18080/notrace; + } + + location /propagate { + otel_trace_context propagate; + proxy_pass http://127.0.0.1:18080/notrace; + } + + location /notrace { + otel_trace off; + add_header "X-Otel-Traceparent" $http_traceparent; + add_header "X-Otel-Tracestate" $http_tracestate; + return 204; + } + } +} + +""" + +TraceContext = namedtuple("TraceContext", ["trace_id", "span_id", "state"]) + +parent_ctx = TraceContext( + trace_id="0af7651916cd43dd8448eb211c80319c", + span_id="b9c7c989f97918e1", + state="congo=ucfJifl5GOE,rojo=00f067aa0ba902b7", +) + + +def trace_headers(ctx): + return ( + { + "Traceparent": f"00-{ctx.trace_id}-{ctx.span_id}-01", + "Tracestate": ctx.state, + } + if ctx + else {"Traceparent": None, "Tracestate": None} + ) + + +def get_attr(span, name): + for value in (a.value for a in span.attributes if a.key == name): + return getattr(value, value.WhichOneof("value")) + + +@pytest.fixture +def client(nginx): + urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + with niquests.Session(multiplexed=True) as s: + yield s + + +def test_http09(trace_service, nginx): + + def get_http09(host, port, path): + with socket.create_connection((host, port)) as sock: + sock.sendall(f"GET {path}\n".encode()) + resp = sock.recv(1024).decode("utf-8") + return resp + + assert get_http09("127.0.0.1", 18080, "/ok") == "OK" + + span = trace_service.get_span() + assert span.name == "/ok" + + +@pytest.mark.parametrize("http_ver", ["1.1", "2.0", "3.0"]) +@pytest.mark.parametrize( + ("path", "status"), + [("/ok", 200), ("/err", 500)], +) +def test_default_attributes(client, trace_service, http_ver, path, status): + scheme, port = ("http", 18080) if http_ver == "1.1" else ("https", 18443) + if http_ver == "3.0": + client.quic_cache_layer.add_domain("127.0.0.1", port) + r = client.get(f"{scheme}://127.0.0.1:{port}{path}", verify=False) + + span = trace_service.get_span() + assert span.name == path + + assert get_attr(span, "http.method") == "GET" + assert get_attr(span, "http.target") == path + assert get_attr(span, "http.route") == path + assert get_attr(span, "http.scheme") == scheme + assert get_attr(span, "http.flavor") == http_ver + assert get_attr(span, "http.user_agent") == ( + f"niquests/{niquests.__version__}" + ) + assert get_attr(span, "http.request_content_length") == 0 + assert get_attr(span, "http.response_content_length") == len(r.text) + assert get_attr(span, "http.status_code") == status + assert get_attr(span, "net.host.name") == "localhost" + assert get_attr(span, "net.host.port") == port + assert get_attr(span, "net.sock.peer.addr") == "127.0.0.1" + assert get_attr(span, "net.sock.peer.port") in range(1024, 65536) + + +def test_custom_attributes(client, trace_service): + assert client.get("http://127.0.0.1:18080/custom").status_code == 200 + + span = trace_service.get_span() + assert span.name == "custom_location" + + assert get_attr(span, "http.request.completion") == "OK" + value = get_attr(span, "http.response.header.content.type") + assert value.values[0].string_value == "text/plain" + assert get_attr(span, "http.request") == "GET /custom HTTP/1.1" + + +def test_trace_off(client, trace_service): + assert client.get("http://127.0.0.1:18080/notrace").status_code == 204 + + time.sleep(0.01) # wait for spans + assert len(trace_service.batches) == 0 + + +@pytest.mark.parametrize("parent", [None, parent_ctx]) +def test_variables(client, trace_service, parent): + r = client.get("http://127.0.0.1:18080/vars", headers=trace_headers(parent)) + + span = trace_service.get_span() + + if parent: + assert span.trace_id.hex() == parent.trace_id + assert span.parent_span_id.hex() == parent.span_id + assert span.trace_state == parent.state + + assert r.headers.get("X-Otel-Trace-Id") == span.trace_id.hex() + assert r.headers.get("X-Otel-Span-Id") == span.span_id.hex() + assert r.headers.get("X-Otel-Parent-Id") or "" == span.parent_span_id.hex() + assert r.headers.get("X-Otel-Parent-Sampled") == ("1" if parent else "0") + + +@pytest.mark.parametrize("parent", [None, parent_ctx]) +@pytest.mark.parametrize( + "path", ["/ignore", "/extract", "/inject", "/propagate"] +) +def test_context(client, trace_service, parent, path): + headers = trace_headers(parent) + + r = client.get(f"http://127.0.0.1:18080{path}", headers=headers) + + span = trace_service.get_span() + + if path in ["/extract", "/propagate"] and parent: + assert span.trace_id.hex() == parent.trace_id + assert span.parent_span_id.hex() == parent.span_id + assert span.trace_state == parent.state + + if path in ["/inject", "/propagate"]: + headers = trace_headers( + TraceContext( + span.trace_id.hex(), + span.span_id.hex(), + span.trace_state or None, + ) + ) + + assert r.headers.get("X-Otel-Traceparent") == headers["Traceparent"] + assert r.headers.get("X-Otel-Tracestate") == headers["Tracestate"] + + +@pytest.mark.parametrize( + "nginx_config", [({"interval": "200ms"})], indirect=True +) +@pytest.mark.parametrize("batch_count", [1, 3]) +def test_batches(client, trace_service, batch_count): + batch_size = 3 + + for _ in range( + batch_count * batch_size + 1 + ): # +1 request to trigger batch sending + assert client.get("http://127.0.0.1:18080/ok").status_code == 200 + + time.sleep(0.01) + + assert len(trace_service.batches) == batch_count + + for batch in trace_service.batches: + assert get_attr(batch[0].resource, "service.name") == "test_service" + assert len(batch[0].scope_spans[0].spans) == batch_size + + time.sleep(0.3) # wait for +1 request to be flushed + trace_service.batches.clear() diff --git a/tests/trace_service.py b/tests/trace_service.py new file mode 100644 index 0000000..9f094f9 --- /dev/null +++ b/tests/trace_service.py @@ -0,0 +1,86 @@ +import concurrent +import grpc +from opentelemetry.proto.collector.trace.v1 import trace_service_pb2 +from opentelemetry.proto.collector.trace.v1 import trace_service_pb2_grpc +import pytest +import subprocess +import time + + +class TraceService(trace_service_pb2_grpc.TraceServiceServicer): + batches = [] + + def Export(self, request, context): + self.batches.append(request.resource_spans) + return trace_service_pb2.ExportTracePartialSuccess() + + def get_span(self): + for _ in range(10): + if len(self.batches): + break + time.sleep(0.001) + + assert len(self.batches) == 1, "No spans received" + span = self.batches[0][0].scope_spans[0].spans.pop() + self.batches.clear() + return span + + +@pytest.fixture(scope="module") +def trace_service(pytestconfig, logger): + server = grpc.server(concurrent.futures.ThreadPoolExecutor()) + trace_service = TraceService() + trace_service_pb2_grpc.add_TraceServiceServicer_to_server( + trace_service, server + ) + listen_addr = f"127.0.0.1:{24317 if pytestconfig.option.otelcol else 14317}" + server.add_insecure_port(listen_addr) + logger.info(f"Starting trace service at {listen_addr}...") + server.start() + yield trace_service + logger.info("Stopping trace service...") + server.stop(grace=None) + + +@pytest.fixture(scope="module") +def otelcol(pytestconfig, testdir, logger, trace_service): + if pytestconfig.option.otelcol is None: + yield + return + + (testdir / "otel-config.yaml").write_text( + """receivers: + otlp: + protocols: + grpc: + endpoint: 127.0.0.1:14317 + +exporters: + otlp: + endpoint: 127.0.0.1:24317 + tls: + insecure: true + +service: + pipelines: + traces: + receivers: [otlp] + exporters: [otlp] + telemetry: + metrics: + # prevent otelcol from opening 8888 port + level: none""" + ) + logger.info("Starting otelcol at 127.0.0.1:14317...") + proc = subprocess.Popen( + [pytestconfig.option.otelcol, "--config", testdir / "otel-config.yaml"] + ) + time.sleep(1) # give some time to get ready + assert proc.poll() is None, "Can't start otelcol" + yield + logger.info("Stopping otelcol...") + proc.terminate() + try: + proc.wait(timeout=5) + except subprocess.TimeoutExpired: + proc.kill() From be30eeffc1761b8ffa9cc5c8b65f6dae6892bac1 Mon Sep 17 00:00:00 2001 From: Eugene Grebenschikov Date: Wed, 18 Dec 2024 16:52:30 -0800 Subject: [PATCH 36/44] Verify HTTP scheme support in endpoint (#60). --- tests/test_otel.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_otel.py b/tests/test_otel.py index 5fbac51..10ce2fe 100644 --- a/tests/test_otel.py +++ b/tests/test_otel.py @@ -21,7 +21,7 @@ http { ssl_certificate_key localhost.key; otel_exporter { - endpoint 127.0.0.1:14317; + endpoint {{ scheme }}127.0.0.1:14317; interval {{ interval or "1ms" }}; batch_size 3; batch_count 3; @@ -239,7 +239,9 @@ def test_context(client, trace_service, parent, path): @pytest.mark.parametrize( - "nginx_config", [({"interval": "200ms"})], indirect=True + "nginx_config", + [({"interval": "200ms", "scheme": "http://"})], + indirect=True, ) @pytest.mark.parametrize("batch_count", [1, 3]) def test_batches(client, trace_service, batch_count): From 1d259542747b41c2fd7f89d6ae17ccfbdfdf6f67 Mon Sep 17 00:00:00 2001 From: Eugene Grebenschikov Date: Wed, 18 Dec 2024 17:23:37 -0800 Subject: [PATCH 37/44] Download the latest otelcol for CI tests. Co-authored-by: Pavel Pautov --- .github/workflows/ubuntu.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 1a8edac..0f328d2 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -36,9 +36,11 @@ jobs: make -j $(nproc) - name: Download otelcol run: | - curl -sLo - https://github.com/\ - open-telemetry/opentelemetry-collector-releases/releases/download/\ - v0.115.1/otelcol_0.115.1_linux_amd64.tar.gz | tar -xzv + LATEST=open-telemetry/opentelemetry-collector-releases/releases/latest + TAG=$(curl -s https://api.github.com/repos/${LATEST} | + jq -r .tag_name) + curl -sLo - https://github.com/${LATEST}/download/\ + otelcol_${TAG:1}_linux_amd64.tar.gz | tar -xzv - name: Install test dependencies run: pip install -r tests/requirements.txt - name: Run tests From c9136f2ec8d76751c35487ba4139e1986074a577 Mon Sep 17 00:00:00 2001 From: Eugene <54681898+jimf5@users.noreply.github.com> Date: Thu, 19 Dec 2024 17:53:38 -0800 Subject: [PATCH 38/44] Verify custom resource attributes support (#32). Co-authored-by: p-pautov <37922380+p-pautov@users.noreply.github.com> --- tests/test_otel.py | 32 +++++++++++++++++++++++++++++--- tests/trace_service.py | 14 +++++++++----- 2 files changed, 38 insertions(+), 8 deletions(-) diff --git a/tests/test_otel.py b/tests/test_otel.py index 10ce2fe..11caa85 100644 --- a/tests/test_otel.py +++ b/tests/test_otel.py @@ -28,7 +28,7 @@ http { } otel_trace on; - otel_service_name test_service; + {{ resource_attrs }} server { listen 127.0.0.1:18443 ssl; @@ -240,7 +240,7 @@ def test_context(client, trace_service, parent, path): @pytest.mark.parametrize( "nginx_config", - [({"interval": "200ms", "scheme": "http://"})], + [{"interval": "200ms", "scheme": "http://"}], indirect=True, ) @pytest.mark.parametrize("batch_count", [1, 3]) @@ -257,8 +257,34 @@ def test_batches(client, trace_service, batch_count): assert len(trace_service.batches) == batch_count for batch in trace_service.batches: - assert get_attr(batch[0].resource, "service.name") == "test_service" + assert ( + get_attr(batch[0].resource, "service.name") + == "unknown_service:nginx" + ) assert len(batch[0].scope_spans[0].spans) == batch_size time.sleep(0.3) # wait for +1 request to be flushed trace_service.batches.clear() + + +@pytest.mark.parametrize( + "nginx_config", + [ + { + "resource_attrs": """ + otel_service_name "test_service"; + otel_resource_attr my.name "my name"; + otel_resource_attr my.service "my service"; + """, + } + ], + indirect=True, +) +def test_custom_resource_attributes(client, trace_service): + assert client.get("http://127.0.0.1:18080/ok").status_code == 200 + + batch = trace_service.get_batch() + + assert get_attr(batch.resource, "service.name") == "test_service" + assert get_attr(batch.resource, "my.name") == "my name" + assert get_attr(batch.resource, "my.service") == "my service" diff --git a/tests/trace_service.py b/tests/trace_service.py index 9f094f9..3b191a1 100644 --- a/tests/trace_service.py +++ b/tests/trace_service.py @@ -14,16 +14,20 @@ class TraceService(trace_service_pb2_grpc.TraceServiceServicer): self.batches.append(request.resource_spans) return trace_service_pb2.ExportTracePartialSuccess() - def get_span(self): + def get_batch(self): for _ in range(10): if len(self.batches): break time.sleep(0.001) + assert len(self.batches) == 1 + assert len(self.batches[0]) == 1 + return self.batches.pop()[0] - assert len(self.batches) == 1, "No spans received" - span = self.batches[0][0].scope_spans[0].spans.pop() - self.batches.clear() - return span + def get_span(self): + batch = self.get_batch() + assert len(batch.scope_spans) == 1 + assert len(batch.scope_spans[0].spans) == 1 + return batch.scope_spans[0].spans.pop() @pytest.fixture(scope="module") From f633a8eef23cdc5f4c4e980605b981cf75595a14 Mon Sep 17 00:00:00 2001 From: Pavel Pautov Date: Thu, 21 Nov 2024 13:57:43 -0800 Subject: [PATCH 39/44] Fail early if "trusted_certificate" is a directory. Previously, the error was caused by enormous std::string allocation. --- src/http_module.cpp | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/http_module.cpp b/src/http_module.cpp index df5702e..bc08b23 100644 --- a/src/http_module.cpp +++ b/src/http_module.cpp @@ -711,7 +711,8 @@ char* addResourceAttr(ngx_conf_t* cf, ngx_command_t* cmd, void* conf) return NGX_CONF_OK; } -char* setTrustedCertificate(ngx_conf_t* cf, ngx_command_t* cmd, void* conf) { +char* setTrustedCertificate(ngx_conf_t* cf, ngx_command_t* cmd, void* conf) +{ auto path = ((ngx_str_t*)cf->args->elts)[1]; auto mcf = getMainConf(cf); @@ -727,11 +728,13 @@ char* setTrustedCertificate(ngx_conf_t* cf, ngx_command_t* cmd, void* conf) { return (char*)NGX_CONF_ERROR; } file.exceptions(std::ios::failbit | std::ios::badbit); - file.seekg(0, std::ios::end); - size_t size = file.tellg(); - mcf->trustedCert.resize(size); + file.peek(); // trigger early error for dirs + + size_t size = file.seekg(0, std::ios::end).tellg(); file.seekg(0); - file.read(&mcf->trustedCert[0], mcf->trustedCert.size()); + + mcf->trustedCert.resize(size); + file.read(&mcf->trustedCert[0], size); } catch (const std::exception& e) { ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "failed to read \"%V\": %s", &path, e.what()); From 88a64bb2c3027be87057efedaa84d0c75d68610e Mon Sep 17 00:00:00 2001 From: Pavel Pautov Date: Thu, 21 Nov 2024 21:46:01 -0800 Subject: [PATCH 40/44] Consolidate transport related parameters into a struct. Also, replace leftover cast with getMainConf(). --- src/batch_exporter.hpp | 4 ++-- src/http_module.cpp | 11 +++++++---- src/trace_service_client.hpp | 16 +++++++++++----- 3 files changed, 20 insertions(+), 11 deletions(-) diff --git a/src/batch_exporter.hpp b/src/batch_exporter.hpp index cb3e075..a2e65b1 100644 --- a/src/batch_exporter.hpp +++ b/src/batch_exporter.hpp @@ -111,10 +111,10 @@ public: int attrSize{0}; }; - BatchExporter(StrView target, bool ssl, const std::string& trustedCert, + BatchExporter(const Target& target, size_t batchSize, size_t batchCount, const std::map& resourceAttrs) : - batchSize(batchSize), client(std::string(target), ssl, trustedCert) + batchSize(batchSize), client(target) { free.reserve(batchCount); while (batchCount-- > 0) { diff --git a/src/http_module.cpp b/src/http_module.cpp index bc08b23..93898b1 100644 --- a/src/http_module.cpp +++ b/src/http_module.cpp @@ -576,10 +576,13 @@ ngx_int_t initWorkerProcess(ngx_cycle_t* cycle) } try { + Target target; + target.endpoint = std::string(toStrView(mcf->endpoint)); + target.ssl = mcf->ssl; + target.trustedCert = mcf->trustedCert; + gExporter.reset(new BatchExporter( - toStrView(mcf->endpoint), - mcf->ssl, - mcf->trustedCert, + target, mcf->batchSize, mcf->batchCount, mcf->resourceAttrs)); @@ -772,7 +775,7 @@ void* createMainConf(ngx_conf_t* cf) char* initMainConf(ngx_conf_t* cf, void* conf) { - auto mcf = (MainConf*)conf; + auto mcf = getMainConf(cf); ngx_conf_init_msec_value(mcf->interval, 5000); ngx_conf_init_size_value(mcf->batchSize, 512); diff --git a/src/trace_service_client.hpp b/src/trace_service_client.hpp index d248f00..485143c 100644 --- a/src/trace_service_client.hpp +++ b/src/trace_service_client.hpp @@ -8,6 +8,12 @@ namespace otel_proto_trace = opentelemetry::proto::collector::trace::v1; +struct Target { + std::string endpoint; + bool ssl; + std::string trustedCert; +}; + class TraceServiceClient { public: typedef otel_proto_trace::ExportTraceServiceRequest Request; @@ -17,18 +23,18 @@ public: typedef std::function ResponseCb; - TraceServiceClient(const std::string& target, bool ssl, - const std::string& trustedCert) + TraceServiceClient(const Target& target) { std::shared_ptr creds; - if (ssl) { + if (target.ssl) { grpc::SslCredentialsOptions options; - options.pem_root_certs = trustedCert; + options.pem_root_certs = target.trustedCert; + creds = grpc::SslCredentials(options); } else { creds = grpc::InsecureChannelCredentials(); } - auto channel = grpc::CreateChannel(target, creds); + auto channel = grpc::CreateChannel(target.endpoint, creds); channel->GetState(true); // trigger 'connecting' state stub = TraceService::NewStub(channel); From a45a594801fbd57657fd821bdd298a39d4575175 Mon Sep 17 00:00:00 2001 From: Pavel Pautov Date: Mon, 2 Dec 2024 21:07:38 -0800 Subject: [PATCH 41/44] Support sending custom headers to export endpoint (fix #62). The headers are configured by "header" directive in "otel_exporter" block, e.g. otel_exporter { endpoint localhost:4317; header X-API-Token "token value"; } --- src/http_module.cpp | 36 +++++++++++++++++++++++++++++++++++- src/trace_service_client.hpp | 23 ++++++++++++++++++++++- 2 files changed, 57 insertions(+), 2 deletions(-) diff --git a/src/http_module.cpp b/src/http_module.cpp index 93898b1..ef77ffb 100644 --- a/src/http_module.cpp +++ b/src/http_module.cpp @@ -30,6 +30,7 @@ struct MainConf : MainConfBase { std::map resourceAttrs; bool ssl; std::string trustedCert; + Target::HeaderVec headers; }; struct SpanAttr { @@ -49,6 +50,7 @@ char* setExporter(ngx_conf_t* cf, ngx_command_t* cmd, void* conf); char* addResourceAttr(ngx_conf_t* cf, ngx_command_t* cmd, void* conf); char* addSpanAttr(ngx_conf_t* cf, ngx_command_t* cmd, void* conf); char* setTrustedCertificate(ngx_conf_t* cf, ngx_command_t* cmd, void* conf); +char* addExporterHeader(ngx_conf_t* cf, ngx_command_t* cmd, void* conf); namespace Propagation { @@ -120,6 +122,10 @@ ngx_command_t gExporterCommands[] = { NGX_CONF_TAKE1, setTrustedCertificate }, + { ngx_string("header"), + NGX_CONF_TAKE2, + addExporterHeader }, + { ngx_string("interval"), NGX_CONF_TAKE1, ngx_conf_set_msec_slot, @@ -580,6 +586,7 @@ ngx_int_t initWorkerProcess(ngx_cycle_t* cycle) target.endpoint = std::string(toStrView(mcf->endpoint)); target.ssl = mcf->ssl; target.trustedCert = mcf->trustedCert; + target.headers = mcf->headers; gExporter.reset(new BatchExporter( target, @@ -651,7 +658,7 @@ char* setExporter(ngx_conf_t* cf, ngx_command_t* cmd, void* conf) continue; } - if (cf->args->nelts != 2) { + if (cf->args->nelts != static_cast(ffs(cmd->type))) { ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "invalid number of arguments in \"%V\" " "directive of \"otel_exporter\"", name); @@ -747,6 +754,33 @@ char* setTrustedCertificate(ngx_conf_t* cf, ngx_command_t* cmd, void* conf) return NGX_CONF_OK; } +char* addExporterHeader(ngx_conf_t* cf, ngx_command_t* cmd, void* conf) +{ + auto args = (ngx_str_t*)cf->args->elts; + + // don't force on users lower case name requirement of gRPC + ngx_strlow(args[1].data, args[1].data, args[1].len); + + try { + // validate header here to avoid runtime assert failure in gRPC + auto name = toStrView(args[1]); + if (!Target::validateHeaderName(name)) { + return (char*)"has invalid header name"; + } + auto value = toStrView(args[2]); + if (!Target::validateHeaderValue(value)) { + return (char*)"has invalid header value"; + } + + getMainConf(cf)->headers.emplace_back(name, value); + } catch (const std::exception& e) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "OTel: %s", e.what()); + return (char*)NGX_CONF_ERROR; + } + + return NGX_CONF_OK; +} + void* createMainConf(ngx_conf_t* cf) { auto cln = ngx_pool_cleanup_add(cf->pool, sizeof(MainConf)); diff --git a/src/trace_service_client.hpp b/src/trace_service_client.hpp index 485143c..4ed92e7 100644 --- a/src/trace_service_client.hpp +++ b/src/trace_service_client.hpp @@ -9,9 +9,24 @@ namespace otel_proto_trace = opentelemetry::proto::collector::trace::v1; struct Target { + typedef std::vector> HeaderVec; + std::string endpoint; bool ssl; std::string trustedCert; + HeaderVec headers; + + static bool validateHeaderName(StrView name) + { + return grpc_header_key_is_legal( + grpc_slice_from_static_buffer(name.data(), name.size())); + } + + static bool validateHeaderValue(StrView value) + { + return grpc_header_nonbin_value_is_legal( + grpc_slice_from_static_buffer(value.data(), value.size())); + } }; class TraceServiceClient { @@ -23,7 +38,7 @@ public: typedef std::function ResponseCb; - TraceServiceClient(const Target& target) + TraceServiceClient(const Target& target) : headers(target.headers) { std::shared_ptr creds; if (target.ssl) { @@ -44,6 +59,10 @@ public: { std::unique_ptr call{new ActiveCall{}}; + for (auto& header : headers) { + call->context.AddMetadata(header.first, header.second); + } + call->request = std::move(req); call->cb = std::move(cb); @@ -113,6 +132,8 @@ private: ResponseCb cb; }; + Target::HeaderVec headers; + std::unique_ptr stub; grpc::CompletionQueue queue; From 9dc4dc2803cc2e7e37f5c73fee7f8d342101aa41 Mon Sep 17 00:00:00 2001 From: Pavel Pautov Date: Thu, 19 Dec 2024 21:28:17 -0800 Subject: [PATCH 42/44] Verify custom exporter headers support (#62). --- tests/test_otel.py | 25 +++++++++++++++++++++++++ tests/trace_service.py | 11 ++++++++--- 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/tests/test_otel.py b/tests/test_otel.py index 11caa85..513c1f1 100644 --- a/tests/test_otel.py +++ b/tests/test_otel.py @@ -25,6 +25,8 @@ http { interval {{ interval or "1ms" }}; batch_size 3; batch_count 3; + + {{ exporter_opts }} } otel_trace on; @@ -288,3 +290,26 @@ def test_custom_resource_attributes(client, trace_service): assert get_attr(batch.resource, "service.name") == "test_service" assert get_attr(batch.resource, "my.name") == "my name" assert get_attr(batch.resource, "my.service") == "my service" + + +@pytest.mark.parametrize( + "nginx_config", + [ + { + "exporter_opts": """ + header X-API-TOKEN api.value; + header Authorization "Basic value"; + """, + } + ], + indirect=True, +) +@pytest.mark.parametrize("trace_service", ["skip_otelcol"], indirect=True) +def test_exporter_headers(client, trace_service): + assert client.get("http://127.0.0.1:18080/ok").status_code == 200 + + assert trace_service.get_span().name == "/ok" + + headers = dict(trace_service.last_metadata) + assert headers["x-api-token"] == "api.value" + assert headers["authorization"] == "Basic value" diff --git a/tests/trace_service.py b/tests/trace_service.py index 3b191a1..f47b104 100644 --- a/tests/trace_service.py +++ b/tests/trace_service.py @@ -12,6 +12,7 @@ class TraceService(trace_service_pb2_grpc.TraceServiceServicer): def Export(self, request, context): self.batches.append(request.resource_spans) + self.last_metadata = context.invocation_metadata() return trace_service_pb2.ExportTracePartialSuccess() def get_batch(self): @@ -31,13 +32,17 @@ class TraceService(trace_service_pb2_grpc.TraceServiceServicer): @pytest.fixture(scope="module") -def trace_service(pytestconfig, logger): +def trace_service(request, pytestconfig, logger): server = grpc.server(concurrent.futures.ThreadPoolExecutor()) trace_service = TraceService() trace_service_pb2_grpc.add_TraceServiceServicer_to_server( trace_service, server ) - listen_addr = f"127.0.0.1:{24317 if pytestconfig.option.otelcol else 14317}" + trace_service.use_otelcol = ( + pytestconfig.option.otelcol + and getattr(request, "param", "") != "skip_otelcol" + ) + listen_addr = f"127.0.0.1:{24317 if trace_service.use_otelcol else 14317}" server.add_insecure_port(listen_addr) logger.info(f"Starting trace service at {listen_addr}...") server.start() @@ -48,7 +53,7 @@ def trace_service(pytestconfig, logger): @pytest.fixture(scope="module") def otelcol(pytestconfig, testdir, logger, trace_service): - if pytestconfig.option.otelcol is None: + if not trace_service.use_otelcol: yield return From f578402f196499edd2a65c31020570fc37e1bdbf Mon Sep 17 00:00:00 2001 From: Eugene Grebenschikov Date: Fri, 20 Dec 2024 17:24:27 -0800 Subject: [PATCH 43/44] Verify export via TLS (#12). Co-authored-by: Pavel Pautov --- tests/conftest.py | 19 ++++++++++--------- tests/test_otel.py | 20 ++++++++++++++++++-- tests/trace_service.py | 20 ++++++++++++++++---- 3 files changed, 44 insertions(+), 15 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 7978759..25933e4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -19,7 +19,7 @@ def pytest_addoption(parser): parser.addoption("--globals", default="") -def self_signed_cert(test_dir, name): +def self_signed_cert(name): k = crypto.PKey() k.generate_key(crypto.TYPE_RSA, 2048) cert = crypto.X509() @@ -29,11 +29,9 @@ def self_signed_cert(test_dir, name): cert.gmtime_adj_notAfter(365 * 86400) # 365 days cert.set_pubkey(k) cert.sign(k, "sha512") - (test_dir / f"{name}.key").write_text( - crypto.dump_privatekey(crypto.FILETYPE_PEM, k).decode("utf-8") - ) - (test_dir / f"{name}.crt").write_text( - crypto.dump_certificate(crypto.FILETYPE_PEM, cert).decode("utf-8") + return ( + crypto.dump_privatekey(crypto.FILETYPE_PEM, k), + crypto.dump_certificate(crypto.FILETYPE_PEM, cert), ) @@ -66,7 +64,7 @@ def nginx_config(request, pytestconfig, testdir, logger): @pytest.fixture(scope="module") -def nginx(testdir, pytestconfig, nginx_config, certs, logger, otelcol): +def nginx(testdir, pytestconfig, nginx_config, cert, logger, otelcol): (testdir / "nginx.conf").write_text(nginx_config) logger.info("Starting nginx...") proc = subprocess.Popen( @@ -96,5 +94,8 @@ def nginx(testdir, pytestconfig, nginx_config, certs, logger, otelcol): @pytest.fixture(scope="module") -def certs(testdir): - self_signed_cert(testdir, "localhost") +def cert(testdir): + key, cert = self_signed_cert("localhost") + (testdir / "localhost.key").write_text(key.decode("utf-8")) + (testdir / "localhost.crt").write_text(cert.decode("utf-8")) + yield (key, cert) diff --git a/tests/test_otel.py b/tests/test_otel.py index 513c1f1..fef771a 100644 --- a/tests/test_otel.py +++ b/tests/test_otel.py @@ -21,7 +21,7 @@ http { ssl_certificate_key localhost.key; otel_exporter { - endpoint {{ scheme }}127.0.0.1:14317; + endpoint {{ endpoint or "127.0.0.1:14317" }}; interval {{ interval or "1ms" }}; batch_size 3; batch_count 3; @@ -242,7 +242,7 @@ def test_context(client, trace_service, parent, path): @pytest.mark.parametrize( "nginx_config", - [{"interval": "200ms", "scheme": "http://"}], + [{"interval": "200ms", "endpoint": "http://127.0.0.1:14317"}], indirect=True, ) @pytest.mark.parametrize("batch_count", [1, 3]) @@ -313,3 +313,19 @@ def test_exporter_headers(client, trace_service): headers = dict(trace_service.last_metadata) assert headers["x-api-token"] == "api.value" assert headers["authorization"] == "Basic value" + + +@pytest.mark.parametrize( + "nginx_config", + [ + { + "endpoint": "https://localhost:14318", + "exporter_opts": "trusted_certificate localhost.crt;", + } + ], + indirect=True, +) +def test_tls_export(client, trace_service): + assert client.get("http://127.0.0.1:18080/ok").status_code == 200 + + assert trace_service.get_span().name == "/ok" diff --git a/tests/trace_service.py b/tests/trace_service.py index f47b104..5ef2bc6 100644 --- a/tests/trace_service.py +++ b/tests/trace_service.py @@ -32,7 +32,7 @@ class TraceService(trace_service_pb2_grpc.TraceServiceServicer): @pytest.fixture(scope="module") -def trace_service(request, pytestconfig, logger): +def trace_service(request, pytestconfig, logger, cert): server = grpc.server(concurrent.futures.ThreadPoolExecutor()) trace_service = TraceService() trace_service_pb2_grpc.add_TraceServiceServicer_to_server( @@ -44,6 +44,10 @@ def trace_service(request, pytestconfig, logger): ) listen_addr = f"127.0.0.1:{24317 if trace_service.use_otelcol else 14317}" server.add_insecure_port(listen_addr) + if not trace_service.use_otelcol: + creds = grpc.ssl_server_credentials([cert]) + server.add_secure_port("127.0.0.1:14318", creds) + listen_addr += " and 127.0.0.1:14318" logger.info(f"Starting trace service at {listen_addr}...") server.start() yield trace_service @@ -52,18 +56,26 @@ def trace_service(request, pytestconfig, logger): @pytest.fixture(scope="module") -def otelcol(pytestconfig, testdir, logger, trace_service): +def otelcol(pytestconfig, testdir, logger, trace_service, cert): if not trace_service.use_otelcol: yield return (testdir / "otel-config.yaml").write_text( - """receivers: + f"""receivers: otlp: protocols: grpc: endpoint: 127.0.0.1:14317 + otlp/tls: + protocols: + grpc: + endpoint: 127.0.0.1:14318 + tls: + cert_file: {testdir}/localhost.crt + key_file: {testdir}/localhost.key + exporters: otlp: endpoint: 127.0.0.1:24317 @@ -73,7 +85,7 @@ exporters: service: pipelines: traces: - receivers: [otlp] + receivers: [otlp, otlp/tls] exporters: [otlp] telemetry: metrics: From 72d8eed53af4c2cd6f3e30a2efe0e38d66f5e176 Mon Sep 17 00:00:00 2001 From: Pavel Pautov Date: Tue, 21 Jan 2025 23:02:39 -0800 Subject: [PATCH 44/44] Fix build against Nginx 1.22 (fix #85). --- src/http_module.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/http_module.cpp b/src/http_module.cpp index ef77ffb..78a5e89 100644 --- a/src/http_module.cpp +++ b/src/http_module.cpp @@ -296,10 +296,10 @@ ngx_int_t setHeader(ngx_http_request_t* r, StrView name, StrView value) return NGX_ERROR; } + *header = {}; header->hash = hash; header->key = toNgxStr(name); header->lowcase_key = header->key.data; - header->next = NULL; } header->value = toNgxStr(value);