diff --git a/CMakeLists.txt b/CMakeLists.txt deleted file mode 100644 index 7a74f0a..0000000 --- a/CMakeLists.txt +++ /dev/null @@ -1,159 +0,0 @@ -cmake_minimum_required(VERSION 3.16.3) -project(nginx-otel) - -set(NGX_OTEL_NGINX_BUILD_DIR "" - CACHE PATH "Nginx build (objs) dir") -set(NGX_OTEL_NGINX_DIR "${NGX_OTEL_NGINX_BUILD_DIR}/.." - CACHE PATH "Nginx source dir") - -set(NGX_OTEL_GRPC e241f37befe7ba4688effd84bfbf99b0f681a2f7 # v1.49.4 - CACHE STRING "gRPC tag to download or 'package' to use preinstalled") -set(NGX_OTEL_SDK 11d5d9e0d8fd8ba876c8994714cc2647479b6574 # v1.11.0 - CACHE STRING "OTel SDK tag to download or 'package' to use preinstalled") -set(NGX_OTEL_PROTO_DIR "" CACHE PATH "OTel proto files root") -set(NGX_OTEL_DEV OFF CACHE BOOL "Enforce compiler warnings") - -if(NOT CMAKE_BUILD_TYPE) - set(CMAKE_BUILD_TYPE RelWithDebInfo) -endif() - -set(CMAKE_CXX_VISIBILITY_PRESET hidden) - -if(NGX_OTEL_GRPC STREQUAL "package") - find_package(protobuf REQUIRED) - find_package(gRPC REQUIRED) -else() - include(FetchContent) - - FetchContent_Declare( - grpc - GIT_REPOSITORY https://github.com/grpc/grpc - GIT_TAG ${NGX_OTEL_GRPC} - GIT_SUBMODULES third_party/protobuf third_party/abseil-cpp third_party/re2 - GIT_SHALLOW ON) - - set(gRPC_USE_PROTO_LITE ON CACHE INTERNAL "") - set(gRPC_INSTALL OFF CACHE INTERNAL "") - set(gRPC_USE_SYSTEMD OFF CACHE INTERNAL "") - set(gRPC_DOWNLOAD_ARCHIVES OFF CACHE INTERNAL "") - set(gRPC_CARES_PROVIDER package CACHE INTERNAL "") - set(gRPC_SSL_PROVIDER package CACHE INTERNAL "") - set(gRPC_ZLIB_PROVIDER package CACHE INTERNAL "") - - set(protobuf_INSTALL OFF CACHE INTERNAL "") - - set(CMAKE_POSITION_INDEPENDENT_CODE ON) - - FetchContent_MakeAvailable(grpc) - - # reconsider once https://github.com/grpc/grpc/issues/36023 is done - target_compile_definitions(grpc PRIVATE GRPC_NO_XDS GRPC_NO_RLS) - - set_property(DIRECTORY ${grpc_SOURCE_DIR} - PROPERTY EXCLUDE_FROM_ALL YES) - - add_library(gRPC::grpc++ ALIAS grpc++) - add_executable(gRPC::grpc_cpp_plugin ALIAS grpc_cpp_plugin) -endif() - -if(NGX_OTEL_SDK STREQUAL "package") - find_package(opentelemetry-cpp REQUIRED) -else() - include(FetchContent) - - FetchContent_Declare( - otelcpp - GIT_REPOSITORY https://github.com/open-telemetry/opentelemetry-cpp - GIT_TAG ${NGX_OTEL_SDK} - GIT_SUBMODULES third_party/opentelemetry-proto - GIT_SHALLOW ON) - - set(BUILD_TESTING OFF CACHE INTERNAL "") - set(WITH_EXAMPLES OFF CACHE INTERNAL "") - - set(CMAKE_POSITION_INDEPENDENT_CODE ON) - set(CMAKE_POLICY_DEFAULT_CMP0063 NEW) - - FetchContent_MakeAvailable(otelcpp) - - set_property(DIRECTORY ${otelcpp_SOURCE_DIR} - PROPERTY EXCLUDE_FROM_ALL YES) - - if(NOT NGX_OTEL_PROTO_DIR) - set(NGX_OTEL_PROTO_DIR - "${otelcpp_SOURCE_DIR}/third_party/opentelemetry-proto") - endif() - - add_library(opentelemetry-cpp::trace ALIAS opentelemetry_trace) -endif() - -set(PROTO_DIR ${NGX_OTEL_PROTO_DIR}) -set(PROTOS - "${PROTO_DIR}/opentelemetry/proto/common/v1/common.proto" - "${PROTO_DIR}/opentelemetry/proto/resource/v1/resource.proto" - "${PROTO_DIR}/opentelemetry/proto/trace/v1/trace.proto" - "${PROTO_DIR}/opentelemetry/proto/collector/trace/v1/trace_service.proto") - -set(PROTO_OUT_DIR "${CMAKE_CURRENT_BINARY_DIR}") -set(PROTO_SOURCES - "${PROTO_OUT_DIR}/opentelemetry/proto/common/v1/common.pb.cc" - "${PROTO_OUT_DIR}/opentelemetry/proto/resource/v1/resource.pb.cc" - "${PROTO_OUT_DIR}/opentelemetry/proto/trace/v1/trace.pb.cc" - "${PROTO_OUT_DIR}/opentelemetry/proto/collector/trace/v1/trace_service.pb.cc" - "${PROTO_OUT_DIR}/opentelemetry/proto/collector/trace/v1/trace_service.grpc.pb.cc") - -# generate protobuf code for lite runtime -add_custom_command( - OUTPUT ${PROTO_SOURCES} - COMMAND protobuf::protoc - --proto_path ${PROTO_DIR} - --cpp_out lite:${PROTO_OUT_DIR} - --grpc_out ${PROTO_OUT_DIR} - --plugin protoc-gen-grpc=$ - ${PROTOS} - # remove inconsequential UTF8 check during serialization to aid performance - COMMAND sed -i.bak -E - -e [[/ ::(PROTOBUF_NAMESPACE_ID|google::protobuf)::internal::WireFormatLite::VerifyUtf8String\(/,/\);/d]] - ${PROTO_SOURCES} - DEPENDS ${PROTOS} protobuf::protoc gRPC::grpc_cpp_plugin - VERBATIM) - -if (NGX_OTEL_DEV) - set(CMAKE_CXX_STANDARD 11) - set(CMAKE_CXX_EXTENSIONS OFF) - - add_compile_options(-Wall -Wtype-limits -Werror) -endif() - -add_library(ngx_otel_module MODULE - src/http_module.cpp - src/grpc_log.cpp - src/modules.c - ${PROTO_SOURCES}) - -# avoid 'lib' prefix in binary name -set_target_properties(ngx_otel_module PROPERTIES PREFIX "") - -# can't use OTel's WITH_ABSEIL until cmake 3.24, as it triggers find_package() -target_compile_definitions(ngx_otel_module PRIVATE HAVE_ABSEIL) - -if (APPLE) - target_link_options(ngx_otel_module PRIVATE -undefined dynamic_lookup) -endif() - -target_include_directories(ngx_otel_module PRIVATE - ${NGX_OTEL_NGINX_BUILD_DIR} - ${NGX_OTEL_NGINX_DIR}/src/core - ${NGX_OTEL_NGINX_DIR}/src/event - ${NGX_OTEL_NGINX_DIR}/src/event/modules - ${NGX_OTEL_NGINX_DIR}/src/event/quic - ${NGX_OTEL_NGINX_DIR}/src/os/unix - ${NGX_OTEL_NGINX_DIR}/src/http - ${NGX_OTEL_NGINX_DIR}/src/http/modules - ${NGX_OTEL_NGINX_DIR}/src/http/v2 - ${NGX_OTEL_NGINX_DIR}/src/http/v3 - ${PROTO_OUT_DIR}) - -target_link_libraries(ngx_otel_module - opentelemetry-cpp::trace - gRPC::grpc++) diff --git a/README.md b/README.md index 1f84fe0..9edb977 100644 --- a/README.md +++ b/README.md @@ -122,7 +122,7 @@ Follow these steps to build the `ngx_otel_module` dynamic module on Ubuntu or De Install build tools and dependencies. ```bash -sudo apt install cmake build-essential libssl-dev zlib1g-dev libpcre3-dev +sudo apt install build-essential libssl-dev zlib1g-dev libpcre3-dev sudo apt install pkg-config libc-ares-dev libre2-dev # for gRPC ``` @@ -134,13 +134,6 @@ For the next step, you will need the `configure` script that is packaged with th git clone https://github.com/nginx/nginx.git ``` -Configure NGINX to generate files necessary for dynamic module compilation. These files will be placed into the `nginx/objs` directory. - -**Important:** If you did not obtain NGINX source code via the clone method in the previous step, you will need to adjust paths in the following commands to conform to your specific directory structure. -```bash -cd nginx -auto/configure --with-compat -``` Exit the NGINX directory and clone the `ngx_otel_module` repository. ```bash @@ -148,14 +141,24 @@ cd .. git clone https://github.com/nginxinc/nginx-otel.git ``` -Configure and build the NGINX OTel module. - -**Important**: replace the path in the `cmake` command with the path to the `nginx/objs` directory from above. +Set up `NGX_OTEL_PROTO_DIR` variable to point it into root of `opentelemetry-proto`: ```bash -cd nginx-otel -mkdir build -cd build -cmake -DNGX_OTEL_NGINX_BUILD_DIR=/path/to/configured/nginx/objs .. +export NGX_OTEL_PROTO_DIR=/opt/local/include +``` + +Configure and build the NGINX OTel module as usual: + +- for a built-in module: +```bash +cd nginx +./configure --with-http_ssl_module --add-module=../nginx-otel +make +``` + +- for a dynamic module: +```bash +cd nginx +./configure --with-http_ssl_module --add-dynamic-module=../nginx-otel make ``` diff --git a/config b/config index 75378e2..2f1da1e 100644 --- a/config +++ b/config @@ -1,9 +1,256 @@ ngx_addon_name=ngx_otel_module +ngx_module_type=HTTP +ngx_module_name=$ngx_addon_name +ngx_module_incs= +ngx_module_deps=" \ + $ngx_addon_dir/src/batch_exporter.hpp \ + $ngx_addon_dir/src/grpc_log.hpp \ + $ngx_addon_dir/src/ngx.hpp \ + $ngx_addon_dir/src/str_view.hpp \ + $ngx_addon_dir/src/trace_context.hpp \ + $ngx_addon_dir/src/trace_service_client.hpp \ +" +ngx_module_srcs=" \ + $ngx_addon_dir/src/grpc_log.cpp \ + $ngx_addon_dir/src/ngx_otel_module.cpp \ + objs/opentelemetry/proto/common/v1/common.pb.cc \ + objs/opentelemetry/proto/resource/v1/resource.pb.cc \ + objs/opentelemetry/proto/trace/v1/trace.pb.cc \ + objs/opentelemetry/proto/collector/trace/v1/trace_service.pb.cc \ + objs/opentelemetry/proto/collector/trace/v1/trace_service.grpc.pb.cc \ +" -cmake -D NGX_OTEL_NGINX_BUILD_DIR=$NGX_OBJS \ - -D CMAKE_LIBRARY_OUTPUT_DIRECTORY=$PWD/$NGX_OBJS \ - -D "CMAKE_C_FLAGS=$NGX_CC_OPT" \ - -D "CMAKE_CXX_FLAGS=$NGX_CC_OPT" \ - -D "CMAKE_MODULE_LINKER_FLAGS=$NGX_LD_OPT" \ - $NGX_OTEL_CMAKE_OPTS \ - -S $ngx_addon_dir -B $NGX_OBJS/otel || exit 1 +ngx_feature="c-ares" +ngx_feature_name="" +ngx_feature_run=no +ngx_feature_incs="#include " +ngx_feature_path="/usr/include" +ngx_feature_libs="-lcares" +ngx_feature_test="ares_version(NULL);" +. auto/feature + +if [ $ngx_found = no ]; then + + # FreeBSD port + + ngx_feature="libcares in /usr/local/" + ngx_feature_path="/usr/local/include" + + if [ $NGX_RPATH = YES ]; then + ngx_feature_libs="-R/usr/local/lib -L/usr/local/lib -lcares" + else + ngx_feature_libs="-L/usr/local/lib -lcares" + fi + + . auto/feature +fi + +if [ $ngx_found = yes ]; then + ngx_module_libs="$ngx_module_libs -lcares" +fi + +unset ngx_found + +### C++ feature test is unavailable in nginx + +CXX=${CXX:-c++} +CXXFLAGS="$CXXFLAGS --std=c++17" +CXX_TEST_FLAGS=${CXX_TEST_FLAGS:---std=c++17} + +autocppfeature() +{ + echo $ngx_n "checking for $ngx_feature ...$ngx_c" + +cat << END >> $NGX_AUTOCONF_ERR +---------------------------------------- +checking for $ngx_feature + +END + + ngx_found=no + + if test -n "$ngx_feature_name"; then + ngx_have_feature=`echo $ngx_feature_name \ + | tr abcdefghijklmnopqrstuvwxyz ABCDEFGHIJKLMNOPQRSTUVWXYZ` + fi + + if test -n "$ngx_feature_path"; then + for ngx_temp in $ngx_feature_path; do + ngx_feature_inc_path="$ngx_feature_inc_path -I $ngx_temp" + done + fi + +cat << END > $NGX_AUTOTEST.cpp + +$ngx_feature_incs + +int main(void) { + $ngx_feature_test; + return 0; +} + +END + + ngx_test="$CXX $CXX_TEST_FLAGS $CXX_AUX_FLAGS $ngx_feature_inc_path \ + -o $NGX_AUTOTEST $NGX_AUTOTEST.cpp $NGX_TEST_LD_OPT $ngx_feature_libs" + + ngx_feature_inc_path= + + eval "/bin/sh -c \"$ngx_test\" >> $NGX_AUTOCONF_ERR 2>&1" + + if [ -x $NGX_AUTOTEST ]; then + + echo " found" + ngx_found=yes + + if test -n "$ngx_feature_name"; then + have=$ngx_have_feature . auto/have + fi + + else + echo " not found" + echo "----------" >> $NGX_AUTOCONF_ERR + cat $NGX_AUTOTEST.cpp >> $NGX_AUTOCONF_ERR + echo "----------" >> $NGX_AUTOCONF_ERR + echo $ngx_test >> $NGX_AUTOCONF_ERR + echo "----------" >> $NGX_AUTOCONF_ERR + fi + + rm -rf $NGX_AUTOTEST* +} + +ngx_feature="re2" +ngx_feature_name="" +ngx_feature_run=no +ngx_feature_incs="#include " +ngx_feature_path="/usr/include" +ngx_feature_libs="-lre2" +ngx_feature_test="RE2::FullMatch(\"hello\", \"h.*o\");" + +autocppfeature + +if [ $ngx_found = no ]; then + + # FreeBSD port + + ngx_feature="re2 in /usr/local/" + ngx_feature_path="/usr/local/include" + + if [ $NGX_RPATH = YES ]; then + ngx_feature_libs="-R/usr/local/lib -L/usr/local/lib -lre2" + else + ngx_feature_libs="-L/usr/local/lib -lre2" + fi + + autocppfeature +fi + +if [ $ngx_found = yes ]; then + ngx_module_libs="$ngx_module_libs -lre2" +fi + +ngx_feature="opentelemetry-cpp" +ngx_feature_name="" +ngx_feature_run=no +ngx_feature_incs="#include + typedef opentelemetry::nostd::string_view StrView;" +ngx_feature_path="/usr/include" +ngx_feature_libs="-lopentelemetry_common" +ngx_feature_test="using namespace std; + StrView str, prefix; + str.substr(0, prefix.size());" + +autocppfeature + +if [ $ngx_found = no ]; then + + # FreeBSD port + + ngx_feature="opentelemetry in /usr/local/" + ngx_feature_path="/usr/local/include" + + if [ $NGX_RPATH = YES ]; then + ngx_feature_libs="-R/usr/local/lib -L/usr/local/lib -lopentelemetry_common" + else + ngx_feature_libs="-L/usr/local/lib -lopentelemetry_common -lopentelemetry_resources -lopentelemetry_trace" + fi + + autocppfeature +fi + +if [ $ngx_found = yes ]; then + ngx_module_libs="$ngx_module_libs -lopentelemetry_common -lopentelemetry_resources -lopentelemetry_trace" +fi + +#if [ ! -d $prefix/include/opentelemetry/proto ]; then +# echo "Need to install opentelemetry-proto." +# exit 2 +#fi + +ngx_feature="protobuf" +ngx_feature_name="" +ngx_feature_run=no +ngx_feature_incs="#include " +ngx_feature_path="/usr/include" +ngx_feature_libs="-lprotobuf" +ngx_feature_test="using namespace google::protobuf; + google::protobuf::internal::VersionString(0);" + +autocppfeature + +if [ $ngx_found = no ]; then + + # FreeBSD port + + ngx_feature="protobuf in /usr/local/" + ngx_feature_path="/usr/local/include" + + if [ $NGX_RPATH = YES ]; then + ngx_feature_libs="-R/usr/local/lib -L/usr/local/lib -lprotobuf" + else + ngx_feature_libs="-L/usr/local/lib -lprotobuf" + fi + + autocppfeature +fi + +if [ $ngx_found = yes ]; then + ngx_module_libs="$ngx_module_libs -lprotobuf" +fi + +ngx_feature="grpc" +ngx_feature_name="" +ngx_feature_run=no +ngx_feature_incs="#include " +ngx_feature_path="/usr/include" +ngx_feature_libs="-lgrpc -lgpr -lgrpc++" +ngx_feature_test="gpr_log_verbosity_init();" + +autocppfeature + +if [ $ngx_found = no ]; then + + # FreeBSD port + + ngx_feature="grpc in /usr/local/" + ngx_feature_path="/usr/local/include" + + if [ $NGX_RPATH = YES ]; then + ngx_feature_libs="-R/usr/local/lib -L/usr/local/lib -lgrpc -lgpr -lgrpc++" + else + ngx_feature_libs="-L/usr/local/lib -lgrpc -lgpr -lgrpc++" + fi + + autocppfeature +fi + +if [ $ngx_found = yes ]; then + ngx_module_libs="$ngx_module_libs -lgrpc -lgpr -lgrpc++" +fi + +ngx_module_libs="$ngx_module_libs -lz -lm -lrt -lssl -lcrypto" + +. auto/module + +CORE_INCS="$CORE_INCS $ngx_feature_path" +OTEL_NGX_SRCS="$ngx_module_srcs" diff --git a/config.make b/config.make index a7cce9f..a066be3 100644 --- a/config.make +++ b/config.make @@ -1,10 +1,45 @@ -cat << END >> $NGX_MAKEFILE +if [ ! "`which protoc 2>/dev/null`" ]; then + echo "Need to install protoc." + exit 2 +else + PROTOC=`which protoc` +fi -modules: ngx_otel_module +if [ ! "`which grpc_cpp_plugin 2>/dev/null`" ]; then + echo "Need to install grpc tools." + exit 2 +else + GRPC_CPP=`which grpc_cpp_plugin` +fi -ngx_otel_module: - \$(MAKE) -C $NGX_OBJS/otel +mkdir -p objs -.PHONY: ngx_otel_module +if [ -z $NGX_OTEL_PROTO_DIR ]; then + echo "Need to set \$NGX_OTEL_PROTO_DIR variable." + exit 2 +fi -END +if [ ! -d $NGX_OTEL_PROTO_DIR ]; then + echo "\$NGX_OTEL_PROTO_DIR is set to unavailable directory." + exit 2 +fi + +find $NGX_OTEL_PROTO_DIR/opentelemetry/proto -type f -name '*.proto' | \ + xargs $PROTOC \ + --proto_path $NGX_OTEL_PROTO_DIR \ + --cpp_out=objs \ + --grpc_out=objs \ + --plugin=protoc-gen-grpc=$GRPC_CPP + +find objs -name '*.pb.cc' | \ + xargs sed -i.bak -e "/ ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String(/,/);/d" + +for src_file in $OTEL_NGX_SRCS; do + obj_file="$NGX_OBJS/addon/src/`basename $src_file .cpp`.o" + echo "$obj_file : CFLAGS += $CXXFLAGS -Wno-missing-field-initializers -Wno-conditional-uninitialized -fPIC -fvisibility=hidden -DHAVE_ABSEIL -Dngx_otel_module_EXPORTS" >> $NGX_MAKEFILE +done + +for src_file in $OTEL_NGX_SRCS; do + obj_file="$NGX_OBJS/addon/v1/`basename $src_file .cc`.o" + echo "$obj_file : CFLAGS += $CXXFLAGS -Wno-missing-field-initializers -Wno-conditional-uninitialized -fPIC -fvisibility=hidden -DHAVE_ABSEIL -Dngx_otel_module_EXPORTS" >> $NGX_MAKEFILE +done diff --git a/src/modules.c b/src/modules.c deleted file mode 100644 index 419d594..0000000 --- a/src/modules.c +++ /dev/null @@ -1,14 +0,0 @@ -#include -#include - -extern ngx_module_t gHttpModule; - -ngx_module_t* ngx_modules[] = { - &gHttpModule, - NULL -}; - -char* ngx_module_names[] = { - "ngx_http_otel_module", - NULL -}; diff --git a/src/http_module.cpp b/src/ngx_otel_module.cpp similarity index 97% rename from src/http_module.cpp rename to src/ngx_otel_module.cpp index df5702e..566376a 100644 --- a/src/http_module.cpp +++ b/src/ngx_otel_module.cpp @@ -8,7 +8,7 @@ #include -extern ngx_module_t gHttpModule; +extern ngx_module_t ngx_otel_module; namespace { @@ -168,18 +168,18 @@ bool iremovePrefix(ngx_str_t* str, StrView p) MainConf* getMainConf(ngx_conf_t* cf) { return static_cast( - (MainConfBase*)ngx_http_conf_get_module_main_conf(cf, gHttpModule)); + (MainConfBase*)ngx_http_conf_get_module_main_conf(cf, ngx_otel_module)); } MainConf* getMainConf(ngx_cycle_t* cycle) { return static_cast( - (MainConfBase*)ngx_http_cycle_get_module_main_conf(cycle, gHttpModule)); + (MainConfBase*)ngx_http_cycle_get_module_main_conf(cycle, ngx_otel_module)); } LocationConf* getLocationConf(ngx_http_request_t* r) { - return (LocationConf*)ngx_http_get_module_loc_conf(r, gHttpModule); + return (LocationConf*)ngx_http_get_module_loc_conf(r, ngx_otel_module); } void cleanupOtelCtx(void* data) @@ -188,7 +188,7 @@ void cleanupOtelCtx(void* data) OtelCtx* getOtelCtx(ngx_http_request_t* r) { - auto ctx = (OtelCtx*)ngx_http_get_module_ctx(r, gHttpModule); + auto ctx = (OtelCtx*)ngx_http_get_module_ctx(r, ngx_otel_module); // restore module context if it was reset by e.g. internal redirect if (ctx == NULL && (r->internal || r->filter_finalize)) { @@ -196,7 +196,7 @@ OtelCtx* getOtelCtx(ngx_http_request_t* r) for (auto cln = r->pool->cleanup; cln; cln = cln->next) { if (cln->handler == cleanupOtelCtx) { ctx = (OtelCtx*)cln->data; - ngx_http_set_ctx(r, ctx, gHttpModule); + ngx_http_set_ctx(r, ctx, ngx_otel_module); break; } } @@ -217,7 +217,7 @@ OtelCtx* createOtelCtx(ngx_http_request_t* r) storage->handler = cleanupOtelCtx; auto ctx = new (storage->data) OtelCtx{}; - ngx_http_set_ctx(r, ctx, gHttpModule); + ngx_http_set_ctx(r, ctx, ngx_otel_module); return ctx; } @@ -932,7 +932,7 @@ char* mergeLocationConf(ngx_conf_t* cf, void* parent, void* child) return NGX_CONF_OK; } -ngx_http_module_t gHttpModuleCtx = { +ngx_http_module_t ngx_otel_moduleCtx = { addVariables, /* preconfiguration */ initModule, /* postconfiguration */ @@ -948,9 +948,9 @@ ngx_http_module_t gHttpModuleCtx = { } -ngx_module_t gHttpModule = { +ngx_module_t ngx_otel_module = { NGX_MODULE_V1, - &gHttpModuleCtx, /* module context */ + &ngx_otel_moduleCtx, /* module context */ gCommands, /* module directives */ NGX_HTTP_MODULE, /* module type */ NULL, /* init master */