diff --git a/docs/AdiakIntegration.rst b/docs/AdiakIntegration.rst new file mode 100644 index 00000000..02216f03 --- /dev/null +++ b/docs/AdiakIntegration.rst @@ -0,0 +1,49 @@ +.. + # Copyright 2021 Lawrence Livermore National Security, LLC and other + # PerfFlowAspect Project Developers. See the top-level LICENSE file for + # details. + # + # SPDX-License-Identifier: LGPL-3.0 + +##################### + Adiak Integration +##################### + +PerfFlowAspect can be built with Adiak, a tool that collects metadata on HPC runs. Learn more about Adiak `here `_. + +PerfFlowAspect integration with Adiak is currently only supported in C/C++. When integrated, the generated PerfFlowAspect `.pfw` trace file contains Adiak metadata. `Note: the trace file must be generated in object format.` + +Adiak must be installed prior to PerfFlowAspect integration. + +************* + C/C++ Build +************* +Adiak is enabled by specifying ``PERFFLOWASPECT_WITH_MPI=On`` along with the path to Adiak's package configuration file, i.e. ``/lib/cmake/adiak>``. + +.. code:: bash + + cmake -DCMAKE_C_COMPILER= \ + -DCMAKE_C_COMPILER= \ + -DPERFFLOWASPECT_WITH_ADIAK=On \ + -Dadiak_DIR=/lib/cmake/adiak> ../ + +Adiak can gather additional metadata related to MPI. The flag ``PERFFLOWASPECT_WITH_MPI=On`` must be specified. + +.. code:: bash + + cmake -DCMAKE_C_COMPILER= \ + -DCMAKE_C_COMPILER= \ + -DPERFFLOWASPECT_WITH_ADIAK=On \ + -Dadiak_DIR=/lib/cmake/adiak> \ + -DPERFFLOWASPECT_WITH_MPI=On ../ + + +*************** + C/C++ Example +*************** +Adiak's metadata can only be displayed in the object format of a PerfFlowAspect `.pfw` trace. Thus, ``PERFFLOW_OPTIONS="log-format=Object"`` must be specified + +.. code:: bash + + PERFFLOW_OPTIONS="log-format=Object" ./smoketest + diff --git a/docs/index.rst b/docs/index.rst index a8de2d9b..debb6311 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -85,6 +85,7 @@ uniformity as to how performance is measured and controlled. BuildingPerfFlowAspect Annotations + AdiakIntegration UpcomingFeatures .. toctree:: diff --git a/src/c/CMakeLists.txt b/src/c/CMakeLists.txt index 8d12a5a8..c7886aaf 100644 --- a/src/c/CMakeLists.txt +++ b/src/c/CMakeLists.txt @@ -11,6 +11,8 @@ else() message(FATAL_ERROR "Unsupported CXX compiler: please use Clang == 18.0") endif() +option(PERFFLOWASPECT_WITH_ADIAK "Build with Adiak" OFF) + include(cmake/CMakeBasics.cmake) include(cmake/Setup3rdParty.cmake) diff --git a/src/c/cmake/Setup3rdParty.cmake b/src/c/cmake/Setup3rdParty.cmake index d7f203a6..b42e5aa8 100644 --- a/src/c/cmake/Setup3rdParty.cmake +++ b/src/c/cmake/Setup3rdParty.cmake @@ -19,3 +19,18 @@ include(cmake/thirdparty/FindOpenSSL.cmake) if(PERFFLOWASPECT_WITH_MULTITHREADS) include(cmake/thirdparty/FindThreads.cmake) endif() + +if(PERFFLOWASPECT_WITH_ADIAK) + if(NOT adiak_DIR) + message(FATAL_ERROR "PFA + Adiak needs explicit adiak_DIR") + endif() + + if (adiak_DIR) + message(STATUS "${adiak_DIR}") + include(cmake/thirdparty/FindAdiak.cmake) + endif() + + if (ADIAK_FOUND) + add_definitions(-DPERFFLOWASPECT_WITH_ADIAK) + endif() +endif() \ No newline at end of file diff --git a/src/c/cmake/thirdparty/FindAdiak.cmake b/src/c/cmake/thirdparty/FindAdiak.cmake new file mode 100644 index 00000000..85e56c5f --- /dev/null +++ b/src/c/cmake/thirdparty/FindAdiak.cmake @@ -0,0 +1,12 @@ +message(STATUS "Looking for Adiak in ${adiak_DIR}") +find_package(adiak REQUIRED + PATHS ${adiak_DIR}/lib/cmake/adiak + NO_DEFAULT_PATH + NO_CMAKE_ENVIRONMENT_PATH + NO_CMAKE_PATH + NO_SYSTEM_ENVIRONMENT_PATH + NO_CMAKE_SYSTEM_PATH) + +message(STATUS "FOUND Adiak: ${adiak_DIR}") +set(ADIAK_FOUND TRUE) +set(PERFFLOWASPECT_ADIAK_ENABLED TRUE) diff --git a/src/c/runtime/CMakeLists.txt b/src/c/runtime/CMakeLists.txt index 12e4fba5..03baac25 100644 --- a/src/c/runtime/CMakeLists.txt +++ b/src/c/runtime/CMakeLists.txt @@ -14,12 +14,22 @@ set(perfflow_runtime_sources include_directories(${JANSSON_INCLUDE_DIRS}) +if(PERFFLOWASPECT_WITH_ADIAK) + include_directories(${adiak_INCLUDE_DIR}) +endif() + add_library(perfflow_runtime SHARED ${perfflow_runtime_sources} ${perfflow_runtime_headers} ) -target_link_libraries(perfflow_runtime ${JANSSON_LIBRARY} OpenSSL::SSL OpenSSL::Crypto) +if (PERFFLOWASPECT_WITH_ADIAK) + target_link_libraries(perfflow_runtime ${JANSSON_LIBRARY} + OpenSSL::SSL OpenSSL::Crypto adiak::adiak) +else() + target_link_libraries(perfflow_runtime ${JANSSON_LIBRARY} + OpenSSL::SSL OpenSSL::Crypto ) +endif() install(TARGETS perfflow_runtime EXPORT perfflow_export diff --git a/src/c/runtime/advice_chrome_tracing.cpp b/src/c/runtime/advice_chrome_tracing.cpp index 4f3ddb0e..4475f1ac 100644 --- a/src/c/runtime/advice_chrome_tracing.cpp +++ b/src/c/runtime/advice_chrome_tracing.cpp @@ -228,6 +228,27 @@ int advice_chrome_tracing_t::flush_if(size_t size) m_ofs << "{" << std::endl; m_ofs << " \"displayTimeUnit\": \"us\"," << std::endl; m_ofs << " \"otherData\": {" << std::endl; +#ifdef PERFFLOWASPECT_WITH_ADIAK + { + const char *key; + json_t *val; + json_t *metadata = get_adiak_statistics(); + size_t total = json_object_size(metadata); + size_t idx = 0; + json_object_foreach(metadata, key, val) + { + char *v = json_dumps(val, JSON_ENCODE_ANY); + m_ofs << " \"" << key << "\": " << v; + free(v); + if (++idx < total) + { + m_ofs << ","; + } + m_ofs << "\n"; + } + json_decref(metadata); + } +#endif m_ofs << "" << std::endl; m_ofs << " }," << std::endl; m_ofs << " \"traceEvents\": [" << std::endl; @@ -663,6 +684,60 @@ long advice_chrome_tracing_t::get_memory_usage() return max_ram_usage; } +#ifdef PERFFLOWASPECT_WITH_ADIAK +void advice_chrome_tracing_t::adiak_cb(const char *name, int cat, + const char *subcat, adiak_value_t *val, adiak_datatype_t *t, void *opaque) +{ + json_t *obj = static_cast(opaque); + json_t *metadata = nullptr; + + switch (t->dtype) + { + case adiak_version: + case adiak_string: + case adiak_catstring: + case adiak_path: + metadata = json_string(reinterpret_cast(val->v_ptr)); + break; + case adiak_long: + case adiak_date: + metadata = json_integer(val->v_long); + break; + case adiak_ulong: + metadata = json_integer(unsigned(val->v_long)); + break; + case adiak_double: + metadata = json_real(val->v_double); + break; + case adiak_int: + metadata = json_integer(val->v_int); + break; + case adiak_uint: + metadata = json_integer(unsigned(val->v_int)); + break; + case adiak_timeval: + { + struct timeval *tv = (struct timeval *)(val->v_ptr); + json_t *tv_obj = json_object(); + json_object_set_new(tv_obj, "tv_sec", json_integer(tv->tv_sec)); + json_object_set_new(tv_obj, "tv_usec", json_integer(tv->tv_usec)); + metadata = tv_obj; + break; + } + default: + return; + } + json_object_set_new(obj, name, metadata); +} + +json_t *advice_chrome_tracing_t::get_adiak_statistics() +{ + json_t *adiak_metadata = json_object(); + adiak_list_namevals(1, adiak_category_all, adiak_cb, adiak_metadata); + return adiak_metadata; +} +#endif + int advice_chrome_tracing_t::with_flow(const char *module, const char *function, const char *flow, @@ -747,6 +822,7 @@ int advice_chrome_tracing_t::before(const char *module, std::string fname; long mem_start; + if (m_enable_logging) { if ((rc = create_event(&event, module, function, my_ts)) < 0) @@ -1022,6 +1098,7 @@ int advice_chrome_tracing_t::after(const char *module, free(json_str); json_decref(event); + return flush_if(FLUSH_SIZE); } else diff --git a/src/c/runtime/advice_chrome_tracing.hpp b/src/c/runtime/advice_chrome_tracing.hpp index 04c13a60..0c6d84d4 100644 --- a/src/c/runtime/advice_chrome_tracing.hpp +++ b/src/c/runtime/advice_chrome_tracing.hpp @@ -16,6 +16,10 @@ #include #include "advice_base.hpp" +#ifdef PERFFLOWASPECT_WITH_ADIAK +#include +#endif + class advice_chrome_tracing_t : public advice_base_t { public: @@ -43,6 +47,11 @@ class advice_chrome_tracing_t : public advice_base_t double get_wall_time(); double get_cpu_time(); long get_memory_usage(); +#ifdef PERFFLOWASPECT_WITH_ADIAK + json_t *get_adiak_statistics(); + static void adiak_cb(const char *name, int cat, const char *subcat, adiak_value_t *val, adiak_datatype_t *t, void *opaque); + +#endif int cannonicalize_perfflow_options (); int parse_perfflow_options (); diff --git a/src/c/test/CMakeLists.txt b/src/c/test/CMakeLists.txt index cb1fa0ee..85cbc83a 100644 --- a/src/c/test/CMakeLists.txt +++ b/src/c/test/CMakeLists.txt @@ -8,11 +8,18 @@ set(SMOKETESTS set(perfflow_deps "-L../runtime -lperfflow_runtime" OpenSSL::Crypto) message(STATUS "Adding CXX unit tests") + +if(PERFFLOWASPECT_WITH_ADIAK) + message(STATUS "Including Adiak in test dependencies") + set(perfflow_deps "${perfflow_deps}" adiak::adiak) +endif() + foreach(TEST ${SMOKETESTS}) message(STATUS " [*] Adding test: ${TEST}") add_executable(${TEST} ${TEST}.cpp) set_source_files_properties(${TEST}.cpp COMPILE_FLAGS "-Xclang -load -Xclang ../weaver/weave/libWeavePass.so -fpass-plugin=../weaver/weave/libWeavePass.so -fPIC") target_link_libraries(${TEST} ${perfflow_deps}) + message(STATUS "${perfflow_deps}") endforeach() # Build Options @@ -30,6 +37,8 @@ else() option(PERFFLOWASPECT_WITH_MPI "Build MPI smoketest" OFF) endif() + + if(PERFFLOWASPECT_WITH_MULTITHREADS) message(STATUS " [*] Adding test: smoketest_MT") add_executable(smoketest_MT smoketest_MT.cpp) diff --git a/src/c/weaver/weave/CMakeLists.txt b/src/c/weaver/weave/CMakeLists.txt index 7945e63f..96ddb95c 100644 --- a/src/c/weaver/weave/CMakeLists.txt +++ b/src/c/weaver/weave/CMakeLists.txt @@ -16,6 +16,13 @@ set_target_properties(WeavePass PROPERTIES target_link_libraries(WeavePass perfflow_parser ${JANSSON_LIB}) +if(PERFFLOWASPECT_WITH_MPI) + include_directories(${MPI_C_INCLUDE_PATH}) + target_link_libraries(WeavePass ${MPI_C_LIBRARIES}) + target_compile_definitions(WeavePass PRIVATE PERFFLOWASPECT_WITH_MPI) +endif() + + add_library(WeavePassPlugin INTERFACE) target_compile_options(WeavePassPlugin INTERFACE "SHELL:$>$>" diff --git a/src/c/weaver/weave/perfflow_weave_common.cpp b/src/c/weaver/weave/perfflow_weave_common.cpp index 8470388f..758f58a7 100644 --- a/src/c/weaver/weave/perfflow_weave_common.cpp +++ b/src/c/weaver/weave/perfflow_weave_common.cpp @@ -26,6 +26,20 @@ bool weave_ns::WeaveCommon::modifyAnnotatedFunctions(Module &m) return false; } +#ifdef PERFFLOWASPECT_WITH_ADIAK + Function *main = m.getFunction("main"); + + if (main != NULL) + { + outs() << "Inserting Adiak?\n"; + insertAdiak(m, *main); + } + else + { + outs() << "No main"; + } +#endif + bool changed = false; if (annotations->getNumOperands() <= 0) @@ -197,6 +211,139 @@ bool weave_ns::WeaveCommon::insertBefore(Module &m, Function &f, StringRef &a, return true; } +#ifdef PERFFLOWASPECT_WITH_ADIAK +bool weave_ns::WeaveCommon::insertAdiak(Module &m, Function &f) +{ + LLVMContext &context = m.getContext(); + + // the adiak_init() call + // return and argument types + Type *voidTy = Type::getVoidTy(context); + Type *int8Ty = Type::getInt8Ty(context); + Type *int32Ty = Type::getInt32Ty(context); + PointerType *voidPtrTy = PointerType::getUnqual(int8Ty); + + // adiak_init function signature + std::vector initArgs = { voidPtrTy }; + FunctionType *adiakInitType = FunctionType::get(voidTy, initArgs, false); + FunctionCallee adiakInit = m.getOrInsertFunction("adiak_init", adiakInitType); + + IRBuilder<> builder(context); + Value *arg; + BasicBlock &entry = f.getEntryBlock(); + builder.SetInsertPoint(&entry, entry.begin()); + +#ifdef PERFFLOWASPECT_WITH_MPI + // find the MPI communicator, MPI_COMM_WORLD + // OpenMPI exposes this as ompi_comm_world (untested) + // MPICH exposes this as a constant 0x44000000 + GlobalVariable *gv = m.getGlobalVariable("ompi_mpi_comm_world"); + if (!gv) + { + gv = m.getGlobalVariable("MPI_COMM_WORLD"); + } + if (gv) + { + // ompi_mpi_comm_world is a pointer to mpi_predefined_communicator_t struct + // the first value holds the actual communicator + StructType *ompCommStruct = cast(gv->getValueType()); + Value *ompCommStructPtr = builder.CreateBitCast(gv, + PointerType::getUnqual(ompCommStruct)); + Value *commPtr = builder.CreateStructGEP(ompCommStruct, ompCommStructPtr, 0); + arg = builder.CreateBitCast(commPtr, voidPtrTy, "mpi_comm_world_void"); + } + else + { + uint64_t mpiValue = 0x44000000; + Value *commVal = ConstantInt::get(int32Ty, mpiValue); + AllocaInst *alloc = builder.CreateAlloca(int32Ty, nullptr, "weave_mpi_comm"); + builder.CreateStore(commVal, alloc); + arg = builder.CreateBitCast( + alloc, + voidPtrTy, + "mpi_comm_world_void" + ); + } + + CallInst *mpi = nullptr; + // find each instruction to see if there is an MPI_Init call instruction + for (BasicBlock &bb : f) + { + for (Instruction &i : bb) + { + if (auto *call = dyn_cast(&i)) + { + if (Function *callee = call->getCalledFunction()) + { + if (callee->getName() == "MPI_Init") + { + mpi = call; + } + } + } + } + if (mpi) { break; } + } + if (mpi) + { + BasicBlock *mpiBlock = mpi->getParent(); + auto insertPos = BasicBlock::iterator(mpi); + ++insertPos; + builder.SetInsertPoint(mpiBlock, insertPos); + } + else + { + arg = Constant::getNullValue(voidPtrTy); + } + builder.CreateCall(adiakInit, {arg}); + +#else + arg = Constant::getNullValue(voidPtrTy); + builder.CreateCall(adiakInit, {arg}); +#endif + + // call adiak_collect_all() + FunctionType *collectType = FunctionType::get(int32Ty, {}, false); + FunctionCallee collectAll = m.getOrInsertFunction("adiak_collect_all", + collectType); + builder.CreateCall(collectAll, {}); + + // call adiak_walltime() + FunctionCallee walltime = m.getOrInsertFunction("adiak_walltime", collectType); + builder.CreateCall(walltime, {}); + + FunctionCallee systime = m.getOrInsertFunction("adiak_systime", collectType); + builder.CreateCall(systime, {}); + + FunctionCallee cputime = m.getOrInsertFunction("adiak_cputime", collectType); + builder.CreateCall(cputime, {}); + + // adiak_fini signature + FunctionType *adiakFinishType = FunctionType::get(voidTy, {}, false); + FunctionCallee adiakFinish = m.getOrInsertFunction("adiak_fini", + adiakFinishType); + + // find all places where the function terminates from a ReturnInst + std::vector returns; + for (BasicBlock &bb : f) + { + if (auto *ret = dyn_cast(bb.getTerminator())) + { + returns.push_back(ret); + } + } + + // insert adiak_fini at those return instructions + for (ReturnInst *ret : returns) + { + builder.SetInsertPoint(ret); + builder.CreateCall(adiakFinish, {}); + } + + return true; +} +#endif + /* * vi:tabstop=4 shiftwidth=4 expandtab */ diff --git a/src/c/weaver/weave/perfflow_weave_common.hpp b/src/c/weaver/weave/perfflow_weave_common.hpp index 672766cf..0e425da3 100644 --- a/src/c/weaver/weave/perfflow_weave_common.hpp +++ b/src/c/weaver/weave/perfflow_weave_common.hpp @@ -22,6 +22,10 @@ #include "llvm/Support/raw_ostream.h" #include "../../parser/perfflow_parser.hpp" +#ifdef PERFFLOWASPECT_WITH_MPI +#include "mpi.h" +#endif + using namespace llvm; namespace weave_ns { @@ -35,7 +39,9 @@ class WeaveCommon bool insertBefore(Module &m, Function &f, StringRef &a, int async, std::string &scope, std::string &flow, std::string pcut); - + #ifdef PERFFLOWASPECT_WITH_ADIAK + bool insertAdiak(Module &m, Function &f); + #endif public: bool modifyAnnotatedFunctions(Module &m);