From f3b404648a5fccf2b368e69a01f81c48a65161cc Mon Sep 17 00:00:00 2001 From: Spencer Scott Greene Date: Fri, 11 Apr 2025 04:02:59 -0700 Subject: [PATCH 01/15] Insert adiak_init() into main functions Here we insert adiak_init every time the module pass encounters a main function. The CMakeLists should be cleaned up to properly integrate Adiak. Need to add in the adiak callback and validate this actually works. --- src/c/cmake/Setup3rdParty.cmake | 4 ++++ src/c/cmake/thirdparty/FindAdiak.cmake | 17 +++++++++++++++ src/c/test/CMakeLists.txt | 9 +++++++- src/c/weaver/weave/perfflow_weave_common.cpp | 23 ++++++++++++++++++++ src/c/weaver/weave/perfflow_weave_common.hpp | 3 ++- 5 files changed, 54 insertions(+), 2 deletions(-) create mode 100644 src/c/cmake/thirdparty/FindAdiak.cmake diff --git a/src/c/cmake/Setup3rdParty.cmake b/src/c/cmake/Setup3rdParty.cmake index d7f203a6..5de28cf3 100644 --- a/src/c/cmake/Setup3rdParty.cmake +++ b/src/c/cmake/Setup3rdParty.cmake @@ -19,3 +19,7 @@ include(cmake/thirdparty/FindOpenSSL.cmake) if(PERFFLOWASPECT_WITH_MULTITHREADS) include(cmake/thirdparty/FindThreads.cmake) endif() + +if(PERFFLOWASPECT_WITH_ADIAK) + include(cmake/thirdparty/FindAdiak.cmake) +endif() \ No newline at end of file diff --git a/src/c/cmake/thirdparty/FindAdiak.cmake b/src/c/cmake/thirdparty/FindAdiak.cmake new file mode 100644 index 00000000..e4ab5645 --- /dev/null +++ b/src/c/cmake/thirdparty/FindAdiak.cmake @@ -0,0 +1,17 @@ +set(adiak_DIR "/g/g14/greene36/Adiak/install/lib/cmake/adiak") + +find_package(adiak REQUIRED) + +message(STATUS "Building Adiak smoketest (PERFFLOWASPECT_WITH_ADIAK == ON)") + +if(adiak_FOUND) + message(STATUS "Adiak found: ${adiak_FOUND}") + message(STATUS "Adiak include directories: ${adiak_INCLUDE_DIRS}") + message(STATUS "Adiak library directories: ${adiak_LIBRARY_DIRS}") + message(STATUS "Adiak libraries: ${adiak_LIBRARIES}") +else() + message(FATAL_ERROR "Adiak not found.") +endif() + +include_directories(${adiak_INCLUDE_DIRS}) +link_directories(${adiak_LIBRARY_DIRS}) \ No newline at end of file diff --git a/src/c/test/CMakeLists.txt b/src/c/test/CMakeLists.txt index cb1fa0ee..4cf6f13a 100644 --- a/src/c/test/CMakeLists.txt +++ b/src/c/test/CMakeLists.txt @@ -5,7 +5,7 @@ set(SMOKETESTS smoketest_class ) -set(perfflow_deps "-L../runtime -lperfflow_runtime" OpenSSL::Crypto) +set(perfflow_deps "-L../runtime -lperfflow_runtime" OpenSSL::Crypto adiak::adiak) message(STATUS "Adding CXX unit tests") foreach(TEST ${SMOKETESTS}) @@ -30,6 +30,13 @@ else() option(PERFFLOWASPECT_WITH_MPI "Build MPI smoketest" OFF) endif() +if(PERFFLOWASPECT_WITH_ADIAK) + message(STATUS " [*] Adding test: smoketest_adiak") + link_directories(${adiak_LIBRARY_DIRS}) +endif() + + + if(PERFFLOWASPECT_WITH_MULTITHREADS) message(STATUS " [*] Adding test: smoketest_MT") add_executable(smoketest_MT smoketest_MT.cpp) diff --git a/src/c/weaver/weave/perfflow_weave_common.cpp b/src/c/weaver/weave/perfflow_weave_common.cpp index 8470388f..acee8d2b 100644 --- a/src/c/weaver/weave/perfflow_weave_common.cpp +++ b/src/c/weaver/weave/perfflow_weave_common.cpp @@ -26,6 +26,16 @@ bool weave_ns::WeaveCommon::modifyAnnotatedFunctions(Module &m) return false; } + Function *main = m.getFunction("main"); + + if (main != NULL) { + outs() << "Inserting Adiak?\n"; + insertAdiak(m, *main); + } + else { + outs () << "No main"; + } + bool changed = false; if (annotations->getNumOperands() <= 0) @@ -197,6 +207,19 @@ bool weave_ns::WeaveCommon::insertBefore(Module &m, Function &f, StringRef &a, return true; } +bool weave_ns::WeaveCommon::insertAdiak(Module &m, Function &f) { + LLVMContext &context = m.getContext(); + + + FunctionCallee adiak_init = m.getOrInsertFunction("adiak_init", FunctionType::get(Type::getVoidTy(context), false)); + BasicBlock &entry = f.getEntryBlock(); + IRBuilder<> builder(&entry, entry.begin()); + + builder.CreateCall(adiak_init); + + return true; +} + /* * vi:tabstop=4 shiftwidth=4 expandtab */ diff --git a/src/c/weaver/weave/perfflow_weave_common.hpp b/src/c/weaver/weave/perfflow_weave_common.hpp index 672766cf..5fac80a0 100644 --- a/src/c/weaver/weave/perfflow_weave_common.hpp +++ b/src/c/weaver/weave/perfflow_weave_common.hpp @@ -35,7 +35,8 @@ class WeaveCommon bool insertBefore(Module &m, Function &f, StringRef &a, int async, std::string &scope, std::string &flow, std::string pcut); - + + bool insertAdiak(Module &m, Function &f); public: bool modifyAnnotatedFunctions(Module &m); From 87e575a5b273663308e9094daf08adfb5e87e819 Mon Sep 17 00:00:00 2001 From: Spencer Scott Greene Date: Wed, 16 Apr 2025 01:12:03 -0700 Subject: [PATCH 02/15] relocate adiak option --- src/c/CMakeLists.txt | 3 +++ src/c/cmake/thirdparty/FindAdiak.cmake | 25 ++++++++++---------- src/c/test/CMakeLists.txt | 5 ++-- src/c/weaver/weave/perfflow_weave_common.cpp | 15 ++++++++---- 4 files changed, 29 insertions(+), 19 deletions(-) diff --git a/src/c/CMakeLists.txt b/src/c/CMakeLists.txt index 8d12a5a8..6fd8b879 100644 --- a/src/c/CMakeLists.txt +++ b/src/c/CMakeLists.txt @@ -15,6 +15,9 @@ include(cmake/CMakeBasics.cmake) include(cmake/Setup3rdParty.cmake) +option(PERFFLOWASPECT_WITH_ADIAK "Build with Adiak" ON) + + include(GNUInstallDirs) set(CMAKE_SKIP_BUILD_RPATH FALSE) set(CMAKE_BUILD_WITH_INSTALL_RPATH FALSE) diff --git a/src/c/cmake/thirdparty/FindAdiak.cmake b/src/c/cmake/thirdparty/FindAdiak.cmake index e4ab5645..cd273423 100644 --- a/src/c/cmake/thirdparty/FindAdiak.cmake +++ b/src/c/cmake/thirdparty/FindAdiak.cmake @@ -1,17 +1,16 @@ set(adiak_DIR "/g/g14/greene36/Adiak/install/lib/cmake/adiak") - find_package(adiak REQUIRED) - +set(perfflow_deps adiak::adiak) message(STATUS "Building Adiak smoketest (PERFFLOWASPECT_WITH_ADIAK == ON)") -if(adiak_FOUND) - message(STATUS "Adiak found: ${adiak_FOUND}") - message(STATUS "Adiak include directories: ${adiak_INCLUDE_DIRS}") - message(STATUS "Adiak library directories: ${adiak_LIBRARY_DIRS}") - message(STATUS "Adiak libraries: ${adiak_LIBRARIES}") -else() - message(FATAL_ERROR "Adiak not found.") -endif() - -include_directories(${adiak_INCLUDE_DIRS}) -link_directories(${adiak_LIBRARY_DIRS}) \ No newline at end of file +#if(adiak_FOUND) +# message(STATUS "Adiak found: ${adiak_FOUND}") +# message(STATUS "Adiak include directories: ${adiak_INCLUDE_DIRS}") +# message(STATUS "Adiak library directories: ${adiak_LIBRARY_DIRS}") +# message(STATUS "Adiak libraries: ${adiak_LIBRARIES}") +#else() +# message(FATAL_ERROR "Adiak not found.") +#endif() +# +#include_directories(${adiak_INCLUDE_DIRS}) +#link_directories(${adiak_LIBRARY_DIRS}) \ No newline at end of file diff --git a/src/c/test/CMakeLists.txt b/src/c/test/CMakeLists.txt index 4cf6f13a..fa3b2c72 100644 --- a/src/c/test/CMakeLists.txt +++ b/src/c/test/CMakeLists.txt @@ -5,7 +5,7 @@ set(SMOKETESTS smoketest_class ) -set(perfflow_deps "-L../runtime -lperfflow_runtime" OpenSSL::Crypto adiak::adiak) +set(perfflow_deps "-L../runtime -lperfflow_runtime" OpenSSL::Crypto) message(STATUS "Adding CXX unit tests") foreach(TEST ${SMOKETESTS}) @@ -31,8 +31,9 @@ else() endif() if(PERFFLOWASPECT_WITH_ADIAK) + find_package(adiak QUIET) message(STATUS " [*] Adding test: smoketest_adiak") - link_directories(${adiak_LIBRARY_DIRS}) + endif() diff --git a/src/c/weaver/weave/perfflow_weave_common.cpp b/src/c/weaver/weave/perfflow_weave_common.cpp index acee8d2b..1f3b27d8 100644 --- a/src/c/weaver/weave/perfflow_weave_common.cpp +++ b/src/c/weaver/weave/perfflow_weave_common.cpp @@ -210,13 +210,20 @@ bool weave_ns::WeaveCommon::insertBefore(Module &m, Function &f, StringRef &a, bool weave_ns::WeaveCommon::insertAdiak(Module &m, Function &f) { LLVMContext &context = m.getContext(); - - FunctionCallee adiak_init = m.getOrInsertFunction("adiak_init", FunctionType::get(Type::getVoidTy(context), false)); + llvm::Type *voidTy = llvm::Type::getVoidTy(context); + llvm::Type *int8Ty = llvm::Type::getInt8Ty(context); + llvm::PointerType *voidPtrTy = llvm::PointerType::getUnqual(int8Ty); + + // function signature + std::vector argTypes = { voidPtrTy }; + llvm::FunctionType *adiakInitType = llvm::FunctionType::get(voidTy, argTypes, false); + + FunctionCallee adiak_init = m.getOrInsertFunction("adiak_init", adiakInitType); + BasicBlock &entry = f.getEntryBlock(); IRBuilder<> builder(&entry, entry.begin()); - + builder.CreateCall(adiak_init); - return true; } From 913ff57f1f67b10fa687886a1e78325aa395d9da Mon Sep 17 00:00:00 2001 From: Spencer Scott Greene Date: Tue, 22 Apr 2025 12:58:06 -0700 Subject: [PATCH 03/15] modified cmake so pfa compiles with adiak --- src/c/CMakeLists.txt | 5 ++--- src/c/cmake/thirdparty/FindAdiak.cmake | 7 ++++--- src/c/test/CMakeLists.txt | 13 +++++++------ 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/src/c/CMakeLists.txt b/src/c/CMakeLists.txt index 6fd8b879..59a9b8a7 100644 --- a/src/c/CMakeLists.txt +++ b/src/c/CMakeLists.txt @@ -11,13 +11,12 @@ else() message(FATAL_ERROR "Unsupported CXX compiler: please use Clang == 18.0") endif() +option(PERFFLOWASPECT_WITH_ADIAK "Build with Adiak" ON) + include(cmake/CMakeBasics.cmake) include(cmake/Setup3rdParty.cmake) -option(PERFFLOWASPECT_WITH_ADIAK "Build with Adiak" ON) - - include(GNUInstallDirs) set(CMAKE_SKIP_BUILD_RPATH FALSE) set(CMAKE_BUILD_WITH_INSTALL_RPATH FALSE) diff --git a/src/c/cmake/thirdparty/FindAdiak.cmake b/src/c/cmake/thirdparty/FindAdiak.cmake index cd273423..e873cb24 100644 --- a/src/c/cmake/thirdparty/FindAdiak.cmake +++ b/src/c/cmake/thirdparty/FindAdiak.cmake @@ -1,8 +1,9 @@ set(adiak_DIR "/g/g14/greene36/Adiak/install/lib/cmake/adiak") find_package(adiak REQUIRED) -set(perfflow_deps adiak::adiak) -message(STATUS "Building Adiak smoketest (PERFFLOWASPECT_WITH_ADIAK == ON)") - +if(adiak_FOUND) + message(STATUS "Building with Adiak") + set(perfflow_deps "${perfflow_deps}" adiak::adiak) +endif() #if(adiak_FOUND) # message(STATUS "Adiak found: ${adiak_FOUND}") # message(STATUS "Adiak include directories: ${adiak_INCLUDE_DIRS}") diff --git a/src/c/test/CMakeLists.txt b/src/c/test/CMakeLists.txt index fa3b2c72..85cbc83a 100644 --- a/src/c/test/CMakeLists.txt +++ b/src/c/test/CMakeLists.txt @@ -8,11 +8,18 @@ set(SMOKETESTS set(perfflow_deps "-L../runtime -lperfflow_runtime" OpenSSL::Crypto) message(STATUS "Adding CXX unit tests") + +if(PERFFLOWASPECT_WITH_ADIAK) + message(STATUS "Including Adiak in test dependencies") + set(perfflow_deps "${perfflow_deps}" adiak::adiak) +endif() + foreach(TEST ${SMOKETESTS}) message(STATUS " [*] Adding test: ${TEST}") add_executable(${TEST} ${TEST}.cpp) set_source_files_properties(${TEST}.cpp COMPILE_FLAGS "-Xclang -load -Xclang ../weaver/weave/libWeavePass.so -fpass-plugin=../weaver/weave/libWeavePass.so -fPIC") target_link_libraries(${TEST} ${perfflow_deps}) + message(STATUS "${perfflow_deps}") endforeach() # Build Options @@ -30,12 +37,6 @@ else() option(PERFFLOWASPECT_WITH_MPI "Build MPI smoketest" OFF) endif() -if(PERFFLOWASPECT_WITH_ADIAK) - find_package(adiak QUIET) - message(STATUS " [*] Adding test: smoketest_adiak") - -endif() - if(PERFFLOWASPECT_WITH_MULTITHREADS) From 74c5a06a1a65297c1e8299d0c415946a284c7090 Mon Sep 17 00:00:00 2001 From: Spencer Scott Greene Date: Tue, 13 May 2025 14:58:45 -0700 Subject: [PATCH 04/15] wip adiak integration pass now initializes adiak, performs the collect all function, and performs a callback to print all collected information. --- src/c/weaver/weave/perfflow_weave_common.cpp | 189 ++++++++++++++++++- src/c/weaver/weave/perfflow_weave_common.hpp | 7 + 2 files changed, 186 insertions(+), 10 deletions(-) diff --git a/src/c/weaver/weave/perfflow_weave_common.cpp b/src/c/weaver/weave/perfflow_weave_common.cpp index 1f3b27d8..2710c7c5 100644 --- a/src/c/weaver/weave/perfflow_weave_common.cpp +++ b/src/c/weaver/weave/perfflow_weave_common.cpp @@ -210,23 +210,192 @@ bool weave_ns::WeaveCommon::insertBefore(Module &m, Function &f, StringRef &a, bool weave_ns::WeaveCommon::insertAdiak(Module &m, Function &f) { LLVMContext &context = m.getContext(); - llvm::Type *voidTy = llvm::Type::getVoidTy(context); - llvm::Type *int8Ty = llvm::Type::getInt8Ty(context); - llvm::PointerType *voidPtrTy = llvm::PointerType::getUnqual(int8Ty); + // the adiak_init() call + // return and argument types + Type *voidTy = Type::getVoidTy(context); + Type *int8Ty = Type::getInt8Ty(context); + Type *int32Ty = Type::getInt32Ty(context); + PointerType *voidPtrTy = PointerType::getUnqual(int8Ty); - // function signature - std::vector argTypes = { voidPtrTy }; - llvm::FunctionType *adiakInitType = llvm::FunctionType::get(voidTy, argTypes, false); - - FunctionCallee adiak_init = m.getOrInsertFunction("adiak_init", adiakInitType); - + // the beginning of main BasicBlock &entry = f.getEntryBlock(); IRBuilder<> builder(&entry, entry.begin()); + + // adiak_init function signature + std::vector initArgs = { voidPtrTy }; + FunctionType *adiakInitType = FunctionType::get(voidTy, initArgs, false); + FunctionCallee adiakInit = m.getOrInsertFunction("adiak_init", adiakInitType); - builder.CreateCall(adiak_init); + // call adiak_init(NULL) + Value *nullPtr = Constant::getNullValue(voidPtrTy); + builder.CreateCall(adiakInit, {nullPtr}); + + // call adiak_collect_all() + std::vector collectArgs = { }; // No arguments + FunctionType *collectType = FunctionType::get(int32Ty, collectArgs, false); + FunctionCallee collectAll = m.getOrInsertFunction("adiak_collect_all", collectType); + builder.CreateCall(collectAll, {}); + + // insert the print callback + Function *cb = printAdiakCallback(m); + + // register cb with adiak_list_namevals + Type *cbTy = cb->getType(); + PointerType *cbPtrTy = PointerType::getUnqual(cbTy); + std::vector listNamevalsArgs = { + int32Ty, + int32Ty, + cbPtrTy, + voidPtrTy + }; + FunctionType *listNamevalsType = FunctionType::get(voidTy, listNamevalsArgs, false); + FunctionCallee listNamevals = m.getOrInsertFunction("adiak_list_namevals", listNamevalsType); + + // call the cb function + Value *adiakVersion = builder.getInt32(1); + Value *categoryAll = builder.getInt32(1); + Value *cbPtr = builder.CreateBitCast(cb, cbPtrTy); + builder.CreateCall(listNamevals, {adiakVersion, categoryAll, cbPtr, nullPtr}); + return true; } +Function* weave_ns::WeaveCommon::printAdiakCallback(Module &m) { + LLVMContext &context = m.getContext(); + Type *voidTy = Type::getVoidTy(context); + Type *int8Ty = Type::getInt8Ty(context); + Type *int32Ty = Type::getInt32Ty(context); + Type *int64Ty = Type::getInt64Ty(context); + StructType *adiakValueTy = StructType::getTypeByName(context, "union.adiak_value_t"); + StructType *datatypeTy = StructType::getTypeByName(context, "struct.adiak_datatype_t"); + if (!adiakValueTy) + adiakValueTy = StructType::create(context, "union.adiak_value_t"); + if (!datatypeTy) + datatypeTy = StructType::create(context, "struct.adiak_datatype_t"); + + datatypeTy->setBody({ + int32Ty, + }, false); + PointerType *voidPtrTy = PointerType::getUnqual(int8Ty); + PointerType *charPtrTy = PointerType::getUnqual(int8Ty); + PointerType *valuePtrTy = PointerType::getUnqual(adiakValueTy); + PointerType *datatypePtrTy = PointerType::getUnqual(datatypeTy); + + + // void (*adiak_nameval_cb_t)(const char *name, + // int category, const char *subcategory, + // adiak_value_t *value, adiak_datatype_t *t, void *opaque_value) + std::vector cbArgs = { + charPtrTy, // const char *name + int32Ty, // adiak_category_t category + charPtrTy, // const char *subcategory + valuePtrTy, // adiak_value_t *value + datatypePtrTy, // adiak_datatype_t *t + voidPtrTy // void *opaque_value + }; + + // callback signature + FunctionType *cbType = llvm::FunctionType::get(voidTy, cbArgs, false); + Function *cb = Function::Create(cbType, Function::ExternalLinkage, "print_callback", &m); + + + // parameter names + auto argIter = cb->arg_begin(); + Value *nameParam = argIter++; + nameParam->setName("name"); + Value *categoryParam = argIter++; + categoryParam->setName("category"); + Value *subParam = argIter++; + subParam->setName("subcategory"); + Value *valueParam = argIter++; + valueParam->setName("value"); + Value *typeParam = argIter++; + typeParam->setName("type"); + Value *argParam = argIter; + argParam->setName("opaque"); + + // function body + BasicBlock *entry = BasicBlock::Create(context, "entry", cb); + BasicBlock *defaultBlock = BasicBlock::Create(context, "default_case", cb); + BasicBlock *uintCase = BasicBlock::Create(context, "uint_case", cb); + BasicBlock *stringCase = BasicBlock::Create(context, "string_case", cb); + BasicBlock *dateCase = BasicBlock::Create(context, "version_case", cb); + + + IRBuilder<> builder(entry); + + // adiak_datatype_t struct. we only care about the first value, which contains the type + Value *dtypePtr = builder.CreateStructGEP( + datatypeTy, + typeParam, + 0, + "dtypePtr" + ); + + Value *dtypeVal = builder.CreateLoad(int32Ty, dtypePtr, "dtype"); + + // find the printf functoin + std::vector printfArgs = { charPtrTy }; + FunctionType *printfType = llvm::FunctionType::get(int32Ty, printfArgs, true); + FunctionCallee printfFunc = m.getOrInsertFunction("printf", printfType); + + Value *formatPrefix = builder.CreateGlobalStringPtr("Adiak: Name: %s, Type: %d, Value: "); + builder.CreateCall(printfFunc, {formatPrefix, nameParam, dtypeVal}); + + // switch to handle different adiak_type_t + SwitchInst *type = builder.CreateSwitch(dtypeVal, defaultBlock, 1); + type->addCase(builder.getInt32(ADIAK_UINT), uintCase); + type->addCase(builder.getInt32(ADIAK_VERSION), stringCase); + type->addCase(builder.getInt32(ADIAK_PATH), stringCase); + type->addCase(builder.getInt32(ADIAK_STRING), stringCase); + type->addCase(builder.getInt32(ADIAK_DATE), dateCase); + + + { + // CASE FOR ADIAK_UINT + builder.SetInsertPoint(uintCase); + + Value *uintPtr = builder.CreateBitCast(valueParam, llvm::PointerType::getUnqual(int32Ty)); + Value *uintVal = builder.CreateLoad(int32Ty, uintPtr); + + Value *uintFmt = builder.CreateGlobalStringPtr("%u\n"); + builder.CreateCall(printfFunc, {uintFmt, uintVal}); + builder.CreateRetVoid(); + } + + { + // CASE FOR ADIAK_STRING, ADIAK_PATH, ADIAK_VERSION + builder.SetInsertPoint(stringCase); + + PointerType *strPtrPtrTy = llvm::PointerType::getUnqual(voidPtrTy); + Value *strPtrPtr = builder.CreateBitCast(valueParam, strPtrPtrTy); + Value *stringVal = builder.CreateLoad(voidPtrTy, strPtrPtr); + + Value *stringFmt = builder.CreateGlobalStringPtr("%s\n"); + builder.CreateCall(printfFunc, {stringFmt, stringVal}); + builder.CreateRetVoid(); + } + + { + // CASE FOR ADIAK_DATE + builder.SetInsertPoint(dateCase); + + Value *datePtr = builder.CreateBitCast(valueParam, llvm::PointerType::getUnqual(int64Ty)); + Value *dateVal = builder.CreateLoad(int64Ty, datePtr); + + Value *dateFmt = builder.CreateGlobalStringPtr("%ld\n"); + builder.CreateCall(printfFunc, {dateFmt, dateVal}); + builder.CreateRetVoid(); + } + + builder.SetInsertPoint(defaultBlock); + Value *unknownFmt = builder.CreateGlobalStringPtr("err: not implemented type\n"); + builder.CreateCall(printfFunc, {unknownFmt}); + builder.CreateRetVoid(); + + return cb; +} + /* * vi:tabstop=4 shiftwidth=4 expandtab */ diff --git a/src/c/weaver/weave/perfflow_weave_common.hpp b/src/c/weaver/weave/perfflow_weave_common.hpp index 5fac80a0..e4f33895 100644 --- a/src/c/weaver/weave/perfflow_weave_common.hpp +++ b/src/c/weaver/weave/perfflow_weave_common.hpp @@ -11,6 +11,12 @@ #ifndef PERFFLOW_WEAVE_COMMON_H #define PERFFLOW_WEAVE_COMMON_H +#define ADIAK_UINT 4 +#define ADIAK_VERSION 8 +#define ADIAK_STRING 9 +#define ADIAK_PATH 11 +#define ADIAK_DATE 6 + #include "llvm/IR/Value.h" #include "llvm/IR/Attributes.h" #include "llvm/IR/Constants.h" @@ -37,6 +43,7 @@ class WeaveCommon int async, std::string &scope, std::string &flow, std::string pcut); bool insertAdiak(Module &m, Function &f); + Function *printAdiakCallback(Module &m); public: bool modifyAnnotatedFunctions(Module &m); From 23ee624293a1d73f153677ab43e6af7b59e71970 Mon Sep 17 00:00:00 2001 From: Spencer Scott Greene Date: Thu, 22 May 2025 13:06:10 -0700 Subject: [PATCH 05/15] modify cmake structure + setup trace modified cmake structure to enable flexibility with adiak install directory similar to how caliper integration works. set up structure in chrome tracing file. --- src/c/CMakeLists.txt | 2 +- src/c/cmake/Setup3rdParty.cmake | 13 ++++++++++- src/c/cmake/thirdparty/FindAdiak.cmake | 29 ++++++++++--------------- src/c/runtime/CMakeLists.txt | 12 +++++++++- src/c/runtime/advice_chrome_tracing.cpp | 11 ++++++++++ src/c/runtime/advice_chrome_tracing.hpp | 7 ++++++ 6 files changed, 54 insertions(+), 20 deletions(-) diff --git a/src/c/CMakeLists.txt b/src/c/CMakeLists.txt index 59a9b8a7..c7886aaf 100644 --- a/src/c/CMakeLists.txt +++ b/src/c/CMakeLists.txt @@ -11,7 +11,7 @@ else() message(FATAL_ERROR "Unsupported CXX compiler: please use Clang == 18.0") endif() -option(PERFFLOWASPECT_WITH_ADIAK "Build with Adiak" ON) +option(PERFFLOWASPECT_WITH_ADIAK "Build with Adiak" OFF) include(cmake/CMakeBasics.cmake) diff --git a/src/c/cmake/Setup3rdParty.cmake b/src/c/cmake/Setup3rdParty.cmake index 5de28cf3..b42e5aa8 100644 --- a/src/c/cmake/Setup3rdParty.cmake +++ b/src/c/cmake/Setup3rdParty.cmake @@ -21,5 +21,16 @@ if(PERFFLOWASPECT_WITH_MULTITHREADS) endif() if(PERFFLOWASPECT_WITH_ADIAK) - include(cmake/thirdparty/FindAdiak.cmake) + if(NOT adiak_DIR) + message(FATAL_ERROR "PFA + Adiak needs explicit adiak_DIR") + endif() + + if (adiak_DIR) + message(STATUS "${adiak_DIR}") + include(cmake/thirdparty/FindAdiak.cmake) + endif() + + if (ADIAK_FOUND) + add_definitions(-DPERFFLOWASPECT_WITH_ADIAK) + endif() endif() \ No newline at end of file diff --git a/src/c/cmake/thirdparty/FindAdiak.cmake b/src/c/cmake/thirdparty/FindAdiak.cmake index e873cb24..85e56c5f 100644 --- a/src/c/cmake/thirdparty/FindAdiak.cmake +++ b/src/c/cmake/thirdparty/FindAdiak.cmake @@ -1,17 +1,12 @@ -set(adiak_DIR "/g/g14/greene36/Adiak/install/lib/cmake/adiak") -find_package(adiak REQUIRED) -if(adiak_FOUND) - message(STATUS "Building with Adiak") - set(perfflow_deps "${perfflow_deps}" adiak::adiak) -endif() -#if(adiak_FOUND) -# message(STATUS "Adiak found: ${adiak_FOUND}") -# message(STATUS "Adiak include directories: ${adiak_INCLUDE_DIRS}") -# message(STATUS "Adiak library directories: ${adiak_LIBRARY_DIRS}") -# message(STATUS "Adiak libraries: ${adiak_LIBRARIES}") -#else() -# message(FATAL_ERROR "Adiak not found.") -#endif() -# -#include_directories(${adiak_INCLUDE_DIRS}) -#link_directories(${adiak_LIBRARY_DIRS}) \ No newline at end of file +message(STATUS "Looking for Adiak in ${adiak_DIR}") +find_package(adiak REQUIRED + PATHS ${adiak_DIR}/lib/cmake/adiak + NO_DEFAULT_PATH + NO_CMAKE_ENVIRONMENT_PATH + NO_CMAKE_PATH + NO_SYSTEM_ENVIRONMENT_PATH + NO_CMAKE_SYSTEM_PATH) + +message(STATUS "FOUND Adiak: ${adiak_DIR}") +set(ADIAK_FOUND TRUE) +set(PERFFLOWASPECT_ADIAK_ENABLED TRUE) diff --git a/src/c/runtime/CMakeLists.txt b/src/c/runtime/CMakeLists.txt index 12e4fba5..03baac25 100644 --- a/src/c/runtime/CMakeLists.txt +++ b/src/c/runtime/CMakeLists.txt @@ -14,12 +14,22 @@ set(perfflow_runtime_sources include_directories(${JANSSON_INCLUDE_DIRS}) +if(PERFFLOWASPECT_WITH_ADIAK) + include_directories(${adiak_INCLUDE_DIR}) +endif() + add_library(perfflow_runtime SHARED ${perfflow_runtime_sources} ${perfflow_runtime_headers} ) -target_link_libraries(perfflow_runtime ${JANSSON_LIBRARY} OpenSSL::SSL OpenSSL::Crypto) +if (PERFFLOWASPECT_WITH_ADIAK) + target_link_libraries(perfflow_runtime ${JANSSON_LIBRARY} + OpenSSL::SSL OpenSSL::Crypto adiak::adiak) +else() + target_link_libraries(perfflow_runtime ${JANSSON_LIBRARY} + OpenSSL::SSL OpenSSL::Crypto ) +endif() install(TARGETS perfflow_runtime EXPORT perfflow_export diff --git a/src/c/runtime/advice_chrome_tracing.cpp b/src/c/runtime/advice_chrome_tracing.cpp index 4f3ddb0e..43da513c 100644 --- a/src/c/runtime/advice_chrome_tracing.cpp +++ b/src/c/runtime/advice_chrome_tracing.cpp @@ -663,6 +663,16 @@ long advice_chrome_tracing_t::get_memory_usage() return max_ram_usage; } +#ifdef PERFFLOWASPECT_WITH_ADIAK +long advice_chrome_tracing_t::get_adiak_statistics() +{ + adiak_datatype_t *t; + adiak_value_t *val; + int cat; + adiak_get_nameval("adiak_version", &t, &val, &cat, NULL); +} +#endif + int advice_chrome_tracing_t::with_flow(const char *module, const char *function, const char *flow, @@ -747,6 +757,7 @@ int advice_chrome_tracing_t::before(const char *module, std::string fname; long mem_start; + if (m_enable_logging) { if ((rc = create_event(&event, module, function, my_ts)) < 0) diff --git a/src/c/runtime/advice_chrome_tracing.hpp b/src/c/runtime/advice_chrome_tracing.hpp index 04c13a60..48fb2cdc 100644 --- a/src/c/runtime/advice_chrome_tracing.hpp +++ b/src/c/runtime/advice_chrome_tracing.hpp @@ -16,6 +16,10 @@ #include #include "advice_base.hpp" +#ifdef PERFFLOWASPECT_WITH_ADIAK +#include +#endif + class advice_chrome_tracing_t : public advice_base_t { public: @@ -43,6 +47,9 @@ class advice_chrome_tracing_t : public advice_base_t double get_wall_time(); double get_cpu_time(); long get_memory_usage(); +#ifdef PERFFLOWASPECT_WITH_ADIAK + long get_adiak_statistics(); +#endif int cannonicalize_perfflow_options (); int parse_perfflow_options (); From e9506ee5b93c1c248a4c00eecfe1f83caffc4432 Mon Sep 17 00:00:00 2001 From: Spencer Scott Greene Date: Tue, 27 May 2025 07:51:40 -0700 Subject: [PATCH 06/15] add adiak_fini calls --- src/c/weaver/weave/perfflow_weave_common.cpp | 52 ++++++++++++++------ src/c/weaver/weave/perfflow_weave_common.hpp | 3 +- 2 files changed, 39 insertions(+), 16 deletions(-) diff --git a/src/c/weaver/weave/perfflow_weave_common.cpp b/src/c/weaver/weave/perfflow_weave_common.cpp index 2710c7c5..e880a38b 100644 --- a/src/c/weaver/weave/perfflow_weave_common.cpp +++ b/src/c/weaver/weave/perfflow_weave_common.cpp @@ -26,6 +26,7 @@ bool weave_ns::WeaveCommon::modifyAnnotatedFunctions(Module &m) return false; } + #ifdef PERFFLOWASPECT_WITH_ADIAK Function *main = m.getFunction("main"); if (main != NULL) { @@ -35,6 +36,7 @@ bool weave_ns::WeaveCommon::modifyAnnotatedFunctions(Module &m) else { outs () << "No main"; } + #endif bool changed = false; @@ -207,6 +209,7 @@ bool weave_ns::WeaveCommon::insertBefore(Module &m, Function &f, StringRef &a, return true; } +#ifdef PERFFLOWASPECT_WITH_ADIAK bool weave_ns::WeaveCommon::insertAdiak(Module &m, Function &f) { LLVMContext &context = m.getContext(); @@ -236,26 +239,44 @@ bool weave_ns::WeaveCommon::insertAdiak(Module &m, Function &f) { FunctionCallee collectAll = m.getOrInsertFunction("adiak_collect_all", collectType); builder.CreateCall(collectAll, {}); + // adiak_fini signature + FunctionType *adiakFinishType = FunctionType::get(voidTy, {}, false); + FunctionCallee adiakFinish = m.getOrInsertFunction("adiak_fini", adiakFinishType); + + // find all places where the function terminates from a ReturnInst + std::vector returns; + for (BasicBlock &bb : f) { + if (auto *ret = dyn_cast(bb.getTerminator())) { + returns.push_back(ret); + } + } + + // insert adiak_fini at those return instructions + for (ReturnInst *ret : returns) { + builder.SetInsertPoint(ret); + builder.CreateCall(adiakFinish, {}); + } + // insert the print callback - Function *cb = printAdiakCallback(m); + // Function *cb = printAdiakCallback(m); // register cb with adiak_list_namevals - Type *cbTy = cb->getType(); - PointerType *cbPtrTy = PointerType::getUnqual(cbTy); - std::vector listNamevalsArgs = { - int32Ty, - int32Ty, - cbPtrTy, - voidPtrTy - }; - FunctionType *listNamevalsType = FunctionType::get(voidTy, listNamevalsArgs, false); - FunctionCallee listNamevals = m.getOrInsertFunction("adiak_list_namevals", listNamevalsType); + // Type *cbTy = cb->getType(); + // PointerType *cbPtrTy = PointerType::getUnqual(cbTy); + // std::vector listNamevalsArgs = { + // int32Ty, + // int32Ty, + // cbPtrTy, + // voidPtrTy + // }; + // FunctionType *listNamevalsType = FunctionType::get(voidTy, listNamevalsArgs, false); + // FunctionCallee listNamevals = m.getOrInsertFunction("adiak_list_namevals", listNamevalsType); // call the cb function - Value *adiakVersion = builder.getInt32(1); - Value *categoryAll = builder.getInt32(1); - Value *cbPtr = builder.CreateBitCast(cb, cbPtrTy); - builder.CreateCall(listNamevals, {adiakVersion, categoryAll, cbPtr, nullPtr}); + // Value *adiakVersion = builder.getInt32(1); + // Value *categoryAll = builder.getInt32(1); + // Value *cbPtr = builder.CreateBitCast(cb, cbPtrTy); + // builder.CreateCall(listNamevals, {adiakVersion, categoryAll, cbPtr, nullPtr}); return true; } @@ -395,6 +416,7 @@ Function* weave_ns::WeaveCommon::printAdiakCallback(Module &m) { return cb; } +#endif /* * vi:tabstop=4 shiftwidth=4 expandtab diff --git a/src/c/weaver/weave/perfflow_weave_common.hpp b/src/c/weaver/weave/perfflow_weave_common.hpp index e4f33895..8ba47c01 100644 --- a/src/c/weaver/weave/perfflow_weave_common.hpp +++ b/src/c/weaver/weave/perfflow_weave_common.hpp @@ -41,9 +41,10 @@ class WeaveCommon bool insertBefore(Module &m, Function &f, StringRef &a, int async, std::string &scope, std::string &flow, std::string pcut); - + #ifdef PERFFLOWASPECT_WITH_ADIAK bool insertAdiak(Module &m, Function &f); Function *printAdiakCallback(Module &m); + #endif public: bool modifyAnnotatedFunctions(Module &m); From c92ddec19a6440e347aa147a22ebd5d054341684 Mon Sep 17 00:00:00 2001 From: Spencer Scott Greene Date: Tue, 27 May 2025 09:40:44 -0700 Subject: [PATCH 07/15] adiak statistics appear in trace! --- src/c/runtime/advice_chrome_tracing.cpp | 29 ++++++++++++++++++++----- src/c/runtime/advice_chrome_tracing.hpp | 2 +- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/src/c/runtime/advice_chrome_tracing.cpp b/src/c/runtime/advice_chrome_tracing.cpp index 43da513c..5da2215e 100644 --- a/src/c/runtime/advice_chrome_tracing.cpp +++ b/src/c/runtime/advice_chrome_tracing.cpp @@ -228,6 +228,20 @@ int advice_chrome_tracing_t::flush_if(size_t size) m_ofs << "{" << std::endl; m_ofs << " \"displayTimeUnit\": \"us\"," << std::endl; m_ofs << " \"otherData\": {" << std::endl; + #ifdef PERFFLOWASPECT_WITH_ADIAK + { + std::string meta_adiak_version = get_adiak_statistics(); + if (!meta_adiak_version.empty()) { + m_ofs << " \"adiak_version\": \"" + << meta_adiak_version + << "\"" + << std::endl; + } + else { + m_ofs << "WHAT!!!" << std::endl; + } + } + #endif m_ofs << "" << std::endl; m_ofs << " }," << std::endl; m_ofs << " \"traceEvents\": [" << std::endl; @@ -664,12 +678,16 @@ long advice_chrome_tracing_t::get_memory_usage() } #ifdef PERFFLOWASPECT_WITH_ADIAK -long advice_chrome_tracing_t::get_adiak_statistics() +std::string advice_chrome_tracing_t::get_adiak_statistics() { - adiak_datatype_t *t; - adiak_value_t *val; - int cat; - adiak_get_nameval("adiak_version", &t, &val, &cat, NULL); + adiak_datatype_t *t = nullptr; + adiak_value_t *val = nullptr; + int cat = 0; + int rc = adiak_get_nameval("adiakversion", &t, &val, &cat, nullptr); + if (rc != 0 || t->dtype != adiak_version) { + return {}; + } + return std::string(static_cast(val->v_ptr)); } #endif @@ -1033,6 +1051,7 @@ int advice_chrome_tracing_t::after(const char *module, free(json_str); json_decref(event); + return flush_if(FLUSH_SIZE); } else diff --git a/src/c/runtime/advice_chrome_tracing.hpp b/src/c/runtime/advice_chrome_tracing.hpp index 48fb2cdc..22584735 100644 --- a/src/c/runtime/advice_chrome_tracing.hpp +++ b/src/c/runtime/advice_chrome_tracing.hpp @@ -48,7 +48,7 @@ class advice_chrome_tracing_t : public advice_base_t double get_cpu_time(); long get_memory_usage(); #ifdef PERFFLOWASPECT_WITH_ADIAK - long get_adiak_statistics(); + std::string get_adiak_statistics(); #endif int cannonicalize_perfflow_options (); From 7b97f2f591c33eac71bcc654c5d07ab4f29d5872 Mon Sep 17 00:00:00 2001 From: Spencer Scott Greene Date: Tue, 27 May 2025 10:44:57 -0700 Subject: [PATCH 08/15] adiak_collect_all is in the trace --- src/c/runtime/advice_chrome_tracing.cpp | 69 ++++++++++++++++++------- src/c/runtime/advice_chrome_tracing.hpp | 4 +- 2 files changed, 54 insertions(+), 19 deletions(-) diff --git a/src/c/runtime/advice_chrome_tracing.cpp b/src/c/runtime/advice_chrome_tracing.cpp index 5da2215e..2df1a2ee 100644 --- a/src/c/runtime/advice_chrome_tracing.cpp +++ b/src/c/runtime/advice_chrome_tracing.cpp @@ -230,16 +230,21 @@ int advice_chrome_tracing_t::flush_if(size_t size) m_ofs << " \"otherData\": {" << std::endl; #ifdef PERFFLOWASPECT_WITH_ADIAK { - std::string meta_adiak_version = get_adiak_statistics(); - if (!meta_adiak_version.empty()) { - m_ofs << " \"adiak_version\": \"" - << meta_adiak_version - << "\"" - << std::endl; - } - else { - m_ofs << "WHAT!!!" << std::endl; + const char *key; + json_t *val; + json_t *metadata = get_adiak_statistics(); + size_t total = json_object_size(metadata); + size_t idx = 0; + json_object_foreach(metadata, key, val) { + char *v = json_dumps(val, JSON_ENCODE_ANY); + m_ofs << " \"" << key << "\": " << v; + free(v); + if (++idx < total) { + m_ofs << ","; + } + m_ofs << "\n"; } + json_decref(metadata); } #endif m_ofs << "" << std::endl; @@ -678,16 +683,44 @@ long advice_chrome_tracing_t::get_memory_usage() } #ifdef PERFFLOWASPECT_WITH_ADIAK -std::string advice_chrome_tracing_t::get_adiak_statistics() -{ - adiak_datatype_t *t = nullptr; - adiak_value_t *val = nullptr; - int cat = 0; - int rc = adiak_get_nameval("adiakversion", &t, &val, &cat, nullptr); - if (rc != 0 || t->dtype != adiak_version) { - return {}; +void advice_chrome_tracing_t::adiak_cb(const char *name, int cat, const char *subcat, adiak_value_t *val, adiak_datatype_t *t, void *opaque) { + json_t *obj = static_cast(opaque); + json_t *metadata = nullptr; + + switch (t->dtype) { + case adiak_version: + case adiak_string: + case adiak_catstring: + case adiak_path: + metadata = json_string(reinterpret_cast(val->v_ptr)); + break; + case adiak_long: + case adiak_date: + metadata = json_integer(val->v_long); + break; + case adiak_double: + metadata = json_real(val->v_double); + break; + default: + return; } - return std::string(static_cast(val->v_ptr)); + json_object_set_new(obj, name, metadata); +} + +json_t *advice_chrome_tracing_t::get_adiak_statistics() +{ + // adiak_datatype_t *t = nullptr; + // adiak_value_t *val = nullptr; + // int cat = 0; + // int rc = adiak_get_nameval("adiakversion", &t, &val, &cat, nullptr); + // if (rc != 0 || t->dtype != adiak_version) { + // return {}; + // } + // return std::string(static_cast(val->v_ptr)); + json_t *adiak_metadata = json_object(); + adiak_list_namevals(1, adiak_category_all, adiak_cb, adiak_metadata); + return adiak_metadata; + } #endif diff --git a/src/c/runtime/advice_chrome_tracing.hpp b/src/c/runtime/advice_chrome_tracing.hpp index 22584735..0c6d84d4 100644 --- a/src/c/runtime/advice_chrome_tracing.hpp +++ b/src/c/runtime/advice_chrome_tracing.hpp @@ -48,7 +48,9 @@ class advice_chrome_tracing_t : public advice_base_t double get_cpu_time(); long get_memory_usage(); #ifdef PERFFLOWASPECT_WITH_ADIAK - std::string get_adiak_statistics(); + json_t *get_adiak_statistics(); + static void adiak_cb(const char *name, int cat, const char *subcat, adiak_value_t *val, adiak_datatype_t *t, void *opaque); + #endif int cannonicalize_perfflow_options (); From 42b2e43166bb601e63347bbfb6d0c96b76e8bae1 Mon Sep 17 00:00:00 2001 From: Spencer Scott Greene Date: Tue, 27 May 2025 13:34:45 -0700 Subject: [PATCH 09/15] wip mpi tracking & cleanup --- src/c/runtime/advice_chrome_tracing.cpp | 16 +- src/c/weaver/weave/perfflow_weave_common.cpp | 211 +++++-------------- src/c/weaver/weave/perfflow_weave_common.hpp | 1 - 3 files changed, 55 insertions(+), 173 deletions(-) diff --git a/src/c/runtime/advice_chrome_tracing.cpp b/src/c/runtime/advice_chrome_tracing.cpp index 2df1a2ee..3b762a0d 100644 --- a/src/c/runtime/advice_chrome_tracing.cpp +++ b/src/c/runtime/advice_chrome_tracing.cpp @@ -698,9 +698,16 @@ void advice_chrome_tracing_t::adiak_cb(const char *name, int cat, const char *su case adiak_date: metadata = json_integer(val->v_long); break; + case adiak_ulong: + metadata = json_integer(unsigned(val->v_long)); + break; case adiak_double: metadata = json_real(val->v_double); break; + case adiak_int: + metadata = json_integer(val->v_int); + case adiak_uint: + metadata = json_integer(unsigned(val->v_int)); default: return; } @@ -709,18 +716,9 @@ void advice_chrome_tracing_t::adiak_cb(const char *name, int cat, const char *su json_t *advice_chrome_tracing_t::get_adiak_statistics() { - // adiak_datatype_t *t = nullptr; - // adiak_value_t *val = nullptr; - // int cat = 0; - // int rc = adiak_get_nameval("adiakversion", &t, &val, &cat, nullptr); - // if (rc != 0 || t->dtype != adiak_version) { - // return {}; - // } - // return std::string(static_cast(val->v_ptr)); json_t *adiak_metadata = json_object(); adiak_list_namevals(1, adiak_category_all, adiak_cb, adiak_metadata); return adiak_metadata; - } #endif diff --git a/src/c/weaver/weave/perfflow_weave_common.cpp b/src/c/weaver/weave/perfflow_weave_common.cpp index e880a38b..1275c804 100644 --- a/src/c/weaver/weave/perfflow_weave_common.cpp +++ b/src/c/weaver/weave/perfflow_weave_common.cpp @@ -221,21 +221,63 @@ bool weave_ns::WeaveCommon::insertAdiak(Module &m, Function &f) { PointerType *voidPtrTy = PointerType::getUnqual(int8Ty); // the beginning of main - BasicBlock &entry = f.getEntryBlock(); - IRBuilder<> builder(&entry, entry.begin()); + // BasicBlock &entry = f.getEntryBlock(); + // IRBuilder<> builder(&entry, entry.begin()); // adiak_init function signature std::vector initArgs = { voidPtrTy }; FunctionType *adiakInitType = FunctionType::get(voidTy, initArgs, false); FunctionCallee adiakInit = m.getOrInsertFunction("adiak_init", adiakInitType); + + GlobalVariable *mpiWorld = m.getNamedGlobal("MPI_COMM_WORLD"); + if (!mpiWorld) { + mpiWorld = new GlobalVariable( + m, + voidPtrTy, + /*isConstant=*/false, + GlobalValue::ExternalLinkage, + /*Initializer=*/nullptr, + "MPI_COMM_WORLD" + ); + } + + CallInst *mpi = nullptr; + // find each instruction to see if there is a call instruction + for (BasicBlock &bb : f) { + for (Instruction &i : bb) { + if (auto *call = dyn_cast(&i)) { + if (Function *callee = call->getCalledFunction()) { + if (callee->getName() == "MPI_Init") { + mpi = call; + } + } + } + } + if (mpi) { + break; + } + } + IRBuilder<> builder(context); + Value *arg; + + if (mpi) { + auto insertPosition = BasicBlock::iterator(mpi); + ++insertPosition; + builder.SetInsertPoint(mpi->getParent(), insertPosition); + arg = builder.CreateLoad(voidPtrTy, mpiWorld, "comm_world"); + } else { + BasicBlock &entry = f.getEntryBlock(); + builder.SetInsertPoint(&entry, entry.begin()); + arg = Constant::getNullValue(voidPtrTy); + } + + builder.CreateCall(adiakInit, {arg}); // call adiak_init(NULL) - Value *nullPtr = Constant::getNullValue(voidPtrTy); - builder.CreateCall(adiakInit, {nullPtr}); + // Value *nullPtr = Constant::getNullValue(voidPtrTy); // call adiak_collect_all() - std::vector collectArgs = { }; // No arguments - FunctionType *collectType = FunctionType::get(int32Ty, collectArgs, false); + FunctionType *collectType = FunctionType::get(int32Ty, {}, false); FunctionCallee collectAll = m.getOrInsertFunction("adiak_collect_all", collectType); builder.CreateCall(collectAll, {}); @@ -257,165 +299,8 @@ bool weave_ns::WeaveCommon::insertAdiak(Module &m, Function &f) { builder.CreateCall(adiakFinish, {}); } - // insert the print callback - // Function *cb = printAdiakCallback(m); - - // register cb with adiak_list_namevals - // Type *cbTy = cb->getType(); - // PointerType *cbPtrTy = PointerType::getUnqual(cbTy); - // std::vector listNamevalsArgs = { - // int32Ty, - // int32Ty, - // cbPtrTy, - // voidPtrTy - // }; - // FunctionType *listNamevalsType = FunctionType::get(voidTy, listNamevalsArgs, false); - // FunctionCallee listNamevals = m.getOrInsertFunction("adiak_list_namevals", listNamevalsType); - - // call the cb function - // Value *adiakVersion = builder.getInt32(1); - // Value *categoryAll = builder.getInt32(1); - // Value *cbPtr = builder.CreateBitCast(cb, cbPtrTy); - // builder.CreateCall(listNamevals, {adiakVersion, categoryAll, cbPtr, nullPtr}); - return true; } - -Function* weave_ns::WeaveCommon::printAdiakCallback(Module &m) { - LLVMContext &context = m.getContext(); - Type *voidTy = Type::getVoidTy(context); - Type *int8Ty = Type::getInt8Ty(context); - Type *int32Ty = Type::getInt32Ty(context); - Type *int64Ty = Type::getInt64Ty(context); - StructType *adiakValueTy = StructType::getTypeByName(context, "union.adiak_value_t"); - StructType *datatypeTy = StructType::getTypeByName(context, "struct.adiak_datatype_t"); - if (!adiakValueTy) - adiakValueTy = StructType::create(context, "union.adiak_value_t"); - if (!datatypeTy) - datatypeTy = StructType::create(context, "struct.adiak_datatype_t"); - - datatypeTy->setBody({ - int32Ty, - }, false); - PointerType *voidPtrTy = PointerType::getUnqual(int8Ty); - PointerType *charPtrTy = PointerType::getUnqual(int8Ty); - PointerType *valuePtrTy = PointerType::getUnqual(adiakValueTy); - PointerType *datatypePtrTy = PointerType::getUnqual(datatypeTy); - - - // void (*adiak_nameval_cb_t)(const char *name, - // int category, const char *subcategory, - // adiak_value_t *value, adiak_datatype_t *t, void *opaque_value) - std::vector cbArgs = { - charPtrTy, // const char *name - int32Ty, // adiak_category_t category - charPtrTy, // const char *subcategory - valuePtrTy, // adiak_value_t *value - datatypePtrTy, // adiak_datatype_t *t - voidPtrTy // void *opaque_value - }; - - // callback signature - FunctionType *cbType = llvm::FunctionType::get(voidTy, cbArgs, false); - Function *cb = Function::Create(cbType, Function::ExternalLinkage, "print_callback", &m); - - - // parameter names - auto argIter = cb->arg_begin(); - Value *nameParam = argIter++; - nameParam->setName("name"); - Value *categoryParam = argIter++; - categoryParam->setName("category"); - Value *subParam = argIter++; - subParam->setName("subcategory"); - Value *valueParam = argIter++; - valueParam->setName("value"); - Value *typeParam = argIter++; - typeParam->setName("type"); - Value *argParam = argIter; - argParam->setName("opaque"); - - // function body - BasicBlock *entry = BasicBlock::Create(context, "entry", cb); - BasicBlock *defaultBlock = BasicBlock::Create(context, "default_case", cb); - BasicBlock *uintCase = BasicBlock::Create(context, "uint_case", cb); - BasicBlock *stringCase = BasicBlock::Create(context, "string_case", cb); - BasicBlock *dateCase = BasicBlock::Create(context, "version_case", cb); - - - IRBuilder<> builder(entry); - - // adiak_datatype_t struct. we only care about the first value, which contains the type - Value *dtypePtr = builder.CreateStructGEP( - datatypeTy, - typeParam, - 0, - "dtypePtr" - ); - - Value *dtypeVal = builder.CreateLoad(int32Ty, dtypePtr, "dtype"); - - // find the printf functoin - std::vector printfArgs = { charPtrTy }; - FunctionType *printfType = llvm::FunctionType::get(int32Ty, printfArgs, true); - FunctionCallee printfFunc = m.getOrInsertFunction("printf", printfType); - - Value *formatPrefix = builder.CreateGlobalStringPtr("Adiak: Name: %s, Type: %d, Value: "); - builder.CreateCall(printfFunc, {formatPrefix, nameParam, dtypeVal}); - - // switch to handle different adiak_type_t - SwitchInst *type = builder.CreateSwitch(dtypeVal, defaultBlock, 1); - type->addCase(builder.getInt32(ADIAK_UINT), uintCase); - type->addCase(builder.getInt32(ADIAK_VERSION), stringCase); - type->addCase(builder.getInt32(ADIAK_PATH), stringCase); - type->addCase(builder.getInt32(ADIAK_STRING), stringCase); - type->addCase(builder.getInt32(ADIAK_DATE), dateCase); - - - { - // CASE FOR ADIAK_UINT - builder.SetInsertPoint(uintCase); - - Value *uintPtr = builder.CreateBitCast(valueParam, llvm::PointerType::getUnqual(int32Ty)); - Value *uintVal = builder.CreateLoad(int32Ty, uintPtr); - - Value *uintFmt = builder.CreateGlobalStringPtr("%u\n"); - builder.CreateCall(printfFunc, {uintFmt, uintVal}); - builder.CreateRetVoid(); - } - - { - // CASE FOR ADIAK_STRING, ADIAK_PATH, ADIAK_VERSION - builder.SetInsertPoint(stringCase); - - PointerType *strPtrPtrTy = llvm::PointerType::getUnqual(voidPtrTy); - Value *strPtrPtr = builder.CreateBitCast(valueParam, strPtrPtrTy); - Value *stringVal = builder.CreateLoad(voidPtrTy, strPtrPtr); - - Value *stringFmt = builder.CreateGlobalStringPtr("%s\n"); - builder.CreateCall(printfFunc, {stringFmt, stringVal}); - builder.CreateRetVoid(); - } - - { - // CASE FOR ADIAK_DATE - builder.SetInsertPoint(dateCase); - - Value *datePtr = builder.CreateBitCast(valueParam, llvm::PointerType::getUnqual(int64Ty)); - Value *dateVal = builder.CreateLoad(int64Ty, datePtr); - - Value *dateFmt = builder.CreateGlobalStringPtr("%ld\n"); - builder.CreateCall(printfFunc, {dateFmt, dateVal}); - builder.CreateRetVoid(); - } - - builder.SetInsertPoint(defaultBlock); - Value *unknownFmt = builder.CreateGlobalStringPtr("err: not implemented type\n"); - builder.CreateCall(printfFunc, {unknownFmt}); - builder.CreateRetVoid(); - - return cb; -} #endif /* diff --git a/src/c/weaver/weave/perfflow_weave_common.hpp b/src/c/weaver/weave/perfflow_weave_common.hpp index 8ba47c01..a43ddc77 100644 --- a/src/c/weaver/weave/perfflow_weave_common.hpp +++ b/src/c/weaver/weave/perfflow_weave_common.hpp @@ -43,7 +43,6 @@ class WeaveCommon int async, std::string &scope, std::string &flow, std::string pcut); #ifdef PERFFLOWASPECT_WITH_ADIAK bool insertAdiak(Module &m, Function &f); - Function *printAdiakCallback(Module &m); #endif public: bool modifyAnnotatedFunctions(Module &m); From 0f2985977c2c9238cc1e4c256e95424e921ed69b Mon Sep 17 00:00:00 2001 From: Spencer Scott Greene Date: Tue, 3 Jun 2025 19:54:51 -0700 Subject: [PATCH 10/15] nearly working adiak integration --- src/c/weaver/weave/CMakeLists.txt | 7 +++ src/c/weaver/weave/perfflow_weave_common.cpp | 60 ++++++++++---------- src/c/weaver/weave/perfflow_weave_common.hpp | 10 ++-- 3 files changed, 41 insertions(+), 36 deletions(-) diff --git a/src/c/weaver/weave/CMakeLists.txt b/src/c/weaver/weave/CMakeLists.txt index 7945e63f..96ddb95c 100644 --- a/src/c/weaver/weave/CMakeLists.txt +++ b/src/c/weaver/weave/CMakeLists.txt @@ -16,6 +16,13 @@ set_target_properties(WeavePass PROPERTIES target_link_libraries(WeavePass perfflow_parser ${JANSSON_LIB}) +if(PERFFLOWASPECT_WITH_MPI) + include_directories(${MPI_C_INCLUDE_PATH}) + target_link_libraries(WeavePass ${MPI_C_LIBRARIES}) + target_compile_definitions(WeavePass PRIVATE PERFFLOWASPECT_WITH_MPI) +endif() + + add_library(WeavePassPlugin INTERFACE) target_compile_options(WeavePassPlugin INTERFACE "SHELL:$>$>" diff --git a/src/c/weaver/weave/perfflow_weave_common.cpp b/src/c/weaver/weave/perfflow_weave_common.cpp index 1275c804..5eb89e22 100644 --- a/src/c/weaver/weave/perfflow_weave_common.cpp +++ b/src/c/weaver/weave/perfflow_weave_common.cpp @@ -220,27 +220,33 @@ bool weave_ns::WeaveCommon::insertAdiak(Module &m, Function &f) { Type *int32Ty = Type::getInt32Ty(context); PointerType *voidPtrTy = PointerType::getUnqual(int8Ty); - // the beginning of main - // BasicBlock &entry = f.getEntryBlock(); - // IRBuilder<> builder(&entry, entry.begin()); - // adiak_init function signature std::vector initArgs = { voidPtrTy }; FunctionType *adiakInitType = FunctionType::get(voidTy, initArgs, false); FunctionCallee adiakInit = m.getOrInsertFunction("adiak_init", adiakInitType); - GlobalVariable *mpiWorld = m.getNamedGlobal("MPI_COMM_WORLD"); - if (!mpiWorld) { - mpiWorld = new GlobalVariable( - m, - voidPtrTy, - /*isConstant=*/false, - GlobalValue::ExternalLinkage, - /*Initializer=*/nullptr, - "MPI_COMM_WORLD" - ); + IRBuilder<> builder(context); + Value *arg; + BasicBlock &entry = f.getEntryBlock(); + builder.SetInsertPoint(&entry, entry.begin()); + + #ifdef PERFFLOWASPECT_WITH_MPI + // find the MPI communicator, MPI_COMM_WORLD + // OpenMPI exposes this as MPI_COMM_WORLD + // MPICH exposes this as a constant 0x44000000 + if (GlobalVariable *gv = m.getNamedGlobal("MPI_COMM_WORLD")) { + arg = builder.CreateLoad(gv->getValueType(), gv); + } + else { + Constant *commVal = ConstantInt::get(int32Ty, 0x44000000); + auto *commGV = new GlobalVariable( + m, + int32Ty, + true, + GlobalValue::PrivateLinkage, commVal, "perfflow_mpi_comm_world"); + arg = builder.CreateBitCast(commGV, voidPtrTy); } - + CallInst *mpi = nullptr; // find each instruction to see if there is a call instruction for (BasicBlock &bb : f) { @@ -254,27 +260,21 @@ bool weave_ns::WeaveCommon::insertAdiak(Module &m, Function &f) { } } if (mpi) { + auto insertPosition = BasicBlock::iterator(mpi); + ++insertPosition; + builder.SetInsertPoint(mpi->getParent(), insertPosition); + builder.CreateCall(adiakInit, {arg}); break; } } - IRBuilder<> builder(context); - Value *arg; - - if (mpi) { - auto insertPosition = BasicBlock::iterator(mpi); - ++insertPosition; - builder.SetInsertPoint(mpi->getParent(), insertPosition); - arg = builder.CreateLoad(voidPtrTy, mpiWorld, "comm_world"); - } else { - BasicBlock &entry = f.getEntryBlock(); - builder.SetInsertPoint(&entry, entry.begin()); + if (!mpi) { arg = Constant::getNullValue(voidPtrTy); + builder.CreateCall(adiakInit, {arg}); } - + #else + arg = Constant::getNullValue(voidPtrTy); builder.CreateCall(adiakInit, {arg}); - - // call adiak_init(NULL) - // Value *nullPtr = Constant::getNullValue(voidPtrTy); + #endif // call adiak_collect_all() FunctionType *collectType = FunctionType::get(int32Ty, {}, false); diff --git a/src/c/weaver/weave/perfflow_weave_common.hpp b/src/c/weaver/weave/perfflow_weave_common.hpp index a43ddc77..0e425da3 100644 --- a/src/c/weaver/weave/perfflow_weave_common.hpp +++ b/src/c/weaver/weave/perfflow_weave_common.hpp @@ -11,12 +11,6 @@ #ifndef PERFFLOW_WEAVE_COMMON_H #define PERFFLOW_WEAVE_COMMON_H -#define ADIAK_UINT 4 -#define ADIAK_VERSION 8 -#define ADIAK_STRING 9 -#define ADIAK_PATH 11 -#define ADIAK_DATE 6 - #include "llvm/IR/Value.h" #include "llvm/IR/Attributes.h" #include "llvm/IR/Constants.h" @@ -28,6 +22,10 @@ #include "llvm/Support/raw_ostream.h" #include "../../parser/perfflow_parser.hpp" +#ifdef PERFFLOWASPECT_WITH_MPI +#include "mpi.h" +#endif + using namespace llvm; namespace weave_ns { From 5f58c111fa1a6310fddc2ed33be4bc8cb3fc882b Mon Sep 17 00:00:00 2001 From: Spencer Scott Greene Date: Wed, 4 Jun 2025 11:06:09 -0700 Subject: [PATCH 11/15] start adiak integration docs --- docs/AdiakIntegration.rst | 49 +++++++++++++++++++++++++++++++++++++++ docs/index.rst | 1 + 2 files changed, 50 insertions(+) create mode 100644 docs/AdiakIntegration.rst diff --git a/docs/AdiakIntegration.rst b/docs/AdiakIntegration.rst new file mode 100644 index 00000000..02216f03 --- /dev/null +++ b/docs/AdiakIntegration.rst @@ -0,0 +1,49 @@ +.. + # Copyright 2021 Lawrence Livermore National Security, LLC and other + # PerfFlowAspect Project Developers. See the top-level LICENSE file for + # details. + # + # SPDX-License-Identifier: LGPL-3.0 + +##################### + Adiak Integration +##################### + +PerfFlowAspect can be built with Adiak, a tool that collects metadata on HPC runs. Learn more about Adiak `here `_. + +PerfFlowAspect integration with Adiak is currently only supported in C/C++. When integrated, the generated PerfFlowAspect `.pfw` trace file contains Adiak metadata. `Note: the trace file must be generated in object format.` + +Adiak must be installed prior to PerfFlowAspect integration. + +************* + C/C++ Build +************* +Adiak is enabled by specifying ``PERFFLOWASPECT_WITH_MPI=On`` along with the path to Adiak's package configuration file, i.e. ``/lib/cmake/adiak>``. + +.. code:: bash + + cmake -DCMAKE_C_COMPILER= \ + -DCMAKE_C_COMPILER= \ + -DPERFFLOWASPECT_WITH_ADIAK=On \ + -Dadiak_DIR=/lib/cmake/adiak> ../ + +Adiak can gather additional metadata related to MPI. The flag ``PERFFLOWASPECT_WITH_MPI=On`` must be specified. + +.. code:: bash + + cmake -DCMAKE_C_COMPILER= \ + -DCMAKE_C_COMPILER= \ + -DPERFFLOWASPECT_WITH_ADIAK=On \ + -Dadiak_DIR=/lib/cmake/adiak> \ + -DPERFFLOWASPECT_WITH_MPI=On ../ + + +*************** + C/C++ Example +*************** +Adiak's metadata can only be displayed in the object format of a PerfFlowAspect `.pfw` trace. Thus, ``PERFFLOW_OPTIONS="log-format=Object"`` must be specified + +.. code:: bash + + PERFFLOW_OPTIONS="log-format=Object" ./smoketest + diff --git a/docs/index.rst b/docs/index.rst index a8de2d9b..debb6311 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -85,6 +85,7 @@ uniformity as to how performance is measured and controlled. BuildingPerfFlowAspect Annotations + AdiakIntegration UpcomingFeatures .. toctree:: From 471b6b6378645d64f5c1291fd49cf1414ca7e82f Mon Sep 17 00:00:00 2001 From: Spencer Scott Greene Date: Fri, 6 Jun 2025 11:35:30 -0700 Subject: [PATCH 12/15] mpi integration --- src/c/weaver/weave/perfflow_weave_common.cpp | 22 ++++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/src/c/weaver/weave/perfflow_weave_common.cpp b/src/c/weaver/weave/perfflow_weave_common.cpp index 5eb89e22..c963312b 100644 --- a/src/c/weaver/weave/perfflow_weave_common.cpp +++ b/src/c/weaver/weave/perfflow_weave_common.cpp @@ -232,19 +232,21 @@ bool weave_ns::WeaveCommon::insertAdiak(Module &m, Function &f) { #ifdef PERFFLOWASPECT_WITH_MPI // find the MPI communicator, MPI_COMM_WORLD - // OpenMPI exposes this as MPI_COMM_WORLD + // OpenMPI exposes this as ompi_comm_world (untested) // MPICH exposes this as a constant 0x44000000 - if (GlobalVariable *gv = m.getNamedGlobal("MPI_COMM_WORLD")) { + if (GlobalVariable *gv = m.getGlobalVariable("ompi_comm_world")) { arg = builder.CreateLoad(gv->getValueType(), gv); } else { - Constant *commVal = ConstantInt::get(int32Ty, 0x44000000); - auto *commGV = new GlobalVariable( - m, - int32Ty, - true, - GlobalValue::PrivateLinkage, commVal, "perfflow_mpi_comm_world"); - arg = builder.CreateBitCast(commGV, voidPtrTy); + AllocaInst *alloc = builder.CreateAlloca(int32Ty, nullptr, "weave_mpi_comm"); + uint64_t mpiValue = 0x44000000; + Value *commVal = ConstantInt::get(int32Ty, mpiValue); + builder.CreateStore(commVal, alloc); + arg = builder.CreateBitCast( + alloc, + voidPtrTy, + "mpi_comm_world_void" + ); } CallInst *mpi = nullptr; @@ -265,8 +267,10 @@ bool weave_ns::WeaveCommon::insertAdiak(Module &m, Function &f) { builder.SetInsertPoint(mpi->getParent(), insertPosition); builder.CreateCall(adiakInit, {arg}); break; + } } + if (!mpi) { arg = Constant::getNullValue(voidPtrTy); builder.CreateCall(adiakInit, {arg}); From 8453c55574fcd0718faab7123befff06bfab1d92 Mon Sep 17 00:00:00 2001 From: Spencer Scott Greene Date: Fri, 6 Jun 2025 14:08:48 -0700 Subject: [PATCH 13/15] nearly working mpi. works with mpich, need to validate openmpi --- src/c/runtime/advice_chrome_tracing.cpp | 10 +++++ src/c/weaver/weave/perfflow_weave_common.cpp | 43 +++++++++++++------- 2 files changed, 38 insertions(+), 15 deletions(-) diff --git a/src/c/runtime/advice_chrome_tracing.cpp b/src/c/runtime/advice_chrome_tracing.cpp index 3b762a0d..f440c956 100644 --- a/src/c/runtime/advice_chrome_tracing.cpp +++ b/src/c/runtime/advice_chrome_tracing.cpp @@ -706,8 +706,18 @@ void advice_chrome_tracing_t::adiak_cb(const char *name, int cat, const char *su break; case adiak_int: metadata = json_integer(val->v_int); + break; case adiak_uint: metadata = json_integer(unsigned(val->v_int)); + break; + case adiak_timeval: { + struct timeval *tv = (struct timeval *)(val->v_ptr); + json_t *tv_obj = json_object(); + json_object_set_new(tv_obj, "tv_sec", json_integer(tv->tv_sec)); + json_object_set_new(tv_obj, "tv_usec", json_integer(tv->tv_usec)); + metadata = tv_obj; + break; + } default: return; } diff --git a/src/c/weaver/weave/perfflow_weave_common.cpp b/src/c/weaver/weave/perfflow_weave_common.cpp index c963312b..23f3d555 100644 --- a/src/c/weaver/weave/perfflow_weave_common.cpp +++ b/src/c/weaver/weave/perfflow_weave_common.cpp @@ -234,13 +234,17 @@ bool weave_ns::WeaveCommon::insertAdiak(Module &m, Function &f) { // find the MPI communicator, MPI_COMM_WORLD // OpenMPI exposes this as ompi_comm_world (untested) // MPICH exposes this as a constant 0x44000000 - if (GlobalVariable *gv = m.getGlobalVariable("ompi_comm_world")) { - arg = builder.CreateLoad(gv->getValueType(), gv); + GlobalVariable *gv = m.getGlobalVariable("ompi_mpi_comm_world"); + if (!gv) { + gv = m.getGlobalVariable("MPI_COMM_WORLD"); + } + if (gv) { + arg = builder.CreateBitCast(gv, voidPtrTy, "mpi_comm_world_void"); } else { - AllocaInst *alloc = builder.CreateAlloca(int32Ty, nullptr, "weave_mpi_comm"); uint64_t mpiValue = 0x44000000; Value *commVal = ConstantInt::get(int32Ty, mpiValue); + AllocaInst *alloc = builder.CreateAlloca(int32Ty, nullptr, "weave_mpi_comm"); builder.CreateStore(commVal, alloc); arg = builder.CreateBitCast( alloc, @@ -250,7 +254,7 @@ bool weave_ns::WeaveCommon::insertAdiak(Module &m, Function &f) { } CallInst *mpi = nullptr; - // find each instruction to see if there is a call instruction + // find each instruction to see if there is an MPI_Init call instruction for (BasicBlock &bb : f) { for (Instruction &i : bb) { if (auto *call = dyn_cast(&i)) { @@ -261,20 +265,19 @@ bool weave_ns::WeaveCommon::insertAdiak(Module &m, Function &f) { } } } - if (mpi) { - auto insertPosition = BasicBlock::iterator(mpi); - ++insertPosition; - builder.SetInsertPoint(mpi->getParent(), insertPosition); - builder.CreateCall(adiakInit, {arg}); - break; - - } + if (mpi) break; } - - if (!mpi) { + if (mpi) { + BasicBlock *mpiBlock = mpi->getParent(); + auto insertPos = BasicBlock::iterator(mpi); + ++insertPos; + builder.SetInsertPoint(mpiBlock, insertPos); + } else { arg = Constant::getNullValue(voidPtrTy); - builder.CreateCall(adiakInit, {arg}); } + + builder.CreateCall(adiakInit, {arg}); + #else arg = Constant::getNullValue(voidPtrTy); builder.CreateCall(adiakInit, {arg}); @@ -285,6 +288,16 @@ bool weave_ns::WeaveCommon::insertAdiak(Module &m, Function &f) { FunctionCallee collectAll = m.getOrInsertFunction("adiak_collect_all", collectType); builder.CreateCall(collectAll, {}); + // call adiak_walltime() + FunctionCallee walltime = m.getOrInsertFunction("adiak_walltime", collectType); + builder.CreateCall(walltime, {}); + + FunctionCallee systime = m.getOrInsertFunction("adiak_systime", collectType); + builder.CreateCall(systime, {}); + + FunctionCallee cputime = m.getOrInsertFunction("adiak_cputime", collectType); + builder.CreateCall(cputime, {}); + // adiak_fini signature FunctionType *adiakFinishType = FunctionType::get(voidTy, {}, false); FunctionCallee adiakFinish = m.getOrInsertFunction("adiak_fini", adiakFinishType); From b4d855587ff14ab020f8ae523b0135f2e20aabe5 Mon Sep 17 00:00:00 2001 From: Spencer Scott Greene Date: Tue, 10 Jun 2025 08:51:28 -0700 Subject: [PATCH 14/15] astyle --- src/c/runtime/advice_chrome_tracing.cpp | 24 ++++-- src/c/weaver/weave/perfflow_weave_common.cpp | 90 +++++++++++++------- 2 files changed, 72 insertions(+), 42 deletions(-) diff --git a/src/c/runtime/advice_chrome_tracing.cpp b/src/c/runtime/advice_chrome_tracing.cpp index f440c956..4475f1ac 100644 --- a/src/c/runtime/advice_chrome_tracing.cpp +++ b/src/c/runtime/advice_chrome_tracing.cpp @@ -228,25 +228,27 @@ int advice_chrome_tracing_t::flush_if(size_t size) m_ofs << "{" << std::endl; m_ofs << " \"displayTimeUnit\": \"us\"," << std::endl; m_ofs << " \"otherData\": {" << std::endl; - #ifdef PERFFLOWASPECT_WITH_ADIAK +#ifdef PERFFLOWASPECT_WITH_ADIAK { const char *key; json_t *val; json_t *metadata = get_adiak_statistics(); size_t total = json_object_size(metadata); size_t idx = 0; - json_object_foreach(metadata, key, val) { + json_object_foreach(metadata, key, val) + { char *v = json_dumps(val, JSON_ENCODE_ANY); m_ofs << " \"" << key << "\": " << v; free(v); - if (++idx < total) { + if (++idx < total) + { m_ofs << ","; } m_ofs << "\n"; } json_decref(metadata); } - #endif +#endif m_ofs << "" << std::endl; m_ofs << " }," << std::endl; m_ofs << " \"traceEvents\": [" << std::endl; @@ -683,11 +685,14 @@ long advice_chrome_tracing_t::get_memory_usage() } #ifdef PERFFLOWASPECT_WITH_ADIAK -void advice_chrome_tracing_t::adiak_cb(const char *name, int cat, const char *subcat, adiak_value_t *val, adiak_datatype_t *t, void *opaque) { - json_t *obj = static_cast(opaque); +void advice_chrome_tracing_t::adiak_cb(const char *name, int cat, + const char *subcat, adiak_value_t *val, adiak_datatype_t *t, void *opaque) +{ + json_t *obj = static_cast(opaque); json_t *metadata = nullptr; - - switch (t->dtype) { + + switch (t->dtype) + { case adiak_version: case adiak_string: case adiak_catstring: @@ -710,7 +715,8 @@ void advice_chrome_tracing_t::adiak_cb(const char *name, int cat, const char *su case adiak_uint: metadata = json_integer(unsigned(val->v_int)); break; - case adiak_timeval: { + case adiak_timeval: + { struct timeval *tv = (struct timeval *)(val->v_ptr); json_t *tv_obj = json_object(); json_object_set_new(tv_obj, "tv_sec", json_integer(tv->tv_sec)); diff --git a/src/c/weaver/weave/perfflow_weave_common.cpp b/src/c/weaver/weave/perfflow_weave_common.cpp index 23f3d555..758f58a7 100644 --- a/src/c/weaver/weave/perfflow_weave_common.cpp +++ b/src/c/weaver/weave/perfflow_weave_common.cpp @@ -26,17 +26,19 @@ bool weave_ns::WeaveCommon::modifyAnnotatedFunctions(Module &m) return false; } - #ifdef PERFFLOWASPECT_WITH_ADIAK +#ifdef PERFFLOWASPECT_WITH_ADIAK Function *main = m.getFunction("main"); - if (main != NULL) { + if (main != NULL) + { outs() << "Inserting Adiak?\n"; insertAdiak(m, *main); } - else { - outs () << "No main"; + else + { + outs() << "No main"; } - #endif +#endif bool changed = false; @@ -210,7 +212,8 @@ bool weave_ns::WeaveCommon::insertBefore(Module &m, Function &f, StringRef &a, } #ifdef PERFFLOWASPECT_WITH_ADIAK -bool weave_ns::WeaveCommon::insertAdiak(Module &m, Function &f) { +bool weave_ns::WeaveCommon::insertAdiak(Module &m, Function &f) +{ LLVMContext &context = m.getContext(); // the adiak_init() call @@ -221,7 +224,7 @@ bool weave_ns::WeaveCommon::insertAdiak(Module &m, Function &f) { PointerType *voidPtrTy = PointerType::getUnqual(int8Ty); // adiak_init function signature - std::vector initArgs = { voidPtrTy }; + std::vector initArgs = { voidPtrTy }; FunctionType *adiakInitType = FunctionType::get(voidTy, initArgs, false); FunctionCallee adiakInit = m.getOrInsertFunction("adiak_init", adiakInitType); @@ -230,62 +233,79 @@ bool weave_ns::WeaveCommon::insertAdiak(Module &m, Function &f) { BasicBlock &entry = f.getEntryBlock(); builder.SetInsertPoint(&entry, entry.begin()); - #ifdef PERFFLOWASPECT_WITH_MPI +#ifdef PERFFLOWASPECT_WITH_MPI // find the MPI communicator, MPI_COMM_WORLD // OpenMPI exposes this as ompi_comm_world (untested) // MPICH exposes this as a constant 0x44000000 GlobalVariable *gv = m.getGlobalVariable("ompi_mpi_comm_world"); - if (!gv) { + if (!gv) + { gv = m.getGlobalVariable("MPI_COMM_WORLD"); } - if (gv) { - arg = builder.CreateBitCast(gv, voidPtrTy, "mpi_comm_world_void"); + if (gv) + { + // ompi_mpi_comm_world is a pointer to mpi_predefined_communicator_t struct + // the first value holds the actual communicator + StructType *ompCommStruct = cast(gv->getValueType()); + Value *ompCommStructPtr = builder.CreateBitCast(gv, + PointerType::getUnqual(ompCommStruct)); + Value *commPtr = builder.CreateStructGEP(ompCommStruct, ompCommStructPtr, 0); + arg = builder.CreateBitCast(commPtr, voidPtrTy, "mpi_comm_world_void"); } - else { + else + { uint64_t mpiValue = 0x44000000; Value *commVal = ConstantInt::get(int32Ty, mpiValue); AllocaInst *alloc = builder.CreateAlloca(int32Ty, nullptr, "weave_mpi_comm"); builder.CreateStore(commVal, alloc); arg = builder.CreateBitCast( - alloc, - voidPtrTy, - "mpi_comm_world_void" - ); + alloc, + voidPtrTy, + "mpi_comm_world_void" + ); } CallInst *mpi = nullptr; // find each instruction to see if there is an MPI_Init call instruction - for (BasicBlock &bb : f) { - for (Instruction &i : bb) { - if (auto *call = dyn_cast(&i)) { - if (Function *callee = call->getCalledFunction()) { - if (callee->getName() == "MPI_Init") { + for (BasicBlock &bb : f) + { + for (Instruction &i : bb) + { + if (auto *call = dyn_cast(&i)) + { + if (Function *callee = call->getCalledFunction()) + { + if (callee->getName() == "MPI_Init") + { mpi = call; } } } } - if (mpi) break; + if (mpi) { break; } } - if (mpi) { + if (mpi) + { BasicBlock *mpiBlock = mpi->getParent(); auto insertPos = BasicBlock::iterator(mpi); ++insertPos; builder.SetInsertPoint(mpiBlock, insertPos); - } else { + } + else + { arg = Constant::getNullValue(voidPtrTy); } - builder.CreateCall(adiakInit, {arg}); - #else +#else arg = Constant::getNullValue(voidPtrTy); builder.CreateCall(adiakInit, {arg}); - #endif +#endif // call adiak_collect_all() FunctionType *collectType = FunctionType::get(int32Ty, {}, false); - FunctionCallee collectAll = m.getOrInsertFunction("adiak_collect_all", collectType); + FunctionCallee collectAll = m.getOrInsertFunction("adiak_collect_all", + collectType); builder.CreateCall(collectAll, {}); // call adiak_walltime() @@ -300,18 +320,22 @@ bool weave_ns::WeaveCommon::insertAdiak(Module &m, Function &f) { // adiak_fini signature FunctionType *adiakFinishType = FunctionType::get(voidTy, {}, false); - FunctionCallee adiakFinish = m.getOrInsertFunction("adiak_fini", adiakFinishType); + FunctionCallee adiakFinish = m.getOrInsertFunction("adiak_fini", + adiakFinishType); // find all places where the function terminates from a ReturnInst - std::vector returns; - for (BasicBlock &bb : f) { - if (auto *ret = dyn_cast(bb.getTerminator())) { + std::vector returns; + for (BasicBlock &bb : f) + { + if (auto *ret = dyn_cast(bb.getTerminator())) + { returns.push_back(ret); } } // insert adiak_fini at those return instructions - for (ReturnInst *ret : returns) { + for (ReturnInst *ret : returns) + { builder.SetInsertPoint(ret); builder.CreateCall(adiakFinish, {}); } From f9d269e2c6dc566bf9d25887270ce91498d5d896 Mon Sep 17 00:00:00 2001 From: Spencer Scott Greene Date: Wed, 22 Oct 2025 09:00:33 -0700 Subject: [PATCH 15/15] cuda wrapper --- src/c/test/CMakeLists.txt | 15 ++++++++++++--- src/c/test/smoketest_cuda_wrapper.cpp | 1 + 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/src/c/test/CMakeLists.txt b/src/c/test/CMakeLists.txt index 85cbc83a..019ba7c7 100644 --- a/src/c/test/CMakeLists.txt +++ b/src/c/test/CMakeLists.txt @@ -57,9 +57,18 @@ endif() if(PERFFLOWASPECT_WITH_CUDA) message(STATUS " [*] Adding test: smoketest_cuda") - set(CUDA_NVCC_FLAGS "-ccbin ${CMAKE_CXX_COMPILER} -Xcompiler=-Xclang -Xcompiler=-load -Xcompiler=-Xclang -Xcompiler=../../../weaver/weave/libWeavePass.so -Xcompiler=-fpass-plugin=../../../weaver/weave/libWeavePass.so") - cuda_add_executable(smoketest_cuda smoketest_cuda_wrapper.cpp smoketest_cuda_kernel.cu) - target_link_libraries(smoketest_cuda ${perfflow_deps} ${CUDA_LIBRARIES}) + list(APPEND CUDA_NVCC_FLAGS + "-ccbin ${CMAKE_CXX_COMPILER}" + ) + cuda_add_executable(smoketest_cuda + smoketest_cuda_wrapper.cpp + smoketest_cuda_kernel.cu + ) + set_source_files_properties(smoketest_cuda_wrapper.cpp COMPILE_FLAGS "-Xclang -load -Xclang ${CMAKE_BINARY_DIR}/weaver/weave/libWeavePass.so -fpass-plugin=${CMAKE_BINARY_DIR}/weaver/weave/libWeavePass.so -fPIC") + target_link_libraries(smoketest_cuda + ${perfflow_deps} + ${CUDA_LIBRARIES} + ) endif() configure_file(t0001-cbinding-basic.t.in diff --git a/src/c/test/smoketest_cuda_wrapper.cpp b/src/c/test/smoketest_cuda_wrapper.cpp index ed325429..0bca947c 100644 --- a/src/c/test/smoketest_cuda_wrapper.cpp +++ b/src/c/test/smoketest_cuda_wrapper.cpp @@ -10,6 +10,7 @@ #include "smoketest_cuda_kernel.cuh" +__attribute__((annotate("@critical_path(pointcut='around')"))) int main(int argc, char *argv[]) { Wrapper::wrapper();