[Offload][OpenMP] Move OMPT out of plugins (llvm#2282)

This PR addresses the issue that we currently cannot build `liboffload`
downstream given the OMPT symbols inside the plugin. Since liboffload is
moving forward upstream, we should enable building it downstream.
This is also a first step towards making the OMPT implementation (more)
acceptable to upstream.

## Problem
liboffload links the plugins but not libomptarget. libomptarget defines
some of the OMPT symbols that are used inside the plugins. As a result
you end up with two error modes: (1) when building offload you get
unresolved symbols linker errors. (2) If you were to add the files that
contains these symbols to the plugin build to address (1), you end up
with a multiply-defined symbols error in the libomptarget build.

## Solution
This patch moves all OMPT symbols out of the plugins and into an OMPT
library with a common interface `GenericProfilerTy` that is used inside
the plugin. The interface is then implemented in the OMPT library as
`OmptProfilerTy` which calls the appropriate existing OMPT infrastructure.
That OMPT library is linked to libomptarget but not liboffload. liboffload
links an empty default implementation for now. This separation was also
a request from upstream when we want to upstream OMPT support.

## Future Work
We need to clean up the whole OMPT build and code anyway, but I 
figured that should not be done in this patch, given that it is already
too large. The clean up should look at moving things into the right places 
and simplify the implementation where possible, given that we statically
link the plugins and all (potentially still existing) dlopen cruft can
go away.

Author: jplehr

offload/CMakeLists.txt

@@ -468,8 +468,7 @@ add_subdirectory(docs)
 # Build target agnostic offloading library.
 add_subdirectory(libomptarget)
 
-# FIXME: Re-enable once OMPT design allows
-# add_subdirectory(liboffload)
+add_subdirectory(liboffload)
 
 # Add tests.
 if(OFFLOAD_INCLUDE_TESTS)

offload/include/OpenMP/OMPT/Interface.h

@@ -473,20 +473,21 @@ template <typename FunctionPairTy, typename AsyncInfoTy, typename... ArgsTy>
 class TracerInterfaceRAII {
 public:
   TracerInterfaceRAII(FunctionPairTy Callbacks, AsyncInfoTy &AsyncInfo,
-                      int TracedDeviceId, ompt_callbacks_t EventType,
-                      ArgsTy... Args)
+                      void *OmptSpecificData, int TracedDeviceId,
+                      ompt_callbacks_t EventType, ArgsTy... Args)
       : Arguments(Args...), beginFunction(std::get<0>(Callbacks)) {
     __tgt_async_info *AI = AsyncInfo;
     if (isTracingEnabled(TracedDeviceId, EventType)) {
       auto Record = begin();
-      // Gets freed in interface.cpp, functions
-      // targetKernel and targetData once launching target operations returns.
-      if (!AI->ProfilerData)
-        AI->ProfilerData = new OmptEventInfoTy();
-      // TODO: Make sure this has the right type
-      auto OEI = reinterpret_cast<OmptEventInfoTy *>(AI->ProfilerData);
-      OEI->TraceRecord = Record;
-      OEI->NumTeams = 0;
+
+      // Access the already allocated profiler data and populate it
+      OmptEventInfoTy *ProfilerData =
+          reinterpret_cast<OmptEventInfoTy *>(OmptSpecificData);
+      ProfilerData->TraceRecord = Record;
+      ProfilerData->NumTeams = 0;
+
+      // Allows to pass down into the plugins via AsyncInfoTy
+      AI->ProfilerData = ProfilerData;
     } else {
       // Actively prevent further tracing of this event
       AI->ProfilerData = nullptr;

offload/include/OpenMP/OMPT/OmptTracing.h

@@ -102,6 +102,13 @@ ompt_set_result_t setTraceEventTy(int DeviceId, unsigned int Enable,
 /// Return thread id
 uint64_t getThreadId();
 
+/// See TracedDevices in OmptDeviceTracing.h
+extern std::map<int32_t, uint64_t> TracedDevices;
+/// Activate tracing on the given device
+void enableDeviceTracing(int DeviceId);
+/// Deactivate tracing on the given device
+void disableDeviceTracing(int DeviceId);
+
 /// Mutexes to serialize invocation of device registration and checks
 extern std::mutex DeviceAccessMutex;
 

offload/include/Shared/APITypes.h

@@ -25,6 +25,7 @@
 #include <variant>
 
 #ifdef OMPT_SUPPORT
+#include "OpenMP/OMPT/OmptEventInfoTy.h"
 #include <omp-tools.h>
 #endif
 #include <mutex>
@@ -94,9 +95,6 @@ struct __tgt_async_info {
   /// Use for sync interface. When false => synchronous execution
   bool ExecAsync = true;
   /// Maintain the actal data for OMPT.
-  /// TODO: Moving forward, this should become a void* to not expose OMPT data
-  /// type in general types
-  // llvm::omp::target::ompt::OmptEventInfoTy *OmptEventInfo = nullptr;
   void *ProfilerData = nullptr;
 };
 

offload/libomptarget/CMakeLists.txt

@@ -85,6 +85,9 @@ endforeach()
 
 target_compile_options(omptarget PRIVATE ${offload_compile_flags})
 target_link_options(omptarget PRIVATE ${offload_link_flags})
+if (OMPT_TARGET_DEFAULT AND LIBOMPTARGET_OMPT_SUPPORT)
+  target_link_libraries(omptarget PRIVATE PluginOmpt)
+endif()
 
 macro(check_plugin_target target)
 if (TARGET omptarget.rtl.${target})

offload/libomptarget/PluginManager.cpp

@@ -643,3 +643,21 @@ Expected<DeviceTy &> PluginManager::getDevice(uint32_t DeviceNo) {
                                        DeviceNo);
   return *DevicePtr;
 }
+
+#ifdef OMPT_SUPPORT
+
+#include "OmptProfiler.h"
+
+std::unique_ptr<llvm::omp::target::plugin::GenericProfilerTy>
+getProfilerToAttach() {
+  return std::make_unique<llvm::omp::target::ompt::OmptProfilerTy>();
+}
+
+#else
+
+std::unique_ptr<llvm::omp::target::plugin::GenericProfilerTy>
+getProfilerToAttach() {
+  return std::make_unique<llvm::omp::target::plugin::GenericProfilerTy>();
+}
+
+#endif

offload/libomptarget/device.cpp

@@ -245,12 +245,18 @@ int32_t DeviceTy::submitData(void *TgtPtrBegin, void *HstPtrBegin, int64_t Size,
           RegionInterface.getCallbacks<ompt_target_data_transfer_to_device>(),
           omp_get_initial_device(), HstPtrBegin, DeviceID, TgtPtrBegin, Size,
           /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);
+      // The Profiler can allocate specific data to be used to pass information
+      // from here to lower parts of the runtime system.
+      // NOTE: It is the responsibility of the programmer to ensure type
+      // compatibility and correct usage of the data. The profiler, however,
+      // OWNS the pointer and frees it at an appropriate time.
+      void *ProfData = RTL->getProfiler()->getProfilerSpecificData();
       // Only if 'TracedDeviceId' is actually traced, AsyncInfo->OmptEventInfo
       // is set and a trace record generated. Otherwise: No OMPT device tracing.
       TracerInterfaceRAII TargetDataSubmitTraceRAII(
           RegionInterface
               .getTraceGenerators<ompt_target_data_transfer_to_device>(),
-          AsyncInfo, /*TracedDeviceId=*/DeviceID,
+          AsyncInfo, ProfData, /*TracedDeviceId=*/DeviceID,
           /*EventType=*/ompt_callback_target_data_op, omp_get_initial_device(),
           HstPtrBegin, DeviceID, TgtPtrBegin, Size,
           /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);)
@@ -275,12 +281,18 @@ int32_t DeviceTy::retrieveData(void *HstPtrBegin, void *TgtPtrBegin,
           RegionInterface.getCallbacks<ompt_target_data_transfer_from_device>(),
           DeviceID, TgtPtrBegin, omp_get_initial_device(), HstPtrBegin, Size,
           /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);
+      // The Profiler can allocate specific data to be used to pass information
+      // from here to lower parts of the runtime system.
+      // NOTE: It is the responsibility of the programmer to ensure type
+      // compatibility and correct usage of the data. The profiler, however,
+      // OWNS the pointer and frees it at an appropriate time.
+      void *ProfData = RTL->getProfiler()->getProfilerSpecificData();
       // Only if 'TracedDeviceId' is actually traced, AsyncInfo->OmptEventInfo
       // is set and a trace record generated. Otherwise: No OMPT device tracing.
       TracerInterfaceRAII TargetDataSubmitTraceRAII(
           RegionInterface
               .getTraceGenerators<ompt_target_data_transfer_from_device>(),
-          AsyncInfo, /*TracedDeviceId=*/DeviceID,
+          AsyncInfo, ProfData, /*TracedDeviceId=*/DeviceID,
           /*EventType=*/ompt_callback_target_data_op, DeviceID, TgtPtrBegin,
           omp_get_initial_device(), HstPtrBegin, Size,
           /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);)
@@ -304,12 +316,18 @@ int32_t DeviceTy::dataExchange(void *SrcPtr, DeviceTy &DstDev, void *DstPtr,
           RegionInterface.getCallbacks<ompt_target_data_transfer_from_device>(),
           RTLDeviceID, SrcPtr, DstDev.RTLDeviceID, DstPtr, Size,
           /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);
+      // The Profiler can allocate specific data to be used to pass information
+      // from here to lower parts of the runtime system.
+      // NOTE: It is the responsibility of the programmer to ensure type
+      // compatibility and correct usage of the data. The profiler, however,
+      // OWNS the pointer and frees it at an appropriate time.
+      void *ProfData = RTL->getProfiler()->getProfilerSpecificData();
       // Only if 'TracedDeviceId' is actually traced, AsyncInfo->OmptEventInfo
       // is set and a trace record generated. Otherwise: No OMPT device tracing.
       TracerInterfaceRAII TargetDataExchangeTraceRAII(
           RegionInterface
               .getTraceGenerators<ompt_target_data_transfer_from_device>(),
-          AsyncInfo, /*TracedDeviceId=*/RTLDeviceID,
+          AsyncInfo, ProfData, /*TracedDeviceId=*/RTLDeviceID,
           /*EventType=*/ompt_callback_target_data_op, RTLDeviceID, SrcPtr,
           DstDev.RTLDeviceID, DstPtr, Size,
           /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);)

offload/libomptarget/interface.cpp

@@ -200,14 +200,6 @@ targetData(ident_t *Loc, int64_t DeviceId, int32_t ArgNum, void **ArgsBase,
       Rc = AsyncInfo.synchronize();
   }
 
-#ifdef OMPT_SUPPORT
-  if (__tgt_async_info *AI = AsyncInfo; AI->ProfilerData) {
-    auto OmptData = reinterpret_cast<OmptEventInfoTy*>(AI->ProfilerData);
-    // These deletes are going to go into the OmptProfiler
-    delete OmptData;
-  }
-#endif
-
   handleTargetOutcome(Rc == OFFLOAD_SUCCESS, Loc);
 }
 
@@ -493,23 +485,6 @@ static inline int targetKernel(ident_t *Loc, int64_t DeviceId, int32_t NumTeams,
     }
   }
 
-#ifdef OMPT_SUPPORT
-  if (__tgt_async_info *AI = AsyncInfo; AI->ProfilerData) {
-    auto OmptData = reinterpret_cast<OmptEventInfoTy*>(AI->ProfilerData);
-    // These deletes are going to go into the OmptProfiler
-    delete OmptData;
-  }
-
-  for (TargetAsyncInfoTy *LocalTAI : TargetAsyncInfos) {
-    AsyncInfoTy &AsyncInfo = *LocalTAI;
-    if (__tgt_async_info *AI = AsyncInfo; AI->ProfilerData) {
-      auto OmptData = reinterpret_cast<OmptEventInfoTy*>(AI->ProfilerData);
-      // These deletes are going to go into the OmptProfiler
-      delete OmptData;
-    }
-  }
-#endif
-
   // Deallocate the multi-device async infos if any were allocated.
   for (TargetAsyncInfoTy *LocalTAI : TargetAsyncInfos)
     delete LocalTAI;

offload/libomptarget/omptarget.cpp

@@ -2101,6 +2101,13 @@ int target(ident_t *Loc, DeviceTy &Device, void *HostPtr,
     InterfaceRAII TargetSubmitRAII(
         RegionInterface.getCallbacks<ompt_callback_target_submit>(), NumTeams);
 
+    // The Profiler can allocate specific data to be used to pass information
+    // from here to lower parts of the runtime system.
+    // NOTE: It is the responsibility of the programmer to ensure type
+    // compatibility and correct usage of the data. The profiler, however, OWNS
+    // the pointer and frees it at an appropriate time.
+    void *ProfData = Device.RTL->getProfiler()->getProfilerSpecificData();
+
     // Calls "begin" for the OMPT trace record and let the plugin
     // enqueue the stop operation for after the kernel is done. The stop
     // operation completes the trace record entry with the information from
@@ -2109,7 +2116,7 @@ int target(ident_t *Loc, DeviceTy &Device, void *HostPtr,
     // set and a trace record generated. Otherwise: No OMPT device tracing.
     TracerInterfaceRAII TargetTraceRAII(
         RegionInterface.getTraceGenerators<ompt_callback_target_submit>(),
-        AsyncInfo, /*TracedDeviceId=*/DeviceId,
+        AsyncInfo, ProfData, /*TracedDeviceId=*/DeviceId,
         /*EventType=*/ompt_callback_target_submit, DeviceId, NumTeams);
 #endif
     Ret = Device.launchKernel(TgtEntryPtr, TgtArgs.data(), TgtOffsets.data(),