Fix VST crashes on Linux (#7987)

Pass winebuild param to disable dynamicbase/aslr
Requires bleeding-edge wine version
See also https://gitlab.winehq.org/wine/wine/-/merge_requests/8786

---------

Co-authored-by: Dalton Messmer <[email protected]>
Co-authored-by: Tres Finocchiaro <[email protected]>

Author: 3 people

plugins/VstBase/CMakeLists.txt

@@ -47,6 +47,25 @@ foreach(var ${export_variables})
 	_export_var_to_external(${var})
 endforeach()
 
+# If we're building wine-powered RemoteVstPlugins for Linux,
+# use a custom winebuild to fix ASLR-related VST crashes.
+# See upstream bug report: https://bugs.winehq.org/show_bug.cgi?id=58480
+if(LMMS_BUILD_LINUX AND (LMMS_HAVE_VST_32 OR LMMS_HAVE_VST_64))
+	ExternalProject_Add(wine
+		GIT_REPOSITORY https://github.com/tresf/wine
+		GIT_TAG 1f8bb63e75baa5c9f901c8f50b4ea9dd69e0baa0
+		GIT_SHALLOW ON
+		SOURCE_DIR "${CMAKE_CURRENT_BINARY_DIR}/wine"
+		SOURCE_SUBDIR tools/winebuild
+		BUILD_ALWAYS ON
+		BUILD_IN_SOURCE ON
+		INSTALL_COMMAND ""
+	)
+	ExternalProject_Get_Property(wine SOURCE_DIR)
+	set(CUSTOM_WINEBUILD_EXECUTABLE "${SOURCE_DIR}/tools/winebuild/winebuild")
+	message(STATUS "Using custom winebuild: ${CUSTOM_WINEBUILD_EXECUTABLE}")
+endif()
+
 # build 32 bit version of RemoteVstPlugin
 if(LMMS_HAVE_VST_32)
 	INCLUDE("${CMAKE_CURRENT_LIST_DIR}/RemoteVstPlugin32.cmake")

plugins/VstBase/RemoteVstPlugin/CMakeLists.txt

@@ -72,18 +72,12 @@ if(MSVC)
 endif()
 
 # If we're building a Windows executable, add linker flags to disable ASLR
-# This is needed because several old VST dlls crash when ASLR is in use (see bug #7830)
+# This is needed because several old VST dlls crash when high entropy ASLR is in use
+# /HIGHENTROPYVA:YES relies on /DYNAMICBASE:YES so this ensures maximum VST compatiblity
 if(MSVC)
 	target_link_options(${EXE_NAME} PRIVATE "/DYNAMICBASE:NO")
-	if(BITNESS EQUAL 64)
-		target_link_options(${EXE_NAME} PRIVATE "/HIGHENTROPYVA:NO")
-	endif()
 elseif(IS_MINGW)
-	if(BITNESS EQUAL 32)
-		target_link_options(${EXE_NAME} PRIVATE "-Wl,--disable-dynamicbase")
-	else()
-		target_link_options(${EXE_NAME} PRIVATE "-Wl,--disable-dynamicbase,--disable-high-entropy-va")
-	endif()
+	target_link_options(${EXE_NAME} PRIVATE "-Wl,--disable-dynamicbase")
 endif()
 
 if(IS_MINGW)

plugins/VstBase/RemoteVstPlugin32.cmake

@@ -34,7 +34,9 @@ ELSEIF(LMMS_BUILD_LINUX)
 		CMAKE_ARGS
 			"${EXTERNALPROJECT_CMAKE_ARGS}"
 			"-DCMAKE_CXX_COMPILER=${WINEGCC}"
-			"-DCMAKE_CXX_FLAGS=-m32"
+			# Pass /DYNAMICBASE:NO to custom winebuild per #7987
+			"-DCMAKE_CXX_FLAGS=-m32 --winebuild \"${CUSTOM_WINEBUILD_EXECUTABLE}\" -Wb,--disable-dynamicbase"
+		DEPENDS wine
 	)
 
 	INSTALL(PROGRAMS "${CMAKE_CURRENT_BINARY_DIR}/../32/RemoteVstPlugin32" "${CMAKE_CURRENT_BINARY_DIR}/../32/RemoteVstPlugin32.exe.so" DESTINATION "${PLUGIN_DIR}/32")

plugins/VstBase/RemoteVstPlugin64.cmake

@@ -9,7 +9,9 @@ ELSEIF(LMMS_BUILD_LINUX)
 		CMAKE_ARGS
 			"${EXTERNALPROJECT_CMAKE_ARGS}"
 			"-DCMAKE_CXX_COMPILER=${WINEGCC}"
-			"-DCMAKE_CXX_FLAGS=${CXX_FLAGS}"
+			# Pass /DYNAMICBASE:NO to custom winebuild per #7987
+			"-DCMAKE_CXX_FLAGS=${CXX_FLAGS} --winebuild \"${CUSTOM_WINEBUILD_EXECUTABLE}\" -Wb,--disable-dynamicbase"
+		DEPENDS wine
 	)
 	INSTALL(PROGRAMS "${CMAKE_CURRENT_BINARY_DIR}/../RemoteVstPlugin64" "${CMAKE_CURRENT_BINARY_DIR}/../RemoteVstPlugin64.exe.so" DESTINATION "${PLUGIN_DIR}")
 ENDIF()