Merge branch 'kaldi-asr:master' into master

Author: a2d8a4v

CMakeLists.txt

@@ -20,9 +20,25 @@ if(CONDA_ROOT)
 
 endif()
 
-include(third_party/get_third_party)
+option(BuildForFedora "Build for Fedora. Means that everything is build with Border tools" NO)
+
+if(BuildForFedora)
+
+    # You also need to install sudo dnf install lapack-devel openfst-devel
+    set(CMAKE_CXX_STANDARD 17)
+    set(CMAKE_CXX_STANDARD_REQUIRED ON)
+    set(CMAKE_CXX_EXTENSIONS OFF)
+
+    #find_package(PkgConfig REQUIRED)
+
+    #pkg_check_modules(FST REQUIRED fst)
+
+else()
+    include(third_party/get_third_party)
+
+    include(cmake/third_party/openfst.cmake)
+endif()
 
-include(cmake/third_party/openfst.cmake)
 
 find_package(PythonInterp)
 if(NOT PYTHON_EXECUTABLE)
@@ -43,8 +59,11 @@ execute_process(COMMAND ${PYTHON_EXECUTABLE}
 )
 unset(IS_LIB_SHARE)
 
-set(CMAKE_CXX_STANDARD 14)
-set(CMAKE_CXX_EXTENSIONS OFF)
+if(BuildForFedora)
+else()
+    set(CMAKE_CXX_STANDARD 14)
+    set(CMAKE_CXX_EXTENSIONS OFF)
+endif()
 set(CMAKE_INSTALL_MESSAGE LAZY) # hide "-- Up-to-date: ..."
 if(BUILD_SHARED_LIBS)
     set(CMAKE_POSITION_INDEPENDENT_CODE ON)
@@ -228,8 +247,19 @@ endif()
 #            PATHS "${CMAKE_CURRENT_SOURCE_DIR}/tools/openfst/include"
 #            REQUIRED)
 
+if(BuildForFedora)
+    # Version used used by Fedora 41 is 1.83
+    # TODO: Detect the right version and put it here.
+    add_definitions(-DOPENFST_VER=18300)
+#    link_directories(/usr/lib64)
+#    include_directories(/usr/include/fst)    
+endif()
+
 link_libraries(fst)
 
+
+
+
 # add all native libraries
 add_subdirectory(src/base) # NOTE, we need to patch the target with version from outside
 set_property(TARGET kaldi-base PROPERTY COMPILE_DEFINITIONS "KALDI_VERSION=\"${KALDI_VERSION}\"")

README.md

@@ -52,6 +52,22 @@ Development pattern for contributors
 Platform specific notes
 -----------------------
 
+### Fedora 41 (and later)
+
+In order to build it on Fedora 41 using the libraries that are provided by the distro, you need to install the development libraries and dependencies with
+
+```
+sudo dnf install lapack-devel openfst-devel
+```
+
+then build the package as follows:
+
+```
+cmake -S ./ -Bbuild/Release -DFETCHCONTENT_FULLY_DISCONNECTED=ON -DBuildForFedora=ON
+cmake --build /home/gerhard/workspace/kaldi/build/Release
+```
+
+
 ### PowerPC 64bits little-endian (ppc64le)
 
 - Kaldi is expected to work out of the box in RHEL >= 7 and Ubuntu >= 16.04 with
@@ -70,6 +86,6 @@ Platform specific notes
 ### Web Assembly
 
 - Kaldi supports cross compiling for Web Assembly for in-browser execution
-  using [emscripten](https://emscripten.org/) and CLAPACK.
-- See [this post](https://gitlab.inria.fr/kaldi.web/kaldi-wasm/-/wikis/build_details.md)
+  using [emscripten](https://emscripten.org) and OpenBLAS
+- See [this repo](https://github.com/msqr1/kaldi-wasm2)
   for a step-by-step description of the build process.

src/configure

@@ -17,7 +17,7 @@
 # ./configure --atlas-root=../tools/ATLAS/build
 # ./configure --use-cuda=no   # disable CUDA detection (will build cpu-only
 #                             # version of kaldi even on CUDA-enabled machine.
-# ./configure --use-cuda --cudatk-dir=/usr/local/cuda/ --cuda-arch=-arch=sm_70
+# ./configure --use-cuda=yes --cudatk-dir=/usr/local/cuda/ --cuda-arch=-arch=sm_70
 #        # Use cuda in /usr/local/cuda and set the arch to sm_70
 # ./configure --static --fst-root=/opt/cross/armv8hf \
 #   --atlas-root=/opt/cross/armv8hf --host=armv8-rpi3-linux-gnueabihf
@@ -69,7 +69,7 @@ Configuration options:
   --version             Display the version of 'configure' and exit
   --static              Build and link against static libraries [default=no]
   --shared              Build and link against shared libraries [default=no]
-  --use-cuda            Build with CUDA [default=yes]
+  --use-cuda            Build with CUDA [default=no]
   --with-cudadecoder    Build with CUDA decoder [default=yes]
   --cudatk-dir=DIR      CUDA toolkit directory
   --cuda-arch=FLAGS     Override the default CUDA_ARCH flags. See:
@@ -894,6 +894,12 @@ if is_set $HOST; then
     fi
   else
     TARGET_ARCH="$HOST"
+    if ! $static_fst || ! $static_math || $dynamic_kaldi; then
+      echo "WARNING: Dynamic libraries are not currently supported when compiling to WASM. Overriding --static, --static-math, and --static-fst."
+    fi
+    dynamic_kaldi=false
+    static_math=true
+    static_fst=true
   fi
 
   HOST_CXX="$HOST-c++"
@@ -1360,14 +1366,6 @@ or try another math library, e.g. --mathlib=OPENBLAS (Kaldi may be slower)."
 ** You can also use other matrix algebra libraries. For information, see:
 **   http://kaldi-asr.org/doc/matrixwrap.html"
     fi
-    if [ -f $OPENBLASROOT/lib/libopenblas.so ]; then
-      OPENBLASLIBDIR=$OPENBLASROOT/lib
-    elif [ -f $OPENBLASROOT/lib64/libopenblas.so ]; then
-      # in REDHAT/CentOS package installs, the library is located here
-      OPENBLASLIBDIR=$OPENBLASROOT/lib64
-    else
-      failure "Expected to find the file $OPENBLASROOT/lib/libopenblas.so"
-    fi
     if [ -f $OPENBLASROOT/include/cblas.h ] ; then
       OPENBLASINCDIR=$OPENBLASROOT/include
     elif [ -f $OPENBLASROOT/include/openblas/cblas.h ] ; then
@@ -1381,18 +1379,35 @@ or try another math library, e.g. --mathlib=OPENBLAS (Kaldi may be slower)."
       echo "** if it is a package-based install)."
       OPENBLASINCDIR="/usr/include"
     fi
-    echo "Your math library seems to be OpenBLAS from $OPENBLASROOT.  Configuring appropriately."
-    # TODO(kkm): Probably, OpenBLAS required libgfortran.so.3 at some point, but
-    # no longer does. *My* linker does not complain about a missing library, but
-    # is it safe to keep the reference if no longer required? Try to figure out
-    # how long ago the dependency was dropped.
     if $static_math; then
+      if [ -f $OPENBLASROOT/lib/libopenblas.a ]; then
+        OPENBLASLIBDIR=$OPENBLASROOT/lib
+      else
+        failure "Expected to find the file $OPENBLASROOT/lib/libopenblas.a"
+      fi
       echo "Configuring static OpenBlas since --static-math=yes"
-      OPENBLASLIBS="-L$OPENBLASLIBDIR -l:libopenblas.a -lgfortran"
+      OPENBLASLIBS="-L$OPENBLASLIBDIR -l:libopenblas.a"
+      # No Fortran for OpenBLAS
+      if [[ "$HOST" != WASM ]]; then
+        OPENBLASLIBS+="-lgfortran"
+      fi
     else
+      if [ -f $OPENBLASROOT/lib/libopenblas.so ]; then
+        OPENBLASLIBDIR=$OPENBLASROOT/lib
+      elif [ -f $OPENBLASROOT/lib64/libopenblas.so ]; then
+        # in REDHAT/CentOS package installs, the library is located here
+        OPENBLASLIBDIR=$OPENBLASROOT/lib64
+      else
+        failure "Expected to find the file $OPENBLASROOT/lib/libopenblas.so"
+      fi
       echo "Configuring dynamically loaded OpenBlas since --static-math=no (the default)"
       OPENBLASLIBS="-L$OPENBLASLIBDIR -lopenblas -lgfortran -Wl,-rpath=$OPENBLASLIBDIR"
     fi
+    echo "Your math library seems to be OpenBLAS from $OPENBLASROOT.  Configuring appropriately."
+    # TODO(kkm): Probably, OpenBLAS required libgfortran.so.3 at some point, but
+    # no longer does. *My* linker does not complain about a missing library, but
+    # is it safe to keep the reference if no longer required? Try to figure out
+    # how long ago the dependency was dropped.
     echo "OPENBLASINC = $OPENBLASINCDIR" >> kaldi.mk
     echo "OPENBLASLIBS = $OPENBLASLIBS" >> kaldi.mk
     echo >> kaldi.mk

src/makefiles/linux_clapack.mk

@@ -44,10 +44,11 @@ ifeq ($(findstring clang,$(COMPILER)),clang)
 CXXFLAGS += -Wno-mismatched-tags
 endif
 
-LDFLAGS = $(EXTRA_LDFLAGS) $(OPENFSTLDFLAGS) -rdynamic
+LDFLAGS = $(EXTRA_LDFLAGS) $(OPENFSTLDFLAGS)
 LDLIBS = $(EXTRA_LDLIBS) $(OPENFSTLIBS) $(CLAPACKLIBS) -lm -ldl
 
 ifneq ($(ARCH), WASM)
+    LDFLAGS += -rdynamic
     CXXFLAGS += -pthread
     LDLIBS += -lpthread
 endif

src/makefiles/linux_openblas.mk

@@ -25,7 +25,7 @@ CXXFLAGS = -std=$(CXXLANGVERSION) -I.. -isystem $(OPENFSTINC) -O1 $(EXTRA_CXXFLA
            -DOPENFST_VER=$(OPENFSTVER) \
            -DKALDI_DOUBLEPRECISION=$(DOUBLE_PRECISION) \
            -DHAVE_EXECINFO_H=1 -DHAVE_CXXABI_H -DHAVE_OPENBLAS -I$(OPENBLASINC) \
-           -msse -msse2 -pthread \
+           -msse -msse2 \
            -g
 
 ifeq ($(KALDI_FLAVOR), dynamic)
@@ -46,5 +46,13 @@ ifeq ($(findstring clang,$(COMPILER)),clang)
 CXXFLAGS += -Wno-mismatched-tags
 endif
 
-LDFLAGS = $(EXTRA_LDFLAGS) $(OPENFSTLDFLAGS) -rdynamic
-LDLIBS = $(EXTRA_LDLIBS) $(OPENFSTLIBS) $(OPENBLASLIBS) -lm -lpthread -ldl
+LDFLAGS = $(EXTRA_LDFLAGS) $(OPENFSTLDFLAGS)
+LDLIBS = $(EXTRA_LDLIBS) $(OPENFSTLIBS) $(OPENBLASLIBS) -lm -ldl
+
+ifneq ($(ARCH), WASM)
+  CXXFLAGS += -pthread
+  LDLIBS += -lpthread
+  LDFLAGS += -rdynamic
+else 
+  CXXFLAGS += -DKALDI_WASM
+endif

src/util/kaldi-thread.cc

@@ -22,8 +22,14 @@
 #include "util/kaldi-thread.h"
 
 namespace kaldi {
-int32 g_num_threads = 8;  // Initialize this global variable.
 
+int32 g_num_threads = 
+#ifdef KALDI_WASM
+0 // Everything will run on the main thread for WASM
+#else 
+8  // Initialize this global variable.
+#endif
+;
 MultiThreadable::~MultiThreadable() {
   // default implementation does nothing
 }

tools/Makefile

@@ -7,7 +7,7 @@ CC ?= gcc        # used for sph2pipe
 
 WGET ?= wget
 
-OPENFST_VERSION ?= 1.8.3
+OPENFST_VERSION ?= 1.8.4
 CUB_VERSION ?= 1.8.0
 # No '?=', since there exists only one version of sph2pipe.
 SPH2PIPE_VERSION = 2.5