QtCreator. CUDA. Make final linking with NVCC

[Sacha_D]

@Gerd said in QtCreator. CUDA. Make final linking with NVCC:

QMAKE_COMPILER_DEFINES

Now I got this (warnings for all .cu files):

nvcc ../src/word-correction-fnn-thrust.cu
nvcc ../src/word-correction-fnn-cuda.cu
nvcc ../src/actfunc.cu
g++: warning: /tmp/tmpxft_000011d4_00000000-12_cu_word-correction-fnn-thrust_dlink.o: linker input file unused because linking not done
g++: warning: /tmp/tmpxft_000011d4_00000000-10_word-correction-fnn-thrust.o: linker input file unused because linking not done
g++: warning: /tmp/tmpxft_000011da_00000000-12_cu_actfunc_dlink.o: linker input file unused because linking not done
g++: warning: /tmp/tmpxft_000011da_00000000-10_actfunc.o: linker input file unused because linking not done
g++: warning: /tmp/tmpxft_000011d9_00000000-12_cu_word-correction-fnn-cuda_dlink.o: linker input file unused because linking not done
g++: warning: /tmp/tmpxft_000011d9_00000000-10_word-correction-fnn-cuda.o: linker input file unused because linking not done
link cuda device-code
Makefile:265: recipe for target '13-word-correction-fnn' failed
nvlink fatal   : Could not open input file 'cuda/debug/13-word-correction-fnn_cu_device_link.o'
make: *** [13-word-correction-fnn] Error 1

[Gerd]

could you show your .pro-file please

[Sacha_D]

@Gerd
.pro file

#-------------------------------------------------
#
# Project created by QtCreator 2019-12-10T19:31:27
#
#-------------------------------------------------

QT       += core gui

greaterThan(QT_MAJOR_VERSION, 4): QT += widgets

TARGET = 13-word-correction-fnn
TEMPLATE = app
# The following define makes your compiler emit warnings if you use
# any feature of Qt which has been marked as deprecated (the exact warnings
# depend on your compiler). Please consult the documentation of the
# deprecated API in order to know how to port your code away from it.
DEFINES += QT_DEPRECATED_WARNINGS

# You can also make your code fail to compile if you use deprecated APIs.
# In order to do so, uncomment the following line.
# You can also select to disable deprecated APIs only up to a certain version of Qt.
#DEFINES += QT_DISABLE_DEPRECATED_BEFORE=0x060000    # disables all the APIs deprecated before Qt 6.0.0


CONFIG += c++11

SOURCES += \
        main.cpp \
        mainwindow.cpp

HEADERS += \
        mainwindow.h \
        word-correction-fnn-cuda.h \
        actfunc.h

FORMS += \
        mainwindow.ui

QMAKE_CXXFLAGS += "-fno-sized-deallocation"

# Cuda sources
CUDA_SOURCES += word-correction-fnn-thrust.cu \
                word-correction-fnn-cuda.cu \
                actfunc.cu
# Project dir and outputs
PROJECT_DIR = $$system(pwd)
OBJECTS_DIR = $$PROJECT_DIR/Obj
DESTDIR = ../bin
include($$PWD/cuda.pri)

cuda.pri file

CUDA_BASE=$$quote($$clean_path($$(CUDA_PATH)))
isEmpty( CUDA_BASE ) {
	error("environment-variable CUDA_PATH is not defined!")
}
CUDA_BIN_PATH=$$CUDA_BASE/bin
CUDA_INC_PATH=$$CUDA_BASE/include
#CUDA_LIB_PATH=$$CUDA_BASE/lib
CUDA_LIB_PATH=$$CUDA_BASE/lib64
#!contains(QMAKE_HOST.arch, x86_64) {
#   CUDA_LIB_PATH=$$CUDA_LIB_PATH/win32
#} else {
#   CUDA_LIB_PATH=$$CUDA_LIB_PATH/x64
#}	

# GPU architecture
CUDA_ARCH = sm_35
CUDA_VARCH = compute_35


# Add the necessary libraries
# CUDA < 9.0
#CUDA_LIBS=cudart_static nppi nppc 
# CUDA >= 10.0
CUDA_LIBS=cuda cudart_static nppc curand cudadevrt


# setting the CUdaCompiler
#QMAKE_CUC = $$CUDA_BIN_PATH/nvcc.exe
QMAKE_CUC = $$CUDA_BIN_PATH/nvcc
win32 {
	!exists($$QMAKE_CUC) {
		warning("can't find cuda compiler($$QMAKE_CUC)")
	}
	!exists($$CUDA_INC_PATH/cuda.h) {
		warning("can't find cuda include ($$CUDA_INC_PATH/cuda.h)")
	}
	!exists($$CUDA_LIB_PATH/cuda.lib) {
		warning("can't find cuda lib ($$CUDA_LIB_PATH/cuda.lib)")
	}
}

# Cuda extra-compiler for handling files specified in the CUDA_SOURCES variable
{
   cu.name = Cuda Sourcefiles
	cu.input = CUDA_SOURCES
	cu.dependency_type = TYPE_C
	cu.CONFIG += no_link
	cu.variable_out = OBJECTS
	isEmpty(QMAKE_CUC) {
      win32:QMAKE_CUC = $$CUDA_BIN_PATH/nvcc.exe
      else:QMAKE_CUC = nvcc
	}

	isEmpty(CU_DIR):CU_DIR = .
	isEmpty(QMAKE_CPP_MOD_CU):QMAKE_CPP_MOD_CU = cu_
	isEmpty(QMAKE_EXT_CPP_CU):QMAKE_EXT_CPP_CU = .cu
	INCLUDEPATH += $$CUDA_INC_PATH

   CONFIG(debug, debug|release) {
      QMAKE_CUFLAGS += $$QMAKE_CXXFLAGS $$QMAKE_CXXFLAGS_DEBUG $$QMAKE_CXXFLAGS_RTTI_ON $$QMAKE_CXXFLAGS_WARN_ON $$QMAKE_CXXFLAGS_STL_ON 
		QMAKE_NVVFLAGS += -G
   } 	
   CONFIG(release, debug|release) {
      QMAKE_CUFLAGS += $$QMAKE_CXXFLAGS $$QMAKE_CXXFLAGS_RELEASE $$QMAKE_CXXFLAGS_RTTI_ON $$QMAKE_CXXFLAGS_WARN_ON $$QMAKE_CXXFLAGS_STL_ON 
   } 	
	#since qt5.9 they use /Zc:rvalueCast- /Zc:inline- in the msvc-mkspecs
	# we have to switch that off!! 
	# else linking __device__ __managed__ vars wont work correctly (in release mode only)
   QMAKE_CUFLAGS = $$replace(QMAKE_CUFLAGS, -Zc:inline, )

        QMAKE_NVVFLAGS += -gencode arch=$$CUDA_VARCH,code=$$CUDA_ARCH
	QMAKE_NVVFLAGS += -rdc=true 
	# -keep for holding intermediat files
	QMAKE_CUEXTRAFLAGS += -Xcompiler $$join(QMAKE_CUFLAGS, ",")
   !contains(QMAKE_HOST.arch, x86_64) {
      ## Windows x86 (32bit) specific build here
      QMAKE_CUEXTRAFLAGS += --machine 32 --debug
   } else {
      ## Windows x64 (64bit) specific build here
      QMAKE_CUEXTRAFLAGS += --machine 64
   }	
   #QMAKE_CUEXTRAFLAGS += $(DEFINES) $(INCPATH) $$join(QMAKE_COMPILER_DEFINES, " -D", -D)  -Xcompiler #/Zc:__cplusplus   this is for Windows
   QMAKE_CUEXTRAFLAGS += $(DEFINES) $(INCPATH) -Xcompiler #/Zc:__cplusplus   this is for Windows

        #QMAKE_CUEXTRAFLAGS += -Xcudafe "--diag_suppress=field_without_dll_interface"      this is for Windows
        #QMAKE_CUEXTRAFLAGS += -Xcudafe "--diag_suppress=code_is_unreachable"         this is for Windows
	#-Xcudafe "--diag_suppress=boolean_controlling_expr_is_constant"

   CONFIG(debug, debug|release) {
      CUDA_OBJ_DIR = cuda/debug
   } else {
      CUDA_OBJ_DIR = cuda/release
   } 
   cu.dependency_type = TYPE_C 
	cu.commands = \"$$QMAKE_CUC\" $$QMAKE_NVVFLAGS $$QMAKE_CUEXTRAFLAGS -c -o $${CUDA_OBJ_DIR}/$${QMAKE_CPP_MOD_CU}${QMAKE_FILE_BASE}$${QMAKE_EXT_OBJ} ${QMAKE_FILE_NAME}$$escape_expand(\\n\\t)
	cu.output = $${CUDA_OBJ_DIR}/$${QMAKE_CPP_MOD_CU}${QMAKE_FILE_BASE}$${QMAKE_EXT_OBJ}
	#silent:cu.commands = @echo nvcc ${QMAKE_FILE_IN} && $$cu.commands
	cu.commands = @echo nvcc ${QMAKE_FILE_IN} && $$cu.commands
   cu.commands = $$replace(cu.commands,-D__cplusplus=199711L,)
	QMAKE_EXTRA_COMPILERS += cu

	build_pass|isEmpty(BUILDS):cuclean.depends = compiler_cu_clean
	else:cuclean.CONFIG += recursive
	QMAKE_EXTRA_TARGETS += cuclean

   # another compiler-entry for linking the device-code
	device_link_target.target = $${CUDA_OBJ_DIR}/$${TARGET}_cu_device_link$${QMAKE_EXT_OBJ}
	#device_link_target.CONFIG += no_check_exist executable
	for(var, CUDA_SOURCES) {
	  var = $$basename(var)
	  var = $$replace(var,"\\.cu",$${QMAKE_EXT_OBJ})
	  CUDEP += $${CUDA_OBJ_DIR}/$${QMAKE_CPP_MOD_CU}$$var
	}
	var = $$basename(CUDEVLINK)
	var = $$replace(var,"\\.cu",$${QMAKE_EXT_OBJ})
   CUDEVLINK_OBJ = $${CUDA_OBJ_DIR}/$${QMAKE_CPP_MOD_CU}$$var

	cu_devlink.output = $${CUDA_OBJ_DIR}/$${TARGET}_cu_device_link$${QMAKE_EXT_OBJ}
	cu_devlink.input = CUDEVLINK
	cu_devlink.depends = $$CUDEP 
	cu_devlink.dependency_type = TYPE_C 
        cu_devlink.commands = @echo "link cuda device-code" && \"$$QMAKE_CUC\" -dlink -w -gencode arch=$$CUDA_VARCH,code=$$CUDA_ARCH $$QMAKE_CUEXTRAFLAGS -o $$device_link_target.target $$CUDEP
	cu_devlink.name = cuda_devlink
	cu_devlink.variable_out = OBJECTS
	cu_devlink.CONFIG = silent
	QMAKE_EXTRA_COMPILERS += cu_devlink

   QMAKE_PRE_LINK += $${cu_devlink.commands}
}

# add the cuda-libraries to the project
LIBS += -L$$CUDA_LIB_PATH
for(lnam, CUDA_LIBS) {
  LIBS+=$$join(lnam, " -l", -l)
}
 

[Gerd]

create an empty file file named "cu_devlink.cu" (or any other name) and put it in a node named "CUDEVLINK"

can't see this in your pro-file...

[Sacha_D]

@Gerd I added. The error is the same.

#...
# Project dir and outputs
PROJECT_DIR = $$system(pwd)
OBJECTS_DIR = $$PROJECT_DIR/Obj
DESTDIR = ../bin

CUDEVLINK = cu_devlink.cu
include($$PWD/cuda.pri)```

[SGaist]

Shouldn't you use CUDA_SOURCES ?

[Gerd]

change this line

QMAKE_CUEXTRAFLAGS += $(DEFINES) $(INCPATH) -Xcompiler #/Zc:__cplusplus   this is for Windows

to

QMAKE_CUEXTRAFLAGS += $(DEFINES) $(INCPATH)

[Sacha_D]

@Gerd Thank you! It works! I also changed:

CUDA_LIBS=cuda cudart_static nppc curand cudadevrt
#to
CUDA_LIBS=cuda cudart nppc curand cudadevrt

[Gerd]

please set this thread to solved than...

[Sacha_D]

@Gerd One more question: where I need to place

| sed \"s/^.*: //\"

to go to error line in .cu file by double clicking in QtCreactor.

[Gerd]

in your first post this was appended to the "cuda_depend_command".
In my solution i havent implemnted a "cu.depend_command". You can add this similar to that in your first post.

[tehfury87]

@Gerd Thank you for these build instructions! Though I only found this post AFTER figuring out the problem. My CUDA code stopped working after upgrading Qt and some other libraries. I was getting invalid symbol and invalid texture errors at runtime, not linking errors. So it took days to track this down. How did you find this?

	#since qt5.9 they use /Zc:rvalueCast- /Zc:inline- in the msvc-mkspecs
	# we have to switch that off!! 
	# else linking __device__ __managed__ vars wont work correctly (in release mode only)
   QMAKE_CUFLAGS = $$replace(QMAKE_CUFLAGS, -Zc:inline, )

Do you think it makes sense to report this change as a bug in qmake? It's easy to work around and I doubt they'd revert it, but it's a frustrating invisible change that breaks things.

Here is the change that broke it:
https://code.qt.io/cgit/qt/qtbase.git/commit/mkspecs/common/msvc-version.conf?h=5.9&id=255d291efd5ed3e193a6340055c35887f687f0ca

To fix the problem and make the build consistent between Qt 5.8 and newer, I added this to my general configuration:

equals(QT_MAJOR_VERSION, 5):lessThan(QT_MINOR_VERSION, 9):*msvc* {
  QMAKE_CXXFLAGS += -Zc:rvalueCast
  QMAKE_CXXFLAGS += -Zc:referenceBinding
  # Need to unset this for CUDA code
  QMAKE_CXXFLAGS += -Zc:inline
}

and this to just the library that uses CUDA:

*msvc* {
  # Fix symbols and textures removed from library
  # https://forum.qt.io/post/584959
  QMAKE_CXXFLAGS -= -Zc:inline
  QMAKE_CXXFLAGS += -Zc:inline-
}