diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index ca91e50bdb10..0de793cf2144 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -11,7 +11,6 @@ config 64BIT
 
 config X86_32
 	def_bool !64BIT
-	select HAVE_FUNCTION_RET_TRACER
 
 config X86_64
 	def_bool 64BIT
@@ -30,6 +29,7 @@ config X86
 	select HAVE_FTRACE_MCOUNT_RECORD
 	select HAVE_DYNAMIC_FTRACE
 	select HAVE_FUNCTION_TRACER
+	select HAVE_FUNCTION_RET_TRACER if X86_32
 	select HAVE_FUNCTION_TRACE_MCOUNT_TEST
 	select HAVE_KVM if ((X86_32 && !X86_VOYAGER && !X86_VISWS && !X86_NUMAQ) || X86_64)
 	select HAVE_ARCH_KGDB if !X86_VOYAGER