mirror of
https://github.com/zebrajr/tensorflow.git
synced 2026-01-15 12:15:41 +00:00
Convert configure to python.
PiperOrigin-RevId: 163114551
This commit is contained in:
A. Unique TensorFlower
committed by
TensorFlower Gardener
TensorFlower Gardener
parent
6f70e57802
commit
73ea287120
814
configure
vendored
814
configure
vendored
@@ -3,816 +3,12 @@
|
||||
set -e
|
||||
set -o pipefail
|
||||
|
||||
MIN_BAZEL_VERSION=0.4.5
|
||||
|
||||
# Find out the absolute path to where ./configure resides
|
||||
pushd `dirname $0` > /dev/null
|
||||
SOURCE_BASE_DIR=`pwd -P`
|
||||
popd > /dev/null
|
||||
|
||||
PLATFORM="$(uname -s | tr 'A-Z' 'a-z')"
|
||||
|
||||
function is_linux() {
|
||||
[[ "${PLATFORM}" == "linux" ]]
|
||||
}
|
||||
|
||||
function is_macos() {
|
||||
[[ "${PLATFORM}" == "darwin" ]]
|
||||
}
|
||||
|
||||
function is_windows() {
|
||||
# On windows, the shell script is actually running in msys
|
||||
[[ "${PLATFORM}" =~ msys_nt*|mingw*|cygwin*|uwin* ]]
|
||||
}
|
||||
|
||||
function is_ppc64le() {
|
||||
[[ "$(uname -m)" == "ppc64le" ]]
|
||||
}
|
||||
|
||||
function sed_in_place() {
|
||||
sed -e $1 $2 > "$2.bak"
|
||||
mv "$2.bak" $2
|
||||
}
|
||||
|
||||
function write_to_bazelrc() {
|
||||
echo "$1" >> .tf_configure.bazelrc
|
||||
}
|
||||
|
||||
function write_action_env_to_bazelrc() {
|
||||
write_to_bazelrc "build --action_env $1=\"$2\""
|
||||
}
|
||||
|
||||
function python_path {
|
||||
"$PYTHON_BIN_PATH" - <<END
|
||||
from __future__ import print_function
|
||||
import site
|
||||
import os
|
||||
|
||||
try:
|
||||
input = raw_input
|
||||
except NameError:
|
||||
pass
|
||||
|
||||
python_paths = []
|
||||
if os.getenv('PYTHONPATH') is not None:
|
||||
python_paths = os.getenv('PYTHONPATH').split(':')
|
||||
try:
|
||||
library_paths = site.getsitepackages()
|
||||
except AttributeError:
|
||||
from distutils.sysconfig import get_python_lib
|
||||
library_paths = [get_python_lib()]
|
||||
all_paths = set(python_paths + library_paths)
|
||||
|
||||
paths = []
|
||||
for path in all_paths:
|
||||
if os.path.isdir(path):
|
||||
paths.append(path)
|
||||
|
||||
print(",".join(paths))
|
||||
END
|
||||
}
|
||||
|
||||
function setup_python {
|
||||
## Set up python-related environment settings:
|
||||
while true; do
|
||||
fromuser=""
|
||||
if [ -z "$PYTHON_BIN_PATH" ]; then
|
||||
default_python_bin_path=$(which python || which python3 || true)
|
||||
read -p "Please specify the location of python. [Default is $default_python_bin_path]: " PYTHON_BIN_PATH
|
||||
fromuser="1"
|
||||
if [ -z "$PYTHON_BIN_PATH" ]; then
|
||||
PYTHON_BIN_PATH=$default_python_bin_path
|
||||
fi
|
||||
fi
|
||||
if [ -e "$PYTHON_BIN_PATH" ]; then
|
||||
break
|
||||
fi
|
||||
echo "Invalid python path. ${PYTHON_BIN_PATH} cannot be found" 1>&2
|
||||
if [ -z "$fromuser" ]; then
|
||||
exit 1
|
||||
fi
|
||||
PYTHON_BIN_PATH=""
|
||||
# Retry
|
||||
done
|
||||
|
||||
if [ -z "$PYTHON_LIB_PATH" ]; then
|
||||
# Split python_path into an array of paths, this allows path containing spaces
|
||||
IFS=',' read -r -a python_lib_path <<< "$(python_path)"
|
||||
|
||||
if [ 1 = "$USE_DEFAULT_PYTHON_LIB_PATH" ]; then
|
||||
PYTHON_LIB_PATH=${python_lib_path[0]}
|
||||
echo "Using python library path: $PYTHON_LIB_PATH"
|
||||
|
||||
else
|
||||
echo "Found possible Python library paths:"
|
||||
for x in "${python_lib_path[@]}"; do
|
||||
echo " $x"
|
||||
done
|
||||
set -- "${python_lib_path[@]}"
|
||||
echo "Please input the desired Python library path to use. Default is [$1]"
|
||||
read b || true
|
||||
if [ "$b" == "" ]; then
|
||||
PYTHON_LIB_PATH=${python_lib_path[0]}
|
||||
echo "Using python library path: $PYTHON_LIB_PATH"
|
||||
else
|
||||
PYTHON_LIB_PATH="$b"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ ! -x "$PYTHON_BIN_PATH" ] || [ -d "$PYTHON_BIN_PATH" ]; then
|
||||
echo "PYTHON_BIN_PATH is not executable. Is it the python binary?"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
local python_major_version
|
||||
python_major_version=$("${PYTHON_BIN_PATH}" -c 'from __future__ import print_function; import sys; print(sys.version_info[0]);' | head -c1)
|
||||
if [ -z "$python_major_version" ]; then
|
||||
echo -e "\n\nERROR: Problem getting python version. Is $PYTHON_BIN_PATH the correct python binary?"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Convert python path to Windows style before writing into bazel.rc
|
||||
if is_windows; then
|
||||
PYTHON_BIN_PATH="$(cygpath -m "$PYTHON_BIN_PATH")"
|
||||
PYTHON_LIB_PATH="$(cygpath -m "$PYTHON_LIB_PATH")"
|
||||
fi
|
||||
|
||||
# Set-up env variables used by python_configure.bzl
|
||||
write_action_env_to_bazelrc "PYTHON_BIN_PATH" "$PYTHON_BIN_PATH"
|
||||
write_action_env_to_bazelrc "PYTHON_LIB_PATH" "$PYTHON_LIB_PATH"
|
||||
write_to_bazelrc "build --define PYTHON_BIN_PATH=\"$PYTHON_BIN_PATH\""
|
||||
write_to_bazelrc "build --define PYTHON_LIB_PATH=\"$PYTHON_LIB_PATH\""
|
||||
write_to_bazelrc "build --force_python=py$python_major_version"
|
||||
write_to_bazelrc "build --host_force_python=py$python_major_version"
|
||||
write_to_bazelrc "build --python${python_major_version}_path=\"$PYTHON_BIN_PATH\""
|
||||
write_to_bazelrc "test --force_python=py$python_major_version"
|
||||
write_to_bazelrc "test --host_force_python=py$python_major_version"
|
||||
write_to_bazelrc "test --define PYTHON_BIN_PATH=\"$PYTHON_BIN_PATH\""
|
||||
write_to_bazelrc "test --define PYTHON_LIB_PATH=\"$PYTHON_LIB_PATH\""
|
||||
write_to_bazelrc "run --define PYTHON_BIN_PATH=\"$PYTHON_BIN_PATH\""
|
||||
write_to_bazelrc "run --define PYTHON_LIB_PATH=\"$PYTHON_LIB_PATH\""
|
||||
|
||||
# Write tools/python_bin_path.sh
|
||||
echo "export PYTHON_BIN_PATH=\"$PYTHON_BIN_PATH\"" > tools/python_bin_path.sh
|
||||
}
|
||||
|
||||
function version {
|
||||
echo "$@" | awk -F. '{ printf("%03d%03d%03d\n", $1,$2,$3); }';
|
||||
}
|
||||
|
||||
|
||||
bazel version > bazel.version
|
||||
set +e
|
||||
curr_bazel_version=$(grep -m 1 'Build label:' bazel.version | cut -d ' ' -f3)
|
||||
set -e
|
||||
rm -f bazel.version
|
||||
|
||||
|
||||
echo "You have bazel $curr_bazel_version installed."
|
||||
if [ -z "$curr_bazel_version" ]; then
|
||||
echo "WARNING: current bazel installation is not a release version."
|
||||
echo "Make sure you are running at least bazel $MIN_BAZEL_VERSION."
|
||||
elif [ "$(version "$MIN_BAZEL_VERSION")" -gt "$(version "$curr_bazel_version")" ]; then
|
||||
echo "Please upgrade your bazel installation to version $MIN_BAZEL_VERSION or higher to build TensorFlow!"
|
||||
echo "Exiting..."
|
||||
exit 1
|
||||
if [ -z "$PYTHON_BIN_PATH" ]; then
|
||||
PYTHON_BIN_PATH=$(which python || which python3 || true)
|
||||
fi
|
||||
|
||||
# This file contains customized config settings.
|
||||
rm -f .tf_configure.bazelrc
|
||||
touch .tf_configure.bazelrc
|
||||
if [[ ! -e .bazelrc ]]; then
|
||||
if [[ -e "${HOME}/.bazelrc" ]]; then
|
||||
echo "import ${HOME}/.bazelrc" >.bazelrc
|
||||
else
|
||||
touch .bazelrc
|
||||
fi
|
||||
fi
|
||||
sed_in_place "/tf_configure/d" .bazelrc
|
||||
echo "import %workspace%/.tf_configure.bazelrc" >> .bazelrc
|
||||
# Set all env variables
|
||||
$PYTHON_BIN_PATH configure.py
|
||||
|
||||
# Delete any leftover BUILD files from the Makefile build, which would interfere
|
||||
# with Bazel parsing.
|
||||
MAKEFILE_DOWNLOAD_DIR=tensorflow/contrib/makefile/downloads
|
||||
if [ -d "${MAKEFILE_DOWNLOAD_DIR}" ]; then
|
||||
find ${MAKEFILE_DOWNLOAD_DIR} -type f -name '*BUILD' -delete
|
||||
fi
|
||||
|
||||
setup_python
|
||||
|
||||
## Set up MKL related environment settings
|
||||
write_to_bazelrc 'build:mkl --define with_mkl_support=true'
|
||||
write_to_bazelrc 'build:mkl --define using_mkl=true'
|
||||
write_to_bazelrc 'build:mkl -c opt'
|
||||
write_to_bazelrc 'build:mkl --copt="-DEIGEN_USE_VML"'
|
||||
echo ""
|
||||
echo "Add \"--config=mkl\" to your bazel command to build with MKL support."
|
||||
echo "Please note that MKL on MacOS or windows is still not supported."
|
||||
echo "If you would like to use a local MKL instead of downloading, please "
|
||||
echo " set the environment variable \"TF_MKL_ROOT\" every time before build."
|
||||
echo ""
|
||||
## End MKL setup
|
||||
|
||||
## Set up architecture-dependent optimization flags.
|
||||
if [ -z "$CC_OPT_FLAGS" ]; then
|
||||
if is_ppc64le; then
|
||||
# gcc on ppc64le does not support -march, use mcpu instead
|
||||
default_cc_opt_flags="-mcpu=native"
|
||||
else
|
||||
default_cc_opt_flags="-march=native"
|
||||
fi
|
||||
read -p "Please specify optimization flags to use during compilation when bazel option "\
|
||||
"\"--config=opt\" is specified [Default is $default_cc_opt_flags]: " CC_OPT_FLAGS
|
||||
if [ -z "$CC_OPT_FLAGS" ]; then
|
||||
CC_OPT_FLAGS=$default_cc_opt_flags
|
||||
fi
|
||||
fi
|
||||
|
||||
if is_windows; then
|
||||
TF_NEED_GCP=0
|
||||
TF_NEED_HDFS=0
|
||||
TF_NEED_JEMALLOC=0
|
||||
TF_NEED_OPENCL=0
|
||||
TF_CUDA_CLANG=0
|
||||
fi
|
||||
|
||||
if is_linux; then
|
||||
while [ "$TF_NEED_JEMALLOC" == "" ]; do
|
||||
read -p "Do you wish to use jemalloc as the malloc implementation? [Y/n] "\
|
||||
INPUT
|
||||
case $INPUT in
|
||||
[Yy]* ) echo "jemalloc enabled"; TF_NEED_JEMALLOC=1;;
|
||||
[Nn]* ) echo "jemalloc disabled"; TF_NEED_JEMALLOC=0;;
|
||||
"" ) echo "jemalloc enabled"; TF_NEED_JEMALLOC=1;;
|
||||
* ) echo "Invalid selection: " $INPUT;;
|
||||
esac
|
||||
done
|
||||
else
|
||||
TF_NEED_JEMALLOC=0
|
||||
fi
|
||||
|
||||
if [[ "$TF_NEED_JEMALLOC" == "1" ]]; then
|
||||
write_to_bazelrc 'build --define with_jemalloc=true'
|
||||
fi
|
||||
|
||||
while [[ "$TF_NEED_GCP" == "" ]]; do
|
||||
read -p "Do you wish to build TensorFlow with "\
|
||||
"Google Cloud Platform support? [y/N] " INPUT
|
||||
case $INPUT in
|
||||
[Yy]* ) echo "Google Cloud Platform support will be enabled for "\
|
||||
"TensorFlow"; TF_NEED_GCP=1;;
|
||||
[Nn]* ) echo "No Google Cloud Platform support will be enabled for "\
|
||||
"TensorFlow"; TF_NEED_GCP=0;;
|
||||
"" ) echo "No Google Cloud Platform support will be enabled for "\
|
||||
"TensorFlow"; TF_NEED_GCP=0;;
|
||||
* ) echo "Invalid selection: " $INPUT;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [[ "$TF_NEED_GCP" == "1" ]]; then
|
||||
write_to_bazelrc 'build --define with_gcp_support=true'
|
||||
fi
|
||||
|
||||
while [[ "$TF_NEED_HDFS" == "" ]]; do
|
||||
read -p "Do you wish to build TensorFlow with "\
|
||||
"Hadoop File System support? [y/N] " INPUT
|
||||
case $INPUT in
|
||||
[Yy]* ) echo "Hadoop File System support will be enabled for "\
|
||||
"TensorFlow"; TF_NEED_HDFS=1;;
|
||||
[Nn]* ) echo "No Hadoop File System support will be enabled for "\
|
||||
"TensorFlow"; TF_NEED_HDFS=0;;
|
||||
"" ) echo "No Hadoop File System support will be enabled for "\
|
||||
"TensorFlow"; TF_NEED_HDFS=0;;
|
||||
* ) echo "Invalid selection: " $INPUT;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [[ "$TF_NEED_HDFS" == "1" ]]; then
|
||||
write_to_bazelrc 'build --define with_hdfs_support=true'
|
||||
fi
|
||||
|
||||
## Enable XLA.
|
||||
while [[ "$TF_ENABLE_XLA" == "" ]]; do
|
||||
read -p "Do you wish to build TensorFlow with the XLA just-in-time compiler (experimental)? [y/N] " INPUT
|
||||
case $INPUT in
|
||||
[Yy]* ) echo "XLA JIT support will be enabled for TensorFlow"; TF_ENABLE_XLA=1;;
|
||||
[Nn]* ) echo "No XLA JIT support will be enabled for TensorFlow"; TF_ENABLE_XLA=0;;
|
||||
"" ) echo "No XLA support will be enabled for TensorFlow"; TF_ENABLE_XLA=0;;
|
||||
* ) echo "Invalid selection: " $INPUT;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [[ "$TF_ENABLE_XLA" == "1" ]]; then
|
||||
write_to_bazelrc 'build --define with_xla_support=true'
|
||||
fi
|
||||
|
||||
# Verbs configuration
|
||||
while [ "$TF_NEED_VERBS" == "" ]; do
|
||||
read -p "Do you wish to build TensorFlow with "\
|
||||
"VERBS support? [y/N] " INPUT
|
||||
case $INPUT in
|
||||
[Yy]* ) echo "VERBS support will be enabled for "\
|
||||
"TensorFlow"; TF_NEED_VERBS=1;;
|
||||
[Nn]* ) echo "No VERBS support will be enabled for "\
|
||||
"TensorFlow"; TF_NEED_VERBS=0;;
|
||||
"" ) echo "No VERBS support will be enabled for "\
|
||||
"TensorFlow"; TF_NEED_VERBS=0;;
|
||||
* ) echo "Invalid selection: " $INPUT;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [[ "$TF_NEED_VERBS" == "1" ]]; then
|
||||
write_to_bazelrc 'build --define with_verbs_support=true'
|
||||
fi
|
||||
|
||||
# Append CC optimization flags to bazel.rc
|
||||
for opt in $CC_OPT_FLAGS; do
|
||||
write_to_bazelrc "build:opt --cxxopt=$opt --copt=$opt"
|
||||
done
|
||||
|
||||
# Run the gen_git_source to create links where bazel can track dependencies for
|
||||
# git hash propagation
|
||||
GEN_GIT_SOURCE=tensorflow/tools/git/gen_git_source.py
|
||||
chmod a+x ${GEN_GIT_SOURCE}
|
||||
"${PYTHON_BIN_PATH}" ${GEN_GIT_SOURCE} --configure "${SOURCE_BASE_DIR}"
|
||||
|
||||
## Set up SYCL-related environment settings
|
||||
while [ "$TF_NEED_OPENCL" == "" ]; do
|
||||
read -p "Do you wish to build TensorFlow with OpenCL support? [y/N] " INPUT
|
||||
case $INPUT in
|
||||
[Yy]* ) echo "OpenCL support will be enabled for TensorFlow"; TF_NEED_OPENCL=1;;
|
||||
[Nn]* ) echo "No OpenCL support will be enabled for TensorFlow"; TF_NEED_OPENCL=0;;
|
||||
"" ) echo "No OpenCL support will be enabled for TensorFlow"; TF_NEED_OPENCL=0;;
|
||||
* ) echo "Invalid selection: " $INPUT;;
|
||||
esac
|
||||
done
|
||||
|
||||
## Set up Cuda-related environment settings
|
||||
|
||||
while [ "$TF_NEED_CUDA" == "" ]; do
|
||||
read -p "Do you wish to build TensorFlow with CUDA support? [y/N] " INPUT
|
||||
case $INPUT in
|
||||
[Yy]* ) echo "CUDA support will be enabled for TensorFlow"; TF_NEED_CUDA=1;;
|
||||
[Nn]* ) echo "No CUDA support will be enabled for TensorFlow"; TF_NEED_CUDA=0;;
|
||||
"" ) echo "No CUDA support will be enabled for TensorFlow"; TF_NEED_CUDA=0;;
|
||||
* ) echo "Invalid selection: " $INPUT;;
|
||||
esac
|
||||
done
|
||||
|
||||
export TF_NEED_CUDA
|
||||
write_action_env_to_bazelrc "TF_NEED_CUDA" "$TF_NEED_CUDA"
|
||||
|
||||
export TF_NEED_OPENCL
|
||||
write_action_env_to_bazelrc "TF_NEED_OPENCL" "$TF_NEED_OPENCL"
|
||||
|
||||
if [ "$TF_NEED_CUDA" == "1" ]; then
|
||||
while [[ "$TF_CUDA_CLANG" == "" ]]; do
|
||||
read -p "Do you want to use clang as CUDA compiler? [y/N] " INPUT
|
||||
case $INPUT in
|
||||
[Yy]* ) echo "Clang will be used as CUDA compiler"; TF_CUDA_CLANG=1;;
|
||||
[Nn]* ) echo "nvcc will be used as CUDA compiler"; TF_CUDA_CLANG=0;;
|
||||
"" ) echo "nvcc will be used as CUDA compiler"; TF_CUDA_CLANG=0;;
|
||||
* ) echo "Invalid selection: " $INPUT;;
|
||||
esac
|
||||
done
|
||||
|
||||
export TF_CUDA_CLANG
|
||||
write_action_env_to_bazelrc "TF_CUDA_CLANG" "$TF_CUDA_CLANG"
|
||||
|
||||
# Set up which clang we should use as the cuda / host compiler.
|
||||
while [[ "$TF_CUDA_CLANG" == "1" ]] && true; do
|
||||
fromuser=""
|
||||
if [ -z "$CLANG_CUDA_COMPILER_PATH" ]; then
|
||||
default_clang_host_compiler_path=$(which clang || true)
|
||||
read -p "Please specify which clang should be used as device and host compiler. [Default is $default_clang_host_compiler_path]: " CLANG_CUDA_COMPILER_PATH
|
||||
fromuser="1"
|
||||
if [ -z "$CLANG_CUDA_COMPILER_PATH" ]; then
|
||||
CLANG_CUDA_COMPILER_PATH="$default_clang_host_compiler_path"
|
||||
fi
|
||||
fi
|
||||
if [ -e "$CLANG_CUDA_COMPILER_PATH" ]; then
|
||||
export CLANG_CUDA_COMPILER_PATH
|
||||
write_action_env_to_bazelrc "CLANG_CUDA_COMPILER_PATH" "$CLANG_CUDA_COMPILER_PATH"
|
||||
break
|
||||
fi
|
||||
echo "Invalid clang path. ${CLANG_CUDA_COMPILER_PATH} cannot be found" 1>&2
|
||||
if [ -z "$fromuser" ]; then
|
||||
exit 1
|
||||
fi
|
||||
CLANG_CUDA_COMPILER_PATH=""
|
||||
# Retry
|
||||
done
|
||||
|
||||
# Find out where the CUDA toolkit is installed
|
||||
while true; do
|
||||
# Configure the Cuda SDK version to use.
|
||||
if [ -z "$TF_CUDA_VERSION" ]; then
|
||||
read -p "Please specify the CUDA SDK version you want to use, e.g. 7.0. [Leave empty to default to CUDA 8.0]: " TF_CUDA_VERSION
|
||||
fi
|
||||
# Set default CUDA version if not set
|
||||
TF_CUDA_VERSION=${TF_CUDA_VERSION:-8.0}
|
||||
|
||||
fromuser=""
|
||||
if [ -z "$CUDA_TOOLKIT_PATH" ]; then
|
||||
default_cuda_path=/usr/local/cuda
|
||||
if is_windows; then
|
||||
if [ -z "$CUDA_PATH" ]; then
|
||||
default_cuda_path="C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v8.0"
|
||||
else
|
||||
default_cuda_path="$(cygpath -m "$CUDA_PATH")"
|
||||
fi
|
||||
elif is_linux; then
|
||||
# If the default doesn't exist, try an alternative default.
|
||||
if [ ! -d $default_cuda_path ] && [ -d /opt/cuda ]; then
|
||||
default_cuda_path=/opt/cuda
|
||||
fi
|
||||
fi
|
||||
read -p "Please specify the location where CUDA $TF_CUDA_VERSION toolkit is installed. Refer to README.md for more details. [Default is $default_cuda_path]: " CUDA_TOOLKIT_PATH
|
||||
fromuser="1"
|
||||
if [ -z "$CUDA_TOOLKIT_PATH" ]; then
|
||||
CUDA_TOOLKIT_PATH="$default_cuda_path"
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ -z "$TF_CUDA_VERSION" ]]; then
|
||||
TF_CUDA_EXT=""
|
||||
else
|
||||
TF_CUDA_EXT=".$TF_CUDA_VERSION"
|
||||
fi
|
||||
|
||||
if is_windows; then
|
||||
CUDA_RT_LIB_PATH="lib/x64/cudart.lib"
|
||||
elif is_linux; then
|
||||
CUDA_RT_LIB_PATH="lib64/libcudart.so${TF_CUDA_EXT}"
|
||||
elif is_macos; then
|
||||
CUDA_RT_LIB_PATH="lib/libcudart${TF_CUDA_EXT}.dylib"
|
||||
fi
|
||||
|
||||
if [ -e "${CUDA_TOOLKIT_PATH}/${CUDA_RT_LIB_PATH}" ]; then
|
||||
export CUDA_TOOLKIT_PATH
|
||||
write_action_env_to_bazelrc "CUDA_TOOLKIT_PATH" "$CUDA_TOOLKIT_PATH"
|
||||
export TF_CUDA_VERSION
|
||||
break
|
||||
fi
|
||||
echo "Invalid path to CUDA $TF_CUDA_VERSION toolkit. ${CUDA_TOOLKIT_PATH}/${CUDA_RT_LIB_PATH} cannot be found"
|
||||
|
||||
if [ -z "$fromuser" ]; then
|
||||
exit 1
|
||||
fi
|
||||
# Retry
|
||||
TF_CUDA_VERSION=""
|
||||
CUDA_TOOLKIT_PATH=""
|
||||
done
|
||||
|
||||
export TF_CUDA_VERSION
|
||||
write_action_env_to_bazelrc "TF_CUDA_VERSION" "$TF_CUDA_VERSION"
|
||||
|
||||
# Set up which gcc nvcc should use as the host compiler
|
||||
# No need to set this on Windows
|
||||
while [[ "$TF_CUDA_CLANG" != "1" ]] && ! is_windows && true; do
|
||||
fromuser=""
|
||||
if [ -z "$GCC_HOST_COMPILER_PATH" ]; then
|
||||
default_gcc_host_compiler_path=$(which gcc || true)
|
||||
cuda_bin_symlink="$CUDA_TOOLKIT_PATH/bin/gcc"
|
||||
if [ -L "$cuda_bin_symlink" ]; then
|
||||
default_gcc_host_compiler_path=$(readlink $cuda_bin_symlink)
|
||||
fi
|
||||
read -p "Please specify which gcc should be used by nvcc as the host compiler. [Default is $default_gcc_host_compiler_path]: " GCC_HOST_COMPILER_PATH
|
||||
fromuser="1"
|
||||
if [ -z "$GCC_HOST_COMPILER_PATH" ]; then
|
||||
GCC_HOST_COMPILER_PATH="$default_gcc_host_compiler_path"
|
||||
fi
|
||||
fi
|
||||
if [ -e "$GCC_HOST_COMPILER_PATH" ]; then
|
||||
export GCC_HOST_COMPILER_PATH
|
||||
write_action_env_to_bazelrc "GCC_HOST_COMPILER_PATH" "$GCC_HOST_COMPILER_PATH"
|
||||
break
|
||||
fi
|
||||
echo "Invalid gcc path. ${GCC_HOST_COMPILER_PATH} cannot be found" 1>&2
|
||||
if [ -z "$fromuser" ]; then
|
||||
exit 1
|
||||
fi
|
||||
GCC_HOST_COMPILER_PATH=""
|
||||
# Retry
|
||||
done
|
||||
|
||||
# Find out where the cuDNN library is installed
|
||||
while true; do
|
||||
# Configure the cuDNN version to use.
|
||||
if [ -z "$TF_CUDNN_VERSION" ]; then
|
||||
read -p "Please specify the cuDNN version you want to use. [Leave empty to default to cuDNN 6.0]: " TF_CUDNN_VERSION
|
||||
fi
|
||||
# Set default CUDNN version if not set
|
||||
TF_CUDNN_VERSION=${TF_CUDNN_VERSION:-6}
|
||||
|
||||
fromuser=""
|
||||
if [ -z "$CUDNN_INSTALL_PATH" ]; then
|
||||
default_cudnn_path=${CUDA_TOOLKIT_PATH}
|
||||
read -p "Please specify the location where cuDNN $TF_CUDNN_VERSION library is installed. Refer to README.md for more details. [Default is $default_cudnn_path]: " CUDNN_INSTALL_PATH
|
||||
fromuser="1"
|
||||
if [ -z "$CUDNN_INSTALL_PATH" ]; then
|
||||
CUDNN_INSTALL_PATH=$default_cudnn_path
|
||||
fi
|
||||
# Result returned from "read" will be used unexpanded. That make "~" unusable.
|
||||
# Going through one more level of expansion to handle that.
|
||||
CUDNN_INSTALL_PATH=`"${PYTHON_BIN_PATH}" -c "import os; print(os.path.realpath(os.path.expanduser('${CUDNN_INSTALL_PATH}')))"`
|
||||
if is_windows; then
|
||||
CUDNN_INSTALL_PATH="$(cygpath -m "$CUDNN_INSTALL_PATH")"
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ -z "$TF_CUDNN_VERSION" ]]; then
|
||||
TF_CUDNN_EXT=""
|
||||
else
|
||||
TF_CUDNN_EXT=".$TF_CUDNN_VERSION"
|
||||
fi
|
||||
|
||||
if is_windows; then
|
||||
CUDA_DNN_LIB_PATH="lib/x64/cudnn.lib"
|
||||
CUDA_DNN_LIB_ALT_PATH="lib/x64/cudnn.lib"
|
||||
elif is_linux; then
|
||||
CUDA_DNN_LIB_PATH="lib64/libcudnn.so${TF_CUDNN_EXT}"
|
||||
CUDA_DNN_LIB_ALT_PATH="libcudnn.so${TF_CUDNN_EXT}"
|
||||
elif is_macos; then
|
||||
CUDA_DNN_LIB_PATH="lib/libcudnn${TF_CUDNN_EXT}.dylib"
|
||||
CUDA_DNN_LIB_ALT_PATH="libcudnn${TF_CUDNN_EXT}.dylib"
|
||||
fi
|
||||
|
||||
if [ -e "$CUDNN_INSTALL_PATH/${CUDA_DNN_LIB_ALT_PATH}" ] || [ -e "$CUDNN_INSTALL_PATH/${CUDA_DNN_LIB_PATH}" ]; then
|
||||
export TF_CUDNN_VERSION
|
||||
write_action_env_to_bazelrc "TF_CUDNN_VERSION" "$TF_CUDNN_VERSION"
|
||||
export CUDNN_INSTALL_PATH
|
||||
write_action_env_to_bazelrc "CUDNN_INSTALL_PATH" "$CUDNN_INSTALL_PATH"
|
||||
break
|
||||
fi
|
||||
|
||||
if is_linux; then
|
||||
if ! type ldconfig > /dev/null 2>&1; then
|
||||
LDCONFIG_BIN=/sbin/ldconfig
|
||||
else
|
||||
LDCONFIG_BIN=ldconfig
|
||||
fi
|
||||
CUDNN_PATH_FROM_LDCONFIG="$($LDCONFIG_BIN -p | sed -n 's/.*libcudnn.so .* => \(.*\)/\1/p')"
|
||||
if [ -e "${CUDNN_PATH_FROM_LDCONFIG}${TF_CUDNN_EXT}" ]; then
|
||||
export TF_CUDNN_VERSION
|
||||
export CUDNN_INSTALL_PATH
|
||||
CUDNN_INSTALL_PATH="$(dirname ${CUDNN_PATH_FROM_LDCONFIG})"
|
||||
write_action_env_to_bazelrc "CUDNN_INSTALL_PATH" "$CUDNN_INSTALL_PATH"
|
||||
break
|
||||
fi
|
||||
fi
|
||||
echo "Invalid path to cuDNN ${CUDNN_VERSION} toolkit. Neither of the following two files can be found:"
|
||||
echo "${CUDNN_INSTALL_PATH}/${CUDA_DNN_LIB_PATH}"
|
||||
echo "${CUDNN_INSTALL_PATH}/${CUDA_DNN_LIB_ALT_PATH}"
|
||||
if is_linux; then
|
||||
echo "${CUDNN_PATH_FROM_LDCONFIG}${TF_CUDNN_EXT}"
|
||||
fi
|
||||
|
||||
if [ -z "$fromuser" ]; then
|
||||
exit 1
|
||||
fi
|
||||
# Retry
|
||||
TF_CUDNN_VERSION=""
|
||||
CUDNN_INSTALL_PATH=""
|
||||
done
|
||||
|
||||
export TF_CUDNN_VERSION
|
||||
write_action_env_to_bazelrc "TF_CUDNN_VERSION" "$TF_CUDNN_VERSION"
|
||||
|
||||
# Configure the compute capabilities that TensorFlow builds for.
|
||||
# Since Cuda toolkit is not backward-compatible, this is not guaranteed to work.
|
||||
function get_native_cuda_compute_capabilities {
|
||||
device_query_bin="$CUDA_TOOLKIT_PATH/extras/demo_suite/deviceQuery" # Also works on Windows without .exe
|
||||
"$device_query_bin" | grep 'Capability' | grep -o '[0-9]*\.[0-9]*' | sed ':a;{N;s/\n/,/};ba'
|
||||
exit 0 # ensure that this function always exit success even if device detection fails, to prevent the whole configure from aborting
|
||||
}
|
||||
while true; do
|
||||
fromuser=""
|
||||
native_cuda_compute_capabilities=$(get_native_cuda_compute_capabilities)
|
||||
default_cuda_compute_capabilities=${native_cuda_compute_capabilities:-"3.5,5.2"}
|
||||
if [ -z "$TF_CUDA_COMPUTE_CAPABILITIES" ]; then
|
||||
cat << EOF
|
||||
Please specify a list of comma-separated Cuda compute capabilities you want to build with.
|
||||
You can find the compute capability of your device at: https://developer.nvidia.com/cuda-gpus.
|
||||
Please note that each additional compute capability significantly increases your build time and binary size.
|
||||
EOF
|
||||
read -p "[Default is: \"$default_cuda_compute_capabilities\"]: " TF_CUDA_COMPUTE_CAPABILITIES
|
||||
fromuser=1
|
||||
fi
|
||||
if [ -z "$TF_CUDA_COMPUTE_CAPABILITIES" ]; then
|
||||
TF_CUDA_COMPUTE_CAPABILITIES=$default_cuda_compute_capabilities
|
||||
fi
|
||||
# Check whether all capabilities from the input is valid
|
||||
COMPUTE_CAPABILITIES=${TF_CUDA_COMPUTE_CAPABILITIES//,/ }
|
||||
ALL_VALID=1
|
||||
for CAPABILITY in $COMPUTE_CAPABILITIES; do
|
||||
if [[ ! "$CAPABILITY" =~ [0-9]+.[0-9]+ ]]; then
|
||||
echo "Invalid compute capability: " $CAPABILITY
|
||||
ALL_VALID=0
|
||||
break
|
||||
fi
|
||||
done
|
||||
if [ "$ALL_VALID" == "0" ]; then
|
||||
if [ -z "$fromuser" ]; then
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
export TF_CUDA_COMPUTE_CAPABILITIES
|
||||
write_action_env_to_bazelrc "TF_CUDA_COMPUTE_CAPABILITIES" "$TF_CUDA_COMPUTE_CAPABILITIES"
|
||||
break
|
||||
fi
|
||||
TF_CUDA_COMPUTE_CAPABILITIES=""
|
||||
done
|
||||
|
||||
if is_windows; then
|
||||
# The following three variables are needed for MSVC toolchain configuration in Bazel
|
||||
export CUDA_PATH="$CUDA_TOOLKIT_PATH"
|
||||
export CUDA_COMPUTE_CAPABILITIES="$TF_CUDA_COMPUTE_CAPABILITIES"
|
||||
export NO_WHOLE_ARCHIVE_OPTION=1
|
||||
write_action_env_to_bazelrc "CUDA_PATH" "$CUDA_PATH"
|
||||
write_action_env_to_bazelrc "CUDA_COMPUTE_CAPABILITIES" "$CUDA_COMPUTE_CAPABILITIES"
|
||||
write_action_env_to_bazelrc "NO_WHOLE_ARCHIVE_OPTION" "1"
|
||||
write_to_bazelrc "build --config=win-cuda"
|
||||
write_to_bazelrc "test --config=win-cuda"
|
||||
else
|
||||
# If CUDA is enabled, always use GPU during build and test.
|
||||
if [ "$TF_CUDA_CLANG" == "1" ]; then
|
||||
write_to_bazelrc "build --config=cuda_clang"
|
||||
write_to_bazelrc "test --config=cuda_clang"
|
||||
else
|
||||
write_to_bazelrc "build --config=cuda"
|
||||
write_to_bazelrc "test --config=cuda"
|
||||
fi
|
||||
fi
|
||||
|
||||
# end of if "$TF_NEED_CUDA" == "1"
|
||||
fi
|
||||
|
||||
# OpenCL configuration
|
||||
|
||||
if [ "$TF_NEED_OPENCL" == "1" ]; then
|
||||
|
||||
# Determine which C++ compiler should be used as the host compiler
|
||||
while true; do
|
||||
fromuser=""
|
||||
if [ -z "$HOST_CXX_COMPILER" ]; then
|
||||
default_cxx_host_compiler=$(which g++ || true)
|
||||
read -p "Please specify which C++ compiler should be used as the host C++ compiler. [Default is $default_cxx_host_compiler]: " HOST_CXX_COMPILER
|
||||
fromuser="1"
|
||||
if [ -z "$HOST_CXX_COMPILER" ]; then
|
||||
HOST_CXX_COMPILER=$default_cxx_host_compiler
|
||||
fi
|
||||
fi
|
||||
if [ -e "$HOST_CXX_COMPILER" ]; then
|
||||
export HOST_CXX_COMPILER
|
||||
write_action_env_to_bazelrc "HOST_CXX_COMPILER" "$HOST_CXX_COMPILER"
|
||||
break
|
||||
fi
|
||||
echo "Invalid C++ compiler path. ${HOST_CXX_COMPILER} cannot be found" 1>&2
|
||||
if [ -z "$fromuser" ]; then
|
||||
exit 1
|
||||
fi
|
||||
HOST_CXX_COMPILER=""
|
||||
# Retry
|
||||
done
|
||||
|
||||
# Determine which C compiler should be used as the host compiler
|
||||
while true; do
|
||||
fromuser=""
|
||||
if [ -z "$HOST_C_COMPILER" ]; then
|
||||
default_c_host_compiler=$(which gcc || true)
|
||||
read -p "Please specify which C compiler should be used as the host C compiler. [Default is $default_c_host_compiler]: " HOST_C_COMPILER
|
||||
fromuser="1"
|
||||
if [ -z "$HOST_C_COMPILER" ]; then
|
||||
HOST_C_COMPILER=$default_c_host_compiler
|
||||
fi
|
||||
fi
|
||||
if [ -e "$HOST_C_COMPILER" ]; then
|
||||
export HOST_C_COMPILER
|
||||
write_action_env_to_bazelrc "HOST_C_COMPILER" "$HOST_C_COMPILER"
|
||||
break
|
||||
fi
|
||||
echo "Invalid C compiler path. ${HOST_C_COMPILER} cannot be found" 1>&2
|
||||
if [ -z "$fromuser" ]; then
|
||||
exit 1
|
||||
fi
|
||||
HOST_C_COMPILER=""
|
||||
# Retry
|
||||
done
|
||||
|
||||
while true; do
|
||||
# Configure the OPENCL version to use.
|
||||
TF_OPENCL_VERSION="1.2"
|
||||
|
||||
# Point to ComputeCpp root
|
||||
if [ -z "$COMPUTECPP_TOOLKIT_PATH" ]; then
|
||||
default_computecpp_toolkit_path=/usr/local/computecpp
|
||||
read -p "Please specify the location where ComputeCpp for SYCL $TF_OPENCL_VERSION is installed. [Default is $default_computecpp_toolkit_path]: " COMPUTECPP_TOOLKIT_PATH
|
||||
fromuser="1"
|
||||
if [ -z "$COMPUTECPP_TOOLKIT_PATH" ]; then
|
||||
COMPUTECPP_TOOLKIT_PATH=$default_computecpp_toolkit_path
|
||||
fi
|
||||
fi
|
||||
|
||||
if is_linux; then
|
||||
SYCL_RT_LIB_PATH="lib/libComputeCpp.so"
|
||||
fi
|
||||
|
||||
if [ -e "${COMPUTECPP_TOOLKIT_PATH}/${SYCL_RT_LIB_PATH}" ]; then
|
||||
export COMPUTECPP_TOOLKIT_PATH
|
||||
write_action_env_to_bazelrc "COMPUTECPP_TOOLKIT_PATH" "$COMPUTECPP_TOOLKIT_PATH"
|
||||
break
|
||||
fi
|
||||
echo "Invalid SYCL $TF_OPENCL_VERSION library path. ${COMPUTECPP_TOOLKIT_PATH}/${SYCL_RT_LIB_PATH} cannot be found"
|
||||
|
||||
if [ -z "$fromuser" ]; then
|
||||
exit 1
|
||||
fi
|
||||
# Retry
|
||||
TF_OPENCL_VERSION=""
|
||||
COMPUTECPP_TOOLKIT_PATH=""
|
||||
done
|
||||
|
||||
# end of if "$TF_NEED_OPENCL" == "1"
|
||||
fi
|
||||
|
||||
|
||||
while [ "$TF_NEED_MPI" == "" ]; do
|
||||
read -p "Do you wish to build TensorFlow with "\
|
||||
"MPI support? [y/N] " INPUT
|
||||
case $INPUT in
|
||||
[Yy]* ) echo "MPI support will be enabled for "\
|
||||
"TensorFlow"; TF_NEED_MPI=1;;
|
||||
[Nn]* ) echo "MPI support will not be enabled for "\
|
||||
"TensorFlow"; TF_NEED_MPI=0;;
|
||||
"" ) echo "MPI support will not be enabled for "\
|
||||
"TensorFlow"; TF_NEED_MPI=0;;
|
||||
* ) echo "Invalid selection: " $INPUT;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Find out where the MPI toolkit is installed
|
||||
while true; do
|
||||
if [ "$TF_NEED_MPI" == "0" ]; then
|
||||
break;
|
||||
fi
|
||||
|
||||
fromuser=""
|
||||
if [ -z "$MPI_HOME" ]; then
|
||||
#Get the base folder by removing the bin path
|
||||
default_mpi_path=$(dirname $(dirname $(which mpirun)) || dirname $(dirname $(which mpiexec)) || true)
|
||||
read -p "Please specify the MPI toolkit folder. [Default is $default_mpi_path]: " MPI_HOME
|
||||
fromuser="1"
|
||||
if [ -z "$MPI_HOME" ]; then
|
||||
MPI_HOME=$default_mpi_path
|
||||
fi
|
||||
fi
|
||||
|
||||
#Check that the include and library folders are where we expect them to be
|
||||
if [ -e "$MPI_HOME/include" ] && [ -e "$MPI_HOME/lib" ]; then
|
||||
break
|
||||
fi
|
||||
|
||||
echo "Invalid path to the MPI Toolkit. ${MPI_HOME}/include or ${MPI_HOME}/lib cannot be found."
|
||||
if [ -z "$fromuser" ]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Retry
|
||||
MPI_HOME=""
|
||||
done
|
||||
|
||||
|
||||
if [ "$TF_NEED_MPI" == "1" ]; then
|
||||
write_to_bazelrc 'build --define with_mpi_support=true'
|
||||
|
||||
#Link the MPI header files
|
||||
ln -sf "${MPI_HOME}/include/mpi.h" third_party/mpi/mpi.h
|
||||
|
||||
|
||||
#Determine if we use OpenMPI or MVAPICH, these require different header files
|
||||
#to be included here to make bazel dependency checker happy
|
||||
|
||||
if [ -e "${MPI_HOME}/include/mpi_portable_platform.h" ]; then
|
||||
#OpenMPI
|
||||
ln -sf "${MPI_HOME}/include/mpi_portable_platform.h" third_party/mpi/
|
||||
sed -i -e "s/MPI_LIB_IS_OPENMPI=False/MPI_LIB_IS_OPENMPI=True/" third_party/mpi/mpi.bzl
|
||||
else
|
||||
#MVAPICH / MPICH
|
||||
ln -sf "${MPI_HOME}/include/mpio.h" third_party/mpi/
|
||||
ln -sf "${MPI_HOME}/include/mpicxx.h" third_party/mpi/
|
||||
sed -i -e "s/MPI_LIB_IS_OPENMPI=True/MPI_LIB_IS_OPENMPI=False/" third_party/mpi/mpi.bzl
|
||||
fi
|
||||
|
||||
|
||||
if [ -e "${MPI_HOME}/lib/libmpi.so" ]; then
|
||||
ln -sf "${MPI_HOME}/lib/libmpi.so" third_party/mpi/
|
||||
else
|
||||
echo "Cannot find the MPI library file in ${MPI_HOME}/lib "
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
|
||||
echo "Configuration finished"
|
||||
echo "Configuration finished"
|
||||
950
configure.py
Normal file
950
configure.py
Normal file
@@ -0,0 +1,950 @@
|
||||
# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
"""configure script to get build parameters from user."""
|
||||
|
||||
from __future__ import absolute_import
|
||||
from __future__ import division
|
||||
from __future__ import print_function
|
||||
|
||||
import errno
|
||||
import os
|
||||
import platform
|
||||
import re
|
||||
import site
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
_TF_BAZELRC = '.tf_configure.bazelrc'
|
||||
_DEFAULT_CUDA_VERSION = '8.0'
|
||||
_DEFAULT_CUDNN_VERSION = '6'
|
||||
_DEFAULT_CUDA_COMPUTE_CAPABILITIES = '3.5,5.2'
|
||||
_DEFAULT_CUDA_PATH = '/usr/local/cuda'
|
||||
_DEFAULT_CUDA_PATH_LINUX = '/opt/cuda'
|
||||
_DEFAULT_CUDA_PATH_WIN = ('C:/Program Files/NVIDIA GPU Computing '
|
||||
'Toolkit/CUDA/v%s' % _DEFAULT_CUDA_VERSION)
|
||||
_TF_OPENCL_VERSION = '1.2'
|
||||
_DEFAULT_COMPUTECPP_TOOLKIT_PATH = '/usr/local/computecpp'
|
||||
|
||||
|
||||
def is_windows():
|
||||
return platform.system() == 'Windows'
|
||||
|
||||
|
||||
def is_linux():
|
||||
return platform.system() == 'Linux'
|
||||
|
||||
|
||||
def is_macos():
|
||||
return platform.system() == 'Darwin'
|
||||
|
||||
|
||||
def is_ppc64le():
|
||||
return platform.machine() == 'ppc64le'
|
||||
|
||||
|
||||
def get_input(question):
|
||||
try:
|
||||
try:
|
||||
answer = raw_input(question)
|
||||
except NameError:
|
||||
answer = input(question) # pylint: disable=bad-builtin
|
||||
except EOFError:
|
||||
answer = ''
|
||||
return answer
|
||||
|
||||
|
||||
def symlink_force(target, link_name):
|
||||
"""Force symlink, equivalent of 'ln -sf'.
|
||||
|
||||
Args:
|
||||
target: items to link to.
|
||||
link_name: name of the link.
|
||||
"""
|
||||
try:
|
||||
os.symlink(target, link_name)
|
||||
except OSError as e:
|
||||
if e.errno == errno.EEXIST:
|
||||
os.remove(link_name)
|
||||
os.symlink(target, link_name)
|
||||
else:
|
||||
raise e
|
||||
|
||||
|
||||
def sed_in_place(filename, old, new):
|
||||
"""Replace old string with new string in file.
|
||||
|
||||
Args:
|
||||
filename: string for filename.
|
||||
old: string to replace.
|
||||
new: new string to replace to.
|
||||
"""
|
||||
with open(filename, 'r') as f:
|
||||
filedata = f.read()
|
||||
newdata = filedata.replace(old, new)
|
||||
with open(filename, 'w') as f:
|
||||
f.write(newdata)
|
||||
|
||||
|
||||
def remove_line_with(filename, token):
|
||||
"""Remove lines that contain token from file.
|
||||
|
||||
Args:
|
||||
filename: string for filename.
|
||||
token: string token to check if to remove a line from file or not.
|
||||
"""
|
||||
with open(filename, 'r') as f:
|
||||
filedata = f.read()
|
||||
|
||||
with open(filename, 'w') as f:
|
||||
for line in filedata.strip().split('\n'):
|
||||
if token not in line:
|
||||
f.write(line + '\n')
|
||||
|
||||
|
||||
def write_to_bazelrc(line):
|
||||
with open(_TF_BAZELRC, 'a') as f:
|
||||
f.write(line + '\n')
|
||||
|
||||
|
||||
def write_action_env_to_bazelrc(var_name, var):
|
||||
write_to_bazelrc('build --action_env %s="%s"' % (var_name, str(var)))
|
||||
|
||||
|
||||
def run_shell(cmd):
|
||||
return subprocess.check_output(cmd, shell=True).decode('UTF-8').strip()
|
||||
|
||||
|
||||
def cygpath(path):
|
||||
"""Convert path from posix to windows."""
|
||||
return run_shell('cygpath -m "%s"' % path)
|
||||
|
||||
|
||||
def get_python_path(environ_cp):
|
||||
"""Get the python site package paths."""
|
||||
python_paths = []
|
||||
if environ_cp.get('PYTHONPATH'):
|
||||
python_paths = environ_cp.get('PYTHONPATH').split(':')
|
||||
try:
|
||||
library_paths = site.getsitepackages()
|
||||
except AttributeError:
|
||||
from distutils.sysconfig import get_python_lib # pylint: disable=g-import-not-at-top
|
||||
library_paths = [get_python_lib()]
|
||||
all_paths = set(python_paths + library_paths)
|
||||
|
||||
paths = []
|
||||
for path in all_paths:
|
||||
if os.path.isdir(path):
|
||||
paths.append(path)
|
||||
return paths
|
||||
|
||||
|
||||
def setup_python(environ_cp):
|
||||
"""Setup python related env variables."""
|
||||
# Get PYTHON_BIN_PATH, default is the current running python.
|
||||
default_python_bin_path = sys.executable
|
||||
ask_python_bin_path = ('Please specify the location of python. [Default is '
|
||||
'%s]: ') % default_python_bin_path
|
||||
while True:
|
||||
python_bin_path = get_from_env_or_user_or_default(
|
||||
environ_cp, 'PYTHON_BIN_PATH', ask_python_bin_path,
|
||||
default_python_bin_path)
|
||||
# Check if the path is valid
|
||||
if (os.path.isfile(python_bin_path) and os.access(
|
||||
python_bin_path, os.X_OK)) or (os.path.isdir(python_bin_path)):
|
||||
break
|
||||
elif not os.path.exists(python_bin_path):
|
||||
print('Invalid python path: %s cannot be found.' % python_bin_path)
|
||||
else:
|
||||
print('%s is not executable. Is it the python binary?' % python_bin_path)
|
||||
environ_cp['PYTHON_BIN_PATH'] = ''
|
||||
|
||||
# Get PYTHON_LIB_PATH
|
||||
python_lib_path = environ_cp.get('PYTHON_LIB_PATH')
|
||||
if not python_lib_path:
|
||||
python_lib_paths = get_python_path(environ_cp)
|
||||
if environ_cp.get('USE_DEFAULT_PYTHON_LIB_PATH') == '1':
|
||||
environ_cp['PYTHON_LIB_PATH'] = python_lib_paths[0]
|
||||
else:
|
||||
print('Found possible Python library paths:\n%s' %
|
||||
'\n'.join(python_lib_paths))
|
||||
default_python_lib_path = python_lib_paths[0]
|
||||
python_lib_path = get_input(
|
||||
'Please input the desired Python library path to use. Default is %s'
|
||||
% python_lib_paths[0])
|
||||
if not python_lib_path:
|
||||
python_lib_path = default_python_lib_path
|
||||
environ_cp['PYTHON_LIB_PATH'] = python_lib_path
|
||||
|
||||
python_major_version = sys.version_info[0]
|
||||
# Convert python path to Windows style before writing into bazel.rc
|
||||
if is_windows():
|
||||
python_bin_path = cygpath(python_bin_path)
|
||||
python_lib_path = cygpath(python_lib_path)
|
||||
|
||||
# Set-up env variables used by python_configure.bzl
|
||||
write_action_env_to_bazelrc('PYTHON_BIN_PATH', python_bin_path)
|
||||
write_action_env_to_bazelrc('PYTHON_LIB_PATH', python_lib_path)
|
||||
write_to_bazelrc('build --define PYTHON_BIN_PATH="%s"' % python_bin_path)
|
||||
write_to_bazelrc('build --define PYTHON_LIB_PATH="%s"' % python_lib_path)
|
||||
write_to_bazelrc('build --force_python=py%s' % python_major_version)
|
||||
write_to_bazelrc('build --host_force_python=py%s' % python_major_version)
|
||||
write_to_bazelrc('build --python%s_path=\"%s"' % (python_major_version,
|
||||
python_bin_path))
|
||||
write_to_bazelrc('test --force_python=py%s' % python_major_version)
|
||||
write_to_bazelrc('test --host_force_python=py%s' % python_major_version)
|
||||
write_to_bazelrc('test --define PYTHON_BIN_PATH="%s"' % python_bin_path)
|
||||
write_to_bazelrc('test --define PYTHON_LIB_PATH="%s"' % python_lib_path)
|
||||
write_to_bazelrc('run --define PYTHON_BIN_PATH="%s"' % python_bin_path)
|
||||
write_to_bazelrc('run --define PYTHON_LIB_PATH="%s"' % python_lib_path)
|
||||
environ_cp['PYTHON_BIN_PATH'] = python_bin_path
|
||||
|
||||
# Write tools/python_bin_path.sh
|
||||
with open('tools/python_bin_path.sh', 'w') as f:
|
||||
f.write('export PYTHON_BIN_PATH="%s"' % python_bin_path)
|
||||
|
||||
|
||||
def reset_tf_configure_bazelrc():
|
||||
"""Reset file that contains customized config settings."""
|
||||
open(_TF_BAZELRC, 'w').close()
|
||||
|
||||
home = os.path.expanduser('~')
|
||||
if not os.path.exists('.bazelrc'):
|
||||
if os.path.exists(os.path.join(home, '.bazelrc')):
|
||||
with open('.bazelrc', 'a') as f:
|
||||
f.write('import %s/.bazelrc\n' % home)
|
||||
else:
|
||||
open('.bazelrc', 'w').close()
|
||||
|
||||
remove_line_with('.bazelrc', 'tf_configure')
|
||||
with open('.bazelrc', 'a') as f:
|
||||
f.write('import %workspace%/.tf_configure.bazelrc\n')
|
||||
|
||||
|
||||
def run_gen_git_source(environ_cp):
|
||||
"""Run the gen_git_source to create links.
|
||||
|
||||
The links are for bazel to track dependencies for git hash propagation.
|
||||
|
||||
Args:
|
||||
environ_cp: copy of the os.environ.
|
||||
"""
|
||||
cmd = '%s tensorflow/tools/git/gen_git_source.py --configure %s' % (
|
||||
environ_cp.get('PYTHON_BIN_PATH'), os.getcwd())
|
||||
os.system(cmd)
|
||||
|
||||
|
||||
def cleanup_makefile():
|
||||
"""Delete any leftover BUILD files from the Makefile build.
|
||||
|
||||
These files could interfere with Bazel parsing.
|
||||
"""
|
||||
makefile_download_dir = 'tensorflow/contrib/makefile/downloads'
|
||||
if os.path.isdir(makefile_download_dir):
|
||||
for root, _, filenames in os.walk(makefile_download_dir):
|
||||
for f in filenames:
|
||||
if f.endswith('BUILD'):
|
||||
os.remove(os.path.join(root, f))
|
||||
|
||||
|
||||
def get_var(environ_cp,
|
||||
var_name,
|
||||
query_item,
|
||||
enabled_by_default,
|
||||
question=None,
|
||||
yes_reply=None,
|
||||
no_reply=None):
|
||||
"""Get boolean input from user.
|
||||
|
||||
If var_name is not set in env, ask user to enable query_item or not. If the
|
||||
response is empty, use the default.
|
||||
|
||||
Args:
|
||||
environ_cp: copy of the os.environ.
|
||||
var_name: string for name of environment variable, e.g. "TF_NEED_HDFS".
|
||||
query_item: string for feature related to the variable, e.g. "Hadoop File
|
||||
System".
|
||||
enabled_by_default: boolean for default behavior.
|
||||
question: optional string for how to ask for user input.
|
||||
yes_reply: optionanl string for reply when feature is enabled.
|
||||
no_reply: optional string for reply when feature is disabled.
|
||||
|
||||
Returns:
|
||||
boolean value of the variable.
|
||||
"""
|
||||
if not question:
|
||||
question = 'Do you wish to build TensorFlow with %s support?' % query_item
|
||||
if not yes_reply:
|
||||
yes_reply = '%s support will be enabled for TensorFlow.' % query_item
|
||||
if not no_reply:
|
||||
no_reply = 'No %s' % yes_reply
|
||||
|
||||
yes_reply += '\n'
|
||||
no_reply += '\n'
|
||||
|
||||
if enabled_by_default:
|
||||
question += ' [Y/n]: '
|
||||
else:
|
||||
question += ' [y/N]: '
|
||||
|
||||
var = environ_cp.get(var_name)
|
||||
while var is None:
|
||||
user_input_origin = get_input(question)
|
||||
user_input = user_input_origin.strip().lower()
|
||||
if user_input == 'y':
|
||||
print(yes_reply)
|
||||
var = True
|
||||
elif user_input == 'n':
|
||||
print(no_reply)
|
||||
var = False
|
||||
elif not user_input:
|
||||
if enabled_by_default:
|
||||
print(yes_reply)
|
||||
var = True
|
||||
else:
|
||||
print(no_reply)
|
||||
var = False
|
||||
else:
|
||||
print('Invalid selection: %s' % user_input_origin)
|
||||
return var
|
||||
|
||||
|
||||
def set_build_var(environ_cp, var_name, query_item, option_name,
|
||||
enabled_by_default):
|
||||
"""Set if query_item will be enabled for the build.
|
||||
|
||||
Ask user if query_item will be enabled. Default is used if no input is given.
|
||||
Set subprocess environment variable and write to .bazelrc if enabled.
|
||||
|
||||
Args:
|
||||
environ_cp: copy of the os.environ.
|
||||
var_name: string for name of environment variable, e.g. "TF_NEED_HDFS".
|
||||
query_item: string for feature related to the variable, e.g. "Hadoop File
|
||||
System".
|
||||
option_name: string for option to define in .bazelrc.
|
||||
enabled_by_default: boolean for default behavior.
|
||||
"""
|
||||
|
||||
var = str(int(get_var(environ_cp, var_name, query_item, enabled_by_default)))
|
||||
environ_cp[var_name] = var
|
||||
if var == '1':
|
||||
write_to_bazelrc('build --define %s=true' % option_name)
|
||||
|
||||
|
||||
def set_action_env_var(environ_cp,
|
||||
var_name,
|
||||
query_item,
|
||||
enabled_by_default,
|
||||
question=None,
|
||||
yes_reply=None,
|
||||
no_reply=None):
|
||||
"""Set boolean action_env variable.
|
||||
|
||||
Ask user if query_item will be enabled. Default is used if no input is given.
|
||||
Set environment variable and write to .bazelrc.
|
||||
|
||||
Args:
|
||||
environ_cp: copy of the os.environ.
|
||||
var_name: string for name of environment variable, e.g. "TF_NEED_HDFS".
|
||||
query_item: string for feature related to the variable, e.g. "Hadoop File
|
||||
System".
|
||||
enabled_by_default: boolean for default behavior.
|
||||
question: optional string for how to ask for user input.
|
||||
yes_reply: optionanl string for reply when feature is enabled.
|
||||
no_reply: optional string for reply when feature is disabled.
|
||||
"""
|
||||
var = int(
|
||||
get_var(environ_cp, var_name, query_item, enabled_by_default, question,
|
||||
yes_reply, no_reply))
|
||||
|
||||
write_action_env_to_bazelrc(var_name, var)
|
||||
environ_cp[var_name] = str(var)
|
||||
|
||||
|
||||
def check_bazel_version(min_version):
|
||||
"""Check installed bezel version is at least min_version.
|
||||
|
||||
Args:
|
||||
min_version: string for minimum bazel version.
|
||||
"""
|
||||
try:
|
||||
curr_version = run_shell('bazel version')
|
||||
except subprocess.CalledProcessError:
|
||||
print('Cannot find bazel. Please install bazel.')
|
||||
sys.exit(0)
|
||||
|
||||
for line in curr_version.split('\n'):
|
||||
if 'Build label: ' in line:
|
||||
curr_version = line.split('Build label: ')[1]
|
||||
break
|
||||
|
||||
min_version_segments = min_version.split('.')
|
||||
curr_version_segments = curr_version.split('.')
|
||||
|
||||
# Check if current bazel version can be detected properly.
|
||||
for seg in curr_version_segments:
|
||||
if not seg.isdigit():
|
||||
print('WARNING: current bazel installation is not a release version.')
|
||||
print('Make sure you are running at least bazel %s' % min_version)
|
||||
return
|
||||
|
||||
min_version_str = ''.join(['%03d' % int(seg) for seg in min_version_segments])
|
||||
curr_version_str = ''.join(
|
||||
['%03d' % int(seg) for seg in curr_version_segments])
|
||||
if int(curr_version_str) < int(min_version_str):
|
||||
print('Please upgrade your bazel installation to version %s or higher to '
|
||||
'build TensorFlow!' % min_version)
|
||||
sys.exit(0)
|
||||
|
||||
|
||||
def set_cc_opt_flags(environ_cp):
|
||||
"""Set up architecture-dependent optimization flags.
|
||||
|
||||
Also append CC optimization flags to bazel.rc..
|
||||
|
||||
Args:
|
||||
environ_cp: copy of the os.environ.
|
||||
"""
|
||||
if is_ppc64le():
|
||||
# gcc on ppc64le does not support -march, use mcpu instead
|
||||
default_cc_opt_flags = '-mcpu=native'
|
||||
else:
|
||||
default_cc_opt_flags = '-march=native'
|
||||
question = ('Please specify optimization flags to use during compilation when'
|
||||
' bazel option "--config=opt" is specified [Default is %s]: '
|
||||
) % default_cc_opt_flags
|
||||
cc_opt_flags = get_from_env_or_user_or_default(environ_cp, 'CC_OPT_FLAGS',
|
||||
question, default_cc_opt_flags)
|
||||
for opt in cc_opt_flags.split():
|
||||
write_to_bazelrc('build:opt --cxxopt=%s --copt=%s' % (opt, opt))
|
||||
|
||||
|
||||
def set_tf_cuda_clang(environ_cp):
|
||||
"""set TF_CUDA_CLANG action_env.
|
||||
|
||||
Args:
|
||||
environ_cp: copy of the os.environ.
|
||||
"""
|
||||
question = 'Do you want to use clang as CUDA compiler?'
|
||||
yes_reply = 'Clang will be used as CUDA compiler.'
|
||||
no_reply = 'nvcc will be used as CUDA compiler.'
|
||||
set_action_env_var(
|
||||
environ_cp,
|
||||
'TF_CUDA_CLANG',
|
||||
None,
|
||||
False,
|
||||
question=question,
|
||||
yes_reply=yes_reply,
|
||||
no_reply=no_reply)
|
||||
|
||||
|
||||
def get_from_env_or_user_or_default(environ_cp, var_name, ask_for_var,
|
||||
var_default):
|
||||
"""Get var_name either from env, or user or default.
|
||||
|
||||
If var_name has been set as environment variable, use the preset value, else
|
||||
ask for user input. If no input is provided, the default is used.
|
||||
|
||||
Args:
|
||||
environ_cp: copy of the os.environ.
|
||||
var_name: string for name of environment variable, e.g. "TF_NEED_HDFS".
|
||||
ask_for_var: string for how to ask for user input.
|
||||
var_default: default value string.
|
||||
|
||||
Returns:
|
||||
string value for var_name
|
||||
"""
|
||||
var = environ_cp.get(var_name)
|
||||
if not var:
|
||||
var = get_input(ask_for_var)
|
||||
if not var:
|
||||
var = var_default
|
||||
return var
|
||||
|
||||
|
||||
def set_clang_cuda_compiler_path(environ_cp):
|
||||
"""Set CLANG_CUDA_COMPILER_PATH."""
|
||||
default_clang_path = run_shell('which clang || true')
|
||||
ask_clang_path = ('Please specify which clang should be used as device and '
|
||||
'host compiler. [Default is %s]: ') % default_clang_path
|
||||
|
||||
while True:
|
||||
clang_cuda_compiler_path = get_from_env_or_user_or_default(
|
||||
environ_cp, 'CLANG_CUDA_COMPILER_PATH', ask_clang_path,
|
||||
default_clang_path)
|
||||
if os.path.exists(clang_cuda_compiler_path):
|
||||
break
|
||||
|
||||
# Reset and retry
|
||||
print('Invalid clang path: %s cannot be found.' % clang_cuda_compiler_path)
|
||||
environ_cp['CLANG_CUDA_COMPILER_PATH'] = ''
|
||||
|
||||
# Set CLANG_CUDA_COMPILER_PATH
|
||||
environ_cp['CLANG_CUDA_COMPILER_PATH'] = clang_cuda_compiler_path
|
||||
write_action_env_to_bazelrc('CLANG_CUDA_COMPILER_PATH',
|
||||
clang_cuda_compiler_path)
|
||||
|
||||
|
||||
def set_gcc_host_compiler_path(environ_cp):
|
||||
"""Set GCC_HOST_COMPILER_PATH."""
|
||||
default_gcc_host_compiler_path = run_shell('which gcc || true')
|
||||
cuda_bin_symlink = '%s/bin/gcc' % environ_cp.get('CUDA_TOOLKIT_PATH')
|
||||
|
||||
if os.path.islink(cuda_bin_symlink):
|
||||
# os.readlink is only available in linux
|
||||
default_gcc_host_compiler_path = run_shell('readlink %s' % cuda_bin_symlink)
|
||||
|
||||
ask_gcc_path = (
|
||||
'Please specify which gcc should be used by nvcc as the '
|
||||
'host compiler. [Default is %s]: ') % default_gcc_host_compiler_path
|
||||
while True:
|
||||
gcc_host_compiler_path = get_from_env_or_user_or_default(
|
||||
environ_cp, 'GCC_HOST_COMPILER_PATH', ask_gcc_path,
|
||||
default_gcc_host_compiler_path)
|
||||
|
||||
if os.path.exists(gcc_host_compiler_path):
|
||||
break
|
||||
|
||||
# Reset and retry
|
||||
print('Invalid gcc path. %s cannot be found' % gcc_host_compiler_path)
|
||||
environ_cp['GCC_HOST_COMPILER_PATH'] = ''
|
||||
|
||||
# Set GCC_HOST_COMPILER_PATH
|
||||
environ_cp['GCC_HOST_COMPILER_PATH'] = gcc_host_compiler_path
|
||||
write_action_env_to_bazelrc('GCC_HOST_COMPILER_PATH', gcc_host_compiler_path)
|
||||
|
||||
|
||||
def set_tf_cuda_version(environ_cp):
|
||||
"""Set CUDA_TOOLKIT_PATH and TF_CUDA_VERSION."""
|
||||
ask_cuda_version = (
|
||||
'Please specify the CUDA SDK version you want to use, '
|
||||
'e.g. 7.0. [Leave empty to default to CUDA %s]: ') % _DEFAULT_CUDA_VERSION
|
||||
|
||||
while True:
|
||||
# Configure the Cuda SDK version to use.
|
||||
tf_cuda_version = get_from_env_or_user_or_default(
|
||||
environ_cp, 'TF_CUDA_VERSION', ask_cuda_version, _DEFAULT_CUDA_VERSION)
|
||||
|
||||
# Find out where the CUDA toolkit is installed
|
||||
default_cuda_path = _DEFAULT_CUDA_PATH
|
||||
if is_windows():
|
||||
default_cuda_path = cygpath(
|
||||
environ_cp.get('CUDA_PATH', _DEFAULT_CUDA_PATH_WIN))
|
||||
elif is_linux():
|
||||
# If the default doesn't exist, try an alternative default.
|
||||
if (not os.path.exists(default_cuda_path)
|
||||
) and os.path.exists(_DEFAULT_CUDA_PATH_LINUX):
|
||||
default_cuda_path = _DEFAULT_CUDA_PATH_LINUX
|
||||
ask_cuda_path = ('Please specify the location where CUDA %s toolkit is'
|
||||
' installed. Refer to README.md for more details. '
|
||||
'[Default is %s]: ') % (tf_cuda_version, default_cuda_path)
|
||||
cuda_toolkit_path = get_from_env_or_user_or_default(
|
||||
environ_cp, 'CUDA_TOOLKIT_PATH', ask_cuda_path, default_cuda_path)
|
||||
|
||||
if is_windows():
|
||||
cuda_rt_lib_path = 'lib/x64/cudart.lib'
|
||||
elif is_linux():
|
||||
cuda_rt_lib_path = 'lib64/libcudart.so.%s' % tf_cuda_version
|
||||
elif is_macos():
|
||||
cuda_rt_lib_path = 'lib/libcudart.%s.dylib' % tf_cuda_version
|
||||
|
||||
cuda_toolkit_path_full = os.path.join(cuda_toolkit_path, cuda_rt_lib_path)
|
||||
if os.path.exists(cuda_toolkit_path_full):
|
||||
break
|
||||
|
||||
# Reset and retry
|
||||
print('Invalid path to CUDA %s toolkit. %s cannot be found' %
|
||||
(tf_cuda_version, cuda_toolkit_path_full))
|
||||
environ_cp['TF_CUDA_VERSION'] = ''
|
||||
environ_cp['CUDA_TOOLKIT_PATH'] = ''
|
||||
|
||||
# Set CUDA_TOOLKIT_PATH and TF_CUDA_VERSION
|
||||
environ_cp['CUDA_TOOLKIT_PATH'] = cuda_toolkit_path
|
||||
write_action_env_to_bazelrc('CUDA_TOOLKIT_PATH', cuda_toolkit_path)
|
||||
environ_cp['TF_CUDA_VERSION'] = tf_cuda_version
|
||||
write_action_env_to_bazelrc('TF_CUDA_VERSION', tf_cuda_version)
|
||||
|
||||
|
||||
def set_tf_cunn_version(environ_cp):
|
||||
"""Set CUDNN_INSTALL_PATH and TF_CUDNN_VERSION."""
|
||||
ask_cudnn_version = (
|
||||
'"Please specify the cuDNN version you want to use. '
|
||||
'[Leave empty to default to cuDNN %s.0]: ') % _DEFAULT_CUDNN_VERSION
|
||||
|
||||
while True:
|
||||
tf_cudnn_version = get_from_env_or_user_or_default(
|
||||
environ_cp, 'TF_CUDNN_VERSION', ask_cudnn_version,
|
||||
_DEFAULT_CUDNN_VERSION)
|
||||
|
||||
default_cudnn_path = environ_cp.get('CUDA_TOOLKIT_PATH')
|
||||
ask_cudnn_path = (r'Please specify the location where cuDNN %s library is '
|
||||
'installed. Refer to README.md for more details. [Default'
|
||||
' is %s]:') % (tf_cudnn_version, default_cudnn_path)
|
||||
cudnn_install_path = get_from_env_or_user_or_default(
|
||||
environ_cp, 'CUDNN_INSTALL_PATH', ask_cudnn_path, default_cudnn_path)
|
||||
|
||||
# Result returned from "read" will be used unexpanded. That make "~"
|
||||
# unusable. Going through one more level of expansion to handle that.
|
||||
cudnn_install_path = os.path.realpath(
|
||||
os.path.expanduser(cudnn_install_path))
|
||||
if is_windows():
|
||||
cudnn_install_path = cygpath(cudnn_install_path)
|
||||
|
||||
if is_windows():
|
||||
cuda_dnn_lib_path = 'lib/x64/cudnn.lib'
|
||||
cuda_dnn_lib_alt_path = 'lib/x64/cudnn.lib'
|
||||
elif is_linux():
|
||||
cuda_dnn_lib_path = 'lib64/libcudnn.so.%s' % tf_cudnn_version
|
||||
cuda_dnn_lib_alt_path = 'libcudnn.so.%s' % tf_cudnn_version
|
||||
elif is_macos():
|
||||
cuda_dnn_lib_path = 'lib/libcudnn.%s.dylib' % tf_cudnn_version
|
||||
cuda_dnn_lib_alt_path = 'libcudnn.%s.dylib' % tf_cudnn_version
|
||||
|
||||
cuda_dnn_lib_path_full = os.path.join(cudnn_install_path, cuda_dnn_lib_path)
|
||||
cuda_dnn_lib_alt_path_full = os.path.join(cudnn_install_path,
|
||||
cuda_dnn_lib_alt_path)
|
||||
if os.path.exists(cuda_dnn_lib_path_full) or os.path.exists(
|
||||
cuda_dnn_lib_alt_path_full):
|
||||
break
|
||||
|
||||
# Try another alternative for Linux
|
||||
if is_linux():
|
||||
if subprocess.call(['which', 'ldconfig']):
|
||||
ldconfig_bin = '/sbin/ldconfig'
|
||||
else:
|
||||
ldconfig_bin = 'ldconfig'
|
||||
cudnn_path_from_ldconfig = run_shell(
|
||||
r'%s -p | sed -n "s/.*libcudnn.so .* => \(.*\)/\\1/p"' % ldconfig_bin)
|
||||
if os.path.exists('%s.%s' % (cudnn_path_from_ldconfig, tf_cudnn_version)):
|
||||
cudnn_install_path = os.path.dirname(cudnn_path_from_ldconfig)
|
||||
break
|
||||
|
||||
# Reset and Retry
|
||||
print(
|
||||
'Invalid path to cuDNN %s toolkit. None of the following files can be '
|
||||
'found:' % tf_cudnn_version)
|
||||
print(cuda_dnn_lib_path_full)
|
||||
print(cuda_dnn_lib_alt_path_full)
|
||||
if is_linux():
|
||||
print('%s.%s' % (cudnn_path_from_ldconfig, tf_cudnn_version))
|
||||
|
||||
environ_cp['TF_CUDNN_VERSION'] = ''
|
||||
|
||||
# Set CUDNN_INSTALL_PATH and TF_CUDNN_VERSION
|
||||
environ_cp['CUDNN_INSTALL_PATH'] = cudnn_install_path
|
||||
write_action_env_to_bazelrc('CUDNN_INSTALL_PATH', cudnn_install_path)
|
||||
environ_cp['TF_CUDNN_VERSION'] = tf_cudnn_version
|
||||
write_action_env_to_bazelrc('TF_CUDNN_VERSION', tf_cudnn_version)
|
||||
|
||||
|
||||
def get_native_cuda_compute_capabilities(environ_cp):
|
||||
"""Get native cuda compute capabilities.
|
||||
|
||||
Args:
|
||||
environ_cp: copy of the os.environ.
|
||||
Returns:
|
||||
string of native cuda compute capabilities, separated by comma.
|
||||
"""
|
||||
device_query_bin = os.path.join(
|
||||
environ_cp.get('CUDA_TOOLKIT_PATH'), 'extras/demo_suite/deviceQuery')
|
||||
cmd = (r'"%s" | grep "Capability" | grep -o "[0-9]*\.[0-9]*" | sed '
|
||||
'":a;{N;s/\\n/,/};ba"') % device_query_bin
|
||||
try:
|
||||
output = run_shell(cmd)
|
||||
except subprocess.CalledProcessError:
|
||||
output = ''
|
||||
return output
|
||||
|
||||
|
||||
def set_tf_cuda_compute_capabilities(environ_cp):
|
||||
"""Set TF_CUDA_COMPUTE_CAPABILITIES."""
|
||||
while True:
|
||||
native_cuda_compute_capabilities = get_native_cuda_compute_capabilities(
|
||||
environ_cp)
|
||||
if not native_cuda_compute_capabilities:
|
||||
default_cuda_compute_capabilities = _DEFAULT_CUDA_COMPUTE_CAPABILITIES
|
||||
else:
|
||||
default_cuda_compute_capabilities = native_cuda_compute_capabilities
|
||||
|
||||
ask_cuda_compute_capabilities = (
|
||||
'Please specify a list of comma-separated '
|
||||
'Cuda compute capabilities you want to '
|
||||
'build with.\nYou can find the compute '
|
||||
'capability of your device at: '
|
||||
'https://developer.nvidia.com/cuda-gpus.\nPlease'
|
||||
' note that each additional compute '
|
||||
'capability significantly increases your '
|
||||
'build time and binary size. [Default is: %s]' %
|
||||
default_cuda_compute_capabilities)
|
||||
tf_cuda_compute_capabilities = get_from_env_or_user_or_default(
|
||||
environ_cp, 'TF_CUDA_COMPUTE_CAPABILITIES',
|
||||
ask_cuda_compute_capabilities, default_cuda_compute_capabilities)
|
||||
# Check whether all capabilities from the input is valid
|
||||
all_valid = True
|
||||
for compute_capability in tf_cuda_compute_capabilities.split(','):
|
||||
if not re.match('[0-9]+.[0-9]+', compute_capability):
|
||||
print('Invalid compute capability: ' % compute_capability)
|
||||
all_valid = False
|
||||
|
||||
if all_valid:
|
||||
break
|
||||
|
||||
# Reset and Retry
|
||||
environ_cp['TF_CUDA_COMPUTE_CAPABILITIES'] = ''
|
||||
|
||||
# Set TF_CUDA_COMPUTE_CAPABILITIES
|
||||
environ_cp['TF_CUDA_COMPUTE_CAPABILITIES'] = tf_cuda_compute_capabilities
|
||||
write_action_env_to_bazelrc('TF_CUDA_COMPUTE_CAPABILITIES',
|
||||
tf_cuda_compute_capabilities)
|
||||
|
||||
|
||||
def set_other_cuda_vars(environ_cp):
|
||||
"""Set other CUDA related variables."""
|
||||
if is_windows():
|
||||
# The following three variables are needed for MSVC toolchain configuration
|
||||
# in Bazel
|
||||
environ_cp['CUDA_PATH'] = environ_cp.get('CUDA_TOOLKIT_PATH')
|
||||
environ_cp['CUDA_COMPUTE_CAPABILITIES'] = environ_cp.get(
|
||||
'TF_CUDA_COMPUTE_CAPABILITIES')
|
||||
environ_cp['NO_WHOLE_ARCHIVE_OPTION'] = 1
|
||||
write_action_env_to_bazelrc('CUDA_PATH', environ_cp.get('CUDA_PATH'))
|
||||
write_action_env_to_bazelrc('CUDA_COMPUTE_CAPABILITIE',
|
||||
environ_cp.get('CUDA_COMPUTE_CAPABILITIE'))
|
||||
write_action_env_to_bazelrc('NO_WHOLE_ARCHIVE_OPTION',
|
||||
environ_cp.get('NO_WHOLE_ARCHIVE_OPTION'))
|
||||
write_to_bazelrc('build --config=win-cuda')
|
||||
write_to_bazelrc('test --config=win-cuda')
|
||||
else:
|
||||
# If CUDA is enabled, always use GPU during build and test.
|
||||
if environ_cp.get('TF_CUDA_CLANG') == '1':
|
||||
write_to_bazelrc('build --config=cuda_clang')
|
||||
write_to_bazelrc('test --config=cuda_clang')
|
||||
else:
|
||||
write_to_bazelrc('build --config=cuda')
|
||||
write_to_bazelrc('test --config=cuda')
|
||||
|
||||
|
||||
def set_host_cxx_compiler(environ_cp):
|
||||
"""Set HOST_CXX_COMPILER."""
|
||||
default_cxx_host_compiler = run_shell('which g++ || true')
|
||||
ask_cxx_host_compiler = (
|
||||
'Please specify which C++ compiler should be used as'
|
||||
' the host C++ compiler. [Default is %s]: ') % default_cxx_host_compiler
|
||||
|
||||
while True:
|
||||
host_cxx_compiler = get_from_env_or_user_or_default(
|
||||
environ_cp, 'HOST_CXX_COMPILER', ask_cxx_host_compiler,
|
||||
default_cxx_host_compiler)
|
||||
if os.path.exists(host_cxx_compiler):
|
||||
break
|
||||
|
||||
# Reset and retry
|
||||
print('Invalid C++ compiler path. %s cannot be found' % host_cxx_compiler)
|
||||
environ_cp['HOST_CXX_COMPILER'] = ''
|
||||
|
||||
# Set HOST_CXX_COMPILER
|
||||
environ_cp['HOST_CXX_COMPILER'] = host_cxx_compiler
|
||||
write_action_env_to_bazelrc('HOST_CXX_COMPILER', host_cxx_compiler)
|
||||
|
||||
|
||||
def set_host_c_compiler(environ_cp):
|
||||
"""Set HOST_C_COMPILER."""
|
||||
default_c_host_compiler = run_shell('which gcc || true')
|
||||
ask_c_host_compiler = (
|
||||
'Please specify which C compiler should be used as the'
|
||||
' host C compiler. [Default is %s]: ') % default_c_host_compiler
|
||||
|
||||
while True:
|
||||
host_c_compiler = get_from_env_or_user_or_default(
|
||||
environ_cp, 'HOST_C_COMPILER', ask_c_host_compiler,
|
||||
default_c_host_compiler)
|
||||
if os.path.exists(host_c_compiler):
|
||||
break
|
||||
|
||||
# Reset and retry
|
||||
print('Invalid C compiler path. %s cannot be found' % host_c_compiler)
|
||||
environ_cp['HOST_C_COMPILER'] = ''
|
||||
|
||||
# Set HOST_C_COMPILER
|
||||
environ_cp['HOST_C_COMPILER'] = host_c_compiler
|
||||
write_action_env_to_bazelrc('HOST_C_COMPILER', host_c_compiler)
|
||||
|
||||
|
||||
def set_computecpp_toolkit_path(environ_cp):
|
||||
"""Set COMPUTECPP_TOOLKIT_PATH."""
|
||||
ask_computecpp_toolkit_path = ('Please specify the location where ComputeCpp '
|
||||
'for SYCL %s is installed. [Default is %s]: '
|
||||
) % (_TF_OPENCL_VERSION,
|
||||
_DEFAULT_COMPUTECPP_TOOLKIT_PATH)
|
||||
|
||||
while True:
|
||||
computecpp_toolkit_path = get_from_env_or_user_or_default(
|
||||
environ_cp, 'COMPUTECPP_TOOLKIT_PATH', ask_computecpp_toolkit_path,
|
||||
_DEFAULT_COMPUTECPP_TOOLKIT_PATH)
|
||||
if is_linux():
|
||||
sycl_rt_lib_path = 'lib/libComputeCpp.so'
|
||||
else:
|
||||
sycl_rt_lib_path = ''
|
||||
|
||||
sycl_rt_lib_path_full = os.path.join(computecpp_toolkit_path,
|
||||
sycl_rt_lib_path)
|
||||
if os.path.exists(sycl_rt_lib_path_full):
|
||||
break
|
||||
|
||||
print('Invalid SYCL %s library path. %s cannot be found' %
|
||||
(_TF_OPENCL_VERSION, sycl_rt_lib_path_full))
|
||||
environ_cp['COMPUTECPP_TOOLKIT_PATH'] = ''
|
||||
|
||||
# Set COMPUTECPP_TOOLKIT_PATH
|
||||
environ_cp['COMPUTECPP_TOOLKIT_PATH'] = computecpp_toolkit_path
|
||||
write_action_env_to_bazelrc('COMPUTECPP_TOOLKIT_PATH',
|
||||
computecpp_toolkit_path)
|
||||
|
||||
|
||||
def set_mpi_home(environ_cp):
|
||||
"""Set MPI_HOME."""
|
||||
cmd = ('dirname $(dirname $(which mpirun)) || dirname $(dirname $(which '
|
||||
'mpiexec)) || true')
|
||||
default_mpi_home = run_shell(cmd)
|
||||
ask_mpi_home = ('Please specify the MPI toolkit folder. [Default is %s]: '
|
||||
) % default_mpi_home
|
||||
while True:
|
||||
mpi_home = get_from_env_or_user_or_default(environ_cp, 'MPI_HOME',
|
||||
ask_mpi_home, default_mpi_home)
|
||||
|
||||
if os.path.exists(os.path.join(mpi_home, 'include')) and os.path.exists(
|
||||
os.path.join(mpi_home, 'lib')):
|
||||
break
|
||||
|
||||
print('Invalid path to the MPI Toolkit. %s or %s cannot be found' %
|
||||
(os.path.join(mpi_home, 'include'),
|
||||
os.path.exists(os.path.join(mpi_home, 'lib'))))
|
||||
environ_cp['MPI_HOME'] = ''
|
||||
|
||||
# Set MPI_HOME
|
||||
environ_cp['MPI_HOME'] = str(mpi_home)
|
||||
|
||||
|
||||
def set_other_mpi_vars(environ_cp):
|
||||
"""Set other MPI related variables."""
|
||||
# Link the MPI header files
|
||||
mpi_home = environ_cp.get('MPI_HOME')
|
||||
symlink_force('%s/include/mpi.h' % mpi_home, 'third_party/mpi/mpi.h')
|
||||
|
||||
# Determine if we use OpenMPI or MVAPICH, these require different header files
|
||||
# to be included here to make bazel dependency checker happy
|
||||
if os.path.exists(os.path.join(mpi_home, 'include/mpi_portable_platform.h')):
|
||||
symlink_force(
|
||||
os.path.join(mpi_home, 'include/mpi_portable_platform.h'),
|
||||
'third_party/mpi/mpi_portable_platform.h')
|
||||
# TODO(gunan): avoid editing files in configure
|
||||
sed_in_place('third_party/mpi/mpi.bzl', 'MPI_LIB_IS_OPENMPI=False',
|
||||
'MPI_LIB_IS_OPENMPI=True')
|
||||
else:
|
||||
# MVAPICH / MPICH
|
||||
symlink_force(
|
||||
os.path.join(mpi_home, 'include/mpio.h'), 'third_party/mpi/mpio.h')
|
||||
symlink_force(
|
||||
os.path.join(mpi_home, 'include/mpicxx.h'), 'third_party/mpi/mpicxx.h')
|
||||
# TODO(gunan): avoid editing files in configure
|
||||
sed_in_place('third_party/mpi/mpi.bzl', 'MPI_LIB_IS_OPENMPI=True',
|
||||
'MPI_LIB_IS_OPENMPI=False')
|
||||
|
||||
if os.path.exists(os.path.join(mpi_home, 'lib/libmpi.so')):
|
||||
symlink_force(
|
||||
os.path.join(mpi_home, 'lib/libmpi.so'), 'third_party/mpi/libmpi.so')
|
||||
else:
|
||||
raise ValueError('Cannot find the MPI library file in %s/lib' % mpi_home)
|
||||
|
||||
|
||||
def set_mkl():
|
||||
write_to_bazelrc('build:mkl --define with_mkl_support=true')
|
||||
write_to_bazelrc('build:mkl --define using_mkl=true')
|
||||
write_to_bazelrc('build:mkl -c opt')
|
||||
write_to_bazelrc('build:mkl --copt="-DEIGEN_USE_VML"')
|
||||
print(
|
||||
'Add "--config=mkl" to your bazel command to build with MKL '
|
||||
'support.\nPlease note that MKL on MacOS or windows is still not '
|
||||
'supported.\nIf you would like to use a local MKL instead of '
|
||||
'downloading, please set the environment variable \"TF_MKL_ROOT\" every '
|
||||
'time before build.')
|
||||
|
||||
|
||||
def main():
|
||||
# Make a copy of os.environ to be clear when functions and getting and setting
|
||||
# environment variables.
|
||||
environ_cp = dict(os.environ)
|
||||
|
||||
check_bazel_version('0.4.5')
|
||||
|
||||
reset_tf_configure_bazelrc()
|
||||
cleanup_makefile()
|
||||
setup_python(environ_cp)
|
||||
run_gen_git_source(environ_cp)
|
||||
|
||||
if is_windows():
|
||||
environ_cp['TF_NEED_GCP'] = '0'
|
||||
environ_cp['TF_NEED_HDFS'] = '0'
|
||||
environ_cp['TF_NEED_JEMALLOC'] = '0'
|
||||
environ_cp['TF_NEED_OPENCL'] = '0'
|
||||
environ_cp['TF_CUDA_CLANG'] = '0'
|
||||
|
||||
if is_macos():
|
||||
environ_cp['TF_NEED_JEMALLOC'] = '0'
|
||||
|
||||
set_build_var(environ_cp, 'TF_NEED_JEMALLOC', 'jemalloc as malloc',
|
||||
'with_jemalloc', True)
|
||||
set_build_var(environ_cp, 'TF_NEED_GCP', 'Google Cloud Platform',
|
||||
'with_gcp_support', False)
|
||||
set_build_var(environ_cp, 'TF_NEED_HDFS', 'Hadoop File System',
|
||||
'with_hdfs_support', False)
|
||||
set_build_var(environ_cp, 'TF_ENABLE_XLA', 'XLA JIT', 'with_xla_support',
|
||||
False)
|
||||
set_build_var(environ_cp, 'TF_NEED_VERBS', 'VERBS', 'with_verbs_support',
|
||||
False)
|
||||
|
||||
set_action_env_var(environ_cp, 'TF_NEED_OPENCL', 'OpenCL', False)
|
||||
if environ_cp.get('TF_NEED_OPENCL') == '1':
|
||||
set_host_cxx_compiler(environ_cp)
|
||||
set_host_c_compiler(environ_cp)
|
||||
set_computecpp_toolkit_path(environ_cp)
|
||||
|
||||
set_action_env_var(environ_cp, 'TF_NEED_CUDA', 'CUDA', False)
|
||||
if environ_cp.get('TF_NEED_CUDA') == '1':
|
||||
set_tf_cuda_version(environ_cp)
|
||||
set_tf_cunn_version(environ_cp)
|
||||
set_tf_cuda_compute_capabilities(environ_cp)
|
||||
|
||||
set_tf_cuda_clang(environ_cp)
|
||||
if environ_cp.get('TF_CUDA_CLANG') == '1':
|
||||
# Set up which clang we should use as the cuda / host compiler.
|
||||
set_clang_cuda_compiler_path(environ_cp)
|
||||
else:
|
||||
# Set up which gcc nvcc should use as the host compiler
|
||||
# No need to set this on Windows
|
||||
if not is_windows():
|
||||
set_gcc_host_compiler_path(environ_cp)
|
||||
set_other_cuda_vars(environ_cp)
|
||||
|
||||
set_build_var(environ_cp, 'TF_NEED_MPI', 'MPI', 'with_mpi_support', False)
|
||||
if environ_cp.get('TF_NEED_MPI') == '1':
|
||||
set_mpi_home(environ_cp)
|
||||
set_other_mpi_vars(environ_cp)
|
||||
|
||||
set_cc_opt_flags(environ_cp)
|
||||
set_mkl()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -56,7 +56,7 @@ else
|
||||
fi
|
||||
|
||||
pushd "${CI_TENSORFLOW_SUBMODULE_PATH:-.}"
|
||||
yes "" | ./configure
|
||||
$PYTHON_BIN_PATH configure.py
|
||||
popd
|
||||
|
||||
# Gather and print build information
|
||||
|
||||
@@ -120,7 +120,7 @@ else
|
||||
fi
|
||||
|
||||
export TF_NEED_CUDA=$IS_GPU
|
||||
yes "" | ./configure
|
||||
${PYTHON_BIN_PATH} configure.py
|
||||
|
||||
# Figure out how many concurrent tests we can run and do run the tests.
|
||||
BAZEL_PARALLEL_TEST_FLAGS=""
|
||||
|
||||
@@ -30,7 +30,7 @@ export TF_NEED_HDFS=0
|
||||
export TF_NEED_CUDA=0
|
||||
# Only running cc tests, python version does not matter.
|
||||
export PYTHON_BIN_PATH=`which python`
|
||||
yes "" | ./configure
|
||||
$PYTHON_BIN_PATH configure.py
|
||||
|
||||
# Run bazel test command. Double test timeouts to avoid flakes.
|
||||
bazel test --test_tag_filters=-no_oss,-gpu,-benchmark-test --test_lang_filters=cc -k \
|
||||
|
||||
@@ -29,7 +29,7 @@ export TF_NEED_GCP=0
|
||||
export TF_NEED_HDFS=0
|
||||
export TF_NEED_CUDA=0
|
||||
export PYTHON_BIN_PATH=`which python2`
|
||||
yes "" | ./configure
|
||||
$PYTHON_BIN_PATH configure.py
|
||||
|
||||
# Run bazel test command. Double test timeouts to avoid flakes.
|
||||
bazel test --test_tag_filters=-no_oss,-oss_serial,-gpu,-benchmark-test --test_lang_filters=py -k \
|
||||
|
||||
@@ -29,7 +29,7 @@ export TF_NEED_GCP=0
|
||||
export TF_NEED_HDFS=0
|
||||
export TF_NEED_CUDA=0
|
||||
export PYTHON_BIN_PATH=`which python3`
|
||||
yes "" | ./configure
|
||||
$PYTHON_BIN_PATH configure.py
|
||||
|
||||
# Run bazel test command. Double test timeouts to avoid flakes.
|
||||
bazel test --test_tag_filters=-no_oss,-oss_serial,-gpu,-benchmark-test -k \
|
||||
|
||||
@@ -29,7 +29,7 @@ export TF_NEED_GCP=0
|
||||
export TF_NEED_HDFS=0
|
||||
export TF_NEED_CUDA=0
|
||||
export PYTHON_BIN_PATH=`which python3`
|
||||
yes "" | ./configure
|
||||
$PYTHON_BIN_PATH configure.py
|
||||
|
||||
# Run bazel test command. Double test timeouts to avoid flakes.
|
||||
bazel test --test_tag_filters=-no_oss,-oss_serial,-gpu,-benchmark-test --test_lang_filters=py -k \
|
||||
|
||||
@@ -32,7 +32,7 @@ export PYTHON_BIN_PATH=`which python3`
|
||||
export TF_NEED_CUDA=1
|
||||
export TF_CUDA_COMPUTE_CAPABILITIES=3.7
|
||||
|
||||
yes "" | ./configure
|
||||
$PYTHON_BIN_PATH configure.py
|
||||
|
||||
# Run bazel test command. Double test timeouts to avoid flakes.
|
||||
bazel test --config=cuda --test_tag_filters=-no_oss,-oss_serial,-no_gpu,-benchmark-test -k \
|
||||
|
||||
@@ -32,7 +32,7 @@ export PYTHON_BIN_PATH=`which python3`
|
||||
export TF_NEED_CUDA=1
|
||||
export TF_CUDA_COMPUTE_CAPABILITIES=3.7
|
||||
|
||||
yes "" | ./configure
|
||||
$PYTHON_BIN_PATH configure.py
|
||||
|
||||
# Run bazel test command. Double test timeouts to avoid flakes.
|
||||
bazel test --config=cuda --test_tag_filters=-no_oss,-oss_serial,-no_gpu,-benchmark-test -k \
|
||||
|
||||
@@ -30,7 +30,7 @@ export TF_NEED_GCP=0
|
||||
export TF_NEED_HDFS=0
|
||||
export TF_NEED_CUDA=0
|
||||
export PYTHON_BIN_PATH=$(which python2)
|
||||
yes "" | ./configure
|
||||
$PYTHON_BIN_PATH configure.py
|
||||
which bazel
|
||||
bazel test --test_tag_filters=-no_oss,-gpu,-benchmark-test,-nomac \
|
||||
--test_timeout 300,450,1200,3600 \
|
||||
|
||||
@@ -33,7 +33,7 @@ export TF_NEED_CUDA=1
|
||||
export TF_ENABLE_XLA=1
|
||||
export TF_CUDA_COMPUTE_CAPABILITIES=3.7
|
||||
|
||||
yes "" | ./configure
|
||||
$PYTHON_BIN_PATH configure.py
|
||||
|
||||
# Run bazel test command. Double test timeouts to avoid flakes.
|
||||
bazel test --config=cuda --test_tag_filters=-no_gpu,-benchmark-test -k \
|
||||
|
||||
Reference in New Issue
Block a user