Files
pytorch/WORKSPACE

Ignoring revisions in .git-blame-ignore-revs. Click here to bypass and see the normal blame view.

365 lines
9.2 KiB
Python
Raw Normal View History

workspace(name = "pytorch")
load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
load("//tools/rules:workspace.bzl", "new_patched_local_repository")
[bazel] enable sccache+nvcc in CI (#95528) Fixes #79348 This change is mostly focused on enabling nvcc+sccache in the PyTorch CI. Along the way we had to do couple tweaks: 1. Split the rules_cc from the rules_cuda that embeeded them before. This is needed in order to apply a different patch to the rules_cc compare to the one that rules_cuda does by default. This is in turn needed because we need to workaround an nvcc behavior where it doesn't send `-iquote xxx` to the host compiler, but it does send `-isystem xxx`. So we workaround this problem with (ab)using `-isystem` instead. Without it we are getting errors like `xxx` is not found. 2. Workaround bug in bazel https://github.com/bazelbuild/bazel/issues/10167 that prevents us from using a straightforward and honest `nvcc` sccache wrapper. Instead we generate ad-hock bazel specific nvcc wrapper that has internal knowledge of the relative bazel paths to local_cuda. This allows us to workaround the issue with CUDA symlinks. Without it we are getting `undeclared inclusion(s) in rule` all over the place for CUDA headers. ## Test plan Green CI build https://github.com/pytorch/pytorch/actions/runs/4267147180/jobs/7428431740 Note that now it says "CUDA" in the sccache output ``` + sccache --show-stats Compile requests 9784 Compile requests executed 6726 Cache hits 6200 Cache hits (C/C++) 6131 Cache hits (CUDA) 69 Cache misses 519 Cache misses (C/C++) 201 Cache misses (CUDA) 318 Cache timeouts 0 Cache read errors 0 Forced recaches 0 Cache write errors 0 Compilation failures 0 Cache errors 7 Cache errors (C/C++) 7 Non-cacheable compilations 0 Non-cacheable calls 2893 Non-compilation calls 165 Unsupported compiler calls 0 Average cache write 0.116 s Average cache read miss 23.722 s Average cache read hit 0.057 s Failed distributed compilations 0 ``` Pull Request resolved: https://github.com/pytorch/pytorch/pull/95528 Approved by: https://github.com/huydhn
2023-02-28 03:51:08 +00:00
http_archive(
name = "rules_cc",
patches = [
"//:tools/rules_cc/cuda_support.patch",
],
strip_prefix = "rules_cc-40548a2974f1aea06215272d9c2b47a14a24e556",
[bazel] enable sccache+nvcc in CI (#95528) Fixes #79348 This change is mostly focused on enabling nvcc+sccache in the PyTorch CI. Along the way we had to do couple tweaks: 1. Split the rules_cc from the rules_cuda that embeeded them before. This is needed in order to apply a different patch to the rules_cc compare to the one that rules_cuda does by default. This is in turn needed because we need to workaround an nvcc behavior where it doesn't send `-iquote xxx` to the host compiler, but it does send `-isystem xxx`. So we workaround this problem with (ab)using `-isystem` instead. Without it we are getting errors like `xxx` is not found. 2. Workaround bug in bazel https://github.com/bazelbuild/bazel/issues/10167 that prevents us from using a straightforward and honest `nvcc` sccache wrapper. Instead we generate ad-hock bazel specific nvcc wrapper that has internal knowledge of the relative bazel paths to local_cuda. This allows us to workaround the issue with CUDA symlinks. Without it we are getting `undeclared inclusion(s) in rule` all over the place for CUDA headers. ## Test plan Green CI build https://github.com/pytorch/pytorch/actions/runs/4267147180/jobs/7428431740 Note that now it says "CUDA" in the sccache output ``` + sccache --show-stats Compile requests 9784 Compile requests executed 6726 Cache hits 6200 Cache hits (C/C++) 6131 Cache hits (CUDA) 69 Cache misses 519 Cache misses (C/C++) 201 Cache misses (CUDA) 318 Cache timeouts 0 Cache read errors 0 Forced recaches 0 Cache write errors 0 Compilation failures 0 Cache errors 7 Cache errors (C/C++) 7 Non-cacheable compilations 0 Non-cacheable calls 2893 Non-compilation calls 165 Unsupported compiler calls 0 Average cache write 0.116 s Average cache read miss 23.722 s Average cache read hit 0.057 s Failed distributed compilations 0 ``` Pull Request resolved: https://github.com/pytorch/pytorch/pull/95528 Approved by: https://github.com/huydhn
2023-02-28 03:51:08 +00:00
urls = [
"https://mirror.bazel.build/github.com/bazelbuild/rules_cc/archive/40548a2974f1aea06215272d9c2b47a14a24e556.tar.gz",
"https://github.com/bazelbuild/rules_cc/archive/40548a2974f1aea06215272d9c2b47a14a24e556.tar.gz",
],
)
http_archive(
name = "rules_cuda",
strip_prefix = "runtime-b1c7cce21ba4661c17ac72421c6a0e2015e7bef3/third_party/rules_cuda",
urls = ["https://github.com/tensorflow/runtime/archive/b1c7cce21ba4661c17ac72421c6a0e2015e7bef3.tar.gz"],
)
http_archive(
name = "platforms",
urls = [
"https://mirror.bazel.build/github.com/bazelbuild/platforms/releases/download/0.0.10/platforms-0.0.10.tar.gz",
# TODO Fix bazel linter to support hashes for release tarballs.
# "https://github.com/bazelbuild/platforms/releases/download/0.0.10/platforms-0.0.10.tar.gz",
],
# sha256 = "218efe8ee736d26a3572663b374a253c012b716d8af0c07e842e82f238a0a7ee",
)
load("@rules_cuda//cuda:dependencies.bzl", "rules_cuda_dependencies")
[bazel] enable sccache+nvcc in CI (#95528) Fixes #79348 This change is mostly focused on enabling nvcc+sccache in the PyTorch CI. Along the way we had to do couple tweaks: 1. Split the rules_cc from the rules_cuda that embeeded them before. This is needed in order to apply a different patch to the rules_cc compare to the one that rules_cuda does by default. This is in turn needed because we need to workaround an nvcc behavior where it doesn't send `-iquote xxx` to the host compiler, but it does send `-isystem xxx`. So we workaround this problem with (ab)using `-isystem` instead. Without it we are getting errors like `xxx` is not found. 2. Workaround bug in bazel https://github.com/bazelbuild/bazel/issues/10167 that prevents us from using a straightforward and honest `nvcc` sccache wrapper. Instead we generate ad-hock bazel specific nvcc wrapper that has internal knowledge of the relative bazel paths to local_cuda. This allows us to workaround the issue with CUDA symlinks. Without it we are getting `undeclared inclusion(s) in rule` all over the place for CUDA headers. ## Test plan Green CI build https://github.com/pytorch/pytorch/actions/runs/4267147180/jobs/7428431740 Note that now it says "CUDA" in the sccache output ``` + sccache --show-stats Compile requests 9784 Compile requests executed 6726 Cache hits 6200 Cache hits (C/C++) 6131 Cache hits (CUDA) 69 Cache misses 519 Cache misses (C/C++) 201 Cache misses (CUDA) 318 Cache timeouts 0 Cache read errors 0 Forced recaches 0 Cache write errors 0 Compilation failures 0 Cache errors 7 Cache errors (C/C++) 7 Non-cacheable compilations 0 Non-cacheable calls 2893 Non-compilation calls 165 Unsupported compiler calls 0 Average cache write 0.116 s Average cache read miss 23.722 s Average cache read hit 0.057 s Failed distributed compilations 0 ``` Pull Request resolved: https://github.com/pytorch/pytorch/pull/95528 Approved by: https://github.com/huydhn
2023-02-28 03:51:08 +00:00
rules_cuda_dependencies(with_rules_cc = False)
load("@rules_cc//cc:repositories.bzl", "rules_cc_toolchains")
rules_cc_toolchains()
http_archive(
name = "bazel_skylib",
urls = [
"https://github.com/bazelbuild/bazel-skylib/releases/download/1.0.2/bazel-skylib-1.0.2.tar.gz",
],
)
http_archive(
name = "pybind11_bazel",
strip_prefix = "pybind11_bazel-b162c7c88a253e3f6b673df0c621aca27596ce6b",
urls = ["https://github.com/pybind/pybind11_bazel/archive/b162c7c88a253e3f6b673df0c621aca27596ce6b.zip"],
)
new_local_repository(
name = "pybind11",
build_file = "@pybind11_bazel//:pybind11.BUILD",
path = "third_party/pybind11",
)
http_archive(
name = "com_github_glog",
build_file_content = """
licenses(['notice'])
load(':bazel/glog.bzl', 'glog_library')
# TODO: figure out why enabling gflags leads to SIGSEV on the logging init
glog_library(with_gflags=0)
""",
strip_prefix = "glog-0.4.0",
urls = [
"https://github.com/google/glog/archive/v0.4.0.tar.gz",
],
)
http_archive(
name = "com_github_gflags_gflags",
strip_prefix = "gflags-2.2.2",
urls = [
"https://github.com/gflags/gflags/archive/v2.2.2.tar.gz",
],
)
http_archive(
name = "com_github_opentelemetry-cpp",
urls = [
"https://github.com/open-telemetry/opentelemetry-cpp/archive/refs/tags/v1.14.2.tar.gz",
],
)
new_local_repository(
name = "gloo",
build_file = "//third_party:gloo.BUILD",
path = "third_party/gloo",
)
new_local_repository(
name = "onnx",
build_file = "//third_party:onnx.BUILD",
path = "third_party/onnx",
)
local_repository(
name = "com_google_protobuf",
path = "third_party/protobuf",
)
new_local_repository(
name = "eigen",
build_file = "//third_party:eigen.BUILD",
path = "third_party/eigen",
)
new_local_repository(
name = "cutlass",
build_file = "//third_party:cutlass.BUILD",
path = "third_party/cutlass",
)
new_local_repository(
name = "fbgemm",
build_file = "//third_party:fbgemm/BUILD.bazel",
path = "third_party/fbgemm",
repo_mapping = {"@cpuinfo": "@org_pytorch_cpuinfo"},
)
new_local_repository(
name = "ideep",
build_file = "//third_party:ideep.BUILD",
path = "third_party/ideep",
)
new_local_repository(
name = "mkl_dnn",
build_file = "//third_party:mkl-dnn.BUILD",
path = "third_party/ideep/mkl-dnn",
)
new_local_repository(
name = "org_pytorch_cpuinfo",
build_file = "//third_party:cpuinfo/BUILD.bazel",
path = "third_party/cpuinfo",
)
new_local_repository(
name = "asmjit",
build_file = "//third_party:fbgemm/third_party/asmjit.BUILD",
path = "third_party/fbgemm/third_party/asmjit",
)
new_local_repository(
name = "sleef",
build_file = "//third_party:sleef.BUILD",
path = "third_party/sleef",
)
new_local_repository(
name = "fmt",
build_file = "//third_party:fmt.BUILD",
path = "third_party/fmt",
)
new_local_repository(
name = "kineto",
build_file = "//third_party:kineto.BUILD",
path = "third_party/kineto",
)
new_local_repository(
name = "opentelemetry-cpp",
build_file = "//third_party::opentelemetry-cpp.BUILD",
path = "third_party/opentelemetry-cpp",
)
new_local_repository(
name = "cpp-httplib",
build_file = "//third_party:cpp-httplib.BUILD",
path = "third_party/cpp-httplib",
)
new_local_repository(
name = "nlohmann",
build_file = "//third_party:nlohmann.BUILD",
path = "third_party/nlohmann",
)
new_local_repository(
name = "tensorpipe",
build_file = "//third_party:tensorpipe.BUILD",
path = "third_party/tensorpipe",
)
http_archive(
name = "mkl",
build_file = "//third_party:mkl.BUILD",
sha256 = "59154b30dd74561e90d547f9a3af26c75b6f4546210888f09c9d4db8f4bf9d4c",
strip_prefix = "lib",
urls = [
"https://anaconda.org/anaconda/mkl/2020.0/download/linux-64/mkl-2020.0-166.tar.bz2",
],
)
http_archive(
name = "mkl_headers",
build_file = "//third_party:mkl_headers.BUILD",
sha256 = "2af3494a4bebe5ddccfdc43bacc80fcd78d14c1954b81d2c8e3d73b55527af90",
urls = [
"https://anaconda.org/anaconda/mkl-include/2020.0/download/linux-64/mkl-include-2020.0-166.tar.bz2",
],
)
http_archive(
name = "rules_python",
# TODO Fix bazel linter to support hashes for release tarballs.
#
# sha256 = "94750828b18044533e98a129003b6a68001204038dc4749f40b195b24c38f49f",
strip_prefix = "rules_python-0.21.0",
url = "https://github.com/bazelbuild/rules_python/releases/download/0.21.0/rules_python-0.21.0.tar.gz",
)
load("@rules_python//python:repositories.bzl", "py_repositories")
py_repositories()
load("@rules_python//python:repositories.bzl", "python_register_toolchains")
python_register_toolchains(
name = "python3_10",
python_version = "3.10",
)
load("@python3_10//:defs.bzl", "interpreter")
load("@rules_python//python:pip.bzl", "pip_parse")
pip_parse(
name = "pip_deps",
python_interpreter_target = interpreter,
requirements_lock = "//:tools/build/bazel/requirements.txt",
)
load("@pip_deps//:requirements.bzl", "install_deps")
install_deps()
load("@pybind11_bazel//:python_configure.bzl", "python_configure")
python_configure(
name = "local_config_python",
python_interpreter_target = interpreter,
)
load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps")
protobuf_deps()
new_local_repository(
name = "cuda",
build_file = "@//third_party:cuda.BUILD",
path = "/usr/local/cuda",
[bazel] GPU-support: add @local_config_cuda and @cuda (#63604) Summary: ## Context We take the first step at tackling the GPU-bazel support by adding bazel external workspaces `local_config_cuda` and `cuda`, where the first one has some hardcoded values and lists of files, and the second one provides a nicer, high-level wrapper that maps into the already expected by pytorch bazel targets that are guarded with `if_cuda` macro. The prefix `local_config_` signifies the fact that we are breaking the bazel hermeticity philosophy by explicitly relaying on the CUDA installation that is present on the machine. ## Testing Notice an important scenario that is unlocked by this change: compilation of cpp code that depends on cuda libraries (i.e. cuda.h and so on). Before: ``` sergei.vorobev@cs-sv7xn77uoy-gpu-1628706590:~/src/pytorch4$ bazelisk build --define=cuda=true //:c10 ERROR: /home/sergei.vorobev/src/pytorch4/tools/config/BUILD:12:1: no such package 'tools/toolchain': BUILD file not found in any of the following directories. Add a BUILD file to a directory to mark it as a package. - /home/sergei.vorobev/src/pytorch4/tools/toolchain and referenced by '//tools/config:cuda_enabled_and_capable' ERROR: While resolving configuration keys for //:c10: Analysis failed ERROR: Analysis of target '//:c10' failed; build aborted: Analysis failed INFO: Elapsed time: 0.259s INFO: 0 processes. FAILED: Build did NOT complete successfully (2 packages loaded, 2 targets configured) ``` After: ``` sergei.vorobev@cs-sv7xn77uoy-gpu-1628706590:~/src/pytorch4$ bazelisk build --define=cuda=true //:c10 INFO: Analyzed target //:c10 (6 packages loaded, 246 targets configured). INFO: Found 1 target... Target //:c10 up-to-date: bazel-bin/libc10.lo bazel-bin/libc10.so INFO: Elapsed time: 0.617s, Critical Path: 0.04s INFO: 0 processes. INFO: Build completed successfully, 1 total action ``` The `//:c10` target is a good testing one for this, because it has such cases where the [glob is different](https://github.com/pytorch/pytorch/blob/075024b9a34904ec3ecdab3704c3bcaa329bdfea/BUILD.bazel#L76-L81), based on do we compile for CUDA or not. ## What is out of scope of this PR This PR is a first in a series of providing the comprehensive GPU bazel build support. Namely, we don't tackle the [cu_library](https://github.com/pytorch/pytorch/blob/11a40ad915d4d3d8551588e303204810887fcf8d/tools/rules/cu.bzl#L2) implementation here. This would be a separate large chunk of work. Pull Request resolved: https://github.com/pytorch/pytorch/pull/63604 Reviewed By: soulitzer Differential Revision: D30442083 Pulled By: malfet fbshipit-source-id: b2a8e4f7e5a25a69b960a82d9e36ba568eb64595
2021-08-27 09:31:36 -07:00
)
new_local_repository(
name = "cudnn",
build_file = "@//third_party:cudnn.BUILD",
path = "/usr/local/cuda",
[bazel] GPU-support: add @local_config_cuda and @cuda (#63604) Summary: ## Context We take the first step at tackling the GPU-bazel support by adding bazel external workspaces `local_config_cuda` and `cuda`, where the first one has some hardcoded values and lists of files, and the second one provides a nicer, high-level wrapper that maps into the already expected by pytorch bazel targets that are guarded with `if_cuda` macro. The prefix `local_config_` signifies the fact that we are breaking the bazel hermeticity philosophy by explicitly relaying on the CUDA installation that is present on the machine. ## Testing Notice an important scenario that is unlocked by this change: compilation of cpp code that depends on cuda libraries (i.e. cuda.h and so on). Before: ``` sergei.vorobev@cs-sv7xn77uoy-gpu-1628706590:~/src/pytorch4$ bazelisk build --define=cuda=true //:c10 ERROR: /home/sergei.vorobev/src/pytorch4/tools/config/BUILD:12:1: no such package 'tools/toolchain': BUILD file not found in any of the following directories. Add a BUILD file to a directory to mark it as a package. - /home/sergei.vorobev/src/pytorch4/tools/toolchain and referenced by '//tools/config:cuda_enabled_and_capable' ERROR: While resolving configuration keys for //:c10: Analysis failed ERROR: Analysis of target '//:c10' failed; build aborted: Analysis failed INFO: Elapsed time: 0.259s INFO: 0 processes. FAILED: Build did NOT complete successfully (2 packages loaded, 2 targets configured) ``` After: ``` sergei.vorobev@cs-sv7xn77uoy-gpu-1628706590:~/src/pytorch4$ bazelisk build --define=cuda=true //:c10 INFO: Analyzed target //:c10 (6 packages loaded, 246 targets configured). INFO: Found 1 target... Target //:c10 up-to-date: bazel-bin/libc10.lo bazel-bin/libc10.so INFO: Elapsed time: 0.617s, Critical Path: 0.04s INFO: 0 processes. INFO: Build completed successfully, 1 total action ``` The `//:c10` target is a good testing one for this, because it has such cases where the [glob is different](https://github.com/pytorch/pytorch/blob/075024b9a34904ec3ecdab3704c3bcaa329bdfea/BUILD.bazel#L76-L81), based on do we compile for CUDA or not. ## What is out of scope of this PR This PR is a first in a series of providing the comprehensive GPU bazel build support. Namely, we don't tackle the [cu_library](https://github.com/pytorch/pytorch/blob/11a40ad915d4d3d8551588e303204810887fcf8d/tools/rules/cu.bzl#L2) implementation here. This would be a separate large chunk of work. Pull Request resolved: https://github.com/pytorch/pytorch/pull/63604 Reviewed By: soulitzer Differential Revision: D30442083 Pulled By: malfet fbshipit-source-id: b2a8e4f7e5a25a69b960a82d9e36ba568eb64595
2021-08-27 09:31:36 -07:00
)
new_local_repository(
name = "cudnn_frontend",
build_file = "@//third_party:cudnn_frontend.BUILD",
path = "third_party/cudnn_frontend/",
)
local_repository(
name = "com_github_google_flatbuffers",
path = "third_party/flatbuffers",
)
local_repository(
name = "google_benchmark",
path = "third_party/benchmark",
)
local_repository(
name = "com_google_googletest",
path = "third_party/googletest",
)
local_repository(
name = "pthreadpool",
path = "third_party/pthreadpool",
repo_mapping = {"@com_google_benchmark": "@google_benchmark"},
)
local_repository(
name = "FXdiv",
path = "third_party/FXdiv",
repo_mapping = {"@com_google_benchmark": "@google_benchmark"},
)
local_repository(
name = "XNNPACK",
path = "third_party/XNNPACK",
repo_mapping = {"@com_google_benchmark": "@google_benchmark"},
)
local_repository(
name = "gemmlowp",
path = "third_party/gemmlowp/gemmlowp",
)
### Unused repos start
# `unused` repos are defined to hide bazel files from submodules of submodules.
# This allows us to run `bazel build //...` and not worry about the submodules madness.
# Otherwise everything traverses recursively and a lot of submodules of submodules have
# they own bazel build files.
local_repository(
name = "unused_tensorpipe_googletest",
path = "third_party/tensorpipe/third_party/googletest",
)
local_repository(
name = "unused_fbgemm",
path = "third_party/fbgemm",
)
local_repository(
name = "unused_ftm_bazel",
path = "third_party/fmt/support/bazel",
)
local_repository(
name = "unused_kineto_fmt_bazel",
path = "third_party/kineto/libkineto/third_party/fmt/support/bazel",
)
local_repository(
name = "unused_kineto_dynolog_googletest",
path = "third_party/kineto/libkineto/third_party/dynolog/third_party/googletest",
)
local_repository(
name = "unused_kineto_dynolog_gflags",
path = "third_party/kineto/libkineto/third_party/dynolog/third_party/gflags",
)
local_repository(
name = "unused_kineto_dynolog_glog",
path = "third_party/kineto/libkineto/third_party/dynolog/third_party/glog",
)
local_repository(
name = "unused_kineto_googletest",
path = "third_party/kineto/libkineto/third_party/googletest",
)
local_repository(
name = "unused_onnx_benchmark",
path = "third_party/onnx/third_party/benchmark",
)
### Unused repos end