diff --git a/.ci/pytorch/create_test_cert.py b/.ci/pytorch/create_test_cert.py index 4e31f97878f..a86facc3b73 100644 --- a/.ci/pytorch/create_test_cert.py +++ b/.ci/pytorch/create_test_cert.py @@ -1,10 +1,10 @@ from datetime import datetime, timedelta from tempfile import mkdtemp -from cryptography.hazmat.primitives import serialization -from cryptography.hazmat.primitives.asymmetric import rsa + from cryptography import x509 +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric import rsa from cryptography.x509.oid import NameOID -from cryptography.hazmat.primitives import hashes temp_dir = mkdtemp() print(temp_dir) @@ -16,37 +16,43 @@ def genrsa(path): key_size=2048, ) with open(path, "wb") as f: - f.write(key.private_bytes( - encoding=serialization.Encoding.PEM, - format=serialization.PrivateFormat.TraditionalOpenSSL, - encryption_algorithm=serialization.NoEncryption(), - )) + f.write( + key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.TraditionalOpenSSL, + encryption_algorithm=serialization.NoEncryption(), + ) + ) return key def create_cert(path, C, ST, L, O, key): - subject = issuer = x509.Name([ - x509.NameAttribute(NameOID.COUNTRY_NAME, C), - x509.NameAttribute(NameOID.STATE_OR_PROVINCE_NAME, ST), - x509.NameAttribute(NameOID.LOCALITY_NAME, L), - x509.NameAttribute(NameOID.ORGANIZATION_NAME, O), - ]) - cert = x509.CertificateBuilder().subject_name( - subject - ).issuer_name( - issuer - ).public_key( - key.public_key() - ).serial_number( - x509.random_serial_number() - ).not_valid_before( - datetime.utcnow() - ).not_valid_after( - # Our certificate will be valid for 10 days - datetime.utcnow() + timedelta(days=10) - ).add_extension( - x509.BasicConstraints(ca=True, path_length=None), critical=True, - ).sign(key, hashes.SHA256()) + subject = issuer = x509.Name( + [ + x509.NameAttribute(NameOID.COUNTRY_NAME, C), + x509.NameAttribute(NameOID.STATE_OR_PROVINCE_NAME, ST), + x509.NameAttribute(NameOID.LOCALITY_NAME, L), + x509.NameAttribute(NameOID.ORGANIZATION_NAME, O), + ] + ) + cert = ( + x509.CertificateBuilder() + .subject_name(subject) + .issuer_name(issuer) + .public_key(key.public_key()) + .serial_number(x509.random_serial_number()) + .not_valid_before(datetime.utcnow()) + .not_valid_after( + # Our certificate will be valid for 10 days + datetime.utcnow() + + timedelta(days=10) + ) + .add_extension( + x509.BasicConstraints(ca=True, path_length=None), + critical=True, + ) + .sign(key, hashes.SHA256()) + ) # Write our certificate out to disk. with open(path, "wb") as f: f.write(cert.public_bytes(serialization.Encoding.PEM)) @@ -54,43 +60,65 @@ def create_cert(path, C, ST, L, O, key): def create_req(path, C, ST, L, O, key): - csr = x509.CertificateSigningRequestBuilder().subject_name(x509.Name([ - # Provide various details about who we are. - x509.NameAttribute(NameOID.COUNTRY_NAME, C), - x509.NameAttribute(NameOID.STATE_OR_PROVINCE_NAME, ST), - x509.NameAttribute(NameOID.LOCALITY_NAME, L), - x509.NameAttribute(NameOID.ORGANIZATION_NAME, O), - ])).sign(key, hashes.SHA256()) + csr = ( + x509.CertificateSigningRequestBuilder() + .subject_name( + x509.Name( + [ + # Provide various details about who we are. + x509.NameAttribute(NameOID.COUNTRY_NAME, C), + x509.NameAttribute(NameOID.STATE_OR_PROVINCE_NAME, ST), + x509.NameAttribute(NameOID.LOCALITY_NAME, L), + x509.NameAttribute(NameOID.ORGANIZATION_NAME, O), + ] + ) + ) + .sign(key, hashes.SHA256()) + ) with open(path, "wb") as f: f.write(csr.public_bytes(serialization.Encoding.PEM)) return csr def sign_certificate_request(path, csr_cert, ca_cert, private_ca_key): - cert = x509.CertificateBuilder().subject_name( - csr_cert.subject - ).issuer_name( - ca_cert.subject - ).public_key( - csr_cert.public_key() - ).serial_number( - x509.random_serial_number() - ).not_valid_before( - datetime.utcnow() - ).not_valid_after( - # Our certificate will be valid for 10 days - datetime.utcnow() + timedelta(days=10) - # Sign our certificate with our private key - ).sign(private_ca_key, hashes.SHA256()) + cert = ( + x509.CertificateBuilder() + .subject_name(csr_cert.subject) + .issuer_name(ca_cert.subject) + .public_key(csr_cert.public_key()) + .serial_number(x509.random_serial_number()) + .not_valid_before(datetime.utcnow()) + .not_valid_after( + # Our certificate will be valid for 10 days + datetime.utcnow() + + timedelta(days=10) + # Sign our certificate with our private key + ) + .sign(private_ca_key, hashes.SHA256()) + ) with open(path, "wb") as f: f.write(cert.public_bytes(serialization.Encoding.PEM)) return cert ca_key = genrsa(temp_dir + "/ca.key") -ca_cert = create_cert(temp_dir + "/ca.pem", "US", "New York", "New York", "Gloo Certificate Authority", ca_key) +ca_cert = create_cert( + temp_dir + "/ca.pem", + "US", + "New York", + "New York", + "Gloo Certificate Authority", + ca_key, +) pkey = genrsa(temp_dir + "/pkey.key") -csr = create_req(temp_dir + "/csr.csr", "US", "California", "San Francisco", "Gloo Testing Company", pkey) +csr = create_req( + temp_dir + "/csr.csr", + "US", + "California", + "San Francisco", + "Gloo Testing Company", + pkey, +) cert = sign_certificate_request(temp_dir + "/cert.pem", csr, ca_cert, ca_key) diff --git a/.ci/pytorch/perf_test/compare_with_baseline.py b/.ci/pytorch/perf_test/compare_with_baseline.py index c756df37872..49b77cbba2a 100644 --- a/.ci/pytorch/perf_test/compare_with_baseline.py +++ b/.ci/pytorch/perf_test/compare_with_baseline.py @@ -1,32 +1,41 @@ -import sys +import argparse import json import math -import argparse +import sys parser = argparse.ArgumentParser() -parser.add_argument('--test-name', dest='test_name', action='store', - required=True, help='test name') -parser.add_argument('--sample-stats', dest='sample_stats', action='store', - required=True, help='stats from sample') -parser.add_argument('--update', action='store_true', - help='whether to update baseline using stats from sample') +parser.add_argument( + "--test-name", dest="test_name", action="store", required=True, help="test name" +) +parser.add_argument( + "--sample-stats", + dest="sample_stats", + action="store", + required=True, + help="stats from sample", +) +parser.add_argument( + "--update", + action="store_true", + help="whether to update baseline using stats from sample", +) args = parser.parse_args() test_name = args.test_name -if 'cpu' in test_name: - backend = 'cpu' -elif 'gpu' in test_name: - backend = 'gpu' +if "cpu" in test_name: + backend = "cpu" +elif "gpu" in test_name: + backend = "gpu" -data_file_path = f'../{backend}_runtime.json' +data_file_path = f"../{backend}_runtime.json" with open(data_file_path) as data_file: data = json.load(data_file) if test_name in data: - mean = float(data[test_name]['mean']) - sigma = float(data[test_name]['sigma']) + mean = float(data[test_name]["mean"]) + sigma = float(data[test_name]["sigma"]) else: # Let the test pass if baseline number doesn't exist mean = sys.maxsize @@ -43,37 +52,39 @@ if math.isnan(mean) or math.isnan(sigma): sample_stats_data = json.loads(args.sample_stats) -sample_mean = float(sample_stats_data['mean']) -sample_sigma = float(sample_stats_data['sigma']) +sample_mean = float(sample_stats_data["mean"]) +sample_sigma = float(sample_stats_data["sigma"]) print("sample mean: ", sample_mean) print("sample sigma: ", sample_sigma) if math.isnan(sample_mean): - raise Exception('''Error: sample mean is NaN''') + raise Exception("""Error: sample mean is NaN""") elif math.isnan(sample_sigma): - raise Exception('''Error: sample sigma is NaN''') + raise Exception("""Error: sample sigma is NaN""") z_value = (sample_mean - mean) / sigma print("z-value: ", z_value) if z_value >= 3: - raise Exception(f'''\n + raise Exception( + f"""\n z-value >= 3, there is high chance of perf regression.\n To reproduce this regression, run `cd .ci/pytorch/perf_test/ && bash {test_name}.sh` on your local machine and compare the runtime before/after your code change. -''') +""" + ) else: print("z-value < 3, no perf regression detected.") if args.update: print("We will use these numbers as new baseline.") - new_data_file_path = f'../new_{backend}_runtime.json' + new_data_file_path = f"../new_{backend}_runtime.json" with open(new_data_file_path) as new_data_file: new_data = json.load(new_data_file) new_data[test_name] = {} - new_data[test_name]['mean'] = sample_mean - new_data[test_name]['sigma'] = max(sample_sigma, sample_mean * 0.1) - with open(new_data_file_path, 'w') as new_data_file: + new_data[test_name]["mean"] = sample_mean + new_data[test_name]["sigma"] = max(sample_sigma, sample_mean * 0.1) + with open(new_data_file_path, "w") as new_data_file: json.dump(new_data, new_data_file, indent=4) diff --git a/.ci/pytorch/perf_test/get_stats.py b/.ci/pytorch/perf_test/get_stats.py index 9e6e72a4da8..fbb2876ae3f 100644 --- a/.ci/pytorch/perf_test/get_stats.py +++ b/.ci/pytorch/perf_test/get_stats.py @@ -1,5 +1,6 @@ -import sys import json +import sys + import numpy sample_data_list = sys.argv[1:] @@ -9,8 +10,8 @@ sample_mean = numpy.mean(sample_data_list) sample_sigma = numpy.std(sample_data_list) data = { - 'mean': sample_mean, - 'sigma': sample_sigma, + "mean": sample_mean, + "sigma": sample_sigma, } print(json.dumps(data)) diff --git a/.ci/pytorch/perf_test/update_commit_hash.py b/.ci/pytorch/perf_test/update_commit_hash.py index ee7fa8a9b09..a27b1dec957 100644 --- a/.ci/pytorch/perf_test/update_commit_hash.py +++ b/.ci/pytorch/perf_test/update_commit_hash.py @@ -1,5 +1,5 @@ -import sys import json +import sys data_file_path = sys.argv[1] commit_hash = sys.argv[2] @@ -7,7 +7,7 @@ commit_hash = sys.argv[2] with open(data_file_path) as data_file: data = json.load(data_file) -data['commit'] = commit_hash +data["commit"] = commit_hash -with open(data_file_path, 'w') as data_file: +with open(data_file_path, "w") as data_file: json.dump(data, data_file) diff --git a/.ci/pytorch/print_sccache_log.py b/.ci/pytorch/print_sccache_log.py index e86b5ddfa94..398501b656b 100644 --- a/.ci/pytorch/print_sccache_log.py +++ b/.ci/pytorch/print_sccache_log.py @@ -9,9 +9,9 @@ for line in lines: # Ignore errors from CPU instruction set, symbol existing testing, # or compilation error formatting ignored_keywords = [ - 'src.c', - 'CheckSymbolExists.c', - 'test_compilation_error_formatting', + "src.c", + "CheckSymbolExists.c", + "test_compilation_error_formatting", ] if all(keyword not in line for keyword in ignored_keywords): print(line) diff --git a/.ci/pytorch/win-test-helpers/run_python_nn_smoketests.py b/.ci/pytorch/win-test-helpers/run_python_nn_smoketests.py index 4d264e1da08..07b78b61511 100755 --- a/.ci/pytorch/win-test-helpers/run_python_nn_smoketests.py +++ b/.ci/pytorch/win-test-helpers/run_python_nn_smoketests.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 -import subprocess import os +import subprocess COMMON_TESTS = [ ( @@ -31,8 +31,7 @@ GPU_TESTS = [ if __name__ == "__main__": - - if 'USE_CUDA' in os.environ and os.environ['USE_CUDA'] == '1': + if "USE_CUDA" in os.environ and os.environ["USE_CUDA"] == "1": TESTS = COMMON_TESTS + GPU_TESTS else: TESTS = COMMON_TESTS @@ -44,8 +43,10 @@ if __name__ == "__main__": try: subprocess.check_call(command_args) except subprocess.CalledProcessError as e: - sdk_root = os.environ.get('WindowsSdkDir', 'C:\\Program Files (x86)\\Windows Kits\\10') - debugger = os.path.join(sdk_root, 'Debuggers', 'x64', 'cdb.exe') + sdk_root = os.environ.get( + "WindowsSdkDir", "C:\\Program Files (x86)\\Windows Kits\\10" + ) + debugger = os.path.join(sdk_root, "Debuggers", "x64", "cdb.exe") if os.path.exists(debugger): command_args = [debugger, "-o", "-c", "~*g; q"] + command_args command_string = " ".join(command_args) diff --git a/.circleci/cimodel/data/binary_build_data.py b/.circleci/cimodel/data/binary_build_data.py index 23191a6f550..2e6f523e02a 100644 --- a/.circleci/cimodel/data/binary_build_data.py +++ b/.circleci/cimodel/data/binary_build_data.py @@ -9,9 +9,10 @@ should be "pruned". from collections import OrderedDict -from cimodel.lib.conf_tree import ConfigNode import cimodel.data.dimensions as dimensions +from cimodel.lib.conf_tree import ConfigNode + LINKING_DIMENSIONS = [ "shared", @@ -26,12 +27,18 @@ DEPS_INCLUSION_DIMENSIONS = [ def get_processor_arch_name(gpu_version): - return "cpu" if not gpu_version else ( - "cu" + gpu_version.strip("cuda") if gpu_version.startswith("cuda") else gpu_version + return ( + "cpu" + if not gpu_version + else ( + "cu" + gpu_version.strip("cuda") + if gpu_version.startswith("cuda") + else gpu_version + ) ) -CONFIG_TREE_DATA = OrderedDict( -) + +CONFIG_TREE_DATA = OrderedDict() # GCC config variants: # @@ -41,8 +48,8 @@ CONFIG_TREE_DATA = OrderedDict( # # Libtorch with new gcc ABI is built with gcc 5.4 on Ubuntu 16.04. LINUX_GCC_CONFIG_VARIANTS = OrderedDict( - manywheel=['devtoolset7'], - conda=['devtoolset7'], + manywheel=["devtoolset7"], + conda=["devtoolset7"], libtorch=[ "devtoolset7", "gcc5.4_cxx11-abi", @@ -63,7 +70,9 @@ class TopLevelNode(ConfigNode): self.props["smoke"] = smoke def get_children(self): - return [OSConfigNode(self, x, c, p) for (x, (c, p)) in self.config_tree_data.items()] + return [ + OSConfigNode(self, x, c, p) for (x, (c, p)) in self.config_tree_data.items() + ] class OSConfigNode(ConfigNode): @@ -85,12 +94,20 @@ class PackageFormatConfigNode(ConfigNode): self.props["python_versions"] = python_versions self.props["package_format"] = package_format - def get_children(self): if self.find_prop("os_name") == "linux": - return [LinuxGccConfigNode(self, v) for v in LINUX_GCC_CONFIG_VARIANTS[self.find_prop("package_format")]] - elif self.find_prop("os_name") == "windows" and self.find_prop("package_format") == "libtorch": - return [WindowsLibtorchConfigNode(self, v) for v in WINDOWS_LIBTORCH_CONFIG_VARIANTS] + return [ + LinuxGccConfigNode(self, v) + for v in LINUX_GCC_CONFIG_VARIANTS[self.find_prop("package_format")] + ] + elif ( + self.find_prop("os_name") == "windows" + and self.find_prop("package_format") == "libtorch" + ): + return [ + WindowsLibtorchConfigNode(self, v) + for v in WINDOWS_LIBTORCH_CONFIG_VARIANTS + ] else: return [ArchConfigNode(self, v) for v in self.find_prop("gpu_versions")] @@ -106,23 +123,29 @@ class LinuxGccConfigNode(ConfigNode): # XXX devtoolset7 on CUDA 9.0 is temporarily disabled # see https://github.com/pytorch/pytorch/issues/20066 - if self.find_prop("gcc_config_variant") == 'devtoolset7': + if self.find_prop("gcc_config_variant") == "devtoolset7": gpu_versions = filter(lambda x: x != "cuda_90", gpu_versions) # XXX disabling conda rocm build since docker images are not there - if self.find_prop("package_format") == 'conda': - gpu_versions = filter(lambda x: x not in dimensions.ROCM_VERSION_LABELS, gpu_versions) + if self.find_prop("package_format") == "conda": + gpu_versions = filter( + lambda x: x not in dimensions.ROCM_VERSION_LABELS, gpu_versions + ) # XXX libtorch rocm build is temporarily disabled - if self.find_prop("package_format") == 'libtorch': - gpu_versions = filter(lambda x: x not in dimensions.ROCM_VERSION_LABELS, gpu_versions) + if self.find_prop("package_format") == "libtorch": + gpu_versions = filter( + lambda x: x not in dimensions.ROCM_VERSION_LABELS, gpu_versions + ) return [ArchConfigNode(self, v) for v in gpu_versions] class WindowsLibtorchConfigNode(ConfigNode): def __init__(self, parent, libtorch_config_variant): - super().__init__(parent, "LIBTORCH_CONFIG_VARIANT=" + str(libtorch_config_variant)) + super().__init__( + parent, "LIBTORCH_CONFIG_VARIANT=" + str(libtorch_config_variant) + ) self.props["libtorch_config_variant"] = libtorch_config_variant @@ -161,11 +184,15 @@ class LinkingVariantConfigNode(ConfigNode): super().__init__(parent, linking_variant) def get_children(self): - return [DependencyInclusionConfigNode(self, v) for v in DEPS_INCLUSION_DIMENSIONS] + return [ + DependencyInclusionConfigNode(self, v) for v in DEPS_INCLUSION_DIMENSIONS + ] class DependencyInclusionConfigNode(ConfigNode): def __init__(self, parent, deps_variant): super().__init__(parent, deps_variant) - self.props["libtorch_variant"] = "-".join([self.parent.get_label(), self.get_label()]) + self.props["libtorch_variant"] = "-".join( + [self.parent.get_label(), self.get_label()] + ) diff --git a/.circleci/cimodel/data/binary_build_definitions.py b/.circleci/cimodel/data/binary_build_definitions.py index 7dccbdc7cbf..7c1ee3694df 100644 --- a/.circleci/cimodel/data/binary_build_definitions.py +++ b/.circleci/cimodel/data/binary_build_definitions.py @@ -1,13 +1,24 @@ from collections import OrderedDict -import cimodel.data.simple.util.branch_filters as branch_filters import cimodel.data.binary_build_data as binary_build_data + +import cimodel.data.simple.util.branch_filters as branch_filters import cimodel.lib.conf_tree as conf_tree import cimodel.lib.miniutils as miniutils -class Conf: - def __init__(self, os, gpu_version, pydistro, parms, smoke, libtorch_variant, gcc_config_variant, libtorch_config_variant): +class Conf: + def __init__( + self, + os, + gpu_version, + pydistro, + parms, + smoke, + libtorch_variant, + gcc_config_variant, + libtorch_config_variant, + ): self.os = os self.gpu_version = gpu_version self.pydistro = pydistro @@ -18,7 +29,11 @@ class Conf: self.libtorch_config_variant = libtorch_config_variant def gen_build_env_parms(self): - elems = [self.pydistro] + self.parms + [binary_build_data.get_processor_arch_name(self.gpu_version)] + elems = ( + [self.pydistro] + + self.parms + + [binary_build_data.get_processor_arch_name(self.gpu_version)] + ) if self.gcc_config_variant is not None: elems.append(str(self.gcc_config_variant)) if self.libtorch_config_variant is not None: @@ -26,7 +41,7 @@ class Conf: return elems def gen_docker_image(self): - if self.gcc_config_variant == 'gcc5.4_cxx11-abi': + if self.gcc_config_variant == "gcc5.4_cxx11-abi": if self.gpu_version is None: return miniutils.quote("pytorch/libtorch-cxx11-builder:cpu") else: @@ -37,30 +52,41 @@ class Conf: if self.gpu_version is None: return miniutils.quote("pytorch/conda-builder:cpu") else: - return miniutils.quote( - f"pytorch/conda-builder:{self.gpu_version}" - ) + return miniutils.quote(f"pytorch/conda-builder:{self.gpu_version}") docker_word_substitution = { "manywheel": "manylinux", "libtorch": "manylinux", } - docker_distro_prefix = miniutils.override(self.pydistro, docker_word_substitution) + docker_distro_prefix = miniutils.override( + self.pydistro, docker_word_substitution + ) # The cpu nightlies are built on the pytorch/manylinux-cuda102 docker image # TODO cuda images should consolidate into tag-base images similar to rocm - alt_docker_suffix = "cuda102" if not self.gpu_version else ( - "rocm:" + self.gpu_version.strip("rocm") if self.gpu_version.startswith("rocm") else self.gpu_version) - docker_distro_suffix = alt_docker_suffix if self.pydistro != "conda" else ( - "cuda" if alt_docker_suffix.startswith("cuda") else "rocm") - return miniutils.quote("pytorch/" + docker_distro_prefix + "-" + docker_distro_suffix) + alt_docker_suffix = ( + "cuda102" + if not self.gpu_version + else ( + "rocm:" + self.gpu_version.strip("rocm") + if self.gpu_version.startswith("rocm") + else self.gpu_version + ) + ) + docker_distro_suffix = ( + alt_docker_suffix + if self.pydistro != "conda" + else ("cuda" if alt_docker_suffix.startswith("cuda") else "rocm") + ) + return miniutils.quote( + "pytorch/" + docker_distro_prefix + "-" + docker_distro_suffix + ) def get_name_prefix(self): return "smoke" if self.smoke else "binary" def gen_build_name(self, build_or_test, nightly): - parts = [self.get_name_prefix(), self.os] + self.gen_build_env_parms() if nightly: @@ -78,7 +104,9 @@ class Conf: def gen_workflow_job(self, phase, upload_phase_dependency=None, nightly=False): job_def = OrderedDict() job_def["name"] = self.gen_build_name(phase, nightly) - job_def["build_environment"] = miniutils.quote(" ".join(self.gen_build_env_parms())) + job_def["build_environment"] = miniutils.quote( + " ".join(self.gen_build_env_parms()) + ) if self.smoke: job_def["requires"] = [ "update_s3_htmls", @@ -116,47 +144,48 @@ class Conf: os_name = miniutils.override(self.os, {"macos": "mac"}) job_name = "_".join([self.get_name_prefix(), os_name, phase]) - return {job_name : job_def} + return {job_name: job_def} def gen_upload_job(self, phase, requires_dependency): """Generate binary_upload job for configuration - Output looks similar to: + Output looks similar to: - - binary_upload: - name: binary_linux_manywheel_3_7m_cu113_devtoolset7_nightly_upload - context: org-member - requires: binary_linux_manywheel_3_7m_cu113_devtoolset7_nightly_test - filters: - branches: - only: - - nightly - tags: - only: /v[0-9]+(\\.[0-9]+)*-rc[0-9]+/ - package_type: manywheel - upload_subfolder: cu113 + - binary_upload: + name: binary_linux_manywheel_3_7m_cu113_devtoolset7_nightly_upload + context: org-member + requires: binary_linux_manywheel_3_7m_cu113_devtoolset7_nightly_test + filters: + branches: + only: + - nightly + tags: + only: /v[0-9]+(\\.[0-9]+)*-rc[0-9]+/ + package_type: manywheel + upload_subfolder: cu113 """ return { - "binary_upload": OrderedDict({ - "name": self.gen_build_name(phase, nightly=True), - "context": "org-member", - "requires": [self.gen_build_name( - requires_dependency, - nightly=True - )], - "filters": branch_filters.gen_filter_dict( - branches_list=["nightly"], - tags_list=[branch_filters.RC_PATTERN], - ), - "package_type": self.pydistro, - "upload_subfolder": binary_build_data.get_processor_arch_name( - self.gpu_version, - ), - }) + "binary_upload": OrderedDict( + { + "name": self.gen_build_name(phase, nightly=True), + "context": "org-member", + "requires": [ + self.gen_build_name(requires_dependency, nightly=True) + ], + "filters": branch_filters.gen_filter_dict( + branches_list=["nightly"], + tags_list=[branch_filters.RC_PATTERN], + ), + "package_type": self.pydistro, + "upload_subfolder": binary_build_data.get_processor_arch_name( + self.gpu_version, + ), + } + ) } -def get_root(smoke, name): +def get_root(smoke, name): return binary_build_data.TopLevelNode( name, binary_build_data.CONFIG_TREE_DATA, @@ -165,7 +194,6 @@ def get_root(smoke, name): def gen_build_env_list(smoke): - root = get_root(smoke, "N/A") config_list = conf_tree.dfs(root) @@ -176,7 +204,8 @@ def gen_build_env_list(smoke): c.find_prop("gpu"), c.find_prop("package_format"), [c.find_prop("pyver")], - c.find_prop("smoke") and not (c.find_prop("os_name") == "macos_arm64"), # don't test arm64 + c.find_prop("smoke") + and not (c.find_prop("os_name") == "macos_arm64"), # don't test arm64 c.find_prop("libtorch_variant"), c.find_prop("gcc_config_variant"), c.find_prop("libtorch_config_variant"), @@ -185,9 +214,11 @@ def gen_build_env_list(smoke): return newlist + def predicate_exclude_macos(config): return config.os == "linux" or config.os == "windows" + def get_nightly_uploads(): configs = gen_build_env_list(False) mylist = [] @@ -197,6 +228,7 @@ def get_nightly_uploads(): return mylist + def get_post_upload_jobs(): return [ { @@ -210,8 +242,8 @@ def get_post_upload_jobs(): }, ] -def get_nightly_tests(): +def get_nightly_tests(): configs = gen_build_env_list(False) filtered_configs = filter(predicate_exclude_macos, configs) diff --git a/.circleci/cimodel/data/dimensions.py b/.circleci/cimodel/data/dimensions.py index 5841b3806b1..403e72981ae 100644 --- a/.circleci/cimodel/data/dimensions.py +++ b/.circleci/cimodel/data/dimensions.py @@ -16,9 +16,4 @@ ROCM_VERSION_LABELS = ["rocm" + v for v in ROCM_VERSIONS] GPU_VERSIONS = [None] + ["cuda" + v for v in CUDA_VERSIONS] + ROCM_VERSION_LABELS -STANDARD_PYTHON_VERSIONS = [ - "3.7", - "3.8", - "3.9", - "3.10" -] +STANDARD_PYTHON_VERSIONS = ["3.7", "3.8", "3.9", "3.10"] diff --git a/.circleci/cimodel/data/pytorch_build_data.py b/.circleci/cimodel/data/pytorch_build_data.py index ebd6e0a3818..1e350ef134b 100644 --- a/.circleci/cimodel/data/pytorch_build_data.py +++ b/.circleci/cimodel/data/pytorch_build_data.py @@ -1,8 +1,7 @@ from cimodel.lib.conf_tree import ConfigNode -CONFIG_TREE_DATA = [ -] +CONFIG_TREE_DATA = [] def get_major_pyver(dotted_version): @@ -96,6 +95,7 @@ class SlowGradcheckConfigNode(TreeConfigNode): def child_constructor(self): return ExperimentalFeatureConfigNode + class PureTorchConfigNode(TreeConfigNode): def modify_label(self, label): return "PURE_TORCH=" + str(label) @@ -117,6 +117,7 @@ class XlaConfigNode(TreeConfigNode): def child_constructor(self): return ImportantConfigNode + class MPSConfigNode(TreeConfigNode): def modify_label(self, label): return "MPS=" + str(label) @@ -254,8 +255,11 @@ class XenialCompilerConfigNode(TreeConfigNode): # noinspection PyMethodMayBeStatic def child_constructor(self): - - return XenialCompilerVersionConfigNode if self.props["compiler_name"] else PyVerConfigNode + return ( + XenialCompilerVersionConfigNode + if self.props["compiler_name"] + else PyVerConfigNode + ) class BionicCompilerConfigNode(TreeConfigNode): @@ -267,8 +271,11 @@ class BionicCompilerConfigNode(TreeConfigNode): # noinspection PyMethodMayBeStatic def child_constructor(self): - - return BionicCompilerVersionConfigNode if self.props["compiler_name"] else PyVerConfigNode + return ( + BionicCompilerVersionConfigNode + if self.props["compiler_name"] + else PyVerConfigNode + ) class XenialCompilerVersionConfigNode(TreeConfigNode): diff --git a/.circleci/cimodel/data/pytorch_build_definitions.py b/.circleci/cimodel/data/pytorch_build_definitions.py index e6e44bd2b5a..b23334bb737 100644 --- a/.circleci/cimodel/data/pytorch_build_definitions.py +++ b/.circleci/cimodel/data/pytorch_build_definitions.py @@ -111,10 +111,10 @@ class Conf: parameters["resource_class"] = resource_class if phase == "build" and self.rocm_version is not None: parameters["resource_class"] = "xlarge" - if hasattr(self, 'filters'): - parameters['filters'] = self.filters + if hasattr(self, "filters"): + parameters["filters"] = self.filters if self.build_only: - parameters['build_only'] = miniutils.quote(str(int(True))) + parameters["build_only"] = miniutils.quote(str(int(True))) return parameters def gen_workflow_job(self, phase): @@ -122,7 +122,6 @@ class Conf: job_def["name"] = self.gen_build_name(phase) if Conf.is_test_phase(phase): - # TODO When merging the caffe2 and pytorch jobs, it might be convenient for a while to make a # caffe2 test job dependent on a pytorch build job. This way we could quickly dedup the repeated # build of pytorch in the caffe2 build job, and just run the caffe2 tests off of a completed @@ -160,6 +159,7 @@ class HiddenConf: def gen_build_name(self, _): return self.name + class DocPushConf: def __init__(self, name, parent_build=None, branch="master"): self.name = name @@ -173,11 +173,13 @@ class DocPushConf: "branch": self.branch, "requires": [self.parent_build], "context": "org-member", - "filters": gen_filter_dict(branches_list=["nightly"], - tags_list=RC_PATTERN) + "filters": gen_filter_dict( + branches_list=["nightly"], tags_list=RC_PATTERN + ), } } + def gen_docs_configs(xenial_parent_config): configs = [] @@ -185,8 +187,9 @@ def gen_docs_configs(xenial_parent_config): HiddenConf( "pytorch_python_doc_build", parent_build=xenial_parent_config, - filters=gen_filter_dict(branches_list=["master", "main", "nightly"], - tags_list=RC_PATTERN), + filters=gen_filter_dict( + branches_list=["master", "main", "nightly"], tags_list=RC_PATTERN + ), ) ) configs.append( @@ -201,8 +204,9 @@ def gen_docs_configs(xenial_parent_config): HiddenConf( "pytorch_cpp_doc_build", parent_build=xenial_parent_config, - filters=gen_filter_dict(branches_list=["master", "main", "nightly"], - tags_list=RC_PATTERN), + filters=gen_filter_dict( + branches_list=["master", "main", "nightly"], tags_list=RC_PATTERN + ), ) ) configs.append( @@ -226,13 +230,11 @@ def gen_tree(): def instantiate_configs(only_slow_gradcheck): - config_list = [] root = get_root() found_configs = conf_tree.dfs(root) for fc in found_configs: - restrict_phases = None distro_name = fc.find_prop("distro_name") compiler_name = fc.find_prop("compiler_name") @@ -351,8 +353,7 @@ def instantiate_configs(only_slow_gradcheck): and compiler_name == "gcc" and fc.find_prop("compiler_version") == "5.4" ): - c.filters = gen_filter_dict(branches_list=r"/.*/", - tags_list=RC_PATTERN) + c.filters = gen_filter_dict(branches_list=r"/.*/", tags_list=RC_PATTERN) c.dependent_tests = gen_docs_configs(c) config_list.append(c) @@ -361,16 +362,13 @@ def instantiate_configs(only_slow_gradcheck): def get_workflow_jobs(only_slow_gradcheck=False): - config_list = instantiate_configs(only_slow_gradcheck) x = [] for conf_options in config_list: - phases = conf_options.restrict_phases or dimensions.PHASES for phase in phases: - # TODO why does this not have a test? if Conf.is_test_phase(phase) and conf_options.cuda_version == "10": continue diff --git a/.circleci/cimodel/data/simple/docker_definitions.py b/.circleci/cimodel/data/simple/docker_definitions.py index 7299b47e786..bfb1d184e36 100644 --- a/.circleci/cimodel/data/simple/docker_definitions.py +++ b/.circleci/cimodel/data/simple/docker_definitions.py @@ -1,39 +1,39 @@ from collections import OrderedDict -from cimodel.lib.miniutils import quote from cimodel.data.simple.util.branch_filters import gen_filter_dict, RC_PATTERN +from cimodel.lib.miniutils import quote + # NOTE: All hardcoded docker image builds have been migrated to GHA -IMAGE_NAMES = [ -] +IMAGE_NAMES = [] # This entry should be an element from the list above # This should contain the image matching the "slow_gradcheck" entry in # pytorch_build_data.py SLOW_GRADCHECK_IMAGE_NAME = "pytorch-linux-xenial-cuda10.2-cudnn7-py3-gcc7" + def get_workflow_jobs(images=IMAGE_NAMES, only_slow_gradcheck=False): """Generates a list of docker image build definitions""" ret = [] for image_name in images: - if image_name.startswith('docker-'): - image_name = image_name.lstrip('docker-') + if image_name.startswith("docker-"): + image_name = image_name.lstrip("docker-") if only_slow_gradcheck and image_name is not SLOW_GRADCHECK_IMAGE_NAME: continue - parameters = OrderedDict({ - "name": quote(f"docker-{image_name}"), - "image_name": quote(image_name), - }) + parameters = OrderedDict( + { + "name": quote(f"docker-{image_name}"), + "image_name": quote(image_name), + } + ) if image_name == "pytorch-linux-xenial-py3.7-gcc5.4": # pushing documentation on tags requires CircleCI to also # build all the dependencies on tags, including this docker image - parameters['filters'] = gen_filter_dict(branches_list=r"/.*/", - tags_list=RC_PATTERN) - ret.append(OrderedDict( - { - "docker_build_job": parameters - } - )) + parameters["filters"] = gen_filter_dict( + branches_list=r"/.*/", tags_list=RC_PATTERN + ) + ret.append(OrderedDict({"docker_build_job": parameters})) return ret diff --git a/.circleci/cimodel/data/simple/ios_definitions.py b/.circleci/cimodel/data/simple/ios_definitions.py index 42aac5d9012..c8215be61b4 100644 --- a/.circleci/cimodel/data/simple/ios_definitions.py +++ b/.circleci/cimodel/data/simple/ios_definitions.py @@ -1,6 +1,6 @@ -from cimodel.data.simple.util.versions import MultiPartVersion -from cimodel.data.simple.util.branch_filters import gen_filter_dict_exclude import cimodel.lib.miniutils as miniutils +from cimodel.data.simple.util.branch_filters import gen_filter_dict_exclude +from cimodel.data.simple.util.versions import MultiPartVersion XCODE_VERSION = MultiPartVersion([12, 5, 1]) @@ -11,7 +11,9 @@ class ArchVariant: self.custom_build_name = custom_build_name def render(self): - extra_parts = [self.custom_build_name] if len(self.custom_build_name) > 0 else [] + extra_parts = ( + [self.custom_build_name] if len(self.custom_build_name) > 0 else [] + ) return "-".join([self.name] + extra_parts).replace("_", "-") @@ -20,7 +22,9 @@ def get_platform(arch_variant_name): class IOSJob: - def __init__(self, xcode_version, arch_variant, is_org_member_context=True, extra_props=None): + def __init__( + self, xcode_version, arch_variant, is_org_member_context=True, extra_props=None + ): self.xcode_version = xcode_version self.arch_variant = arch_variant self.is_org_member_context = is_org_member_context @@ -29,11 +33,15 @@ class IOSJob: def gen_name_parts(self): version_parts = self.xcode_version.render_dots_or_parts("-") build_variant_suffix = self.arch_variant.render() - return [ - "ios", - ] + version_parts + [ - build_variant_suffix, - ] + return ( + [ + "ios", + ] + + version_parts + + [ + build_variant_suffix, + ] + ) def gen_job_name(self): return "-".join(self.gen_name_parts()) @@ -59,8 +67,12 @@ class IOSJob: WORKFLOW_DATA = [ - IOSJob(XCODE_VERSION, ArchVariant("x86_64"), is_org_member_context=False, extra_props={ - "lite_interpreter": miniutils.quote(str(int(True)))}), + IOSJob( + XCODE_VERSION, + ArchVariant("x86_64"), + is_org_member_context=False, + extra_props={"lite_interpreter": miniutils.quote(str(int(True)))}, + ), # IOSJob(XCODE_VERSION, ArchVariant("arm64"), extra_props={ # "lite_interpreter": miniutils.quote(str(int(True)))}), # IOSJob(XCODE_VERSION, ArchVariant("arm64", "metal"), extra_props={ @@ -69,9 +81,15 @@ WORKFLOW_DATA = [ # IOSJob(XCODE_VERSION, ArchVariant("arm64", "custom-ops"), extra_props={ # "op_list": "mobilenetv2.yaml", # "lite_interpreter": miniutils.quote(str(int(True)))}), - IOSJob(XCODE_VERSION, ArchVariant("x86_64", "coreml"), is_org_member_context=False, extra_props={ - "use_coreml": miniutils.quote(str(int(True))), - "lite_interpreter": miniutils.quote(str(int(True)))}), + IOSJob( + XCODE_VERSION, + ArchVariant("x86_64", "coreml"), + is_org_member_context=False, + extra_props={ + "use_coreml": miniutils.quote(str(int(True))), + "lite_interpreter": miniutils.quote(str(int(True))), + }, + ), # IOSJob(XCODE_VERSION, ArchVariant("arm64", "coreml"), extra_props={ # "use_coreml": miniutils.quote(str(int(True))), # "lite_interpreter": miniutils.quote(str(int(True)))}), diff --git a/.circleci/cimodel/data/simple/mobile_definitions.py b/.circleci/cimodel/data/simple/mobile_definitions.py index 8c5924af386..d9cee4895d3 100644 --- a/.circleci/cimodel/data/simple/mobile_definitions.py +++ b/.circleci/cimodel/data/simple/mobile_definitions.py @@ -2,17 +2,14 @@ PyTorch Mobile PR builds (use linux host toolchain + mobile build options) """ -import cimodel.lib.miniutils as miniutils import cimodel.data.simple.util.branch_filters +import cimodel.lib.miniutils as miniutils class MobileJob: def __init__( - self, - docker_image, - docker_requires, - variant_parts, - is_master_only=False): + self, docker_image, docker_requires, variant_parts, is_master_only=False + ): self.docker_image = docker_image self.docker_requires = docker_requires self.variant_parts = variant_parts @@ -40,13 +37,14 @@ class MobileJob: } if self.is_master_only: - props_dict["filters"] = cimodel.data.simple.util.branch_filters.gen_filter_dict() + props_dict[ + "filters" + ] = cimodel.data.simple.util.branch_filters.gen_filter_dict() return [{"pytorch_linux_build": props_dict}] -WORKFLOW_DATA = [ -] +WORKFLOW_DATA = [] def get_workflow_jobs(): diff --git a/.circleci/cimodel/data/simple/nightly_ios.py b/.circleci/cimodel/data/simple/nightly_ios.py index f75bcb4bfe2..4f00f617332 100644 --- a/.circleci/cimodel/data/simple/nightly_ios.py +++ b/.circleci/cimodel/data/simple/nightly_ios.py @@ -3,11 +3,7 @@ import cimodel.lib.miniutils as miniutils class IOSNightlyJob: - def __init__(self, - variant, - is_full_jit=False, - is_upload=False): - + def __init__(self, variant, is_full_jit=False, is_upload=False): self.variant = variant self.is_full_jit = is_full_jit self.is_upload = is_upload @@ -16,19 +12,24 @@ class IOSNightlyJob: return "upload" if self.is_upload else "build" def get_common_name_pieces(self, sep): - extra_name_suffix = [self.get_phase_name()] if self.is_upload else [] extra_name = ["full_jit"] if self.is_full_jit else [] - common_name_pieces = [ - "ios", - ] + extra_name + [ - ] + ios_definitions.XCODE_VERSION.render_dots_or_parts(sep) + [ - "nightly", - self.variant, - "build", - ] + extra_name_suffix + common_name_pieces = ( + [ + "ios", + ] + + extra_name + + [] + + ios_definitions.XCODE_VERSION.render_dots_or_parts(sep) + + [ + "nightly", + self.variant, + "build", + ] + + extra_name_suffix + ) return common_name_pieces @@ -37,10 +38,14 @@ class IOSNightlyJob: def gen_tree(self): build_configs = BUILD_CONFIGS_FULL_JIT if self.is_full_jit else BUILD_CONFIGS - extra_requires = [x.gen_job_name() for x in build_configs] if self.is_upload else [] + extra_requires = ( + [x.gen_job_name() for x in build_configs] if self.is_upload else [] + ) props_dict = { - "build_environment": "-".join(["libtorch"] + self.get_common_name_pieces(".")), + "build_environment": "-".join( + ["libtorch"] + self.get_common_name_pieces(".") + ), "requires": extra_requires, "context": "org-member", "filters": {"branches": {"only": "nightly"}}, @@ -56,11 +61,13 @@ class IOSNightlyJob: if self.is_full_jit: props_dict["lite_interpreter"] = miniutils.quote(str(int(False))) - template_name = "_".join([ - "binary", - "ios", - self.get_phase_name(), - ]) + template_name = "_".join( + [ + "binary", + "ios", + self.get_phase_name(), + ] + ) return [{template_name: props_dict}] @@ -75,10 +82,14 @@ BUILD_CONFIGS_FULL_JIT = [ IOSNightlyJob("arm64", is_full_jit=True), ] -WORKFLOW_DATA = BUILD_CONFIGS + BUILD_CONFIGS_FULL_JIT + [ - IOSNightlyJob("binary", is_full_jit=False, is_upload=True), - IOSNightlyJob("binary", is_full_jit=True, is_upload=True), -] +WORKFLOW_DATA = ( + BUILD_CONFIGS + + BUILD_CONFIGS_FULL_JIT + + [ + IOSNightlyJob("binary", is_full_jit=False, is_upload=True), + IOSNightlyJob("binary", is_full_jit=True, is_upload=True), + ] +) def get_workflow_jobs(): diff --git a/.circleci/cimodel/data/simple/util/branch_filters.py b/.circleci/cimodel/data/simple/util/branch_filters.py index e87d0045636..2a4d752894e 100644 --- a/.circleci/cimodel/data/simple/util/branch_filters.py +++ b/.circleci/cimodel/data/simple/util/branch_filters.py @@ -15,10 +15,7 @@ RC_PATTERN = r"/v[0-9]+(\.[0-9]+)*-rc[0-9]+/" MAC_IOS_EXCLUSION_LIST = ["nightly", "postnightly"] -def gen_filter_dict( - branches_list=NON_PR_BRANCH_LIST, - tags_list=None -): +def gen_filter_dict(branches_list=NON_PR_BRANCH_LIST, tags_list=None): """Generates a filter dictionary for use with CircleCI's job filter""" filter_dict = { "branches": { diff --git a/.circleci/cimodel/data/simple/util/docker_constants.py b/.circleci/cimodel/data/simple/util/docker_constants.py index 985466718ea..9a96b9f9f2a 100644 --- a/.circleci/cimodel/data/simple/util/docker_constants.py +++ b/.circleci/cimodel/data/simple/util/docker_constants.py @@ -1,11 +1,13 @@ AWS_DOCKER_HOST = "308535385114.dkr.ecr.us-east-1.amazonaws.com" + def gen_docker_image(container_type): return ( "/".join([AWS_DOCKER_HOST, "pytorch", container_type]), f"docker-{container_type}", ) + def gen_docker_image_requires(image_name): return [f"docker-{image_name}"] diff --git a/.circleci/cimodel/data/simple/util/versions.py b/.circleci/cimodel/data/simple/util/versions.py index 518feb2e386..bc3a9e3d684 100644 --- a/.circleci/cimodel/data/simple/util/versions.py +++ b/.circleci/cimodel/data/simple/util/versions.py @@ -12,7 +12,9 @@ class MultiPartVersion: with the prefix string. """ if self.parts: - return [self.prefix + str(self.parts[0])] + [str(part) for part in self.parts[1:]] + return [self.prefix + str(self.parts[0])] + [ + str(part) for part in self.parts[1:] + ] else: return [self.prefix] diff --git a/.circleci/cimodel/lib/conf_tree.py b/.circleci/cimodel/lib/conf_tree.py index db2a30f29c2..495b134d99f 100644 --- a/.circleci/cimodel/lib/conf_tree.py +++ b/.circleci/cimodel/lib/conf_tree.py @@ -1,5 +1,5 @@ from dataclasses import dataclass, field -from typing import Optional, Dict +from typing import Dict, Optional def X(val): @@ -19,6 +19,7 @@ class Ver: """ Represents a product with a version number """ + name: str version: str = "" @@ -28,7 +29,7 @@ class Ver: @dataclass class ConfigNode: - parent: Optional['ConfigNode'] + parent: Optional["ConfigNode"] node_name: str props: Dict[str, str] = field(default_factory=dict) @@ -40,7 +41,11 @@ class ConfigNode: return [] def get_parents(self): - return (self.parent.get_parents() + [self.parent.get_label()]) if self.parent else [] + return ( + (self.parent.get_parents() + [self.parent.get_label()]) + if self.parent + else [] + ) def get_depth(self): return len(self.get_parents()) @@ -69,13 +74,13 @@ class ConfigNode: def dfs_recurse( - node, - leaf_callback=lambda x: None, - discovery_callback=lambda x, y, z: None, - child_callback=lambda x, y: None, - sibling_index=0, - sibling_count=1): - + node, + leaf_callback=lambda x: None, + discovery_callback=lambda x, y, z: None, + child_callback=lambda x, y: None, + sibling_index=0, + sibling_count=1, +): discovery_callback(node, sibling_index, sibling_count) node_children = node.get_children() @@ -96,7 +101,6 @@ def dfs_recurse( def dfs(toplevel_config_node): - config_list = [] def leaf_callback(node): diff --git a/.circleci/cimodel/lib/miniyaml.py b/.circleci/cimodel/lib/miniyaml.py index 40ffbdbb92e..70199e92309 100644 --- a/.circleci/cimodel/lib/miniyaml.py +++ b/.circleci/cimodel/lib/miniyaml.py @@ -25,7 +25,6 @@ def render(fh, data, depth, is_list_member=False): indentation = " " * INDENTATION_WIDTH * depth if is_dict(data): - tuples = list(data.items()) if type(data) is not OrderedDict: tuples.sort() diff --git a/.circleci/codegen_validation/normalize_yaml_fragment.py b/.circleci/codegen_validation/normalize_yaml_fragment.py index 0778404e8bb..32899cc729c 100755 --- a/.circleci/codegen_validation/normalize_yaml_fragment.py +++ b/.circleci/codegen_validation/normalize_yaml_fragment.py @@ -2,10 +2,11 @@ import os import sys + import yaml # Need to import modules that lie on an upward-relative path -sys.path.append(os.path.join(sys.path[0], '..')) +sys.path.append(os.path.join(sys.path[0], "..")) import cimodel.lib.miniyaml as miniyaml diff --git a/.circleci/ensure-consistency.py b/.circleci/ensure-consistency.py index 972ededb69f..44f5299cbaf 100755 --- a/.circleci/ensure-consistency.py +++ b/.circleci/ensure-consistency.py @@ -21,7 +21,6 @@ Please re-run the "%s" script in the "%s" directory and commit the result. See " def check_consistency(): - _, temp_filename = tempfile.mkstemp("-generated-config.yml") with open(temp_filename, "w") as fh: @@ -30,7 +29,10 @@ def check_consistency(): try: subprocess.check_call(["cmp", temp_filename, CHECKED_IN_FILE]) except subprocess.CalledProcessError: - sys.exit(ERROR_MESSAGE_TEMPLATE % (CHECKED_IN_FILE, REGENERATION_SCRIPT, PARENT_DIR, README_PATH)) + sys.exit( + ERROR_MESSAGE_TEMPLATE + % (CHECKED_IN_FILE, REGENERATION_SCRIPT, PARENT_DIR, README_PATH) + ) finally: os.remove(temp_filename) diff --git a/.circleci/generate_config_yml.py b/.circleci/generate_config_yml.py index d1ef439941d..2d0c93eacd4 100755 --- a/.circleci/generate_config_yml.py +++ b/.circleci/generate_config_yml.py @@ -10,10 +10,11 @@ import shutil import sys from collections import namedtuple +import cimodel.data.simple.anaconda_prune_defintions + import cimodel.data.simple.docker_definitions import cimodel.data.simple.mobile_definitions import cimodel.data.simple.nightly_ios -import cimodel.data.simple.anaconda_prune_defintions import cimodel.lib.miniutils as miniutils import cimodel.lib.miniyaml as miniyaml @@ -82,15 +83,19 @@ def _for_all_items(items, functor) -> None: def filter_master_only_jobs(items): def _is_main_or_master_item(item): - filters = item.get('filters', None) - branches = filters.get('branches', None) if filters is not None else None - branches_only = branches.get('only', None) if branches is not None else None - return ('main' in branches_only or 'master' in branches_only) if branches_only is not None else False + filters = item.get("filters", None) + branches = filters.get("branches", None) if filters is not None else None + branches_only = branches.get("only", None) if branches is not None else None + return ( + ("main" in branches_only or "master" in branches_only) + if branches_only is not None + else False + ) master_deps = set() def _save_requires_if_master(item_type, item): - requires = item.get('requires', None) + requires = item.get("requires", None) item_name = item.get("name", None) if not isinstance(requires, list): return @@ -107,9 +112,9 @@ def filter_master_only_jobs(items): item_name = item_name.strip('"') if item_name is not None else None if not _is_main_or_master_item(item) and item_name not in master_deps: return None - if 'filters' in item: + if "filters" in item: item = item.copy() - item.pop('filters') + item.pop("filters") return {item_type: item} # Scan of dependencies twice to pick up nested required jobs @@ -123,12 +128,12 @@ def generate_required_docker_images(items): required_docker_images = set() def _requires_docker_image(item_type, item): - requires = item.get('requires', None) + requires = item.get("requires", None) if not isinstance(requires, list): return for requirement in requires: - requirement = requirement.replace('"', '') - if requirement.startswith('docker-'): + requirement = requirement.replace('"', "") + if requirement.startswith("docker-"): required_docker_images.add(requirement) _for_all_items(items, _requires_docker_image) @@ -191,5 +196,4 @@ def stitch_sources(output_filehandle): if __name__ == "__main__": - stitch_sources(sys.stdout) diff --git a/.circleci/scripts/trigger_azure_pipeline.py b/.circleci/scripts/trigger_azure_pipeline.py index 9dc9dff2d54..fc15b5810e4 100644 --- a/.circleci/scripts/trigger_azure_pipeline.py +++ b/.circleci/scripts/trigger_azure_pipeline.py @@ -1,12 +1,13 @@ # Documentation: https://docs.microsoft.com/en-us/rest/api/azure/devops/build/?view=azure-devops-rest-6.0 -import re import json import os +import re import sys -import requests import time +import requests + AZURE_PIPELINE_BASE_URL = "https://aiinfra.visualstudio.com/PyTorch/" AZURE_DEVOPS_PAT_BASE64 = os.environ.get("AZURE_DEVOPS_PAT_BASE64_SECRET", "") PIPELINE_ID = "911" @@ -19,54 +20,68 @@ build_base_url = AZURE_PIPELINE_BASE_URL + "_apis/build/builds?api-version=6.0" s = requests.Session() s.headers.update({"Authorization": "Basic " + AZURE_DEVOPS_PAT_BASE64}) + def submit_build(pipeline_id, project_id, source_branch, source_version): print("Submitting build for branch: " + source_branch) print("Commit SHA1: ", source_version) - run_build_raw = s.post(build_base_url, json={ - "definition": {"id": pipeline_id}, - "project": {"id": project_id}, - "sourceBranch": source_branch, - "sourceVersion": source_version - }) + run_build_raw = s.post( + build_base_url, + json={ + "definition": {"id": pipeline_id}, + "project": {"id": project_id}, + "sourceBranch": source_branch, + "sourceVersion": source_version, + }, + ) try: run_build_json = run_build_raw.json() except json.decoder.JSONDecodeError as e: print(e) - print("Failed to parse the response. Check if the Azure DevOps PAT is incorrect or expired.") + print( + "Failed to parse the response. Check if the Azure DevOps PAT is incorrect or expired." + ) sys.exit(-1) - build_id = run_build_json['id'] + build_id = run_build_json["id"] print("Submitted bulid: " + str(build_id)) - print("Bulid URL: " + run_build_json['url']) + print("Bulid URL: " + run_build_json["url"]) return build_id + def get_build(_id): - get_build_url = AZURE_PIPELINE_BASE_URL + f"/_apis/build/builds/{_id}?api-version=6.0" + get_build_url = ( + AZURE_PIPELINE_BASE_URL + f"/_apis/build/builds/{_id}?api-version=6.0" + ) get_build_raw = s.get(get_build_url) return get_build_raw.json() + def get_build_logs(_id): - get_build_logs_url = AZURE_PIPELINE_BASE_URL + f"/_apis/build/builds/{_id}/logs?api-version=6.0" + get_build_logs_url = ( + AZURE_PIPELINE_BASE_URL + f"/_apis/build/builds/{_id}/logs?api-version=6.0" + ) get_build_logs_raw = s.get(get_build_logs_url) return get_build_logs_raw.json() + def get_log_content(url): resp = s.get(url) return resp.text + def wait_for_build(_id): build_detail = get_build(_id) - build_status = build_detail['status'] + build_status = build_detail["status"] - while build_status == 'notStarted': - print('Waiting for run to start: ' + str(_id)) + while build_status == "notStarted": + print("Waiting for run to start: " + str(_id)) sys.stdout.flush() try: build_detail = get_build(_id) - build_status = build_detail['status'] + build_status = build_detail["status"] except Exception as e: print("Error getting build") print(e) @@ -76,7 +91,7 @@ def wait_for_build(_id): print("Bulid started: ", str(_id)) handled_logs = set() - while build_status == 'inProgress': + while build_status == "inProgress": try: print("Waiting for log: " + str(_id)) logs = get_build_logs(_id) @@ -86,38 +101,39 @@ def wait_for_build(_id): time.sleep(30) continue - for log in logs['value']: - log_id = log['id'] + for log in logs["value"]: + log_id = log["id"] if log_id in handled_logs: continue handled_logs.add(log_id) - print('Fetching log: \n' + log['url']) + print("Fetching log: \n" + log["url"]) try: - log_content = get_log_content(log['url']) + log_content = get_log_content(log["url"]) print(log_content) except Exception as e: print("Error getting log content") print(e) sys.stdout.flush() build_detail = get_build(_id) - build_status = build_detail['status'] + build_status = build_detail["status"] time.sleep(30) - build_result = build_detail['result'] + build_result = build_detail["result"] print("Bulid status: " + build_status) print("Bulid result: " + build_result) return build_status, build_result -if __name__ == '__main__': + +if __name__ == "__main__": # Convert the branch name for Azure DevOps - match = re.search(r'pull/(\d+)', TARGET_BRANCH) + match = re.search(r"pull/(\d+)", TARGET_BRANCH) if match is not None: pr_num = match.group(1) - SOURCE_BRANCH = f'refs/pull/{pr_num}/head' + SOURCE_BRANCH = f"refs/pull/{pr_num}/head" else: - SOURCE_BRANCH = f'refs/heads/{TARGET_BRANCH}' + SOURCE_BRANCH = f"refs/heads/{TARGET_BRANCH}" MAX_RETRY = 2 retry = MAX_RETRY @@ -126,7 +142,7 @@ if __name__ == '__main__': build_id = submit_build(PIPELINE_ID, PROJECT_ID, SOURCE_BRANCH, TARGET_COMMIT) build_status, build_result = wait_for_build(build_id) - if build_result != 'succeeded': + if build_result != "succeeded": retry = retry - 1 if retry > 0: print("Retrying... remaining attempt: " + str(retry)) diff --git a/.lintrunner.toml b/.lintrunner.toml index b0ccdd31d5c..39f2c20a5b2 100644 --- a/.lintrunner.toml +++ b/.lintrunner.toml @@ -908,58 +908,12 @@ exclude_patterns = [ 'third_party/**/*.pyi', # These files are all grandfathered in, feel free to remove from this list # as necessary - '.ci/pytorch/create_test_cert.py', - '.ci/pytorch/fake_numpy/numpy.py', - '.ci/pytorch/perf_test/compare_with_baseline.py', - '.ci/pytorch/perf_test/get_stats.py', - '.ci/pytorch/perf_test/update_commit_hash.py', - '.ci/pytorch/print_sccache_log.py', - '.ci/pytorch/win-test-helpers/run_python_nn_smoketests.py', - '.circleci/cimodel/__init__.py', - '.circleci/cimodel/data/__init__.py', - '.circleci/cimodel/data/binary_build_data.py', - '.circleci/cimodel/data/binary_build_definitions.py', - '.circleci/cimodel/data/dimensions.py', - '.circleci/cimodel/data/pytorch_build_data.py', - '.circleci/cimodel/data/pytorch_build_definitions.py', - '.circleci/cimodel/data/simple/__init__.py', - '.circleci/cimodel/data/simple/anaconda_prune_defintions.py', - '.circleci/cimodel/data/simple/docker_definitions.py', - '.circleci/cimodel/data/simple/ios_definitions.py', - '.circleci/cimodel/data/simple/macos_definitions.py', - '.circleci/cimodel/data/simple/mobile_definitions.py', - '.circleci/cimodel/data/simple/nightly_ios.py', - '.circleci/cimodel/data/simple/util/__init__.py', - '.circleci/cimodel/data/simple/util/branch_filters.py', - '.circleci/cimodel/data/simple/util/docker_constants.py', - '.circleci/cimodel/data/simple/util/versions.py', - '.circleci/cimodel/lib/__init__.py', - '.circleci/cimodel/lib/conf_tree.py', - '.circleci/cimodel/lib/miniutils.py', - '.circleci/cimodel/lib/miniyaml.py', - '.circleci/codegen_validation/normalize_yaml_fragment.py', - '.circleci/ensure-consistency.py', - '.circleci/generate_config_yml.py', - '.circleci/scripts/trigger_azure_pipeline.py', - 'android/pytorch_android/generate_test_torchscripts.py', - 'android/test_app/make_assets.py', - 'android/test_app/make_assets_custom.py', 'aten/src/ATen/function_wrapper.py', 'aten/src/ATen/native/quantized/cpu/qnnpack/configure.py', 'aten/src/ATen/native/quantized/cpu/qnnpack/deps/clog/configure.py', 'aten/src/ATen/native/quantized/cpu/qnnpack/generate-wrapper.py', 'aten/src/ATen/native/transformers/cuda/mem_eff_attention/kernels/generate_kernels.py', 'aten/src/ATen/nnapi/codegen.py', - 'binaries/bench_gen/bench_gen.py', - 'docs/caffe2/process.py', - 'docs/cpp/source/conf.py', - 'docs/source/conf.py', - 'docs/source/scripts/build_activation_images.py', - 'docs/source/scripts/build_opsets.py', - 'docs/source/scripts/build_quantization_configs.py', - 'docs/source/scripts/exportdb/generate_example_rst.py', - 'docs/source/scripts/onnx/build_onnx_diagnostics_rules_md.py', - 'docs/source/scripts/onnx/build_onnx_supported_aten_op_csv_table.py', 'functorch/__init__.py', 'functorch/_src/__init__.py', 'functorch/_src/aot_autograd/__init__.py', @@ -1010,25 +964,6 @@ exclude_patterns = [ 'functorch/notebooks/_src/plot_jacobians_and_hessians.py', 'functorch/notebooks/_src/plot_per_sample_gradients.py', 'functorch/op_analysis/gen_data.py', - 'ios/TestApp/benchmark/coreml_backend.py', - 'ios/TestApp/benchmark/trace_model.py', - 'ios/TestApp/custom_build/custom_build.py', - 'modules/detectron/upsample_nearest_op_test.py', - 'mypy_plugins/check_mypy_version.py', - 'scripts/analysis/format_test_csv.py', - 'scripts/diagnose_protobuf.py', - 'scripts/get_python_cmake_flags.py', - 'scripts/jit/log_extract.py', - 'scripts/model_zoo/update-caffe2-models.py', - 'scripts/model_zoo/update-models-from-caffe2.py', - 'scripts/release_notes/apply_categories.py', - 'scripts/release_notes/categorize.py', - 'scripts/release_notes/classifier.py', - 'scripts/release_notes/commitlist.py', - 'scripts/release_notes/common.py', - 'scripts/release_notes/namespace_check.py', - 'scripts/release_notes/test_release_notes.py', - 'setup.py', 'test/_nvfuser/__init__.py', 'test/_nvfuser/test_dynamo.py', 'test/_nvfuser/test_python_frontend.py', diff --git a/android/pytorch_android/generate_test_torchscripts.py b/android/pytorch_android/generate_test_torchscripts.py index 897c430c01f..c3c9518517a 100644 --- a/android/pytorch_android/generate_test_torchscripts.py +++ b/android/pytorch_android/generate_test_torchscripts.py @@ -1,18 +1,21 @@ +from typing import Dict, List, Optional, Tuple + import torch from torch import Tensor -from typing import Dict, List, Tuple, Optional OUTPUT_DIR = "src/androidTest/assets/" + def scriptAndSave(module, fileName): - print('-' * 80) + print("-" * 80) script_module = torch.jit.script(module) print(script_module.graph) outputFileName = OUTPUT_DIR + fileName # note that the lite interpreter model can also be used in full JIT script_module._save_for_lite_interpreter(outputFileName) print("Saved to " + outputFileName) - print('=' * 80) + print("=" * 80) + class Test(torch.jit.ScriptModule): @torch.jit.script_method @@ -73,7 +76,9 @@ class Test(torch.jit.ScriptModule): return res @torch.jit.script_method - def tupleIntSumReturnTuple(self, input: Tuple[int, int, int]) -> Tuple[Tuple[int, int, int], int]: + def tupleIntSumReturnTuple( + self, input: Tuple[int, int, int] + ) -> Tuple[Tuple[int, int, int], int]: sum = 0 for x in input: sum += x @@ -114,7 +119,7 @@ class Test(torch.jit.ScriptModule): @torch.jit.script_method def conv2d(self, x: Tensor, w: Tensor, toChannelsLast: bool) -> Tensor: r = torch.nn.functional.conv2d(x, w) - if (toChannelsLast): + if toChannelsLast: r = r.contiguous(memory_format=torch.channels_last) else: r = r.contiguous() @@ -132,4 +137,5 @@ class Test(torch.jit.ScriptModule): def contiguousChannelsLast3d(self, x: Tensor) -> Tensor: return x.contiguous(memory_format=torch.channels_last_3d) + scriptAndSave(Test(), "test.pt") diff --git a/android/test_app/make_assets.py b/android/test_app/make_assets.py index 8f9c8749199..eaeece4f2f7 100644 --- a/android/test_app/make_assets.py +++ b/android/test_app/make_assets.py @@ -5,12 +5,20 @@ print(torch.version.__version__) resnet18 = torchvision.models.resnet18(pretrained=True) resnet18.eval() -resnet18_traced = torch.jit.trace(resnet18, torch.rand(1, 3, 224, 224)).save("app/src/main/assets/resnet18.pt") +resnet18_traced = torch.jit.trace(resnet18, torch.rand(1, 3, 224, 224)).save( + "app/src/main/assets/resnet18.pt" +) resnet50 = torchvision.models.resnet50(pretrained=True) resnet50.eval() -torch.jit.trace(resnet50, torch.rand(1, 3, 224, 224)).save("app/src/main/assets/resnet50.pt") +torch.jit.trace(resnet50, torch.rand(1, 3, 224, 224)).save( + "app/src/main/assets/resnet50.pt" +) -mobilenet2q = torchvision.models.quantization.mobilenet_v2(pretrained=True, quantize=True) +mobilenet2q = torchvision.models.quantization.mobilenet_v2( + pretrained=True, quantize=True +) mobilenet2q.eval() -torch.jit.trace(mobilenet2q, torch.rand(1, 3, 224, 224)).save("app/src/main/assets/mobilenet2q.pt") +torch.jit.trace(mobilenet2q, torch.rand(1, 3, 224, 224)).save( + "app/src/main/assets/mobilenet2q.pt" +) diff --git a/android/test_app/make_assets_custom.py b/android/test_app/make_assets_custom.py index 8da813ef009..8fe9bc57c4e 100644 --- a/android/test_app/make_assets_custom.py +++ b/android/test_app/make_assets_custom.py @@ -21,5 +21,5 @@ traced_script_module.save("MobileNetV2.pt") # Dump root ops used by the model (for custom build optimization). ops = torch.jit.export_opnames(traced_script_module) -with open('MobileNetV2.yaml', 'w') as output: +with open("MobileNetV2.yaml", "w") as output: yaml.dump(ops, output) diff --git a/binaries/bench_gen/bench_gen.py b/binaries/bench_gen/bench_gen.py index aab941cf1cd..7523e76f8b1 100755 --- a/binaries/bench_gen/bench_gen.py +++ b/binaries/bench_gen/bench_gen.py @@ -3,13 +3,14 @@ import argparse import ast +from caffe2.python import brew, workspace + from caffe2.python.model_helper import ModelHelper from caffe2.python.predictor import mobile_exporter -from caffe2.python import workspace, brew def parse_kwarg(kwarg_str): - key, value = kwarg_str.split('=') + key, value = kwarg_str.split("=") try: value = ast.literal_eval(value) except ValueError: @@ -30,7 +31,7 @@ def main(args): iters = int(args.instances) for i in range(iters): - input_blob_name = input_name + (str(i) if i > 0 and args.chain else '') + input_blob_name = input_name + (str(i) if i > 0 and args.chain else "") output_blob_name = output_name + str(i + 1) add_op = getattr(brew, op_type) add_op(model, input_blob_name, output_blob_name, **kwargs) @@ -39,9 +40,7 @@ def main(args): workspace.RunNetOnce(model.param_init_net) - init_net, predict_net = mobile_exporter.Export( - workspace, model.net, model.params - ) + init_net, predict_net = mobile_exporter.Export(workspace, model.net, model.params) if args.debug: print("init_net:") @@ -51,40 +50,69 @@ def main(args): for op in predict_net.op: print(" ", op.type, op.input, "-->", op.output) - with open(args.predict_net, 'wb') as f: + with open(args.predict_net, "wb") as f: f.write(predict_net.SerializeToString()) - with open(args.init_net, 'wb') as f: + with open(args.init_net, "wb") as f: f.write(init_net.SerializeToString()) if __name__ == "__main__": parser = argparse.ArgumentParser( - description="Utility to generate Caffe2 benchmark models.") + description="Utility to generate Caffe2 benchmark models." + ) parser.add_argument("operator", help="Caffe2 operator to benchmark.") - parser.add_argument("-b", "--blob", - help="Instantiate a blob --blob name=dim1,dim2,dim3", - action='append') + parser.add_argument( + "-b", + "--blob", + help="Instantiate a blob --blob name=dim1,dim2,dim3", + action="append", + ) parser.add_argument("--context", help="Context to run on.", default="CPU") - parser.add_argument("--kwargs", help="kwargs to pass to operator.", - nargs="*", type=parse_kwarg, default=[]) - parser.add_argument("--init-net", "--init_net", help="Output initialization net.", - default="init_net.pb") - parser.add_argument("--predict-net", "--predict_net", help="Output prediction net.", - default="predict_net.pb") - parser.add_argument("--benchmark-name", "--benchmark_name", - help="Name of the benchmark network", - default="benchmark") - parser.add_argument("--input-name", "--input_name", help="Name of the input blob.", - default="data") - parser.add_argument("--output-name", "--output_name", help="Name of the output blob.", - default="output") - parser.add_argument("--instances", - help="Number of instances to run the operator.", - default="1") - parser.add_argument("-d", "--debug", help="Print debug information.", - action='store_true') - parser.add_argument("-c", "--chain", - help="Chain ops together (create data dependencies)", - action='store_true') + parser.add_argument( + "--kwargs", + help="kwargs to pass to operator.", + nargs="*", + type=parse_kwarg, + default=[], + ) + parser.add_argument( + "--init-net", + "--init_net", + help="Output initialization net.", + default="init_net.pb", + ) + parser.add_argument( + "--predict-net", + "--predict_net", + help="Output prediction net.", + default="predict_net.pb", + ) + parser.add_argument( + "--benchmark-name", + "--benchmark_name", + help="Name of the benchmark network", + default="benchmark", + ) + parser.add_argument( + "--input-name", "--input_name", help="Name of the input blob.", default="data" + ) + parser.add_argument( + "--output-name", + "--output_name", + help="Name of the output blob.", + default="output", + ) + parser.add_argument( + "--instances", help="Number of instances to run the operator.", default="1" + ) + parser.add_argument( + "-d", "--debug", help="Print debug information.", action="store_true" + ) + parser.add_argument( + "-c", + "--chain", + help="Chain ops together (create data dependencies)", + action="store_true", + ) args = parser.parse_args() main(args) diff --git a/docs/caffe2/process.py b/docs/caffe2/process.py index 4a59eec388d..ca9ebd97316 100644 --- a/docs/caffe2/process.py +++ b/docs/caffe2/process.py @@ -6,18 +6,20 @@ import os import shutil + # Module caffe2...caffe2.python.control_test def insert(originalfile, first_line, description): with open(originalfile) as f: f1 = f.readline() - if(f1.find(first_line) < 0): + if f1.find(first_line) < 0: docs = first_line + description + f1 - with open('newfile.txt', 'w') as f2: + with open("newfile.txt", "w") as f2: f2.write(docs) f2.write(f.read()) - os.rename('newfile.txt', originalfile) + os.rename("newfile.txt", originalfile) else: - print('already inserted') + print("already inserted") + # move up from /caffe2_root/doxygen os.chdir("..") @@ -28,7 +30,11 @@ os.system("git checkout caffe2/python/.") for root, dirs, files in os.walk("."): for file in files: - if (file.endswith(".py") and not file.endswith("_test.py") and not file.endswith("__.py")): + if ( + file.endswith(".py") + and not file.endswith("_test.py") + and not file.endswith("__.py") + ): filepath = os.path.join(root, file) print("filepath: " + filepath) directory = os.path.dirname(filepath)[2:] diff --git a/docs/cpp/source/conf.py b/docs/cpp/source/conf.py index 2b94cfdb505..917d370b2d9 100644 --- a/docs/cpp/source/conf.py +++ b/docs/cpp/source/conf.py @@ -19,6 +19,7 @@ # enabled on nightlies (and not trunk or on PRs) due to OOM errors in CI. # See https://github.com/pytorch/pytorch/issues/79992. import os + # sys.path.insert(0, os.path.abspath('.')) import textwrap @@ -27,22 +28,17 @@ import textwrap # If your documentation needs a minimal Sphinx version, state it here. # -needs_sphinx = '3.1.2' -run_doxygen = os.environ.get('RUN_DOXYGEN', "false") == "true" +needs_sphinx = "3.1.2" +run_doxygen = os.environ.get("RUN_DOXYGEN", "false") == "true" # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ - 'sphinx.ext.intersphinx', -] + ([ - 'breathe', - 'exhale' -] if run_doxygen else []) + "sphinx.ext.intersphinx", +] + (["breathe", "exhale"] if run_doxygen else []) -intersphinx_mapping = { - 'pytorch': ('https://pytorch.org/docs/main', None) -} +intersphinx_mapping = {"pytorch": ("https://pytorch.org/docs/main", None)} # Setup absolute paths for communicating with breathe / exhale where # items are expected / should be trimmed by. @@ -50,13 +46,13 @@ intersphinx_mapping = { this_file_dir = os.path.abspath(os.path.dirname(__file__)) doxygen_xml_dir = os.path.join( os.path.dirname(this_file_dir), # {repo_root}/docs/cpp - 'build', # {repo_root}/docs/cpp/build - 'xml' # {repo_root}/docs/cpp/build/xml + "build", # {repo_root}/docs/cpp/build + "xml", # {repo_root}/docs/cpp/build/xml ) repo_root = os.path.dirname( # {repo_root} - os.path.dirname( # {repo_root}/docs - os.path.dirname( # {repo_root}/docs/cpp - this_file_dir # {repo_root}/docs/cpp/source + os.path.dirname( # {repo_root}/docs + os.path.dirname( # {repo_root}/docs/cpp + this_file_dir # {repo_root}/docs/cpp/source ) ) ) @@ -98,16 +94,18 @@ exhale_args = { ############################################################################ # Main library page layout example configuration. # ############################################################################ - "afterTitleDescription": textwrap.dedent(''' + "afterTitleDescription": textwrap.dedent( + """ Welcome to the developer reference for the PyTorch C++ API. - '''), + """ + ), } # Tell sphinx what the primary language being documented is. -primary_domain = 'cpp' +primary_domain = "cpp" # Tell sphinx what the pygments highlight language should be. -highlight_language = 'cpp' +highlight_language = "cpp" # Add any paths that contain templates here, relative to this directory. # templates_path = ['_templates'] @@ -116,15 +114,15 @@ highlight_language = 'cpp' # You can specify multiple suffix as a list of string: # # source_suffix = ['.rst', '.md'] -source_suffix = '.rst' +source_suffix = ".rst" # The master toctree document. -master_doc = 'index' +master_doc = "index" # General information about the project. -project = 'PyTorch' -copyright = '2022, PyTorch Contributors' -author = 'PyTorch Contributors' +project = "PyTorch" +copyright = "2022, PyTorch Contributors" +author = "PyTorch Contributors" # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -132,10 +130,10 @@ author = 'PyTorch Contributors' # # The short X.Y version. # TODO: change to [:2] at v1.0 -version = 'main' +version = "main" # The full version, including alpha/beta/rc tags. # TODO: verify this works as expected -release = 'main' +release = "main" # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -150,7 +148,7 @@ language = None exclude_patterns = [] # The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' +pygments_style = "sphinx" # If true, `todo` and `todoList` produce output, else they produce nothing. todo_include_todos = True @@ -160,30 +158,30 @@ todo_include_todos = True # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -html_theme = 'pytorch_sphinx_theme' +html_theme = "pytorch_sphinx_theme" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. # html_theme_options = { - 'canonical_url': 'https://pytorch.org/docs/stable/', - 'pytorch_project': 'docs', - 'collapse_navigation': False, - 'display_version': True, - 'logo_only': True, + "canonical_url": "https://pytorch.org/docs/stable/", + "pytorch_project": "docs", + "collapse_navigation": False, + "display_version": True, + "logo_only": True, } # NOTE: sharing python docs resources html_logo = os.path.join( - repo_root, 'docs', 'source', '_static', 'img', 'pytorch-logo-dark-unstable.png' + repo_root, "docs", "source", "_static", "img", "pytorch-logo-dark-unstable.png" ) # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". # NOTE: sharing python docs resources -html_static_path = [os.path.join(repo_root, 'docs', 'cpp', 'source', '_static')] +html_static_path = [os.path.join(repo_root, "docs", "cpp", "source", "_static")] # Called automatically by Sphinx, making this `conf.py` an "extension". @@ -191,14 +189,15 @@ def setup(app): # NOTE: in Sphinx 1.8+ `html_css_files` is an official configuration value # and can be moved outside of this function (and the setup(app) function # can be deleted). - html_css_files = ['cpp_theme.css'] + html_css_files = ["cpp_theme.css"] # In Sphinx 1.8 it was renamed to `add_css_file`, 1.7 and prior it is # `add_stylesheet` (deprecated in 1.8). - add_css = getattr(app, 'add_css_file', app.add_stylesheet) + add_css = getattr(app, "add_css_file", app.add_stylesheet) for css_file in html_css_files: add_css(css_file) + # -- Options for HTMLHelp output ------------------------------------------ # Output file base name for HTML help builder. @@ -211,15 +210,12 @@ latex_elements = { # The paper size ('letterpaper' or 'a4paper'). # # 'papersize': 'letterpaper', - # The font size ('10pt', '11pt' or '12pt'). # # 'pointsize': '10pt', - # Additional stuff for the LaTeX preamble. # # 'preamble': '', - # Latex figure (float) alignment # # 'figure_align': 'htbp', @@ -229,8 +225,13 @@ latex_elements = { # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - (master_doc, 'pytorch.tex', 'PyTorch Documentation', - 'Torch Contributors', 'manual'), + ( + master_doc, + "pytorch.tex", + "PyTorch Documentation", + "Torch Contributors", + "manual", + ), ] @@ -238,10 +239,7 @@ latex_documents = [ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). -man_pages = [ - (master_doc, 'PyTorch', 'PyTorch Documentation', - [author], 1) -] +man_pages = [(master_doc, "PyTorch", "PyTorch Documentation", [author], 1)] # -- Options for Texinfo output ------------------------------------------- @@ -250,7 +248,13 @@ man_pages = [ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - (master_doc, 'PyTorch', 'PyTorch Documentation', - author, 'PyTorch', 'One line description of project.', - 'Miscellaneous'), + ( + master_doc, + "PyTorch", + "PyTorch Documentation", + author, + "PyTorch", + "One line description of project.", + "Miscellaneous", + ), ] diff --git a/docs/source/conf.py b/docs/source/conf.py index 8fec5f16f98..6039a5b8265 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -16,10 +16,11 @@ # documentation root, use os.path.abspath to make it absolute, like shown here. # import os -from os import path -import re + # import sys import pkgutil +import re +from os import path # source code directory, relative to this file, for sphinx-autobuild # sys.path.insert(0, os.path.abspath('../..')) @@ -30,9 +31,10 @@ try: import torchvision # noqa: F401 except ImportError: import warnings + warnings.warn('unable to load "torchvision" package') -RELEASE = os.environ.get('RELEASE', False) +RELEASE = os.environ.get("RELEASE", False) import pytorch_sphinx_theme @@ -40,25 +42,25 @@ import pytorch_sphinx_theme # If your documentation needs a minimal Sphinx version, state it here. # -needs_sphinx = '3.1.2' +needs_sphinx = "3.1.2" # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ - 'sphinx.ext.autodoc', - 'sphinx.ext.autosummary', - 'sphinx.ext.doctest', - 'sphinx.ext.intersphinx', - 'sphinx.ext.todo', - 'sphinx.ext.coverage', - 'sphinx.ext.napoleon', - 'sphinx.ext.viewcode', - 'sphinxcontrib.katex', - 'sphinx.ext.autosectionlabel', - 'sphinx_copybutton', - 'sphinx_panels', - 'myst_parser', + "sphinx.ext.autodoc", + "sphinx.ext.autosummary", + "sphinx.ext.doctest", + "sphinx.ext.intersphinx", + "sphinx.ext.todo", + "sphinx.ext.coverage", + "sphinx.ext.napoleon", + "sphinx.ext.viewcode", + "sphinxcontrib.katex", + "sphinx.ext.autosectionlabel", + "sphinx_copybutton", + "sphinx_panels", + "myst_parser", ] # build the templated autosummary files @@ -82,7 +84,7 @@ katex_prerender = True napoleon_use_ivar = True # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +templates_path = ["_templates"] # TODO: document these and remove them from here. @@ -343,15 +345,15 @@ coverage_ignore_classes = [ # You can specify multiple suffix as a list of string: # # source_suffix = ['.rst', '.md'] -source_suffix = '.rst' +source_suffix = ".rst" # The master toctree document. -master_doc = 'index' +master_doc = "index" # General information about the project. -project = 'PyTorch' -copyright = '2023, PyTorch Contributors' -author = 'PyTorch Contributors' +project = "PyTorch" +copyright = "2023, PyTorch Contributors" +author = "PyTorch Contributors" torch_version = str(torch.__version__) # The version info for the project you're documenting, acts as replacement for @@ -360,10 +362,10 @@ torch_version = str(torch.__version__) # # The short X.Y version. # TODO: change to [:2] at v1.0 -version = 'main (' + torch_version + ' )' +version = "main (" + torch_version + " )" # The full version, including alpha/beta/rc tags. # TODO: verify this works as expected -release = 'main' +release = "main" # Customized html_title here. # Default is " ".join(project, release, "documentation") if not set @@ -371,7 +373,7 @@ if RELEASE: # Turn 1.11.0aHASH into 1.11 # Note: the release candidates should no longer have the aHASH suffix, but in any # case we wish to leave only major.minor, even for rc builds. - version = '.'.join(torch_version.split('.')[:2]) + version = ".".join(torch_version.split(".")[:2]) html_title = " ".join((project, version, "documentation")) release = version @@ -388,7 +390,7 @@ language = "en" exclude_patterns = [] # The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' +pygments_style = "sphinx" # If true, `todo` and `todoList` produce output, else they produce nothing. todo_include_todos = True @@ -397,10 +399,10 @@ todo_include_todos = True autodoc_inherit_docstrings = False # Show type hints in the description -autodoc_typehints = 'description' +autodoc_typehints = "description" # Add parameter types if the parameter is documented in the docstring -autodoc_typehints_description_target = 'documented_params' +autodoc_typehints_description_target = "documented_params" # Type aliases for common types # Sphinx type aliases only works with Postponed Evaluation of Annotations @@ -442,7 +444,7 @@ autodoc_docstring_signature = True # # -html_theme = 'pytorch_sphinx_theme' +html_theme = "pytorch_sphinx_theme" html_theme_path = [pytorch_sphinx_theme.get_html_theme_path()] # Theme options are theme-specific and customize the look and feel of a theme @@ -450,30 +452,31 @@ html_theme_path = [pytorch_sphinx_theme.get_html_theme_path()] # documentation. html_theme_options = { - 'pytorch_project': 'docs', - 'canonical_url': 'https://pytorch.org/docs/stable/', - 'collapse_navigation': False, - 'display_version': True, - 'logo_only': True, - 'analytics_id': 'GTM-T8XT4PS', + "pytorch_project": "docs", + "canonical_url": "https://pytorch.org/docs/stable/", + "collapse_navigation": False, + "display_version": True, + "logo_only": True, + "analytics_id": "GTM-T8XT4PS", } -html_logo = '_static/img/pytorch-logo-dark-unstable.png' +html_logo = "_static/img/pytorch-logo-dark-unstable.png" if RELEASE: - html_logo = '_static/img/pytorch-logo-dark.svg' + html_logo = "_static/img/pytorch-logo-dark.svg" # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +html_static_path = ["_static"] html_css_files = [ - 'css/jit.css', + "css/jit.css", ] from sphinx.ext.coverage import CoverageBuilder + def coverage_post_process(app, exception): if exception is not None: return @@ -483,14 +486,16 @@ def coverage_post_process(app, exception): return if not torch.distributed.is_available(): - raise RuntimeError("The coverage tool cannot run with a version " - "of PyTorch that was built with USE_DISTRIBUTED=0 " - "as this module's API changes.") + raise RuntimeError( + "The coverage tool cannot run with a version " + "of PyTorch that was built with USE_DISTRIBUTED=0 " + "as this module's API changes." + ) # These are all the modules that have "automodule" in an rst file # These modules are the ones for which coverage is checked # Here, we make sure that no module is missing from that list - modules = app.env.domaindata['py']['modules'] + modules = app.env.domaindata["py"]["modules"] # We go through all the torch submodules and make sure they are # properly tested @@ -507,8 +512,9 @@ def coverage_post_process(app, exception): if "torch" not in modules: missing.add("torch") - for _, modname, ispkg in pkgutil.walk_packages(path=torch.__path__, - prefix=torch.__name__ + '.'): + for _, modname, ispkg in pkgutil.walk_packages( + path=torch.__path__, prefix=torch.__name__ + "." + ): if ispkg and is_not_internal(modname): if modname not in modules: missing.add(modname) @@ -517,16 +523,18 @@ def coverage_post_process(app, exception): if missing: mods = ", ".join(missing) - output.append(f"\nYou added the following module(s) to the PyTorch namespace '{mods}' " - "but they have no corresponding entry in a doc .rst file. You should " - "either make sure that the .rst file that contains the module's documentation " - "properly contains either '.. automodule:: mod_name' (if you do not want " - "the paragraph added by the automodule, you can simply use '.. py:module:: mod_name') " - " or make the module private (by appending an '_' at the beginning of its name).") + output.append( + f"\nYou added the following module(s) to the PyTorch namespace '{mods}' " + "but they have no corresponding entry in a doc .rst file. You should " + "either make sure that the .rst file that contains the module's documentation " + "properly contains either '.. automodule:: mod_name' (if you do not want " + "the paragraph added by the automodule, you can simply use '.. py:module:: mod_name') " + " or make the module private (by appending an '_' at the beginning of its name)." + ) # The output file is hard-coded by the coverage tool # Our CI is setup to fail if any line is added to this file - output_file = path.join(app.outdir, 'python.txt') + output_file = path.join(app.outdir, "python.txt") if output: with open(output_file, "a") as f: @@ -561,21 +569,21 @@ def process_docstring(app, what_, name, obj, options, lines): https://www.sphinx-doc.org/en/master/usage/extensions/autodoc.html """ import re + remove_directives = [ # Remove all xdoctest directives - re.compile(r'\s*>>>\s*#\s*x?doctest:\s*.*'), - re.compile(r'\s*>>>\s*#\s*x?doc:\s*.*'), + re.compile(r"\s*>>>\s*#\s*x?doctest:\s*.*"), + re.compile(r"\s*>>>\s*#\s*x?doc:\s*.*"), ] filtered_lines = [ - line for line in lines - if not any(pat.match(line) for pat in remove_directives) + line for line in lines if not any(pat.match(line) for pat in remove_directives) ] # Modify the lines inplace lines[:] = filtered_lines # make sure there is a blank line at the end if lines and lines[-1].strip(): - lines.append('') + lines.append("") # Called automatically by Sphinx, making this `conf.py` an "extension". @@ -584,17 +592,18 @@ def setup(app): # and can be moved outside of this function (and the setup(app) function # can be deleted). html_css_files = [ - 'https://cdn.jsdelivr.net/npm/katex@0.10.0-beta/dist/katex.min.css' + "https://cdn.jsdelivr.net/npm/katex@0.10.0-beta/dist/katex.min.css" ] # In Sphinx 1.8 it was renamed to `add_css_file`, 1.7 and prior it is # `add_stylesheet` (deprecated in 1.8). - add_css = getattr(app, 'add_css_file', app.add_stylesheet) + add_css = getattr(app, "add_css_file", app.add_stylesheet) for css_file in html_css_files: add_css(css_file) app.connect("build-finished", coverage_post_process) - app.connect('autodoc-process-docstring', process_docstring) + app.connect("autodoc-process-docstring", process_docstring) + # From PyTorch 1.5, we now use autogenerated files to document classes and # functions. This breaks older references since @@ -608,13 +617,14 @@ def setup(app): from sphinx.writers import html, html5 + def replace(Klass): old_call = Klass.visit_reference def visit_reference(self, node): - if 'refuri' in node and 'generated' in node.get('refuri'): - ref = node.get('refuri') - ref_anchor = ref.split('#') + if "refuri" in node and "generated" in node.get("refuri"): + ref = node.get("refuri") + ref_anchor = ref.split("#") if len(ref_anchor) > 1: # Only add the id if the node href and the text match, # i.e. the href is "torch.flip#torch.flip" and the content is @@ -622,18 +632,20 @@ def replace(Klass): # to autogenerated content anchor = ref_anchor[1] txt = node.parent.astext() - if txt == anchor or txt == anchor.split('.')[-1]: + if txt == anchor or txt == anchor.split(".")[-1]: self.body.append(f'

') return old_call(self, node) + Klass.visit_reference = visit_reference + replace(html.HTMLTranslator) replace(html5.HTML5Translator) # -- Options for HTMLHelp output ------------------------------------------ # Output file base name for HTML help builder. -htmlhelp_basename = 'PyTorchdoc' +htmlhelp_basename = "PyTorchdoc" # -- Options for LaTeX output --------------------------------------------- @@ -642,15 +654,12 @@ latex_elements = { # The paper size ('letterpaper' or 'a4paper'). # # 'papersize': 'letterpaper', - # The font size ('10pt', '11pt' or '12pt'). # # 'pointsize': '10pt', - # Additional stuff for the LaTeX preamble. # # 'preamble': '', - # Latex figure (float) alignment # # 'figure_align': 'htbp', @@ -660,8 +669,13 @@ latex_elements = { # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - (master_doc, 'pytorch.tex', 'PyTorch Documentation', - 'Torch Contributors', 'manual'), + ( + master_doc, + "pytorch.tex", + "PyTorch Documentation", + "Torch Contributors", + "manual", + ), ] @@ -669,10 +683,7 @@ latex_documents = [ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). -man_pages = [ - (master_doc, 'PyTorch', 'PyTorch Documentation', - [author], 1) -] +man_pages = [(master_doc, "PyTorch", "PyTorch Documentation", [author], 1)] # -- Options for Texinfo output ------------------------------------------- @@ -681,36 +692,43 @@ man_pages = [ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - (master_doc, 'PyTorch', 'PyTorch Documentation', - author, 'PyTorch', 'One line description of project.', - 'Miscellaneous'), + ( + master_doc, + "PyTorch", + "PyTorch Documentation", + author, + "PyTorch", + "One line description of project.", + "Miscellaneous", + ), ] # Example configuration for intersphinx: refer to the Python standard library. intersphinx_mapping = { - 'python': ('https://docs.python.org/3', None), - 'numpy': ('https://numpy.org/doc/stable', None), + "python": ("https://docs.python.org/3", None), + "numpy": ("https://numpy.org/doc/stable", None), } +import sphinx.ext.doctest + # -- A patch that prevents Sphinx from cross-referencing ivar tags ------- # See http://stackoverflow.com/a/41184353/3343043 from docutils import nodes -from sphinx.util.docfields import TypedField from sphinx import addnodes -import sphinx.ext.doctest +from sphinx.util.docfields import TypedField # Without this, doctest adds any example with a `>>>` as a test -doctest_test_doctest_blocks = '' +doctest_test_doctest_blocks = "" doctest_default_flags = sphinx.ext.doctest.doctest.ELLIPSIS -doctest_global_setup = ''' +doctest_global_setup = """ import torch try: import torchvision except ImportError: torchvision = None -''' +""" def patched_make_field(self, types, domain, items, **kw): @@ -720,43 +738,51 @@ def patched_make_field(self, types, domain, items, **kw): # type: (List, unicode, Tuple) -> nodes.field def handle_item(fieldarg, content): par = nodes.paragraph() - par += addnodes.literal_strong('', fieldarg) # Patch: this line added + par += addnodes.literal_strong("", fieldarg) # Patch: this line added # par.extend(self.make_xrefs(self.rolename, domain, fieldarg, # addnodes.literal_strong)) if fieldarg in types: - par += nodes.Text(' (') + par += nodes.Text(" (") # NOTE: using .pop() here to prevent a single type node to be # inserted twice into the doctree, which leads to # inconsistencies later when references are resolved fieldtype = types.pop(fieldarg) if len(fieldtype) == 1 and isinstance(fieldtype[0], nodes.Text): typename = fieldtype[0].astext() - builtin_types = ['int', 'long', 'float', 'bool', 'type'] + builtin_types = ["int", "long", "float", "bool", "type"] for builtin_type in builtin_types: - pattern = fr'(?>> |\.\.\. ' +copybutton_prompt_text = r">>> |\.\.\. " copybutton_prompt_is_regexp = True diff --git a/docs/source/scripts/build_activation_images.py b/docs/source/scripts/build_activation_images.py index 94f0416bc1b..cce571531d1 100644 --- a/docs/source/scripts/build_activation_images.py +++ b/docs/source/scripts/build_activation_images.py @@ -6,8 +6,9 @@ online tutorials. from pathlib import Path -import torch import matplotlib + +import torch from matplotlib import pyplot as plt matplotlib.use("Agg") diff --git a/docs/source/scripts/build_opsets.py b/docs/source/scripts/build_opsets.py index 2ab913fe85a..65b35be7d93 100644 --- a/docs/source/scripts/build_opsets.py +++ b/docs/source/scripts/build_opsets.py @@ -1,10 +1,10 @@ import os +from collections import OrderedDict from pathlib import Path import torch -from collections import OrderedDict -from torchgen.gen import parse_native_yaml import torch._prims as prims +from torchgen.gen import parse_native_yaml ROOT = Path(__file__).absolute().parent.parent.parent.parent NATIVE_FUNCTION_YAML_PATH = ROOT / Path("aten/src/ATen/native/native_functions.yaml") @@ -38,7 +38,6 @@ def get_aten(): def get_prims(): op_schema_pairs = [] for op_name in prims.__all__: - op_overload = getattr(prims, op_name, None) if not isinstance(op_overload, torch._ops.OpOverload): @@ -53,6 +52,7 @@ def get_prims(): return op_schema_pairs + def main(): aten_ops_list = get_aten() prims_ops_list = get_prims() @@ -70,5 +70,5 @@ def main(): f.write(f'"``{name}``","{schema}"\n') -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/docs/source/scripts/build_quantization_configs.py b/docs/source/scripts/build_quantization_configs.py index 6ab4fd433ef..bf405662040 100644 --- a/docs/source/scripts/build_quantization_configs.py +++ b/docs/source/scripts/build_quantization_configs.py @@ -3,33 +3,35 @@ This script will generate default values of quantization configs. These are for use in the documentation. """ +import os.path + import torch from torch.ao.quantization.backend_config import get_native_backend_config_dict from torch.ao.quantization.backend_config.utils import ( entry_to_pretty_str, remove_boolean_dispatch_from_name, ) -import os.path # Create a directory for the images, if it doesn't exist QUANTIZATION_BACKEND_CONFIG_IMAGE_PATH = os.path.join( - os.path.realpath(os.path.join(__file__, "..")), - "quantization_backend_configs" + os.path.realpath(os.path.join(__file__, "..")), "quantization_backend_configs" ) if not os.path.exists(QUANTIZATION_BACKEND_CONFIG_IMAGE_PATH): os.mkdir(QUANTIZATION_BACKEND_CONFIG_IMAGE_PATH) -output_path = os.path.join(QUANTIZATION_BACKEND_CONFIG_IMAGE_PATH, "default_backend_config.txt") +output_path = os.path.join( + QUANTIZATION_BACKEND_CONFIG_IMAGE_PATH, "default_backend_config.txt" +) with open(output_path, "w") as f: native_backend_config_dict = get_native_backend_config_dict() - configs = native_backend_config_dict['configs'] + configs = native_backend_config_dict["configs"] def _sort_key_func(entry): - pattern = entry['pattern'] + pattern = entry["pattern"] while isinstance(pattern, tuple): pattern = pattern[-1] @@ -49,8 +51,8 @@ with open(output_path, "w") as f: # to be next to each other, so convert to all lower case # and remove the underscores, and compare the last part # of the string - pattern_str_normalized = pattern.lower().replace('_', '') - key = pattern_str_normalized.split('.')[-1] + pattern_str_normalized = pattern.lower().replace("_", "") + key = pattern_str_normalized.split(".")[-1] return key configs.sort(key=_sort_key_func) diff --git a/docs/source/scripts/exportdb/generate_example_rst.py b/docs/source/scripts/exportdb/generate_example_rst.py index 96b6aae37f5..88697f629d2 100644 --- a/docs/source/scripts/exportdb/generate_example_rst.py +++ b/docs/source/scripts/exportdb/generate_example_rst.py @@ -159,13 +159,14 @@ def generate_rst(): tag_to_modules = {} support_level_to_modules = {} for example_case in example_cases.values(): - doc_contents = generate_example_rst(example_case) for tag in example_case.tags: tag_to_modules.setdefault(tag, []).append(doc_contents) - support_level_to_modules.setdefault(example_case.support_level, []).append(doc_contents) + support_level_to_modules.setdefault(example_case.support_level, []).append( + doc_contents + ) generate_tag_rst(tag_to_modules) generate_index_rst(example_cases, tag_to_modules, support_level_to_modules) diff --git a/docs/source/scripts/onnx/build_onnx_supported_aten_op_csv_table.py b/docs/source/scripts/onnx/build_onnx_supported_aten_op_csv_table.py index e4b4eb11fdf..88e055a8cfe 100644 --- a/docs/source/scripts/onnx/build_onnx_supported_aten_op_csv_table.py +++ b/docs/source/scripts/onnx/build_onnx_supported_aten_op_csv_table.py @@ -5,6 +5,7 @@ docs/source/onnx_supported_aten_list.rst. """ import os + from torch.onnx import _onnx_supported_ops # Constants diff --git a/ios/TestApp/benchmark/coreml_backend.py b/ios/TestApp/benchmark/coreml_backend.py index 347ac5ca1b8..140080b2e83 100644 --- a/ios/TestApp/benchmark/coreml_backend.py +++ b/ios/TestApp/benchmark/coreml_backend.py @@ -1,11 +1,8 @@ import torch import torchvision -from torch.backends._coreml.preprocess import ( - CompileSpec, - TensorSpec, - CoreMLComputeUnit, -) +from torch.backends._coreml.preprocess import CompileSpec, CoreMLComputeUnit, TensorSpec + def mobilenetv2_spec(): return { diff --git a/ios/TestApp/benchmark/trace_model.py b/ios/TestApp/benchmark/trace_model.py index ef82d92fed6..39158f04498 100644 --- a/ios/TestApp/benchmark/trace_model.py +++ b/ios/TestApp/benchmark/trace_model.py @@ -7,5 +7,7 @@ model.eval() example = torch.rand(1, 3, 224, 224) traced_script_module = torch.jit.trace(model, example) optimized_scripted_module = optimize_for_mobile(traced_script_module) -torch.jit.save(optimized_scripted_module, '../models/model.pt') -exported_optimized_scripted_module = optimized_scripted_module._save_for_lite_interpreter("../models/model_lite.ptl") +torch.jit.save(optimized_scripted_module, "../models/model.pt") +exported_optimized_scripted_module = ( + optimized_scripted_module._save_for_lite_interpreter("../models/model_lite.ptl") +) diff --git a/ios/TestApp/custom_build/custom_build.py b/ios/TestApp/custom_build/custom_build.py index f6980de5938..d009252c297 100644 --- a/ios/TestApp/custom_build/custom_build.py +++ b/ios/TestApp/custom_build/custom_build.py @@ -7,5 +7,5 @@ model.eval() example = torch.rand(1, 3, 224, 224) traced_script_module = torch.jit.trace(model, example) ops = torch.jit.export_opnames(traced_script_module) -with open('mobilenetv2.yaml', 'w') as output: +with open("mobilenetv2.yaml", "w") as output: yaml.dump(ops, output) diff --git a/modules/detectron/upsample_nearest_op_test.py b/modules/detectron/upsample_nearest_op_test.py index 96209cb80cb..276d50474d1 100644 --- a/modules/detectron/upsample_nearest_op_test.py +++ b/modules/detectron/upsample_nearest_op_test.py @@ -1,4 +1,3 @@ - import unittest import caffe2.python.hypothesis_test_util as hu @@ -17,7 +16,7 @@ class TestUpsampleNearestOp(hu.HypothesisTestCase): H=st.integers(10, 300), W=st.integers(10, 300), scale=st.integers(1, 3), - **hu.gcs + **hu.gcs, ) @settings(deadline=None, max_examples=20) def test_upsample_nearest_op(self, N, H, W, scale, gc, dc): diff --git a/mypy_plugins/check_mypy_version.py b/mypy_plugins/check_mypy_version.py index 7ef19ef22b0..4fa72e85577 100644 --- a/mypy_plugins/check_mypy_version.py +++ b/mypy_plugins/check_mypy_version.py @@ -7,18 +7,21 @@ from mypy.plugin import Plugin def get_correct_mypy_version(): # there's probably a more elegant way to do this - match, = re.finditer( - r'mypy==(\d+(?:\.\d+)*)', - (Path(__file__).parent.parent / '.ci' / 'docker' / 'requirements-ci.txt').read_text(), + (match,) = re.finditer( + r"mypy==(\d+(?:\.\d+)*)", + ( + Path(__file__).parent.parent / ".ci" / "docker" / "requirements-ci.txt" + ).read_text(), ) - version, = match.groups() + (version,) = match.groups() return version def plugin(version: str): correct_version = get_correct_mypy_version() if version != correct_version: - print(f'''\ + print( + f"""\ You are using mypy version {version}, which is not supported in the PyTorch repo. Please switch to mypy version {correct_version}. @@ -29,5 +32,7 @@ For example, if you installed mypy via pip, run this: Or if you installed mypy via conda, run this: conda install -c conda-forge mypy={correct_version} -''', file=sys.stderr) +""", + file=sys.stderr, + ) return Plugin diff --git a/scripts/analysis/format_test_csv.py b/scripts/analysis/format_test_csv.py index f91419342d2..da796f01fdd 100644 --- a/scripts/analysis/format_test_csv.py +++ b/scripts/analysis/format_test_csv.py @@ -15,13 +15,14 @@ onto your local file system is to send it to GitHub Gist: See also scripts/analysis/run_test_csv.sh """ +import argparse import csv import subprocess import sys -import argparse -parser = argparse.ArgumentParser(description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter) +parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter +) parser.add_argument("--log-url", type=str, default="", help="URL of raw logs") parser.add_argument("file", help="pytest CSV file to format") args = parser.parse_args() diff --git a/scripts/diagnose_protobuf.py b/scripts/diagnose_protobuf.py index bc643b39792..30bdbb2c2fe 100644 --- a/scripts/diagnose_protobuf.py +++ b/scripts/diagnose_protobuf.py @@ -15,44 +15,45 @@ Usage: import os import re -from subprocess import Popen, PIPE +from subprocess import PIPE, Popen # Get python protobuf version. try: import google.protobuf + python_version = google.protobuf.__version__ python_protobuf_installed = True except ImportError: print("DEBUG: cannot find python protobuf install.") python_protobuf_installed = False -if os.name == 'nt': - protoc_name = 'protoc.exe' +if os.name == "nt": + protoc_name = "protoc.exe" else: - protoc_name = 'protoc' + protoc_name = "protoc" try: - p = Popen([protoc_name, '--version'], stdout=PIPE, stderr=PIPE) + p = Popen([protoc_name, "--version"], stdout=PIPE, stderr=PIPE) out, err = p.communicate() except: - print('DEBUG: did not find protoc binary.') - print('DEBUG: out: ' + out) - print('DEBUG: err: ' + err) + print("DEBUG: did not find protoc binary.") + print("DEBUG: out: " + out) + print("DEBUG: err: " + err) native_protobuf_installed = False else: if p.returncode: - print('DEBUG: protoc returned a non-zero return code.') - print('DEBUG: out: ' + out) - print('DEBUG: err: ' + err) + print("DEBUG: protoc returned a non-zero return code.") + print("DEBUG: out: " + out) + print("DEBUG: err: " + err) native_protobuf_installed = False else: - tmp = re.search(r'\d\.\d\.\d', out) + tmp = re.search(r"\d\.\d\.\d", out) if tmp: native_version = tmp.group(0) native_protobuf_installed = True else: - print('DEBUG: cannot parse protoc version string.') - print('DEBUG: out: ' + out) + print("DEBUG: cannot parse protoc version string.") + print("DEBUG: out: " + out) native_protobuf_installed = False PYTHON_PROTOBUF_NOT_INSTALLED = """ @@ -74,7 +75,9 @@ VERSION_MISMATCH = """ Your python protobuf is of version {py_ver} but your native protoc version is of version {native_ver}. This will cause the installation to produce incompatible protobuf files. This is bad in general - consider installing the same version. -""".format(py_ver=python_version, native_ver=native_version) +""".format( + py_ver=python_version, native_ver=native_version +) # Now, give actual recommendations if not python_protobuf_installed: @@ -87,4 +90,4 @@ if python_protobuf_installed and native_protobuf_installed: if python_version != native_version: print(VERSION_MISMATCH) else: - print('All looks good.') + print("All looks good.") diff --git a/scripts/get_python_cmake_flags.py b/scripts/get_python_cmake_flags.py index 05562510d05..a1aad03888f 100644 --- a/scripts/get_python_cmake_flags.py +++ b/scripts/get_python_cmake_flags.py @@ -13,14 +13,12 @@ # - - -import sysconfig import sys +import sysconfig flags = [ - f'-DPYTHON_EXECUTABLE:FILEPATH={sys.executable}', + f"-DPYTHON_EXECUTABLE:FILEPATH={sys.executable}", f"-DPYTHON_INCLUDE_DIR={sysconfig.get_path('include')}", ] -print(' '.join(flags), end='') +print(" ".join(flags), end="") diff --git a/scripts/jit/log_extract.py b/scripts/jit/log_extract.py index 61e3172fe0b..14c45049a68 100644 --- a/scripts/jit/log_extract.py +++ b/scripts/jit/log_extract.py @@ -1,10 +1,17 @@ import argparse import functools import traceback -from torch.utils.jit.log_extract import extract_ir, load_graph_and_inputs, run_baseline_no_fusion, run_nnc, run_nvfuser -from typing import List, Tuple, Callable, Optional +from typing import Callable, List, Optional, Tuple -''' +from torch.utils.jit.log_extract import ( + extract_ir, + load_graph_and_inputs, + run_baseline_no_fusion, + run_nnc, + run_nvfuser, +) + +""" Usage: 1. Run your script and pipe into a log file PYTORCH_JIT_LOG_LEVEL=">>graph_fuser" python3 my_test.py &> log.txt @@ -15,10 +22,14 @@ You can also extract the list of extracted IR: log_extract.py log.txt --output Passing in --graphs 0 2 will only run graphs 0 and 2 -''' +""" -def test_runners(graphs: List[str], runners: List[Tuple[str, Callable]], graph_set: Optional[List[int]]): +def test_runners( + graphs: List[str], + runners: List[Tuple[str, Callable]], + graph_set: Optional[List[int]], +): for i, ir in enumerate(graphs): _, inputs = load_graph_and_inputs(ir) if graph_set and i not in graph_set: @@ -33,7 +44,9 @@ def test_runners(graphs: List[str], runners: List[Tuple[str, Callable]], graph_s result = runner_fn(ir, inputs) if prev_result: improvement = (prev_result / result - 1) * 100 - print(f"{runner_name} : {result:.6f} ms improvement over {prev_runner_name}: improvement: {improvement:.2f}%") + print( + f"{runner_name} : {result:.6f} ms improvement over {prev_runner_name}: improvement: {improvement:.2f}%" + ) else: print(f"{runner_name} : {result:.6f} ms") prev_result = result @@ -47,32 +60,66 @@ def run(): description="Extracts torchscript IR from log files and, optionally, benchmarks it or outputs the IR" ) parser.add_argument("filename", help="Filename of log file") - parser.add_argument("--nvfuser", dest="nvfuser", action="store_true", help="benchmark nvfuser") - parser.add_argument("--no-nvfuser", dest="nvfuser", action="store_false", help="DON'T benchmark nvfuser") + parser.add_argument( + "--nvfuser", dest="nvfuser", action="store_true", help="benchmark nvfuser" + ) + parser.add_argument( + "--no-nvfuser", + dest="nvfuser", + action="store_false", + help="DON'T benchmark nvfuser", + ) parser.set_defaults(nvfuser=False) - parser.add_argument("--nnc-static", dest="nnc_static", action="store_true", help="benchmark nnc static") - parser.add_argument("--no-nnc-static", dest="nnc_static", action="store_false", help="DON'T benchmark nnc static") + parser.add_argument( + "--nnc-static", + dest="nnc_static", + action="store_true", + help="benchmark nnc static", + ) + parser.add_argument( + "--no-nnc-static", + dest="nnc_static", + action="store_false", + help="DON'T benchmark nnc static", + ) parser.set_defaults(nnc_static=False) - parser.add_argument("--nnc-dynamic", dest="nnc_dynamic", action="store_true", help="nnc with dynamic shapes") + parser.add_argument( + "--nnc-dynamic", + dest="nnc_dynamic", + action="store_true", + help="nnc with dynamic shapes", + ) parser.add_argument( "--no-nnc-dynamic", dest="nnc_dynamic", action="store_false", - help="DONT't benchmark nnc with dynamic shapes") + help="DONT't benchmark nnc with dynamic shapes", + ) parser.set_defaults(nnc_dynamic=False) - - parser.add_argument("--baseline", dest="baseline", action="store_true", help="benchmark baseline") - parser.add_argument("--no-baseline", dest="baseline", action="store_false", help="DON'T benchmark baseline") + parser.add_argument( + "--baseline", dest="baseline", action="store_true", help="benchmark baseline" + ) + parser.add_argument( + "--no-baseline", + dest="baseline", + action="store_false", + help="DON'T benchmark baseline", + ) parser.set_defaults(baseline=False) - parser.add_argument("--output", dest="output", action="store_true", help="Output graph IR") - parser.add_argument("--no-output", dest="output", action="store_false", help="DON'T output graph IR") + parser.add_argument( + "--output", dest="output", action="store_true", help="Output graph IR" + ) + parser.add_argument( + "--no-output", dest="output", action="store_false", help="DON'T output graph IR" + ) parser.set_defaults(output=False) - parser.add_argument('--graphs', nargs="+", type=int, help="Run only specified graph indices") - + parser.add_argument( + "--graphs", nargs="+", type=int, help="Run only specified graph indices" + ) args = parser.parse_args() graphs = extract_ir(args.filename) @@ -97,8 +144,9 @@ def run(): for i, ir in enumerate(graphs): if graph_set and i not in graph_set: continue - quoted.append("\"\"\"" + ir + "\"\"\"") + quoted.append('"""' + ir + '"""') print("[" + ", ".join(quoted) + "]") + if __name__ == "__main__": run() diff --git a/scripts/model_zoo/update-caffe2-models.py b/scripts/model_zoo/update-caffe2-models.py index 9f48920e6a8..aad5ec58e5d 100755 --- a/scripts/model_zoo/update-caffe2-models.py +++ b/scripts/model_zoo/update-caffe2-models.py @@ -8,7 +8,12 @@ import tempfile from urllib.request import urlretrieve -from caffe2.python.models.download import downloadFromURLToFile, getURLFromName, deleteDirectory +from caffe2.python.models.download import ( + deleteDirectory, + downloadFromURLToFile, + getURLFromName, +) + class SomeClass: # largely copied from @@ -17,13 +22,12 @@ class SomeClass: model_dir = self._caffe2_model_dir(model) assert not os.path.exists(model_dir) os.makedirs(model_dir) - for f in ['predict_net.pb', 'init_net.pb', 'value_info.json']: + for f in ["predict_net.pb", "init_net.pb", "value_info.json"]: url = getURLFromName(model, f) dest = os.path.join(model_dir, f) try: try: - downloadFromURLToFile(url, dest, - show_progress=False) + downloadFromURLToFile(url, dest, show_progress=False) except TypeError: # show_progress not supported prior to # Caffe2 78c014e752a374d905ecfb465d44fa16e02a28f1 @@ -36,13 +40,13 @@ class SomeClass: exit(1) def _caffe2_model_dir(self, model): - caffe2_home = os.path.expanduser('~/.caffe2') - models_dir = os.path.join(caffe2_home, 'models') + caffe2_home = os.path.expanduser("~/.caffe2") + models_dir = os.path.join(caffe2_home, "models") return os.path.join(models_dir, model) def _onnx_model_dir(self, model): - onnx_home = os.path.expanduser('~/.onnx') - models_dir = os.path.join(onnx_home, 'models') + onnx_home = os.path.expanduser("~/.onnx") + models_dir = os.path.join(onnx_home, "models") model_dir = os.path.join(models_dir, model) return model_dir, os.path.dirname(model_dir) @@ -53,45 +57,44 @@ class SomeClass: if os.path.exists(model_dir): return os.makedirs(model_dir) - url = f'https://s3.amazonaws.com/download.onnx/models/{model}.tar.gz' + url = f"https://s3.amazonaws.com/download.onnx/models/{model}.tar.gz" # On Windows, NamedTemporaryFile cannot be opened for a # second time download_file = tempfile.NamedTemporaryFile(delete=False) try: download_file.close() - print(f'Start downloading model {model} from {url}') + print(f"Start downloading model {model} from {url}") urlretrieve(url, download_file.name) - print('Done') + print("Done") with tarfile.open(download_file.name) as t: t.extractall(models_dir) except Exception as e: - print(f'Failed to prepare data for model {model}: {e}') + print(f"Failed to prepare data for model {model}: {e}") raise finally: os.remove(download_file.name) -models = [ - 'bvlc_alexnet', - 'densenet121', - 'inception_v1', - 'inception_v2', - 'resnet50', +models = [ + "bvlc_alexnet", + "densenet121", + "inception_v1", + "inception_v2", + "resnet50", # TODO currently onnx can't translate squeezenet :( # 'squeezenet', - - 'vgg16', - + "vgg16", # TODO currently vgg19 doesn't work in the CI environment, # possibly due to OOM # 'vgg19' ] + def download_models(): sc = SomeClass() for model in models: - print('update-caffe2-models.py: downloading', model) + print("update-caffe2-models.py: downloading", model) caffe2_model_dir = sc._caffe2_model_dir(model) onnx_model_dir, onnx_models_dir = sc._onnx_model_dir(model) if not os.path.exists(caffe2_model_dir): @@ -99,61 +102,74 @@ def download_models(): if not os.path.exists(onnx_model_dir): sc._prepare_model_data(model) + def generate_models(): sc = SomeClass() for model in models: - print('update-caffe2-models.py: generating', model) + print("update-caffe2-models.py: generating", model) caffe2_model_dir = sc._caffe2_model_dir(model) onnx_model_dir, onnx_models_dir = sc._onnx_model_dir(model) - subprocess.check_call(['echo', model]) - with open(os.path.join(caffe2_model_dir, 'value_info.json'), 'r') as f: + subprocess.check_call(["echo", model]) + with open(os.path.join(caffe2_model_dir, "value_info.json"), "r") as f: value_info = f.read() - subprocess.check_call([ - 'convert-caffe2-to-onnx', - '--caffe2-net-name', model, - '--caffe2-init-net', os.path.join(caffe2_model_dir, 'init_net.pb'), - '--value-info', value_info, - '-o', os.path.join(onnx_model_dir, 'model.pb'), - os.path.join(caffe2_model_dir, 'predict_net.pb') - ]) - subprocess.check_call([ - 'tar', - '-czf', - model + '.tar.gz', - model - ], cwd=onnx_models_dir) + subprocess.check_call( + [ + "convert-caffe2-to-onnx", + "--caffe2-net-name", + model, + "--caffe2-init-net", + os.path.join(caffe2_model_dir, "init_net.pb"), + "--value-info", + value_info, + "-o", + os.path.join(onnx_model_dir, "model.pb"), + os.path.join(caffe2_model_dir, "predict_net.pb"), + ] + ) + subprocess.check_call( + ["tar", "-czf", model + ".tar.gz", model], cwd=onnx_models_dir + ) + def upload_models(): sc = SomeClass() for model in models: - print('update-caffe2-models.py: uploading', model) + print("update-caffe2-models.py: uploading", model) onnx_model_dir, onnx_models_dir = sc._onnx_model_dir(model) - subprocess.check_call([ - 'aws', - 's3', - 'cp', - model + '.tar.gz', - f"s3://download.onnx/models/{model}.tar.gz", - '--acl', 'public-read' - ], cwd=onnx_models_dir) + subprocess.check_call( + [ + "aws", + "s3", + "cp", + model + ".tar.gz", + f"s3://download.onnx/models/{model}.tar.gz", + "--acl", + "public-read", + ], + cwd=onnx_models_dir, + ) + def cleanup(): sc = SomeClass() for model in models: onnx_model_dir, onnx_models_dir = sc._onnx_model_dir(model) - os.remove(os.path.join(os.path.dirname(onnx_model_dir), model + '.tar.gz')) + os.remove(os.path.join(os.path.dirname(onnx_model_dir), model + ".tar.gz")) -if __name__ == '__main__': + +if __name__ == "__main__": try: - subprocess.check_call(['aws', 'sts', 'get-caller-identity']) + subprocess.check_call(["aws", "sts", "get-caller-identity"]) except: - print('update-caffe2-models.py: please run `aws configure` manually to set up credentials') + print( + "update-caffe2-models.py: please run `aws configure` manually to set up credentials" + ) sys.exit(1) - if sys.argv[1] == 'download': + if sys.argv[1] == "download": download_models() - if sys.argv[1] == 'generate': + if sys.argv[1] == "generate": generate_models() - elif sys.argv[1] == 'upload': + elif sys.argv[1] == "upload": upload_models() - elif sys.argv[1] == 'cleanup': + elif sys.argv[1] == "cleanup": cleanup() diff --git a/scripts/model_zoo/update-models-from-caffe2.py b/scripts/model_zoo/update-models-from-caffe2.py index 537fc03a60a..a75683abed6 100644 --- a/scripts/model_zoo/update-models-from-caffe2.py +++ b/scripts/model_zoo/update-models-from-caffe2.py @@ -1,26 +1,29 @@ #! /usr/bin/env python3 -import onnx.backend - import argparse -import caffe2.python.workspace as c2_workspace import glob import json -import numpy as np -import onnx -import caffe2.python.onnx.frontend -import caffe2.python.onnx.backend import os import shutil import tarfile import tempfile -import boto3 - from urllib.request import urlretrieve -from caffe2.python.models.download import downloadFromURLToFile, getURLFromName, deleteDirectory +import boto3 +import caffe2.python.onnx.backend +import caffe2.python.onnx.frontend +import caffe2.python.workspace as c2_workspace +import numpy as np +import onnx +import onnx.backend from caffe2.proto import caffe2_pb2 + +from caffe2.python.models.download import ( + deleteDirectory, + downloadFromURLToFile, + getURLFromName, +) from onnx import numpy_helper @@ -46,25 +49,31 @@ Examples: def upload_onnx_model(model_name, zoo_dir, backup=False, only_local=False): if only_local: - print('No uploading in local only mode.') + print("No uploading in local only mode.") return model_dir = os.path.join(zoo_dir, model_name) - suffix = '-backup' if backup else '' + suffix = "-backup" if backup else "" if backup: - print(f'Backing up the previous version of ONNX model {model_name}...') - rel_file_name = f'{model_name}{suffix}.tar.gz' + print(f"Backing up the previous version of ONNX model {model_name}...") + rel_file_name = f"{model_name}{suffix}.tar.gz" abs_file_name = os.path.join(zoo_dir, rel_file_name) - print(f'Compressing {model_name} model to {abs_file_name}') - with tarfile.open(abs_file_name, 'w:gz') as f: + print(f"Compressing {model_name} model to {abs_file_name}") + with tarfile.open(abs_file_name, "w:gz") as f: f.add(model_dir, arcname=model_name) file_size = os.stat(abs_file_name).st_size - print(f'Uploading {abs_file_name} ({float(file_size) / 1024 / 1024} MB) to s3 cloud...') - client = boto3.client('s3', 'us-east-1') + print( + f"Uploading {abs_file_name} ({float(file_size) / 1024 / 1024} MB) to s3 cloud..." + ) + client = boto3.client("s3", "us-east-1") transfer = boto3.s3.transfer.S3Transfer(client) - transfer.upload_file(abs_file_name, 'download.onnx', f'models/latest/{rel_file_name}', - extra_args={'ACL': 'public-read'}) + transfer.upload_file( + abs_file_name, + "download.onnx", + f"models/latest/{rel_file_name}", + extra_args={"ACL": "public-read"}, + ) - print(f'Successfully uploaded {rel_file_name} to s3!') + print(f"Successfully uploaded {rel_file_name} to s3!") def download_onnx_model(model_name, zoo_dir, use_cache=True, only_local=False): @@ -75,19 +84,22 @@ def download_onnx_model(model_name, zoo_dir, use_cache=True, only_local=False): return else: shutil.rmtree(model_dir) - url = f'https://s3.amazonaws.com/download.onnx/models/latest/{model_name}.tar.gz' + url = f"https://s3.amazonaws.com/download.onnx/models/latest/{model_name}.tar.gz" download_file = tempfile.NamedTemporaryFile(delete=False) try: download_file.close() - print('Downloading ONNX model {} from {} and save in {} ...\n'.format( - model_name, url, download_file.name)) + print( + "Downloading ONNX model {} from {} and save in {} ...\n".format( + model_name, url, download_file.name + ) + ) urlretrieve(url, download_file.name) with tarfile.open(download_file.name) as t: - print(f'Extracting ONNX model {model_name} to {zoo_dir} ...\n') + print(f"Extracting ONNX model {model_name} to {zoo_dir} ...\n") t.extractall(zoo_dir) except Exception as e: - print(f'Failed to download/backup data for ONNX model {model_name}: {e}') + print(f"Failed to download/backup data for ONNX model {model_name}: {e}") if not os.path.exists(model_dir): os.makedirs(model_dir) finally: @@ -106,13 +118,12 @@ def download_caffe2_model(model_name, zoo_dir, use_cache=True): shutil.rmtree(model_dir) os.makedirs(model_dir) - for f in ['predict_net.pb', 'init_net.pb', 'value_info.json']: + for f in ["predict_net.pb", "init_net.pb", "value_info.json"]: url = getURLFromName(model_name, f) dest = os.path.join(model_dir, f) try: try: - downloadFromURLToFile(url, dest, - show_progress=False) + downloadFromURLToFile(url, dest, show_progress=False) except TypeError: # show_progress not supported prior to # Caffe2 78c014e752a374d905ecfb465d44fa16e02a28f1 @@ -129,20 +140,22 @@ def caffe2_to_onnx(caffe2_model_name, caffe2_model_dir): caffe2_init_proto = caffe2_pb2.NetDef() caffe2_predict_proto = caffe2_pb2.NetDef() - with open(os.path.join(caffe2_model_dir, 'init_net.pb'), 'rb') as f: + with open(os.path.join(caffe2_model_dir, "init_net.pb"), "rb") as f: caffe2_init_proto.ParseFromString(f.read()) - caffe2_init_proto.name = f'{caffe2_model_name}_init' - with open(os.path.join(caffe2_model_dir, 'predict_net.pb'), 'rb') as f: + caffe2_init_proto.name = f"{caffe2_model_name}_init" + with open(os.path.join(caffe2_model_dir, "predict_net.pb"), "rb") as f: caffe2_predict_proto.ParseFromString(f.read()) caffe2_predict_proto.name = caffe2_model_name - with open(os.path.join(caffe2_model_dir, 'value_info.json'), 'rb') as f: + with open(os.path.join(caffe2_model_dir, "value_info.json"), "rb") as f: value_info = json.loads(f.read()) - print(f'Converting Caffe2 model {caffe2_model_name} in {caffe2_model_dir} to ONNX format') + print( + f"Converting Caffe2 model {caffe2_model_name} in {caffe2_model_dir} to ONNX format" + ) onnx_model = caffe2.python.onnx.frontend.caffe2_net_to_onnx_model( init_net=caffe2_init_proto, predict_net=caffe2_predict_proto, - value_info=value_info + value_info=value_info, ) return onnx_model, caffe2_init_proto, caffe2_predict_proto @@ -163,7 +176,10 @@ def tensortype_to_ndarray(tensor_type): def generate_test_input_data(onnx_model, scale): - real_inputs_names = list({input.name for input in onnx_model.graph.input} - {init.name for init in onnx_model.graph.initializer}) + real_inputs_names = list( + {input.name for input in onnx_model.graph.input} + - {init.name for init in onnx_model.graph.initializer} + ) real_inputs = [] for name in real_inputs_names: for input in onnx_model.graph.input: @@ -180,7 +196,7 @@ def generate_test_input_data(onnx_model, scale): def generate_test_output_data(caffe2_init_net, caffe2_predict_net, inputs): p = c2_workspace.Predictor(caffe2_init_net, caffe2_predict_net) - inputs_map = {input[0]:input[1] for input in inputs} + inputs_map = {input[0]: input[1] for input in inputs} output = p.run(inputs_map) c2_workspace.ResetWorkspace() @@ -200,37 +216,53 @@ def onnx_verify(onnx_model, inputs, ref_outputs): model_mapping = { - 'bvlc_alexnet': 'bvlc_alexnet', - 'bvlc_googlenet': 'bvlc_googlenet', - 'bvlc_reference_caffenet': 'bvlc_reference_caffenet', - 'bvlc_reference_rcnn_ilsvrc13': 'bvlc_reference_rcnn_ilsvrc13', - 'densenet121': 'densenet121', + "bvlc_alexnet": "bvlc_alexnet", + "bvlc_googlenet": "bvlc_googlenet", + "bvlc_reference_caffenet": "bvlc_reference_caffenet", + "bvlc_reference_rcnn_ilsvrc13": "bvlc_reference_rcnn_ilsvrc13", + "densenet121": "densenet121", #'finetune_flickr_style': 'finetune_flickr_style', - 'inception_v1': 'inception_v1', - 'inception_v2': 'inception_v2', - 'resnet50': 'resnet50', - 'shufflenet': 'shufflenet', - 'squeezenet': 'squeezenet_old', + "inception_v1": "inception_v1", + "inception_v2": "inception_v2", + "resnet50": "resnet50", + "shufflenet": "shufflenet", + "squeezenet": "squeezenet_old", #'vgg16': 'vgg16', - 'vgg19': 'vgg19', - 'zfnet512': 'zfnet512', + "vgg19": "vgg19", + "zfnet512": "zfnet512", } - -if __name__ == '__main__': - parser = argparse.ArgumentParser(description='Update the ONNX models.') - parser.add_argument('-v', action="store_true", default=False, help="verbose") - parser.add_argument("--local-dir", type=str, default=os.path.expanduser('~'), - help="local dir to store Caffe2 and ONNX models") - parser.add_argument("--no-cache", action="store_true", default=False, - help="whether use local ONNX models") - parser.add_argument('--clean-test-data', action="store_true", default=False, - help="remove the old test data") - parser.add_argument('--add-test-data', type=int, default=0, - help="add new test data") - parser.add_argument('--only-local', action="store_true", default=False, - help="no upload including backup") +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Update the ONNX models.") + parser.add_argument("-v", action="store_true", default=False, help="verbose") + parser.add_argument( + "--local-dir", + type=str, + default=os.path.expanduser("~"), + help="local dir to store Caffe2 and ONNX models", + ) + parser.add_argument( + "--no-cache", + action="store_true", + default=False, + help="whether use local ONNX models", + ) + parser.add_argument( + "--clean-test-data", + action="store_true", + default=False, + help="remove the old test data", + ) + parser.add_argument( + "--add-test-data", type=int, default=0, help="add new test data" + ) + parser.add_argument( + "--only-local", + action="store_true", + default=False, + help="no upload including backup", + ) args = parser.parse_args() delete_test_data = args.clean_test_data @@ -245,87 +277,99 @@ if __name__ == '__main__': for onnx_model_name in model_mapping: c2_model_name = model_mapping[onnx_model_name] - print(f'####### Processing ONNX model {onnx_model_name} ({c2_model_name} in Caffe2) #######') + print( + f"####### Processing ONNX model {onnx_model_name} ({c2_model_name} in Caffe2) #######" + ) download_caffe2_model(c2_model_name, caffe2_zoo_dir, use_cache=use_cache) - download_onnx_model(onnx_model_name, onnx_zoo_dir, use_cache=use_cache, only_local=only_local) + download_onnx_model( + onnx_model_name, onnx_zoo_dir, use_cache=use_cache, only_local=only_local + ) onnx_model_dir = os.path.join(onnx_zoo_dir, onnx_model_name) if delete_test_data: - print('Deleting all the existing test data...') + print("Deleting all the existing test data...") # NB: For now, we don't delete the npz files. - #for f in glob.glob(os.path.join(onnx_model_dir, '*.npz')): + # for f in glob.glob(os.path.join(onnx_model_dir, '*.npz')): # os.remove(f) - for f in glob.glob(os.path.join(onnx_model_dir, 'test_data_set*')): + for f in glob.glob(os.path.join(onnx_model_dir, "test_data_set*")): shutil.rmtree(f) - onnx_model, c2_init_net, c2_predict_net = caffe2_to_onnx(c2_model_name, os.path.join(caffe2_zoo_dir, c2_model_name)) + onnx_model, c2_init_net, c2_predict_net = caffe2_to_onnx( + c2_model_name, os.path.join(caffe2_zoo_dir, c2_model_name) + ) - print(f'Deleteing old ONNX {onnx_model_name} model...') - for f in glob.glob(os.path.join(onnx_model_dir, 'model*'.format(onnx_model_name))): + print(f"Deleteing old ONNX {onnx_model_name} model...") + for f in glob.glob( + os.path.join(onnx_model_dir, "model*".format(onnx_model_name)) + ): os.remove(f) - print(f'Serializing generated ONNX {onnx_model_name} model ...') - with open(os.path.join(onnx_model_dir, 'model.onnx'), 'wb') as file: + print(f"Serializing generated ONNX {onnx_model_name} model ...") + with open(os.path.join(onnx_model_dir, "model.onnx"), "wb") as file: file.write(onnx_model.SerializeToString()) - print(f'Verifying model {onnx_model_name} with ONNX model checker...') + print(f"Verifying model {onnx_model_name} with ONNX model checker...") onnx.checker.check_model(onnx_model) total_existing_data_set = 0 - print(f'Verifying model {onnx_model_name} with existing test data...') - for f in glob.glob(os.path.join(onnx_model_dir, '*.npz')): - test_data = np.load(f, encoding='bytes') - inputs = list(test_data['inputs']) - ref_outputs = list(test_data['outputs']) + print(f"Verifying model {onnx_model_name} with existing test data...") + for f in glob.glob(os.path.join(onnx_model_dir, "*.npz")): + test_data = np.load(f, encoding="bytes") + inputs = list(test_data["inputs"]) + ref_outputs = list(test_data["outputs"]) onnx_verify(onnx_model, inputs, ref_outputs) total_existing_data_set += 1 - for f in glob.glob(os.path.join(onnx_model_dir, 'test_data_set*')): + for f in glob.glob(os.path.join(onnx_model_dir, "test_data_set*")): inputs = [] - inputs_num = len(glob.glob(os.path.join(f, 'input_*.pb'))) + inputs_num = len(glob.glob(os.path.join(f, "input_*.pb"))) for i in range(inputs_num): tensor = onnx.TensorProto() - with open(os.path.join(f, f'input_{i}.pb'), 'rb') as pf: + with open(os.path.join(f, f"input_{i}.pb"), "rb") as pf: tensor.ParseFromString(pf.read()) inputs.append(numpy_helper.to_array(tensor)) ref_outputs = [] - ref_outputs_num = len(glob.glob(os.path.join(f, 'output_*.pb'))) + ref_outputs_num = len(glob.glob(os.path.join(f, "output_*.pb"))) for i in range(ref_outputs_num): tensor = onnx.TensorProto() - with open(os.path.join(f, f'output_{i}.pb'), 'rb') as pf: + with open(os.path.join(f, f"output_{i}.pb"), "rb") as pf: tensor.ParseFromString(pf.read()) ref_outputs.append(numpy_helper.to_array(tensor)) onnx_verify(onnx_model, inputs, ref_outputs) total_existing_data_set += 1 starting_index = 0 - while os.path.exists(os.path.join(onnx_model_dir, f'test_data_set_{starting_index}')): + while os.path.exists( + os.path.join(onnx_model_dir, f"test_data_set_{starting_index}") + ): starting_index += 1 if total_existing_data_set == 0 and add_test_data == 0: add_test_data = 3 total_existing_data_set = 3 - print(f'Generating {add_test_data} sets of new test data...') + print(f"Generating {add_test_data} sets of new test data...") for i in range(starting_index, add_test_data + starting_index): - data_dir = os.path.join(onnx_model_dir, f'test_data_set_{i}') + data_dir = os.path.join(onnx_model_dir, f"test_data_set_{i}") os.makedirs(data_dir) inputs = generate_test_input_data(onnx_model, 255) ref_outputs = generate_test_output_data(c2_init_net, c2_predict_net, inputs) onnx_verify(onnx_model, inputs, ref_outputs) for index, input in enumerate(inputs): tensor = numpy_helper.from_array(input[1]) - with open(os.path.join(data_dir, f'input_{index}.pb'), 'wb') as file: + with open(os.path.join(data_dir, f"input_{index}.pb"), "wb") as file: file.write(tensor.SerializeToString()) for index, output in enumerate(ref_outputs): tensor = numpy_helper.from_array(output) - with open(os.path.join(data_dir, f'output_{index}.pb'), 'wb') as file: + with open(os.path.join(data_dir, f"output_{index}.pb"), "wb") as file: file.write(tensor.SerializeToString()) del onnx_model del c2_init_net del c2_predict_net - upload_onnx_model(onnx_model_name, onnx_zoo_dir, backup=False, only_local=only_local) + upload_onnx_model( + onnx_model_name, onnx_zoo_dir, backup=False, only_local=only_local + ) - print('\n\n') + print("\n\n") diff --git a/scripts/release_notes/apply_categories.py b/scripts/release_notes/apply_categories.py index b656f83c1e1..5fd2928c89c 100644 --- a/scripts/release_notes/apply_categories.py +++ b/scripts/release_notes/apply_categories.py @@ -2,9 +2,10 @@ # base commitlist . Useful if you are refactoring any code # but want to keep the previous data on categories -import commitlist import csv +import commitlist + category_csv = "results/category_data.csv" commitlist_csv = "results/commitlist.csv" diff --git a/scripts/release_notes/categorize.py b/scripts/release_notes/categorize.py index e7be5a5315a..0187637e7b8 100644 --- a/scripts/release_notes/categorize.py +++ b/scripts/release_notes/categorize.py @@ -1,27 +1,44 @@ import argparse import os import textwrap -from common import topics, get_commit_data_cache -from commitlist import CommitList - -# Imports for working with classi -from classifier import CommitClassifier, CategoryConfig, XLMR_BASE, get_author_map, get_file_map, CommitClassifierInputs -import common -import torch from pathlib import Path +import common +import torch + +# Imports for working with classi +from classifier import ( + CategoryConfig, + CommitClassifier, + CommitClassifierInputs, + get_author_map, + get_file_map, + XLMR_BASE, +) +from commitlist import CommitList +from common import get_commit_data_cache, topics + + class Categorizer: - def __init__(self, path, category='Uncategorized', use_classifier:bool = False): + def __init__(self, path, category="Uncategorized", use_classifier: bool = False): self.cache = get_commit_data_cache() self.commits = CommitList.from_existing(path) if use_classifier: print("Using a classifier to aid with categorization.") device = "cuda" if torch.cuda.is_available() else "cpu" classifier_config = CategoryConfig(common.categories) - author_map = get_author_map(Path("results/classifier"), regen_data=False, assert_stored=True) - file_map = get_file_map(Path("results/classifier"), regen_data=False, assert_stored=True) - self.classifier = CommitClassifier(XLMR_BASE, author_map, file_map, classifier_config).to(device) - self.classifier.load_state_dict(torch.load(Path("results/classifier/commit_classifier.pt"))) + author_map = get_author_map( + Path("results/classifier"), regen_data=False, assert_stored=True + ) + file_map = get_file_map( + Path("results/classifier"), regen_data=False, assert_stored=True + ) + self.classifier = CommitClassifier( + XLMR_BASE, author_map, file_map, classifier_config + ).to(device) + self.classifier.load_state_dict( + torch.load(Path("results/classifier/commit_classifier.pt")) + ) self.classifier.eval() else: self.classifier = None @@ -37,7 +54,9 @@ class Categorizer: while i < len(commits): cur_commit = commits[i] next_commit = commits[i + 1] if i + 1 < len(commits) else None - jump_to = self.handle_commit(cur_commit, already_done + i + 1, total_commits, commits) + jump_to = self.handle_commit( + cur_commit, already_done + i + 1, total_commits, commits + ) # Increment counter if jump_to is not None: @@ -51,15 +70,17 @@ class Categorizer: return self.cache.get(commit.commit_hash) def potential_reverts_of(self, commit, commits): - submodule_update_str = ['Update TensorPipe submodule', - 'Updating submodules', - 'Automated submodule update'] + submodule_update_str = [ + "Update TensorPipe submodule", + "Updating submodules", + "Automated submodule update", + ] if any(a in commit.title for a in submodule_update_str): return [] features = self.features(commit) - if 'Reverted' in features.labels: - reasons = {'GithubBot': "Reverted"} + if "Reverted" in features.labels: + reasons = {"GithubBot": "Reverted"} else: reasons = {} @@ -67,15 +88,20 @@ class Categorizer: # -8 to remove the (#35011) cleaned_title = commit.title[:-10] # NB: the index + 2 is sketch - reasons.update({(index + 2 + delta): cand for delta, cand in enumerate(commits[index + 1:]) - if cleaned_title in cand.title and - commit.commit_hash != cand.commit_hash}) + reasons.update( + { + (index + 2 + delta): cand + for delta, cand in enumerate(commits[index + 1 :]) + if cleaned_title in cand.title + and commit.commit_hash != cand.commit_hash + } + ) return reasons def handle_commit(self, commit, i, total, commits): potential_reverts = self.potential_reverts_of(commit, commits) if potential_reverts: - potential_reverts = f'!!!POTENTIAL REVERTS!!!: {potential_reverts}' + potential_reverts = f"!!!POTENTIAL REVERTS!!!: {potential_reverts}" else: potential_reverts = "" @@ -83,22 +109,27 @@ class Categorizer: if self.classifier is not None: # Some commits don't have authors: author = features.author if features.author else "Unknown" - files = ' '.join(features.files_changed) - classifier_input = CommitClassifierInputs(title=[features.title], files=[files], author=[author]) - classifier_category = self.classifier.get_most_likely_category_name(classifier_input)[0] + files = " ".join(features.files_changed) + classifier_input = CommitClassifierInputs( + title=[features.title], files=[files], author=[author] + ) + classifier_category = self.classifier.get_most_likely_category_name( + classifier_input + )[0] else: classifier_category = commit.category breaking_alarm = "" - if 'module: bc-breaking' in features.labels: + if "module: bc-breaking" in features.labels: breaking_alarm += "\n!!!!!! BC BREAKING !!!!!!" - if 'module: deprecation' in features.labels: + if "module: deprecation" in features.labels: breaking_alarm += "\n!!!!!! DEPRECATION !!!!!!" - os.system('clear') - view = textwrap.dedent(f'''\ + os.system("clear") + view = textwrap.dedent( + f"""\ [{i}/{total}] ================================================================================ {features.title} @@ -115,38 +146,37 @@ Current category: {commit.category} Select from: {', '.join(common.categories)} - ''') + """ + ) print(view) cat_choice = None while cat_choice is None: print("Enter category: ") - value = input(f'{classifier_category} ').strip() + value = input(f"{classifier_category} ").strip() if len(value) == 0: # The user just pressed enter and likes the default value cat_choice = classifier_category continue - choices = [cat for cat in common.categories - if cat.startswith(value)] + choices = [cat for cat in common.categories if cat.startswith(value)] if len(choices) != 1: - print(f'Possible matches: {choices}, try again') + print(f"Possible matches: {choices}, try again") continue cat_choice = choices[0] - print(f'\nSelected: {cat_choice}') - print(f'\nCurrent topic: {commit.topic}') - print(f'''Select from: {', '.join(topics)}''') + print(f"\nSelected: {cat_choice}") + print(f"\nCurrent topic: {commit.topic}") + print(f"""Select from: {', '.join(topics)}""") topic_choice = None while topic_choice is None: - value = input('topic> ').strip() + value = input("topic> ").strip() if len(value) == 0: topic_choice = commit.topic continue - choices = [cat for cat in topics - if cat.startswith(value)] + choices = [cat for cat in topics if cat.startswith(value)] if len(choices) != 1: - print(f'Possible matches: {choices}, try again') + print(f"Possible matches: {choices}, try again") continue topic_choice = choices[0] - print(f'\nSelected: {topic_choice}') + print(f"\nSelected: {topic_choice}") self.update_commit(commit, cat_choice, topic_choice) return None @@ -157,18 +187,30 @@ Select from: {', '.join(common.categories)} commit.topic = topic self.commits.write_result() + def main(): - parser = argparse.ArgumentParser(description='Tool to help categorize commits') - parser.add_argument('--category', type=str, default='Uncategorized', - help='Which category to filter by. "Uncategorized", None, or a category name') - parser.add_argument('--file', help='The location of the commits CSV', - default='results/commitlist.csv') - parser.add_argument('--use_classifier', action='store_true', help="Whether or not to use a classifier to aid in categorization.") + parser = argparse.ArgumentParser(description="Tool to help categorize commits") + parser.add_argument( + "--category", + type=str, + default="Uncategorized", + help='Which category to filter by. "Uncategorized", None, or a category name', + ) + parser.add_argument( + "--file", + help="The location of the commits CSV", + default="results/commitlist.csv", + ) + parser.add_argument( + "--use_classifier", + action="store_true", + help="Whether or not to use a classifier to aid in categorization.", + ) args = parser.parse_args() categorizer = Categorizer(args.file, args.category, args.use_classifier) categorizer.categorize() -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/scripts/release_notes/classifier.py b/scripts/release_notes/classifier.py index 1a3a55626bb..b906c3849fd 100644 --- a/scripts/release_notes/classifier.py +++ b/scripts/release_notes/classifier.py @@ -1,20 +1,20 @@ import argparse -from pathlib import Path -import torch -import torchtext -from torchtext.functional import to_tensor -import torch.nn as nn -import torch.nn.functional as F -from typing import List, Dict -import pandas as pd -from dataclasses import dataclass import math import pickle import random -from tqdm import tqdm +from dataclasses import dataclass from itertools import chain +from pathlib import Path +from typing import Dict, List import common +import pandas as pd +import torch +import torch.nn as nn +import torch.nn.functional as F +import torchtext +from torchtext.functional import to_tensor +from tqdm import tqdm XLMR_BASE = torchtext.models.XLMR_BASE_ENCODER @@ -24,6 +24,7 @@ device = "cuda" if torch.cuda.is_available() else "cpu" HAS_IMBLEARN = False try: import imblearn + HAS_IMBLEARN = True except ImportError: HAS_IMBLEARN = False @@ -37,12 +38,14 @@ UNKNOWN_TOKEN = "" def truncate_file(file: Path, max_len: int = 5): - return ('/').join(file.parts[:max_len]) + return ("/").join(file.parts[:max_len]) def build_file_set(all_files: List[Path], max_len: int): truncated_files = [truncate_file(file, max_len) for file in all_files] return set(truncated_files) + + @dataclass class CommitClassifierInputs: title: List[str] @@ -62,7 +65,13 @@ class CategoryConfig: class CommitClassifier(nn.Module): - def __init__(self, encoder_base: torchtext.models.XLMR_BASE_ENCODER, author_map: Dict[str, int], file_map: [str, int], config: CategoryConfig): + def __init__( + self, + encoder_base: torchtext.models.XLMR_BASE_ENCODER, + author_map: Dict[str, int], + file_map: [str, int], + config: CategoryConfig, + ): super().__init__() self.encoder = encoder_base.get_model().requires_grad_(False) self.transform = encoder_base.transform() @@ -72,7 +81,9 @@ class CommitClassifier(nn.Module): self.num_authors = len(author_map) self.num_files = len(file_map) self.embedding_table = nn.Embedding(self.num_authors, config.embedding_dim) - self.file_embedding_bag = nn.EmbeddingBag(self.num_files, config.file_embedding_dim, mode='sum') + self.file_embedding_bag = nn.EmbeddingBag( + self.num_files, config.file_embedding_dim, mode="sum" + ) self.dense_title = nn.Linear(config.input_dim, config.inner_dim) self.dense_files = nn.Linear(config.file_embedding_dim, config.inner_dim) self.dense_author = nn.Linear(config.embedding_dim, config.inner_dim) @@ -97,10 +108,22 @@ class CommitClassifier(nn.Module): files: list[str] = input_batch.files batch_file_indexes = [] for file in files: - paths = [truncate_file(Path(file_part), MAX_LEN_FILE) for file_part in file.split(" ")] - batch_file_indexes.append([self.file_map.get(file, self.file_map[UNKNOWN_TOKEN]) for file in paths]) + paths = [ + truncate_file(Path(file_part), MAX_LEN_FILE) + for file_part in file.split(" ") + ] + batch_file_indexes.append( + [ + self.file_map.get(file, self.file_map[UNKNOWN_TOKEN]) + for file in paths + ] + ) - flat_indexes = torch.tensor(list(chain.from_iterable(batch_file_indexes)), dtype=torch.long, device=device) + flat_indexes = torch.tensor( + list(chain.from_iterable(batch_file_indexes)), + dtype=torch.long, + device=device, + ) offsets = [0] offsets.extend(len(files) for files in batch_file_indexes[:-1]) offsets = torch.tensor(offsets, dtype=torch.long, device=device) @@ -114,7 +137,10 @@ class CommitClassifier(nn.Module): # Add author embedding authors: List[str] = input_batch.author - author_ids = [self.author_map.get(author, self.author_map[UNKNOWN_TOKEN]) for author in authors] + author_ids = [ + self.author_map.get(author, self.author_map[UNKNOWN_TOKEN]) + for author in authors + ] author_ids = torch.tensor(author_ids).to(device) author_embed = self.embedding_table(author_ids) author_embed = self.dense_author(author_embed) @@ -138,18 +164,24 @@ class CommitClassifier(nn.Module): def get_train_val_data(data_folder: Path, regen_data: bool, train_percentage=0.95): - if not regen_data and Path(data_folder / "train_df.csv").exists() and Path(data_folder / "val_df.csv").exists(): + if ( + not regen_data + and Path(data_folder / "train_df.csv").exists() + and Path(data_folder / "val_df.csv").exists() + ): train_data = pd.read_csv(data_folder / "train_df.csv") val_data = pd.read_csv(data_folder / "val_df.csv") return train_data, val_data else: print("Train, Val, Test Split not found generating from scratch.") commit_list_df = pd.read_csv(data_folder / "commitlist.csv") - test_df = commit_list_df[commit_list_df['category'] == 'Uncategorized'] - all_train_df = commit_list_df[commit_list_df['category'] != 'Uncategorized'] + test_df = commit_list_df[commit_list_df["category"] == "Uncategorized"] + all_train_df = commit_list_df[commit_list_df["category"] != "Uncategorized"] # We are going to drop skip from training set since it is so imbalanced - print("We are removing skip categories, YOU MIGHT WANT TO CHANGE THIS, BUT THIS IS A MORE HELPFUL CLASSIFIER FOR LABELING.") - all_train_df = all_train_df[all_train_df['category'] != 'skip'] + print( + "We are removing skip categories, YOU MIGHT WANT TO CHANGE THIS, BUT THIS IS A MORE HELPFUL CLASSIFIER FOR LABELING." + ) + all_train_df = all_train_df[all_train_df["category"] != "skip"] all_train_df = all_train_df.sample(frac=1).reset_index(drop=True) split_index = math.floor(train_percentage * len(all_train_df)) train_df = all_train_df[:split_index] @@ -165,30 +197,32 @@ def get_train_val_data(data_folder: Path, regen_data: bool, train_percentage=0.9 def get_author_map(data_folder: Path, regen_data, assert_stored=False): if not regen_data and Path(data_folder / "author_map.pkl").exists(): - with open(data_folder / "author_map.pkl", 'rb') as f: + with open(data_folder / "author_map.pkl", "rb") as f: return pickle.load(f) else: if assert_stored: raise FileNotFoundError( - "Author map not found, you are loading for inference you need to have an author map!") + "Author map not found, you are loading for inference you need to have an author map!" + ) print("Regenerating Author Map") all_data = pd.read_csv(data_folder / "commitlist.csv") authors = all_data.author.unique().tolist() authors.append(UNKNOWN_TOKEN) author_map = {author: i for i, author in enumerate(authors)} - with open(data_folder / "author_map.pkl", 'wb') as f: + with open(data_folder / "author_map.pkl", "wb") as f: pickle.dump(author_map, f) return author_map - def get_file_map(data_folder: Path, regen_data, assert_stored=False): if not regen_data and Path(data_folder / "file_map.pkl").exists(): - with open(data_folder / "file_map.pkl", 'rb') as f: + with open(data_folder / "file_map.pkl", "rb") as f: return pickle.load(f) else: if assert_stored: - raise FileNotFoundError("File map not found, you are loading for inference you need to have a file map!") + raise FileNotFoundError( + "File map not found, you are loading for inference you need to have a file map!" + ) print("Regenerating File Map") all_data = pd.read_csv(data_folder / "commitlist.csv") # Lets explore files @@ -201,10 +235,11 @@ def get_file_map(data_folder: Path, regen_data, assert_stored=False): all_files.append(Path(UNKNOWN_TOKEN)) file_set = build_file_set(all_files, MAX_LEN_FILE) file_map = {file: i for i, file in enumerate(file_set)} - with open(data_folder / "file_map.pkl", 'wb') as f: + with open(data_folder / "file_map.pkl", "wb") as f: pickle.dump(file_map, f) return file_map + # Generate a dataset for training @@ -222,7 +257,9 @@ def generate_batch(batch): files = list(files) author = list(author) category = list(category) - targets = torch.tensor([common.categories.index(cat) for cat in category]).to(device) + targets = torch.tensor([common.categories.index(cat) for cat in category]).to( + device + ) return CommitClassifierInputs(title, files, author), targets @@ -251,6 +288,7 @@ def balance_dataset(dataset: List): category = [common.categories.index(cat) for cat in category] inpt_data = list(zip(title, files, author)) from imblearn.over_sampling import RandomOverSampler + # from imblearn.under_sampling import RandomUnderSampler rus = RandomOverSampler(random_state=42) X, y = rus.fit_resample(inpt_data, category) @@ -265,6 +303,7 @@ def balance_dataset(dataset: List): def gen_class_weights(dataset: List): from collections import Counter + epsilon = 1e-1 title, files, author, category = zip(*dataset) category = [common.categories.index(cat) for cat in category] @@ -274,8 +313,13 @@ def gen_class_weights(dataset: List): least_common = counter.most_common()[-percentile_33:] smoothed_top = sum(i[1] + epsilon for i in most_common) / len(most_common) smoothed_bottom = sum(i[1] + epsilon for i in least_common) / len(least_common) // 3 - class_weights = torch.tensor([1.0 / (min(max(counter[i], smoothed_bottom), smoothed_top) + epsilon) - for i in range(len(common.categories))], device=device) + class_weights = torch.tensor( + [ + 1.0 / (min(max(counter[i], smoothed_bottom), smoothed_top) + epsilon) + for i in range(len(common.categories)) + ], + device=device, + ) return class_weights @@ -287,7 +331,9 @@ def train(save_path: Path, data_folder: Path, regen_data: bool, resample: bool): classifier_config = CategoryConfig(common.categories) author_map = get_author_map(data_folder, regen_data) file_map = get_file_map(data_folder, regen_data) - commit_classifier = CommitClassifier(XLMR_BASE, author_map, file_map, classifier_config).to(device) + commit_classifier = CommitClassifier( + XLMR_BASE, author_map, file_map, classifier_config + ).to(device) # Lets train this bag of bits class_weights = gen_class_weights(train_zip_list) @@ -320,7 +366,9 @@ def train(save_path: Path, data_folder: Path, regen_data: bool, resample: bool): start = end val_l = eval_step(val_batch, commit_classifier, loss) - tqdm.write(f"Finished epoch {i} with a train loss of: {l.item()} and a val_loss of: {val_l.item()}") + tqdm.write( + f"Finished epoch {i} with a train loss of: {l.item()} and a val_loss of: {val_l.item()}" + ) with torch.no_grad(): commit_classifier.eval() @@ -335,22 +383,39 @@ def train(save_path: Path, data_folder: Path, regen_data: bool, resample: bool): def main(): - parser = argparse.ArgumentParser(description='Tool to create a classifier for helping to categorize commits') + parser = argparse.ArgumentParser( + description="Tool to create a classifier for helping to categorize commits" + ) - parser.add_argument('--train', action='store_true', help='Train a new classifier') + parser.add_argument("--train", action="store_true", help="Train a new classifier") parser.add_argument("--commit_data_folder", default="results/classifier/") - parser.add_argument('--save_path', default='results/classifier/commit_classifier.pt') - parser.add_argument('--regen_data', action='store_true', - help="Regenerate the training data, helps if labeld more examples and want to re-train.") - parser.add_argument('--resample', action='store_true', - help="Resample the training data to be balanced. (Only works if imblearn is installed.)") + parser.add_argument( + "--save_path", default="results/classifier/commit_classifier.pt" + ) + parser.add_argument( + "--regen_data", + action="store_true", + help="Regenerate the training data, helps if labeld more examples and want to re-train.", + ) + parser.add_argument( + "--resample", + action="store_true", + help="Resample the training data to be balanced. (Only works if imblearn is installed.)", + ) args = parser.parse_args() if args.train: - train(Path(args.save_path), Path(args.commit_data_folder), args.regen_data, args.resample) + train( + Path(args.save_path), + Path(args.commit_data_folder), + args.regen_data, + args.resample, + ) return - print("Currently this file only trains a new classifier please pass in --train to train a new classifier") + print( + "Currently this file only trains a new classifier please pass in --train to train a new classifier" + ) if __name__ == "__main__": diff --git a/scripts/release_notes/commitlist.py b/scripts/release_notes/commitlist.py index 99c2235946a..69310d69e2c 100644 --- a/scripts/release_notes/commitlist.py +++ b/scripts/release_notes/commitlist.py @@ -1,16 +1,23 @@ import argparse -from common import run, topics, get_features, frontend_categories -from collections import defaultdict -import os -from pathlib import Path import csv -import pprint -import common -from common import get_commit_data_cache, features_to_dict -import re import dataclasses +import os +import pprint +import re +from collections import defaultdict +from pathlib import Path from typing import List +import common +from common import ( + features_to_dict, + frontend_categories, + get_commit_data_cache, + get_features, + run, + topics, +) + """ Example Usages @@ -25,6 +32,8 @@ Update the existing commitlist to commit bfcb687b9c. python commitlist.py --update-to bfcb687b9c """ + + @dataclasses.dataclass(frozen=False) class Commit: commit_hash: str @@ -43,10 +52,14 @@ class Commit: merge_into: str = None def __repr__(self): - return f'Commit({self.commit_hash}, {self.category}, {self.topic}, {self.title})' + return ( + f"Commit({self.commit_hash}, {self.category}, {self.topic}, {self.title})" + ) + commit_fields = tuple(f.name for f in dataclasses.fields(Commit)) + class CommitList: # NB: Private ctor. Use `from_existing` or `create_new`. def __init__(self, path: str, commits: List[Commit]): @@ -61,7 +74,9 @@ class CommitList: @staticmethod def create_new(path, base_version, new_version): if os.path.exists(path): - raise ValueError('Attempted to create a new commitlist but one exists already!') + raise ValueError( + "Attempted to create a new commitlist but one exists already!" + ) commits = CommitList.get_commits_between(base_version, new_version) return CommitList(path, commits) @@ -83,11 +98,12 @@ class CommitList: @staticmethod def write_to_disk_static(path, commit_list): os.makedirs(Path(path).parent, exist_ok=True) - with open(path, 'w') as csvfile: + with open(path, "w") as csvfile: writer = csv.writer(csvfile) writer.writerow(commit_fields) for commit in commit_list: writer.writerow(dataclasses.astuple(commit)) + @staticmethod def keywordInFile(file, keywords): for key in keywords: @@ -105,16 +121,27 @@ class CommitList: pr_link = f"https://github.com/pytorch/pytorch/pull/{features['pr_number']}" else: pr_link = None - files_changed_str = ' '.join(features['files_changed']) - return Commit(commit_hash, category, topic, features["title"], files_changed_str, pr_link, features["author"], a1, a2, a3) + files_changed_str = " ".join(features["files_changed"]) + return Commit( + commit_hash, + category, + topic, + features["title"], + files_changed_str, + pr_link, + features["author"], + a1, + a2, + a3, + ) @staticmethod def category_remapper(category: str) -> str: if category in frontend_categories: - category = category + '_frontend' + category = category + "_frontend" return category - if category == 'Meta API': - category = 'composability' + if category == "Meta API": + category = "composability" return category if category in common.quantization.categories: category = common.quantization.name @@ -135,20 +162,20 @@ class CommitList: optional[str] """ pairs = [ - ('[dynamo]', 'dynamo'), - ('[torchdynamo]', 'dynamo'), - ('[torchinductor]', 'inductor'), - ('[inductor]', 'inductor'), - ('[codemod', 'skip'), - ('[profiler]', 'profiler'), - ('[functorch]', 'functorch'), - ('[autograd]', 'autograd_frontend'), - ('[quantization]', 'quantization'), - ('[nn]', 'nn_frontend'), - ('[complex]', 'complex_frontend'), - ('[mps]', 'mps'), - ('[optimizer]', 'optimizer_frontend'), - ('[xla]', 'xla'), + ("[dynamo]", "dynamo"), + ("[torchdynamo]", "dynamo"), + ("[torchinductor]", "inductor"), + ("[inductor]", "inductor"), + ("[codemod", "skip"), + ("[profiler]", "profiler"), + ("[functorch]", "functorch"), + ("[autograd]", "autograd_frontend"), + ("[quantization]", "quantization"), + ("[nn]", "nn_frontend"), + ("[complex]", "complex_frontend"), + ("[mps]", "mps"), + ("[optimizer]", "optimizer_frontend"), + ("[xla]", "xla"), ] title_lower = title.lower() for bracket, category in pairs: @@ -158,123 +185,188 @@ class CommitList: @staticmethod def categorize(features): - title = features['title'] - labels = features['labels'] - category = 'Uncategorized' - topic = 'Untopiced' + title = features["title"] + labels = features["labels"] + category = "Uncategorized" + topic = "Untopiced" # Revert commits are merged directly to master with no associated PR number - if features['pr_number'] is None: + if features["pr_number"] is None: if title.startswith("Revert"): - return 'skip', topic + return "skip", topic # We ask contributors to label their PR's appropriately # when they're first landed. # Check if the labels are there first. already_categorized = already_topiced = False for label in labels: - if label.startswith('release notes: '): - category = label.split('release notes: ', 1)[1] + if label.startswith("release notes: "): + category = label.split("release notes: ", 1)[1] category = CommitList.category_remapper(category) already_categorized = True - if label.startswith('topic: '): - topic = label.split('topic: ', 1)[1] + if label.startswith("topic: "): + topic = label.split("topic: ", 1)[1] already_topiced = True if already_categorized and already_topiced: return category, topic # update this to check if each file starts with caffe2 - if 'caffe2' in title: - return 'caffe2', topic - if 'Reverted' in labels: - return 'skip', topic - if 'bc_breaking' in labels: - topic = 'bc-breaking' - if 'module: deprecation' in labels: - topic = 'deprecation' + if "caffe2" in title: + return "caffe2", topic + if "Reverted" in labels: + return "skip", topic + if "bc_breaking" in labels: + topic = "bc-breaking" + if "module: deprecation" in labels: + topic = "deprecation" found_bracket_category = CommitList.bracket_category_matcher(title) if found_bracket_category: return found_bracket_category, topic - files_changed = features['files_changed'] + files_changed = features["files_changed"] for file in files_changed: file_lowercase = file.lower() - if CommitList.keywordInFile(file, ['docker/', '.circleci', '.github', '.jenkins', '.ci', '.azure_pipelines']): - category = 'releng' + if CommitList.keywordInFile( + file, + [ + "docker/", + ".circleci", + ".github", + ".jenkins", + ".ci", + ".azure_pipelines", + ], + ): + category = "releng" break # datapipe(s), torch/utils/data, test_{dataloader, datapipe} - if CommitList.keywordInFile(file, ['torch/utils/data', 'test_dataloader', 'test_datapipe']): - category = 'dataloader_frontend' + if CommitList.keywordInFile( + file, ["torch/utils/data", "test_dataloader", "test_datapipe"] + ): + category = "dataloader_frontend" break - if CommitList.keywordInFile(file, ['torch/csrc/api', 'test/cpp/api']): - category = 'cpp_frontend' + if CommitList.keywordInFile(file, ["torch/csrc/api", "test/cpp/api"]): + category = "cpp_frontend" break - if CommitList.keywordInFile(file, ['distributed', 'c10d']): - category = 'distributed' + if CommitList.keywordInFile(file, ["distributed", "c10d"]): + category = "distributed" break - if ('vulkan' in file_lowercase): - category = 'vulkan' + if "vulkan" in file_lowercase: + category = "vulkan" break - if ('Foreach' in file_lowercase): - category = 'foreach_frontend' + if "Foreach" in file_lowercase: + category = "foreach_frontend" break - if 'onnx' in file_lowercase: - category = 'onnx' + if "onnx" in file_lowercase: + category = "onnx" break - if CommitList.keywordInFile(file, ['torch/fx', 'test_fx']): - category = 'fx' + if CommitList.keywordInFile(file, ["torch/fx", "test_fx"]): + category = "fx" break - if CommitList.keywordInFile(file, ['torch/ao', 'test/ao']): + if CommitList.keywordInFile(file, ["torch/ao", "test/ao"]): category = common.quantization.name break # torch/quantization, test/quantization, aten/src/ATen/native/quantized, torch/nn/{quantized, quantizable} - if CommitList.keywordInFile(file, ['torch/quantization', 'test/quantization', 'aten/src/ATen/native/quantized', 'torch/nn/quantiz']): + if CommitList.keywordInFile( + file, + [ + "torch/quantization", + "test/quantization", + "aten/src/ATen/native/quantized", + "torch/nn/quantiz", + ], + ): category = common.quantization.name break - if CommitList.keywordInFile(file, ['torch/package', 'test/package']): - category = 'package' + if CommitList.keywordInFile(file, ["torch/package", "test/package"]): + category = "package" break - if CommitList.keywordInFile(file, ['torch/csrc/jit/mobile', 'aten/src/ATen/native/metal', 'test/mobile', 'torch/backends/_nnapi/', 'test/test_nnapi.py']): - category = 'mobile' + if CommitList.keywordInFile( + file, + [ + "torch/csrc/jit/mobile", + "aten/src/ATen/native/metal", + "test/mobile", + "torch/backends/_nnapi/", + "test/test_nnapi.py", + ], + ): + category = "mobile" break - if CommitList.keywordInFile(file, ['aten/src/ATen/native/LinearAlgebra.cpp', 'test/test_linalg.py', 'torch/linalg']): - category = 'linalg_frontend' + if CommitList.keywordInFile( + file, + [ + "aten/src/ATen/native/LinearAlgebra.cpp", + "test/test_linalg.py", + "torch/linalg", + ], + ): + category = "linalg_frontend" break - if CommitList.keywordInFile(file, ['torch/sparse', 'aten/src/ATen/native/sparse', 'torch/_masked/__init__.py']): - category = 'sparse_frontend' + if CommitList.keywordInFile( + file, + [ + "torch/sparse", + "aten/src/ATen/native/sparse", + "torch/_masked/__init__.py", + ], + ): + category = "sparse_frontend" break - if CommitList.keywordInFile(file, ['tools/autograd']): - category = 'autograd_frontend' + if CommitList.keywordInFile(file, ["tools/autograd"]): + category = "autograd_frontend" break - if CommitList.keywordInFile(file, ['test/test_nn.py', 'test/test_module.py', 'torch/nn/modules', 'torch/nn/functional.py']): - category = 'nn_frontend' + if CommitList.keywordInFile( + file, + [ + "test/test_nn.py", + "test/test_module.py", + "torch/nn/modules", + "torch/nn/functional.py", + ], + ): + category = "nn_frontend" break - if CommitList.keywordInFile(file, ['torch/csrc/jit', 'torch/jit']): - category = 'jit' + if CommitList.keywordInFile(file, ["torch/csrc/jit", "torch/jit"]): + category = "jit" break - if CommitList.keywordInFile(file, ['torch/_meta_registrations.py', 'torch/_decomp', 'torch/_prims', 'torch/_refs']): - category = 'composability' + if CommitList.keywordInFile( + file, + [ + "torch/_meta_registrations.py", + "torch/_decomp", + "torch/_prims", + "torch/_refs", + ], + ): + category = "composability" break - if CommitList.keywordInFile(file, ['torch/_dynamo']): - category = 'dynamo' + if CommitList.keywordInFile(file, ["torch/_dynamo"]): + category = "dynamo" break - if CommitList.keywordInFile(file, ['torch/_inductor']): - category = 'inductor' + if CommitList.keywordInFile(file, ["torch/_inductor"]): + category = "inductor" break else: # Below are some extra quick checks that aren't necessarily file-path related, # but I found that to catch a decent number of extra commits. - if len(files_changed) > 0 and all(f_name.endswith(('.cu', '.cuh')) for f_name in files_changed): - category = 'cuda' - elif '[PyTorch Edge]' in title: - category = 'mobile' - elif len(files_changed) == 1 and 'torch/testing/_internal/common_methods_invocations.py' in files_changed[0]: + if len(files_changed) > 0 and all( + f_name.endswith((".cu", ".cuh")) for f_name in files_changed + ): + category = "cuda" + elif "[PyTorch Edge]" in title: + category = "mobile" + elif ( + len(files_changed) == 1 + and "torch/testing/_internal/common_methods_invocations.py" + in files_changed[0] + ): # when this is the only file changed, it's almost always an OpInfo change. - category = 'python_frontend' - elif len(files_changed) == 1 and 'torch/_torch_docs.py' in files_changed[0]: + category = "python_frontend" + elif len(files_changed) == 1 and "torch/_torch_docs.py" in files_changed[0]: # individual torch_docs changes are usually for python ops - category = 'python_frontend' + category = "python_frontend" # If we couldn't find a category but the topic is not user facing we can skip these: if category == "Uncategorized" and topic == "not user facing": @@ -284,18 +376,18 @@ class CommitList: @staticmethod def get_commits_between(base_version, new_version): - cmd = f'git merge-base {base_version} {new_version}' + cmd = f"git merge-base {base_version} {new_version}" rc, merge_base, _ = run(cmd) assert rc == 0 # Returns a list of something like # b33e38ec47 Allow a higher-precision step type for Vec256::arange (#34555) - cmd = f'git log --reverse --oneline {merge_base}..{new_version}' + cmd = f"git log --reverse --oneline {merge_base}..{new_version}" rc, commits, _ = run(cmd) assert rc == 0 - log_lines = commits.split('\n') - hashes, titles = zip(*[log_line.split(' ', 1) for log_line in log_lines]) + log_lines = commits.split("\n") + hashes, titles = zip(*[log_line.split(" ", 1) for log_line in log_lines]) return [CommitList.gen_commit(commit_hash) for commit_hash in hashes] def filter(self, *, category=None, topic=None): @@ -322,38 +414,47 @@ def create_new(path, base_version, new_version): commits = CommitList.create_new(path, base_version, new_version) commits.write_result() + def update_existing(path, new_version): commits = CommitList.from_existing(path) commits.update_to(new_version) commits.write_result() + def rerun_with_new_filters(path): current_commits = CommitList.from_existing(path) for i, commit in enumerate(current_commits.commits): current_category = commit.category - if current_category == 'Uncategorized' or current_category not in common.categories: + if ( + current_category == "Uncategorized" + or current_category not in common.categories + ): feature_item = get_commit_data_cache().get(commit.commit_hash) features = features_to_dict(feature_item) category, topic = CommitList.categorize(features) - current_commits.commits[i] = dataclasses.replace(commit, category=category, topic=topic) + current_commits.commits[i] = dataclasses.replace( + commit, category=category, topic=topic + ) current_commits.write_result() + def get_hash_or_pr_url(commit: Commit): # cdc = get_commit_data_cache() pr_link = commit.pr_link if pr_link is None: return commit.commit_hash else: - regex = r'https://github.com/pytorch/pytorch/pull/([0-9]+)' + regex = r"https://github.com/pytorch/pytorch/pull/([0-9]+)" matches = re.findall(regex, pr_link) if len(matches) == 0: return commit.commit_hash - return f'[#{matches[0]}]({pr_link})' + return f"[#{matches[0]}]({pr_link})" + def to_markdown(commit_list: CommitList, category): def cleanup_title(commit): - match = re.match(r'(.*) \(#\d+\)', commit.title) + match = re.match(r"(.*) \(#\d+\)", commit.title) if match is None: return commit.title return match.group(1) @@ -364,23 +465,30 @@ def to_markdown(commit_list: CommitList, category): merge_mapping[commit.merge_into].append(commit) cdc = get_commit_data_cache() - lines = [f'\n## {category}\n'] + lines = [f"\n## {category}\n"] for topic in topics: - lines.append(f'### {topic}\n') + lines.append(f"### {topic}\n") commits = commit_list.filter(category=category, topic=topic) - if '_' in topic: - commits.extend(commit_list.filter(category=category, topic=topic.replace('_', ' '))) - if ' ' in topic: - commits.extend(commit_list.filter(category=category, topic=topic.replace(' ', '_'))) + if "_" in topic: + commits.extend( + commit_list.filter(category=category, topic=topic.replace("_", " ")) + ) + if " " in topic: + commits.extend( + commit_list.filter(category=category, topic=topic.replace(" ", "_")) + ) for commit in commits: if commit.merge_into: continue all_related_commits = merge_mapping[commit.commit_hash] + [commit] - commit_list_md = ", ".join(get_hash_or_pr_url(c) for c in all_related_commits) - result = f'- {cleanup_title(commit)} ({commit_list_md})\n' + commit_list_md = ", ".join( + get_hash_or_pr_url(c) for c in all_related_commits + ) + result = f"- {cleanup_title(commit)} ({commit_list_md})\n" lines.append(result) return lines + def get_markdown_header(category): header = f""" # Release Notes worksheet {category} @@ -404,29 +512,37 @@ The categories below are as follows: * Developers: All commits that are not end-user facing but still impact people that compile from source, develop into pytorch, extend pytorch, etc """ - return [header, ] + return [ + header, + ] def main(): - parser = argparse.ArgumentParser(description='Tool to create a commit list') + parser = argparse.ArgumentParser(description="Tool to create a commit list") group = parser.add_mutually_exclusive_group(required=True) - group.add_argument('--create-new', '--create_new', nargs=2) - group.add_argument('--update-to', '--update_to') + group.add_argument("--create-new", "--create_new", nargs=2) + group.add_argument("--update-to", "--update_to") # I found this flag useful when experimenting with adding new auto-categorizing filters. # After running commitlist.py the first time, if you add any new filters in this file, # re-running with "rerun_with_new_filters" will update the existing commitlist.csv file, # but only affect the rows that were previously marked as "Uncategorized" - group.add_argument('--rerun-with-new-filters', '--rerun_with_new_filters', action='store_true') - group.add_argument('--stat', action='store_true') - group.add_argument('--export-markdown', '--export_markdown', action='store_true') - group.add_argument('--export-csv-categories', '--export_csv_categories', action='store_true') - parser.add_argument('--path', default='results/commitlist.csv') + group.add_argument( + "--rerun-with-new-filters", "--rerun_with_new_filters", action="store_true" + ) + group.add_argument("--stat", action="store_true") + group.add_argument("--export-markdown", "--export_markdown", action="store_true") + group.add_argument( + "--export-csv-categories", "--export_csv_categories", action="store_true" + ) + parser.add_argument("--path", default="results/commitlist.csv") args = parser.parse_args() if args.create_new: create_new(args.path, args.create_new[0], args.create_new[1]) - print("Finished creating new commit list. Results have been saved to results/commitlist.csv") + print( + "Finished creating new commit list. Results have been saved to results/commitlist.csv" + ) return if args.update_to: update_existing(args.path, args.update_to) @@ -445,7 +561,7 @@ def main(): categories = list(commits.stat().keys()) for category in categories: print(f"Exporting {category}...") - filename = f'results/export/result_{category}.csv' + filename = f"results/export/result_{category}.csv" CommitList.write_to_disk_static(filename, commits.filter(category=category)) return @@ -456,12 +572,13 @@ def main(): print(f"Exporting {category}...") lines = get_markdown_header(category) lines += to_markdown(commits, category) - filename = f'results/export/result_{category}.md' + filename = f"results/export/result_{category}.md" os.makedirs(os.path.dirname(filename), exist_ok=True) - with open(filename, 'w') as f: + with open(filename, "w") as f: f.writelines(lines) return raise AssertionError() -if __name__ == '__main__': + +if __name__ == "__main__": main() diff --git a/scripts/release_notes/common.py b/scripts/release_notes/common.py index 26317e8ddad..53734d78245 100644 --- a/scripts/release_notes/common.py +++ b/scripts/release_notes/common.py @@ -1,135 +1,141 @@ -from collections import namedtuple -from pathlib import Path -import locale -import subprocess -import re -import requests -import os import json +import locale +import os +import re +import subprocess +from collections import namedtuple from dataclasses import dataclass +from pathlib import Path + +import requests + @dataclass class CategoryGroup: name: str categories: list + frontend_categories = [ - 'meta', - 'nn', - 'linalg', - 'cpp', - 'python', - 'complex', - 'vmap', - 'autograd', - 'build', - 'memory_format', - 'foreach', - 'dataloader', - 'sparse', - 'nested tensor', - 'optimizer' + "meta", + "nn", + "linalg", + "cpp", + "python", + "complex", + "vmap", + "autograd", + "build", + "memory_format", + "foreach", + "dataloader", + "sparse", + "nested tensor", + "optimizer", ] pytorch_2_categories = [ - 'dynamo', - 'inductor', + "dynamo", + "inductor", ] # These will all get mapped to quantization quantization = CategoryGroup( name="quantization", categories=[ - 'quantization', - 'AO frontend', - 'AO Pruning', ] + "quantization", + "AO frontend", + "AO Pruning", + ], ) # Distributed has a number of release note labels we want to map to one distributed = CategoryGroup( name="distributed", categories=[ - 'distributed', - 'distributed (c10d)', - 'distributed (composable)', - 'distributed (ddp)', - 'distributed (fsdp)', - 'distributed (rpc)', - 'distributed (sharded)', - ] + "distributed", + "distributed (c10d)", + "distributed (composable)", + "distributed (ddp)", + "distributed (fsdp)", + "distributed (rpc)", + "distributed (sharded)", + ], ) -categories = [ - 'Uncategorized', - 'lazy', - 'hub', - 'mobile', - 'jit', - 'visualization', - 'onnx', - 'caffe2', - 'amd', - 'rocm', - 'cuda', - 'cpu', - 'cudnn', - 'xla', - 'benchmark', - 'profiler', - 'performance_as_product', - 'package', - 'dispatcher', - 'releng', - 'fx', - 'code_coverage', - 'vulkan', - 'skip', - 'composability', - # 2.0 release - 'mps', - 'intel', - 'functorch', - 'gnn', - 'distributions', - 'serialization', - ] + [f'{category}_frontend' for category in frontend_categories] + pytorch_2_categories + [quantization.name] + [distributed.name] +categories = ( + [ + "Uncategorized", + "lazy", + "hub", + "mobile", + "jit", + "visualization", + "onnx", + "caffe2", + "amd", + "rocm", + "cuda", + "cpu", + "cudnn", + "xla", + "benchmark", + "profiler", + "performance_as_product", + "package", + "dispatcher", + "releng", + "fx", + "code_coverage", + "vulkan", + "skip", + "composability", + # 2.0 release + "mps", + "intel", + "functorch", + "gnn", + "distributions", + "serialization", + ] + + [f"{category}_frontend" for category in frontend_categories] + + pytorch_2_categories + + [quantization.name] + + [distributed.name] +) topics = [ - 'bc_breaking', - 'deprecations', - 'new_features', - 'improvements', - 'bug_fixes', - 'performance', - 'docs', - 'devs', - 'Untopiced', + "bc_breaking", + "deprecations", + "new_features", + "improvements", + "bug_fixes", + "performance", + "docs", + "devs", + "Untopiced", "not user facing", "security", ] -Features = namedtuple('Features', [ - 'title', - 'body', - 'pr_number', - 'files_changed', - 'labels', - 'author', - 'accepters' -]) +Features = namedtuple( + "Features", + ["title", "body", "pr_number", "files_changed", "labels", "author", "accepters"], +) def dict_to_features(dct): return Features( - title=dct['title'], - body=dct['body'], - pr_number=dct['pr_number'], - files_changed=dct['files_changed'], - labels=dct['labels'], - author=dct['author'], - accepters=tuple(dct['accepters'])) + title=dct["title"], + body=dct["body"], + pr_number=dct["pr_number"], + files_changed=dct["files_changed"], + labels=dct["labels"], + author=dct["author"], + accepters=tuple(dct["accepters"]), + ) def features_to_dict(features): @@ -138,8 +144,9 @@ def features_to_dict(features): def run(command): """Returns (return-code, stdout, stderr)""" - p = subprocess.Popen(command, stdout=subprocess.PIPE, - stderr=subprocess.PIPE, shell=True) + p = subprocess.Popen( + command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True + ) output, err = p.communicate() rc = p.returncode enc = locale.getpreferredencoding() @@ -149,45 +156,46 @@ def run(command): def commit_body(commit_hash): - cmd = f'git log -n 1 --pretty=format:%b {commit_hash}' + cmd = f"git log -n 1 --pretty=format:%b {commit_hash}" ret, out, err = run(cmd) return out if ret == 0 else None def commit_title(commit_hash): - cmd = f'git log -n 1 --pretty=format:%s {commit_hash}' + cmd = f"git log -n 1 --pretty=format:%s {commit_hash}" ret, out, err = run(cmd) return out if ret == 0 else None def commit_files_changed(commit_hash): - cmd = f'git diff-tree --no-commit-id --name-only -r {commit_hash}' + cmd = f"git diff-tree --no-commit-id --name-only -r {commit_hash}" ret, out, err = run(cmd) - return out.split('\n') if ret == 0 else None + return out.split("\n") if ret == 0 else None def parse_pr_number(body, commit_hash, title): - regex = r'Pull Request resolved: https://github.com/pytorch/pytorch/pull/([0-9]+)' + regex = r"Pull Request resolved: https://github.com/pytorch/pytorch/pull/([0-9]+)" matches = re.findall(regex, body) if len(matches) == 0: - if 'revert' not in title.lower() and 'updating submodules' not in title.lower(): - print(f'[{commit_hash}: {title}] Could not parse PR number, ignoring PR') + if "revert" not in title.lower() and "updating submodules" not in title.lower(): + print(f"[{commit_hash}: {title}] Could not parse PR number, ignoring PR") return None if len(matches) > 1: - print(f'[{commit_hash}: {title}] Got two PR numbers, using the first one') + print(f"[{commit_hash}: {title}] Got two PR numbers, using the first one") return matches[0] return matches[0] def get_ghstack_token(): - pattern = 'github_oauth = (.*)' - with open(Path('~/.ghstackrc').expanduser(), 'r+') as f: + pattern = "github_oauth = (.*)" + with open(Path("~/.ghstackrc").expanduser(), "r+") as f: config = f.read() matches = re.findall(pattern, config) if len(matches) == 0: raise RuntimeError("Can't find a github oauth token") return matches[0] + def get_token(): env_token = os.environ.get("GITHUB_TOKEN") if env_token is not None: @@ -196,20 +204,27 @@ def get_token(): else: return get_ghstack_token() + token = get_token() headers = {"Authorization": f"token {token}"} + def run_query(query): - request = requests.post('https://api.github.com/graphql', json={'query': query}, headers=headers) + request = requests.post( + "https://api.github.com/graphql", json={"query": query}, headers=headers + ) if request.status_code == 200: return request.json() else: - raise Exception(f"Query failed to run by returning code of {request.status_code}. {request.json()}") + raise Exception( + f"Query failed to run by returning code of {request.status_code}. {request.json()}" + ) def github_data(pr_number): - query = """ + query = ( + """ { repository(owner: "pytorch", name: "pytorch") { pullRequest(number: %s ) { @@ -233,14 +248,16 @@ def github_data(pr_number): } } } - """ % pr_number + """ + % pr_number + ) query = run_query(query) - if query.get('errors'): - raise Exception(query['errors']) - edges = query['data']['repository']['pullRequest']['labels']['edges'] - labels = [edge['node']['name'] for edge in edges] - author = query['data']['repository']['pullRequest']['author']['login'] - nodes = query['data']['repository']['pullRequest']['reviews']['nodes'] + if query.get("errors"): + raise Exception(query["errors"]) + edges = query["data"]["repository"]["pullRequest"]["labels"]["edges"] + labels = [edge["node"]["name"] for edge in edges] + author = query["data"]["repository"]["pullRequest"]["author"]["login"] + nodes = query["data"]["repository"]["pullRequest"]["reviews"]["nodes"] # using set to dedup multiple accepts from same accepter accepters = {node["author"]["login"] for node in nodes} @@ -253,7 +270,8 @@ def get_features(commit_hash): title, body, files_changed = ( commit_title(commit_hash), commit_body(commit_hash), - commit_files_changed(commit_hash)) + commit_files_changed(commit_hash), + ) pr_number = parse_pr_number(body, commit_hash, title) labels = [] author = "" @@ -266,12 +284,14 @@ def get_features(commit_hash): _commit_data_cache = None -def get_commit_data_cache(path='results/data.json'): + +def get_commit_data_cache(path="results/data.json"): global _commit_data_cache if _commit_data_cache is None: _commit_data_cache = _CommitDataCache(path) return _commit_data_cache + class _CommitDataCache: def __init__(self, path): self.path = path @@ -289,13 +309,12 @@ class _CommitDataCache: return self.data[commit] def read_from_disk(self): - with open(self.path, 'r') as f: + with open(self.path, "r") as f: data = json.load(f) - data = {commit: dict_to_features(dct) - for commit, dct in data.items()} + data = {commit: dict_to_features(dct) for commit, dct in data.items()} return data def write_to_disk(self): data = {commit: features._asdict() for commit, features in self.data.items()} - with open(self.path, 'w') as f: + with open(self.path, "w") as f: json.dump(data, f) diff --git a/scripts/release_notes/namespace_check.py b/scripts/release_notes/namespace_check.py index 1b9a91c12f8..64f9642703f 100644 --- a/scripts/release_notes/namespace_check.py +++ b/scripts/release_notes/namespace_check.py @@ -1,7 +1,8 @@ import argparse -import torch -from os import path import json +from os import path + +import torch # Import all utils so that getattr below can find them from torch.utils import bottleneck, checkpoint, model_zoo @@ -29,6 +30,7 @@ all_submod_list = [ "utils.model_zoo", ] + def get_content(submod): mod = torch if submod: @@ -38,10 +40,12 @@ def get_content(submod): content = dir(mod) return content + def namespace_filter(data): out = {d for d in data if d[0] != "_"} return out + def run(args, submod): print(f"## Processing torch.{submod}") prev_filename = f"prev_data_{submod}.json" @@ -87,20 +91,30 @@ def run(args, submod): print(prev_content - new_content) print("") + def main(): - parser = argparse.ArgumentParser(description='Tool to check namespace content changes') + parser = argparse.ArgumentParser( + description="Tool to check namespace content changes" + ) group = parser.add_mutually_exclusive_group(required=True) - group.add_argument('--prev-version', action='store_true') - group.add_argument('--new-version', action='store_true') - group.add_argument('--compare', action='store_true') + group.add_argument("--prev-version", action="store_true") + group.add_argument("--new-version", action="store_true") + group.add_argument("--compare", action="store_true") group = parser.add_mutually_exclusive_group() - group.add_argument('--submod', default='', help='part of the submodule to check') - group.add_argument('--all-submod', action='store_true', help='collects data for all main submodules') - - parser.add_argument('--show-all', action='store_true', help='show all the diff, not just public APIs') + group.add_argument("--submod", default="", help="part of the submodule to check") + group.add_argument( + "--all-submod", + action="store_true", + help="collects data for all main submodules", + ) + parser.add_argument( + "--show-all", + action="store_true", + help="show all the diff, not just public APIs", + ) args = parser.parse_args() @@ -113,5 +127,5 @@ def main(): run(args, mod) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/scripts/release_notes/test_release_notes.py b/scripts/release_notes/test_release_notes.py index 8bd32eee13f..4c21a82f8b9 100644 --- a/scripts/release_notes/test_release_notes.py +++ b/scripts/release_notes/test_release_notes.py @@ -1,26 +1,36 @@ -import unittest import tempfile +import unittest + from commitlist import CommitList + class TestCommitList(unittest.TestCase): def test_create_new(self): with tempfile.TemporaryDirectory() as tempdir: - commit_list_path = f'{tempdir}/commitlist.csv' - commit_list = CommitList.create_new(commit_list_path, 'v1.5.0', '6000dca5df') + commit_list_path = f"{tempdir}/commitlist.csv" + commit_list = CommitList.create_new( + commit_list_path, "v1.5.0", "6000dca5df" + ) self.assertEqual(len(commit_list.commits), 33) - self.assertEqual(commit_list.commits[0].commit_hash, '7335f079ab') - self.assertTrue(commit_list.commits[0].title.startswith('[pt][quant] qmul and qadd')) - self.assertEqual(commit_list.commits[-1].commit_hash, '6000dca5df') - self.assertTrue(commit_list.commits[-1].title.startswith('[nomnigraph] Copy device option when customize ')) + self.assertEqual(commit_list.commits[0].commit_hash, "7335f079ab") + self.assertTrue( + commit_list.commits[0].title.startswith("[pt][quant] qmul and qadd") + ) + self.assertEqual(commit_list.commits[-1].commit_hash, "6000dca5df") + self.assertTrue( + commit_list.commits[-1].title.startswith( + "[nomnigraph] Copy device option when customize " + ) + ) def test_read_write(self): with tempfile.TemporaryDirectory() as tempdir: - commit_list_path = f'{tempdir}/commitlist.csv' - initial = CommitList.create_new(commit_list_path, 'v1.5.0', '7543e7e558') + commit_list_path = f"{tempdir}/commitlist.csv" + initial = CommitList.create_new(commit_list_path, "v1.5.0", "7543e7e558") initial.write_to_disk() expected = CommitList.from_existing(commit_list_path) - expected.commits[-2].category = 'foobar' + expected.commits[-2].category = "foobar" expected.write_to_disk() commit_list = CommitList.from_existing(commit_list_path) @@ -29,17 +39,17 @@ class TestCommitList(unittest.TestCase): def test_update_to(self): with tempfile.TemporaryDirectory() as tempdir: - commit_list_path = f'{tempdir}/commitlist.csv' - initial = CommitList.create_new(commit_list_path, 'v1.5.0', '7543e7e558') - initial.commits[-2].category = 'foobar' + commit_list_path = f"{tempdir}/commitlist.csv" + initial = CommitList.create_new(commit_list_path, "v1.5.0", "7543e7e558") + initial.commits[-2].category = "foobar" self.assertEqual(len(initial.commits), 2143) initial.write_to_disk() commit_list = CommitList.from_existing(commit_list_path) - commit_list.update_to('5702a28b26') + commit_list.update_to("5702a28b26") self.assertEqual(len(commit_list.commits), 2143 + 4) self.assertEqual(commit_list.commits[-5], initial.commits[-1]) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/setup.py b/setup.py index 41e9ad6e1ac..e169d401c49 100644 --- a/setup.py +++ b/setup.py @@ -216,39 +216,44 @@ # By default, It is only enabled on Windows. import sys -if sys.platform == 'win32' and sys.maxsize.bit_length() == 31: - print("32-bit Windows Python runtime is not supported. Please switch to 64-bit Python.") + +if sys.platform == "win32" and sys.maxsize.bit_length() == 31: + print( + "32-bit Windows Python runtime is not supported. Please switch to 64-bit Python." + ) sys.exit(-1) import platform + python_min_version = (3, 8, 0) -python_min_version_str = '.'.join(map(str, python_min_version)) +python_min_version_str = ".".join(map(str, python_min_version)) if sys.version_info < python_min_version: - print("You are using Python {}. Python >={} is required.".format(platform.python_version(), # noqa: UP032 - python_min_version_str)) + print( + f"You are using Python {platform.python_version()}. Python >={python_min_version_str} is required." + ) sys.exit(-1) -from setuptools import setup, Extension, find_packages +import filecmp +import glob +import importlib +import json +import os +import shutil +import subprocess +import sysconfig +import time from collections import defaultdict -from setuptools.dist import Distribution + import setuptools.command.build_ext import setuptools.command.install import setuptools.command.sdist -import filecmp -import shutil -import subprocess -import os -import json -import glob -import importlib -import time -import sysconfig +from setuptools import Extension, find_packages, setup +from setuptools.dist import Distribution from tools.build_pytorch_libs import build_caffe2 -from tools.setup_helpers.env import (IS_WINDOWS, IS_DARWIN, IS_LINUX, - build_type) -from tools.setup_helpers.cmake import CMake from tools.generate_torch_version import get_torch_version +from tools.setup_helpers.cmake import CMake +from tools.setup_helpers.env import build_type, IS_DARWIN, IS_LINUX, IS_WINDOWS ################################################################################ # Parameters parsed from environment @@ -263,31 +268,34 @@ RERUN_CMAKE = False CMAKE_ONLY = False filtered_args = [] for i, arg in enumerate(sys.argv): - if arg == '--cmake': + if arg == "--cmake": RERUN_CMAKE = True continue - if arg == '--cmake-only': + if arg == "--cmake-only": # Stop once cmake terminates. Leave users a chance to adjust build # options. CMAKE_ONLY = True continue - if arg == 'rebuild' or arg == 'build': - arg = 'build' # rebuild is gone, make it build + if arg == "rebuild" or arg == "build": + arg = "build" # rebuild is gone, make it build EMIT_BUILD_WARNING = True if arg == "--": filtered_args += sys.argv[i:] break - if arg == '-q' or arg == '--quiet': + if arg == "-q" or arg == "--quiet": VERBOSE_SCRIPT = False - if arg in ['clean', 'egg_info', 'sdist']: + if arg in ["clean", "egg_info", "sdist"]: RUN_BUILD_DEPS = False filtered_args.append(arg) sys.argv = filtered_args if VERBOSE_SCRIPT: + def report(*args): print(*args) + else: + def report(*args): pass @@ -303,25 +311,25 @@ caffe2_build_dir = os.path.join(cwd, "build") # CMAKE: full path to python library if IS_WINDOWS: cmake_python_library = "{}/libs/python{}.lib".format( - sysconfig.get_config_var("prefix"), - sysconfig.get_config_var("VERSION")) + sysconfig.get_config_var("prefix"), sysconfig.get_config_var("VERSION") + ) # Fix virtualenv builds if not os.path.exists(cmake_python_library): cmake_python_library = "{}/libs/python{}.lib".format( - sys.base_prefix, - sysconfig.get_config_var("VERSION")) + sys.base_prefix, sysconfig.get_config_var("VERSION") + ) else: cmake_python_library = "{}/{}".format( - sysconfig.get_config_var("LIBDIR"), - sysconfig.get_config_var("INSTSONAME")) + sysconfig.get_config_var("LIBDIR"), sysconfig.get_config_var("INSTSONAME") + ) cmake_python_include_dir = sysconfig.get_path("include") ################################################################################ # Version, create_version_file, and package_name ################################################################################ -package_name = os.getenv('TORCH_PACKAGE_NAME', 'torch') -package_type = os.getenv('PACKAGE_TYPE', 'wheel') +package_name = os.getenv("TORCH_PACKAGE_NAME", "torch") +package_type = os.getenv("PACKAGE_TYPE", "wheel") version = get_torch_version() report(f"Building wheel {package_name}-{version}") @@ -330,15 +338,27 @@ cmake = CMake() def get_submodule_folders(): git_modules_path = os.path.join(cwd, ".gitmodules") - default_modules_path = [os.path.join(third_party_path, name) for name in [ - "gloo", "cpuinfo", "tbb", "onnx", - "foxi", "QNNPACK", "fbgemm", "cutlass" - ]] + default_modules_path = [ + os.path.join(third_party_path, name) + for name in [ + "gloo", + "cpuinfo", + "tbb", + "onnx", + "foxi", + "QNNPACK", + "fbgemm", + "cutlass", + ] + ] if not os.path.exists(git_modules_path): return default_modules_path with open(git_modules_path) as f: - return [os.path.join(cwd, line.split("=", 1)[1].strip()) for line in - f.readlines() if line.strip().startswith("path")] + return [ + os.path.join(cwd, line.split("=", 1)[1].strip()) + for line in f.readlines() + if line.strip().startswith("path") + ] def check_submodules(): @@ -349,7 +369,9 @@ def check_submodules(): sys.exit(1) def not_exists_or_empty(folder): - return not os.path.exists(folder) or (os.path.isdir(folder) and len(os.listdir(folder)) == 0) + return not os.path.exists(folder) or ( + os.path.isdir(folder) and len(os.listdir(folder)) == 0 + ) if bool(os.getenv("USE_SYSTEM_LIBS", False)): return @@ -357,21 +379,37 @@ def check_submodules(): # If none of the submodule folders exists, try to initialize them if all(not_exists_or_empty(folder) for folder in folders): try: - print(' --- Trying to initialize submodules') + print(" --- Trying to initialize submodules") start = time.time() - subprocess.check_call(["git", "submodule", "update", "--init", "--recursive"], cwd=cwd) + subprocess.check_call( + ["git", "submodule", "update", "--init", "--recursive"], cwd=cwd + ) end = time.time() - print(f' --- Submodule initialization took {end - start:.2f} sec') + print(f" --- Submodule initialization took {end - start:.2f} sec") except Exception: - print(' --- Submodule initalization failed') - print('Please run:\n\tgit submodule update --init --recursive') + print(" --- Submodule initalization failed") + print("Please run:\n\tgit submodule update --init --recursive") sys.exit(1) for folder in folders: - check_for_files(folder, ["CMakeLists.txt", "Makefile", "setup.py", "LICENSE", "LICENSE.md", "LICENSE.txt"]) - check_for_files(os.path.join(third_party_path, 'fbgemm', 'third_party', - 'asmjit'), ['CMakeLists.txt']) - check_for_files(os.path.join(third_party_path, 'onnx', 'third_party', - 'benchmark'), ['CMakeLists.txt']) + check_for_files( + folder, + [ + "CMakeLists.txt", + "Makefile", + "setup.py", + "LICENSE", + "LICENSE.md", + "LICENSE.txt", + ], + ) + check_for_files( + os.path.join(third_party_path, "fbgemm", "third_party", "asmjit"), + ["CMakeLists.txt"], + ) + check_for_files( + os.path.join(third_party_path, "onnx", "third_party", "benchmark"), + ["CMakeLists.txt"], + ) # Windows has very bad support for symbolic links. @@ -380,9 +418,12 @@ def mirror_files_into_torchgen(): # (new_path, orig_path) # Directories are OK and are recursively mirrored. paths = [ - ('torchgen/packaged/ATen/native/native_functions.yaml', 'aten/src/ATen/native/native_functions.yaml'), - ('torchgen/packaged/ATen/native/tags.yaml', 'aten/src/ATen/native/tags.yaml'), - ('torchgen/packaged/ATen/templates', 'aten/src/ATen/templates'), + ( + "torchgen/packaged/ATen/native/native_functions.yaml", + "aten/src/ATen/native/native_functions.yaml", + ), + ("torchgen/packaged/ATen/native/tags.yaml", "aten/src/ATen/native/tags.yaml"), + ("torchgen/packaged/ATen/templates", "aten/src/ATen/templates"), ] for new_path, orig_path in paths: # Create the dirs involved in new_path if they don't exist @@ -401,37 +442,42 @@ def mirror_files_into_torchgen(): continue raise RuntimeError("Check the file paths in `mirror_files_into_torchgen()`") + # all the work we need to do _before_ setup runs def build_deps(): - report('-- Building version ' + version) + report("-- Building version " + version) check_submodules() - check_pydep('yaml', 'pyyaml') + check_pydep("yaml", "pyyaml") - build_caffe2(version=version, - cmake_python_library=cmake_python_library, - build_python=True, - rerun_cmake=RERUN_CMAKE, - cmake_only=CMAKE_ONLY, - cmake=cmake) + build_caffe2( + version=version, + cmake_python_library=cmake_python_library, + build_python=True, + rerun_cmake=RERUN_CMAKE, + cmake_only=CMAKE_ONLY, + cmake=cmake, + ) if CMAKE_ONLY: - report('Finished running cmake. Run "ccmake build" or ' - '"cmake-gui build" to adjust build options and ' - '"python setup.py install" to build.') + report( + 'Finished running cmake. Run "ccmake build" or ' + '"cmake-gui build" to adjust build options and ' + '"python setup.py install" to build.' + ) sys.exit() # Use copies instead of symbolic files. # Windows has very poor support for them. sym_files = [ - 'tools/shared/_utils_internal.py', - 'torch/utils/benchmark/utils/valgrind_wrapper/callgrind.h', - 'torch/utils/benchmark/utils/valgrind_wrapper/valgrind.h', + "tools/shared/_utils_internal.py", + "torch/utils/benchmark/utils/valgrind_wrapper/callgrind.h", + "torch/utils/benchmark/utils/valgrind_wrapper/valgrind.h", ] orig_files = [ - 'torch/_utils_internal.py', - 'third_party/valgrind-headers/callgrind.h', - 'third_party/valgrind-headers/valgrind.h', + "torch/_utils_internal.py", + "third_party/valgrind-headers/callgrind.h", + "third_party/valgrind-headers/valgrind.h", ] for sym_file, orig_file in zip(sym_files, orig_files): same = False @@ -443,48 +489,53 @@ def build_deps(): if not same: shutil.copyfile(orig_file, sym_file) + ################################################################################ # Building dependent libraries ################################################################################ -missing_pydep = ''' +missing_pydep = """ Missing build dependency: Unable to `import {importname}`. Please install it via `conda install {module}` or `pip install {module}` -'''.strip() +""".strip() def check_pydep(importname, module): try: importlib.import_module(importname) except ImportError as e: - raise RuntimeError(missing_pydep.format(importname=importname, module=module)) from e + raise RuntimeError( + missing_pydep.format(importname=importname, module=module) + ) from e class build_ext(setuptools.command.build_ext.build_ext): - # Copy libiomp5.dylib inside the wheel package on OS X def _embed_libiomp(self): - - lib_dir = os.path.join(self.build_lib, 'torch', 'lib') - libtorch_cpu_path = os.path.join(lib_dir, 'libtorch_cpu.dylib') + lib_dir = os.path.join(self.build_lib, "torch", "lib") + libtorch_cpu_path = os.path.join(lib_dir, "libtorch_cpu.dylib") if not os.path.exists(libtorch_cpu_path): return # Parse libtorch_cpu load commands - otool_cmds = subprocess.check_output(['otool', '-l', libtorch_cpu_path]).decode('utf-8').split('\n') + otool_cmds = ( + subprocess.check_output(["otool", "-l", libtorch_cpu_path]) + .decode("utf-8") + .split("\n") + ) rpaths, libs = [], [] for idx, line in enumerate(otool_cmds): - if line.strip() == 'cmd LC_LOAD_DYLIB': + if line.strip() == "cmd LC_LOAD_DYLIB": lib_name = otool_cmds[idx + 2].strip() - assert lib_name.startswith('name ') - libs.append(lib_name.split(' ', 1)[1].rsplit('(', 1)[0][:-1]) + assert lib_name.startswith("name ") + libs.append(lib_name.split(" ", 1)[1].rsplit("(", 1)[0][:-1]) - if line.strip() == 'cmd LC_RPATH': + if line.strip() == "cmd LC_RPATH": rpath = otool_cmds[idx + 2].strip() - assert rpath.startswith('path ') - rpaths.append(rpath.split(' ', 1)[1].rsplit('(', 1)[0][:-1]) + assert rpath.startswith("path ") + rpaths.append(rpath.split(" ", 1)[1].rsplit("(", 1)[0][:-1]) - omp_lib_name = 'libiomp5.dylib' - if os.path.join('@rpath', omp_lib_name) not in libs: + omp_lib_name = "libiomp5.dylib" + if os.path.join("@rpath", omp_lib_name) not in libs: return # Copy libiomp5 from rpath locations @@ -492,7 +543,7 @@ class build_ext(setuptools.command.build_ext.build_ext): source_lib = os.path.join(rpath, omp_lib_name) if not os.path.exists(source_lib): continue - target_lib = os.path.join(self.build_lib, 'torch', 'lib', omp_lib_name) + target_lib = os.path.join(self.build_lib, "torch", "lib", omp_lib_name) self.copy_file(source_lib, target_lib) break @@ -500,91 +551,113 @@ class build_ext(setuptools.command.build_ext.build_ext): # Report build options. This is run after the build completes so # `CMakeCache.txt` exists and we can get an # accurate report on what is used and what is not. cmake_cache_vars = defaultdict(lambda: False, cmake.get_cmake_cache_variables()) - if cmake_cache_vars['USE_NUMPY']: - report('-- Building with NumPy bindings') + if cmake_cache_vars["USE_NUMPY"]: + report("-- Building with NumPy bindings") else: - report('-- NumPy not found') - if cmake_cache_vars['USE_CUDNN']: - report('-- Detected cuDNN at ' + - cmake_cache_vars['CUDNN_LIBRARY'] + ', ' + cmake_cache_vars['CUDNN_INCLUDE_DIR']) + report("-- NumPy not found") + if cmake_cache_vars["USE_CUDNN"]: + report( + "-- Detected cuDNN at " + + cmake_cache_vars["CUDNN_LIBRARY"] + + ", " + + cmake_cache_vars["CUDNN_INCLUDE_DIR"] + ) else: - report('-- Not using cuDNN') - if cmake_cache_vars['USE_CUDA']: - report('-- Detected CUDA at ' + cmake_cache_vars['CUDA_TOOLKIT_ROOT_DIR']) + report("-- Not using cuDNN") + if cmake_cache_vars["USE_CUDA"]: + report("-- Detected CUDA at " + cmake_cache_vars["CUDA_TOOLKIT_ROOT_DIR"]) else: - report('-- Not using CUDA') - if cmake_cache_vars['USE_MKLDNN']: - report('-- Using MKLDNN') - if cmake_cache_vars['USE_MKLDNN_ACL']: - report('-- Using Compute Library for the Arm architecture with MKLDNN') + report("-- Not using CUDA") + if cmake_cache_vars["USE_MKLDNN"]: + report("-- Using MKLDNN") + if cmake_cache_vars["USE_MKLDNN_ACL"]: + report("-- Using Compute Library for the Arm architecture with MKLDNN") else: - report('-- Not using Compute Library for the Arm architecture with MKLDNN') - if cmake_cache_vars['USE_MKLDNN_CBLAS']: - report('-- Using CBLAS in MKLDNN') + report( + "-- Not using Compute Library for the Arm architecture with MKLDNN" + ) + if cmake_cache_vars["USE_MKLDNN_CBLAS"]: + report("-- Using CBLAS in MKLDNN") else: - report('-- Not using CBLAS in MKLDNN') + report("-- Not using CBLAS in MKLDNN") else: - report('-- Not using MKLDNN') - if cmake_cache_vars['USE_NCCL'] and cmake_cache_vars['USE_SYSTEM_NCCL']: - report('-- Using system provided NCCL library at {}, {}'.format(cmake_cache_vars['NCCL_LIBRARIES'], - cmake_cache_vars['NCCL_INCLUDE_DIRS'])) - elif cmake_cache_vars['USE_NCCL']: - report('-- Building NCCL library') + report("-- Not using MKLDNN") + if cmake_cache_vars["USE_NCCL"] and cmake_cache_vars["USE_SYSTEM_NCCL"]: + report( + "-- Using system provided NCCL library at {}, {}".format( + cmake_cache_vars["NCCL_LIBRARIES"], + cmake_cache_vars["NCCL_INCLUDE_DIRS"], + ) + ) + elif cmake_cache_vars["USE_NCCL"]: + report("-- Building NCCL library") else: - report('-- Not using NCCL') - if cmake_cache_vars['USE_DISTRIBUTED']: + report("-- Not using NCCL") + if cmake_cache_vars["USE_DISTRIBUTED"]: if IS_WINDOWS: - report('-- Building without distributed package') + report("-- Building without distributed package") else: - report('-- Building with distributed package: ') - report(' -- USE_TENSORPIPE={}'.format(cmake_cache_vars['USE_TENSORPIPE'])) - report(' -- USE_GLOO={}'.format(cmake_cache_vars['USE_GLOO'])) - report(' -- USE_MPI={}'.format(cmake_cache_vars['USE_OPENMPI'])) + report("-- Building with distributed package: ") + report( + " -- USE_TENSORPIPE={}".format(cmake_cache_vars["USE_TENSORPIPE"]) + ) + report(" -- USE_GLOO={}".format(cmake_cache_vars["USE_GLOO"])) + report(" -- USE_MPI={}".format(cmake_cache_vars["USE_OPENMPI"])) else: - report('-- Building without distributed package') - if cmake_cache_vars['STATIC_DISPATCH_BACKEND']: - report('-- Using static dispatch with backend {}'.format(cmake_cache_vars['STATIC_DISPATCH_BACKEND'])) - if cmake_cache_vars['USE_LIGHTWEIGHT_DISPATCH']: - report('-- Using lightweight dispatch') - if cmake_cache_vars['BUILD_EXECUTORCH']: - report('-- Building Executorch') + report("-- Building without distributed package") + if cmake_cache_vars["STATIC_DISPATCH_BACKEND"]: + report( + "-- Using static dispatch with backend {}".format( + cmake_cache_vars["STATIC_DISPATCH_BACKEND"] + ) + ) + if cmake_cache_vars["USE_LIGHTWEIGHT_DISPATCH"]: + report("-- Using lightweight dispatch") + if cmake_cache_vars["BUILD_EXECUTORCH"]: + report("-- Building Executorch") - if cmake_cache_vars['USE_ITT']: - report('-- Using ITT') + if cmake_cache_vars["USE_ITT"]: + report("-- Using ITT") else: - report('-- Not using ITT') + report("-- Not using ITT") - if cmake_cache_vars['BUILD_NVFUSER']: - report('-- Building nvfuser') + if cmake_cache_vars["BUILD_NVFUSER"]: + report("-- Building nvfuser") else: - report('-- Not Building nvfuser') + report("-- Not Building nvfuser") # Do not use clang to compile extensions if `-fstack-clash-protection` is defined # in system CFLAGS - c_flags = str(os.getenv('CFLAGS', '')) - if IS_LINUX and '-fstack-clash-protection' in c_flags and 'clang' in os.environ.get('CC', ''): - os.environ['CC'] = str(os.environ['CC']) + c_flags = str(os.getenv("CFLAGS", "")) + if ( + IS_LINUX + and "-fstack-clash-protection" in c_flags + and "clang" in os.environ.get("CC", "") + ): + os.environ["CC"] = str(os.environ["CC"]) # It's an old-style class in Python 2.7... setuptools.command.build_ext.build_ext.run(self) - if IS_DARWIN and package_type != 'conda': + if IS_DARWIN and package_type != "conda": self._embed_libiomp() # Copy the essential export library to compile C++ extensions. if IS_WINDOWS: build_temp = self.build_temp - ext_filename = self.get_ext_filename('_C') - lib_filename = '.'.join(ext_filename.split('.')[:-1]) + '.lib' + ext_filename = self.get_ext_filename("_C") + lib_filename = ".".join(ext_filename.split(".")[:-1]) + ".lib" export_lib = os.path.join( - build_temp, 'torch', 'csrc', lib_filename).replace('\\', '/') + build_temp, "torch", "csrc", lib_filename + ).replace("\\", "/") build_lib = self.build_lib - target_lib = os.path.join( - build_lib, 'torch', 'lib', '_C.lib').replace('\\', '/') + target_lib = os.path.join(build_lib, "torch", "lib", "_C.lib").replace( + "\\", "/" + ) # Create "torch/lib" directory if not exists. # (It is not created yet in "develop" mode.) @@ -604,9 +677,9 @@ class build_ext(setuptools.command.build_ext.build_ext): # "install" command by default. # We only make this copy for Caffe2's pybind extensions caffe2_pybind_exts = [ - 'caffe2.python.caffe2_pybind11_state', - 'caffe2.python.caffe2_pybind11_state_gpu', - 'caffe2.python.caffe2_pybind11_state_hip', + "caffe2.python.caffe2_pybind11_state", + "caffe2.python.caffe2_pybind11_state_gpu", + "caffe2.python.caffe2_pybind11_state_hip", ] i = 0 while i < len(self.extensions): @@ -618,7 +691,11 @@ class build_ext(setuptools.command.build_ext.build_ext): filename = self.get_ext_filename(fullname) report(f"\nCopying extension {ext.name}") - relative_site_packages = sysconfig.get_path('purelib').replace(sysconfig.get_path('data'), '').lstrip(os.path.sep) + relative_site_packages = ( + sysconfig.get_path("purelib") + .replace(sysconfig.get_path("data"), "") + .lstrip(os.path.sep) + ) src = os.path.join("torch", relative_site_packages, filename) if not os.path.exists(src): report(f"{src} does not exist") @@ -666,7 +743,6 @@ class build_ext(setuptools.command.build_ext.build_ext): setuptools.command.build_ext.build_ext.build_extensions(self) - def get_outputs(self): outputs = setuptools.command.build_ext.build_ext.get_outputs(self) outputs.append(os.path.join(self.build_lib, "caffe2")) @@ -677,29 +753,29 @@ class build_ext(setuptools.command.build_ext.build_ext): def load(filename): with open(filename) as f: return json.load(f) - ninja_files = glob.glob('build/*compile_commands.json') - cmake_files = glob.glob('torch/lib/build/*/compile_commands.json') - all_commands = [entry - for f in ninja_files + cmake_files - for entry in load(f)] + + ninja_files = glob.glob("build/*compile_commands.json") + cmake_files = glob.glob("torch/lib/build/*/compile_commands.json") + all_commands = [entry for f in ninja_files + cmake_files for entry in load(f)] # cquery does not like c++ compiles that start with gcc. # It forgets to include the c++ header directories. # We can work around this by replacing the gcc calls that python # setup.py generates with g++ calls instead for command in all_commands: - if command['command'].startswith("gcc "): - command['command'] = "g++ " + command['command'][4:] + if command["command"].startswith("gcc "): + command["command"] = "g++ " + command["command"][4:] new_contents = json.dumps(all_commands, indent=2) - contents = '' - if os.path.exists('compile_commands.json'): - with open('compile_commands.json') as f: + contents = "" + if os.path.exists("compile_commands.json"): + with open("compile_commands.json") as f: contents = f.read() if contents != new_contents: - with open('compile_commands.json', 'w') as f: + with open("compile_commands.json", "w") as f: f.write(new_contents) + class concat_license_files: """Merge LICENSE and LICENSES_BUNDLED.txt as a context manager @@ -708,9 +784,10 @@ class concat_license_files: is a single license file in the sdist and wheels with all of the necessary licensing info. """ + def __init__(self, include_files=False): - self.f1 = 'LICENSE' - self.f2 = 'third_party/LICENSES_BUNDLED.txt' + self.f1 = "LICENSE" + self.f2 = "third_party/LICENSES_BUNDLED.txt" self.include_files = include_files def __enter__(self): @@ -726,15 +803,15 @@ class concat_license_files: with open(self.f1) as f1: self.bsd_text = f1.read() - with open(self.f1, 'a') as f1: - f1.write('\n\n') - create_bundled(os.path.relpath(third_party_path), f1, - include_files=self.include_files) - + with open(self.f1, "a") as f1: + f1.write("\n\n") + create_bundled( + os.path.relpath(third_party_path), f1, include_files=self.include_files + ) def __exit__(self, exception_type, exception_value, traceback): """Restore content of f1""" - with open(self.f1, 'w') as f: + with open(self.f1, "w") as f: f.write(self.bsd_text) @@ -748,7 +825,8 @@ except ImportError: else: # Need to create the proper LICENSE.txt for the wheel class wheel_concatenate(bdist_wheel): - """ check submodules on sdist to prevent incomplete tarballs """ + """check submodules on sdist to prevent incomplete tarballs""" + def run(self): with concat_license_files(include_files=True): super().run() @@ -771,10 +849,11 @@ class clean(setuptools.Command): def run(self): import glob import re - with open('.gitignore') as f: + + with open(".gitignore") as f: ignores = f.read() - pat = re.compile(r'^#( BEGIN NOT-CLEAN-FILES )?') - for wildcard in filter(None, ignores.split('\n')): + pat = re.compile(r"^#( BEGIN NOT-CLEAN-FILES )?") + for wildcard in filter(None, ignores.split("\n")): match = pat.match(wildcard) if match: if match.group(1): @@ -783,7 +862,7 @@ class clean(setuptools.Command): # Ignore lines which begin with '#'. else: # Don't remove absolute paths from the system - wildcard = wildcard.lstrip('./') + wildcard = wildcard.lstrip("./") for filename in glob.glob(wildcard): try: @@ -825,161 +904,166 @@ def configure_extension_build(): if IS_WINDOWS: # /NODEFAULTLIB makes sure we only link to DLL runtime # and matches the flags set for protobuf and ONNX - extra_link_args = ['/NODEFAULTLIB:LIBCMT.LIB'] + extra_link_args = ["/NODEFAULTLIB:LIBCMT.LIB"] # /MD links against DLL runtime # and matches the flags set for protobuf and ONNX # /EHsc is about standard C++ exception handling - extra_compile_args = ['/MD', '/FS', '/EHsc'] + extra_compile_args = ["/MD", "/FS", "/EHsc"] else: extra_link_args = [] extra_compile_args = [ - '-Wall', - '-Wextra', - '-Wno-strict-overflow', - '-Wno-unused-parameter', - '-Wno-missing-field-initializers', - '-Wno-unknown-pragmas', + "-Wall", + "-Wextra", + "-Wno-strict-overflow", + "-Wno-unused-parameter", + "-Wno-missing-field-initializers", + "-Wno-unknown-pragmas", # This is required for Python 2 declarations that are deprecated in 3. - '-Wno-deprecated-declarations', + "-Wno-deprecated-declarations", # Python 2.6 requires -fno-strict-aliasing, see # http://legacy.python.org/dev/peps/pep-3123/ # We also depend on it in our code (even Python 3). - '-fno-strict-aliasing', + "-fno-strict-aliasing", # Clang has an unfixed bug leading to spurious missing # braces warnings, see # https://bugs.llvm.org/show_bug.cgi?id=21629 - '-Wno-missing-braces', + "-Wno-missing-braces", ] library_dirs.append(lib_path) main_compile_args = [] - main_libraries = ['torch_python'] + main_libraries = ["torch_python"] main_link_args = [] main_sources = ["torch/csrc/stub.c"] - if cmake_cache_vars['USE_CUDA']: - library_dirs.append( - os.path.dirname(cmake_cache_vars['CUDA_CUDA_LIB'])) + if cmake_cache_vars["USE_CUDA"]: + library_dirs.append(os.path.dirname(cmake_cache_vars["CUDA_CUDA_LIB"])) if build_type.is_debug(): if IS_WINDOWS: - extra_compile_args.append('/Z7') - extra_link_args.append('/DEBUG:FULL') + extra_compile_args.append("/Z7") + extra_link_args.append("/DEBUG:FULL") else: - extra_compile_args += ['-O0', '-g'] - extra_link_args += ['-O0', '-g'] + extra_compile_args += ["-O0", "-g"] + extra_link_args += ["-O0", "-g"] if build_type.is_rel_with_deb_info(): if IS_WINDOWS: - extra_compile_args.append('/Z7') - extra_link_args.append('/DEBUG:FULL') + extra_compile_args.append("/Z7") + extra_link_args.append("/DEBUG:FULL") else: - extra_compile_args += ['-g'] - extra_link_args += ['-g'] + extra_compile_args += ["-g"] + extra_link_args += ["-g"] # special CUDA 11.7 package that requires installation of cuda runtime, cudnn and cublas - pytorch_extra_install_requirements = os.getenv("PYTORCH_EXTRA_INSTALL_REQUIREMENTS", "") + pytorch_extra_install_requirements = os.getenv( + "PYTORCH_EXTRA_INSTALL_REQUIREMENTS", "" + ) if pytorch_extra_install_requirements: - report(f"pytorch_extra_install_requirements: {pytorch_extra_install_requirements}") + report( + f"pytorch_extra_install_requirements: {pytorch_extra_install_requirements}" + ) extra_install_requires += pytorch_extra_install_requirements.split("|") - # Cross-compile for M1 if IS_DARWIN: - macos_target_arch = os.getenv('CMAKE_OSX_ARCHITECTURES', '') - if macos_target_arch in ['arm64', 'x86_64']: - macos_sysroot_path = os.getenv('CMAKE_OSX_SYSROOT') + macos_target_arch = os.getenv("CMAKE_OSX_ARCHITECTURES", "") + if macos_target_arch in ["arm64", "x86_64"]: + macos_sysroot_path = os.getenv("CMAKE_OSX_SYSROOT") if macos_sysroot_path is None: - macos_sysroot_path = subprocess.check_output([ - 'xcrun', '--show-sdk-path', '--sdk', 'macosx' - ]).decode('utf-8').strip() - extra_compile_args += ['-arch', macos_target_arch, '-isysroot', macos_sysroot_path] - extra_link_args += ['-arch', macos_target_arch] - + macos_sysroot_path = ( + subprocess.check_output( + ["xcrun", "--show-sdk-path", "--sdk", "macosx"] + ) + .decode("utf-8") + .strip() + ) + extra_compile_args += [ + "-arch", + macos_target_arch, + "-isysroot", + macos_sysroot_path, + ] + extra_link_args += ["-arch", macos_target_arch] def make_relative_rpath_args(path): if IS_DARWIN: - return ['-Wl,-rpath,@loader_path/' + path] + return ["-Wl,-rpath,@loader_path/" + path] elif IS_WINDOWS: return [] else: - return ['-Wl,-rpath,$ORIGIN/' + path] + return ["-Wl,-rpath,$ORIGIN/" + path] ################################################################################ # Declare extensions and package ################################################################################ extensions = [] - excludes = ['tools', 'tools.*'] - if not cmake_cache_vars['BUILD_CAFFE2']: - excludes.extend(['caffe2', 'caffe2.*']) - if not cmake_cache_vars['BUILD_FUNCTORCH']: - excludes.extend(['functorch', 'functorch.*']) - if not cmake_cache_vars['BUILD_NVFUSER']: - excludes.extend(['nvfuser', 'nvfuser.*']) + excludes = ["tools", "tools.*"] + if not cmake_cache_vars["BUILD_CAFFE2"]: + excludes.extend(["caffe2", "caffe2.*"]) + if not cmake_cache_vars["BUILD_FUNCTORCH"]: + excludes.extend(["functorch", "functorch.*"]) + if not cmake_cache_vars["BUILD_NVFUSER"]: + excludes.extend(["nvfuser", "nvfuser.*"]) packages = find_packages(exclude=excludes) - C = Extension("torch._C", - libraries=main_libraries, - sources=main_sources, - language='c', - extra_compile_args=main_compile_args + extra_compile_args, - include_dirs=[], - library_dirs=library_dirs, - extra_link_args=extra_link_args + main_link_args + make_relative_rpath_args('lib')) + C = Extension( + "torch._C", + libraries=main_libraries, + sources=main_sources, + language="c", + extra_compile_args=main_compile_args + extra_compile_args, + include_dirs=[], + library_dirs=library_dirs, + extra_link_args=extra_link_args + + main_link_args + + make_relative_rpath_args("lib"), + ) extensions.append(C) # These extensions are built by cmake and copied manually in build_extensions() # inside the build_ext implementation - if cmake_cache_vars['BUILD_CAFFE2']: + if cmake_cache_vars["BUILD_CAFFE2"]: extensions.append( - Extension( - name='caffe2.python.caffe2_pybind11_state', - sources=[]), + Extension(name="caffe2.python.caffe2_pybind11_state", sources=[]), ) - if cmake_cache_vars['USE_CUDA']: + if cmake_cache_vars["USE_CUDA"]: extensions.append( - Extension( - name='caffe2.python.caffe2_pybind11_state_gpu', - sources=[]), + Extension(name="caffe2.python.caffe2_pybind11_state_gpu", sources=[]), ) - if cmake_cache_vars['USE_ROCM']: + if cmake_cache_vars["USE_ROCM"]: extensions.append( - Extension( - name='caffe2.python.caffe2_pybind11_state_hip', - sources=[]), + Extension(name="caffe2.python.caffe2_pybind11_state_hip", sources=[]), ) - if cmake_cache_vars['BUILD_FUNCTORCH']: + if cmake_cache_vars["BUILD_FUNCTORCH"]: extensions.append( - Extension( - name='functorch._C', - sources=[]), + Extension(name="functorch._C", sources=[]), ) - if cmake_cache_vars['BUILD_NVFUSER']: + if cmake_cache_vars["BUILD_NVFUSER"]: extensions.append( - Extension( - name='nvfuser._C', - sources=[]), + Extension(name="nvfuser._C", sources=[]), ) cmdclass = { - 'bdist_wheel': wheel_concatenate, - 'build_ext': build_ext, - 'clean': clean, - 'install': install, - 'sdist': sdist, + "bdist_wheel": wheel_concatenate, + "build_ext": build_ext, + "clean": clean, + "install": install, + "sdist": sdist, } entry_points = { - 'console_scripts': [ - 'convert-caffe2-to-onnx = caffe2.python.onnx.bin.conversion:caffe2_to_onnx', - 'convert-onnx-to-caffe2 = caffe2.python.onnx.bin.conversion:onnx_to_caffe2', - 'torchrun = torch.distributed.run:main', + "console_scripts": [ + "convert-caffe2-to-onnx = caffe2.python.onnx.bin.conversion:caffe2_to_onnx", + "convert-onnx-to-caffe2 = caffe2.python.onnx.bin.conversion:onnx_to_caffe2", + "torchrun = torch.distributed.run:main", ] } return extensions, cmdclass, packages, entry_points, extra_install_requires + # post run, warnings, printed at the end to make them more visible build_update_message = """ It is no longer necessary to use the 'build' or 'rebuild' targets @@ -994,44 +1078,47 @@ build_update_message = """ def print_box(msg): - lines = msg.split('\n') + lines = msg.split("\n") size = max(len(l) + 1 for l in lines) - print('-' * (size + 2)) + print("-" * (size + 2)) for l in lines: - print('|{}{}|'.format(l, ' ' * (size - len(l)))) - print('-' * (size + 2)) + print("|{}{}|".format(l, " " * (size - len(l)))) + print("-" * (size + 2)) def main(): # the list of runtime dependencies required by this built package install_requires = [ - 'filelock', - 'typing-extensions', - 'sympy', - 'networkx', - 'jinja2', - 'fsspec', + "filelock", + "typing-extensions", + "sympy", + "networkx", + "jinja2", + "fsspec", ] - extras_require = { - 'opt-einsum': ['opt-einsum>=3.3'] - } - if platform.system() == 'Linux': + extras_require = {"opt-einsum": ["opt-einsum>=3.3"]} + if platform.system() == "Linux": cmake_cache_vars = get_cmake_cache_vars() - if cmake_cache_vars['USE_ROCM']: + if cmake_cache_vars["USE_ROCM"]: triton_text_file = "triton-rocm.txt" triton_package_name = "pytorch-triton-rocm" else: triton_text_file = "triton.txt" triton_package_name = "pytorch-triton" - triton_pin_file = os.path.join(cwd, ".ci", "docker", "ci_commit_pins", triton_text_file) + triton_pin_file = os.path.join( + cwd, ".ci", "docker", "ci_commit_pins", triton_text_file + ) triton_version_file = os.path.join(cwd, ".ci", "docker", "triton_version.txt") if os.path.exists(triton_pin_file) and os.path.exists(triton_version_file): with open(triton_pin_file) as f: triton_pin = f.read().strip() with open(triton_version_file) as f: triton_version = f.read().strip() - extras_require['dynamo'] = [triton_package_name + '==' + triton_version + '+' + triton_pin[:10], 'jinja2'] + extras_require["dynamo"] = [ + triton_package_name + "==" + triton_version + "+" + triton_pin[:10], + "jinja2", + ] # Parse the command line and check the arguments before we proceed with # building deps and setup. We need to set values so `--help` works. @@ -1048,7 +1135,13 @@ def main(): if RUN_BUILD_DEPS: build_deps() - extensions, cmdclass, packages, entry_points, extra_install_requires = configure_extension_build() + ( + extensions, + cmdclass, + packages, + entry_points, + extra_install_requires, + ) = configure_extension_build() install_requires += extra_install_requires @@ -1058,215 +1151,223 @@ def main(): version_range_max = max(sys.version_info[1], 10) + 1 torch_package_data = [ - 'py.typed', - 'bin/*', - 'test/*', - '*.pyi', - '_C/*.pyi', - 'cuda/*.pyi', - 'fx/*.pyi', - 'optim/*.pyi', - 'autograd/*.pyi', - 'nn/*.pyi', - 'nn/modules/*.pyi', - 'nn/parallel/*.pyi', - 'utils/data/*.pyi', - 'utils/data/datapipes/*.pyi', - 'lib/*.so*', - 'lib/*.dylib*', - 'lib/*.dll', - 'lib/*.lib', - 'lib/*.pdb', - 'lib/torch_shm_manager', - 'lib/*.h', - 'include/*.h', - 'include/ATen/*.h', - 'include/ATen/cpu/*.h', - 'include/ATen/cpu/vec/vec256/*.h', - 'include/ATen/cpu/vec/vec256/vsx/*.h', - 'include/ATen/cpu/vec/vec512/*.h', - 'include/ATen/cpu/vec/*.h', - 'include/ATen/core/*.h', - 'include/ATen/cuda/*.cuh', - 'include/ATen/cuda/*.h', - 'include/ATen/cuda/detail/*.cuh', - 'include/ATen/cuda/detail/*.h', - 'include/ATen/cudnn/*.h', - 'include/ATen/functorch/*.h', - 'include/ATen/ops/*.h', - 'include/ATen/hip/*.cuh', - 'include/ATen/hip/*.h', - 'include/ATen/hip/detail/*.cuh', - 'include/ATen/hip/detail/*.h', - 'include/ATen/hip/impl/*.h', - 'include/ATen/mps/*.h', - 'include/ATen/miopen/*.h', - 'include/ATen/detail/*.h', - 'include/ATen/native/*.h', - 'include/ATen/native/cpu/*.h', - 'include/ATen/native/cuda/*.h', - 'include/ATen/native/cuda/*.cuh', - 'include/ATen/native/hip/*.h', - 'include/ATen/native/hip/*.cuh', - 'include/ATen/native/mps/*.h', - 'include/ATen/native/quantized/*.h', - 'include/ATen/native/quantized/cpu/*.h', - 'include/ATen/quantized/*.h', - 'include/caffe2/serialize/*.h', - 'include/c10/*.h', - 'include/c10/macros/*.h', - 'include/c10/core/*.h', - 'include/ATen/core/boxing/*.h', - 'include/ATen/core/boxing/impl/*.h', - 'include/ATen/core/dispatch/*.h', - 'include/ATen/core/op_registration/*.h', - 'include/c10/core/impl/*.h', - 'include/c10/core/impl/cow/*.h', - 'include/c10/util/*.h', - 'include/c10/cuda/*.h', - 'include/c10/cuda/impl/*.h', - 'include/c10/hip/*.h', - 'include/c10/hip/impl/*.h', - 'include/torch/*.h', - 'include/torch/csrc/*.h', - 'include/torch/csrc/api/include/torch/*.h', - 'include/torch/csrc/api/include/torch/data/*.h', - 'include/torch/csrc/api/include/torch/data/dataloader/*.h', - 'include/torch/csrc/api/include/torch/data/datasets/*.h', - 'include/torch/csrc/api/include/torch/data/detail/*.h', - 'include/torch/csrc/api/include/torch/data/samplers/*.h', - 'include/torch/csrc/api/include/torch/data/transforms/*.h', - 'include/torch/csrc/api/include/torch/detail/*.h', - 'include/torch/csrc/api/include/torch/detail/ordered_dict.h', - 'include/torch/csrc/api/include/torch/nn/*.h', - 'include/torch/csrc/api/include/torch/nn/functional/*.h', - 'include/torch/csrc/api/include/torch/nn/options/*.h', - 'include/torch/csrc/api/include/torch/nn/modules/*.h', - 'include/torch/csrc/api/include/torch/nn/modules/container/*.h', - 'include/torch/csrc/api/include/torch/nn/parallel/*.h', - 'include/torch/csrc/api/include/torch/nn/utils/*.h', - 'include/torch/csrc/api/include/torch/optim/*.h', - 'include/torch/csrc/api/include/torch/optim/schedulers/*.h', - 'include/torch/csrc/api/include/torch/serialize/*.h', - 'include/torch/csrc/autograd/*.h', - 'include/torch/csrc/autograd/functions/*.h', - 'include/torch/csrc/autograd/generated/*.h', - 'include/torch/csrc/autograd/utils/*.h', - 'include/torch/csrc/cuda/*.h', - 'include/torch/csrc/distributed/c10d/*.h', - 'include/torch/csrc/distributed/c10d/*.hpp', - 'include/torch/csrc/distributed/rpc/*.h', - 'include/torch/csrc/distributed/autograd/context/*.h', - 'include/torch/csrc/distributed/autograd/functions/*.h', - 'include/torch/csrc/distributed/autograd/rpc_messages/*.h', - 'include/torch/csrc/dynamo/eval_frame.h', - 'include/torch/csrc/inductor/*.h', - 'include/torch/csrc/jit/*.h', - 'include/torch/csrc/jit/backends/*.h', - 'include/torch/csrc/jit/generated/*.h', - 'include/torch/csrc/jit/passes/*.h', - 'include/torch/csrc/jit/passes/quantization/*.h', - 'include/torch/csrc/jit/passes/utils/*.h', - 'include/torch/csrc/jit/runtime/*.h', - 'include/torch/csrc/jit/ir/*.h', - 'include/torch/csrc/jit/frontend/*.h', - 'include/torch/csrc/jit/api/*.h', - 'include/torch/csrc/jit/serialization/*.h', - 'include/torch/csrc/jit/python/*.h', - 'include/torch/csrc/jit/mobile/*.h', - 'include/torch/csrc/jit/testing/*.h', - 'include/torch/csrc/jit/tensorexpr/*.h', - 'include/torch/csrc/jit/tensorexpr/operators/*.h', - 'include/torch/csrc/jit/codegen/cuda/*.h', - 'include/torch/csrc/jit/codegen/cuda/ops/*.h', - 'include/torch/csrc/jit/codegen/cuda/scheduler/*.h', - 'include/torch/csrc/onnx/*.h', - 'include/torch/csrc/profiler/*.h', - 'include/torch/csrc/profiler/orchestration/*.h', - 'include/torch/csrc/profiler/stubs/*.h', - 'include/torch/csrc/utils/*.h', - 'include/torch/csrc/tensor/*.h', - 'include/torch/csrc/lazy/backend/*.h', - 'include/torch/csrc/lazy/core/*.h', - 'include/torch/csrc/lazy/core/internal_ops/*.h', - 'include/torch/csrc/lazy/core/ops/*.h', - 'include/torch/csrc/lazy/python/python_util.h', - 'include/torch/csrc/lazy/ts_backend/*.h', - 'include/pybind11/*.h', - 'include/pybind11/detail/*.h', - 'include/TH/*.h*', - 'include/TH/generic/*.h*', - 'include/THC/*.cuh', - 'include/THC/*.h*', - 'include/THC/generic/*.h', - 'include/THH/*.cuh', - 'include/THH/*.h*', - 'include/THH/generic/*.h', - 'include/sleef.h', - '_inductor/codegen/*.cpp', - '_inductor/codegen/*.h', - 'share/cmake/ATen/*.cmake', - 'share/cmake/Caffe2/*.cmake', - 'share/cmake/Caffe2/public/*.cmake', - 'share/cmake/Caffe2/Modules_CUDA_fix/*.cmake', - 'share/cmake/Caffe2/Modules_CUDA_fix/upstream/*.cmake', - 'share/cmake/Caffe2/Modules_CUDA_fix/upstream/FindCUDA/*.cmake', - 'share/cmake/Gloo/*.cmake', - 'share/cmake/Tensorpipe/*.cmake', - 'share/cmake/Torch/*.cmake', - 'utils/benchmark/utils/*.cpp', - 'utils/benchmark/utils/valgrind_wrapper/*.cpp', - 'utils/benchmark/utils/valgrind_wrapper/*.h', - 'utils/model_dump/skeleton.html', - 'utils/model_dump/code.js', - 'utils/model_dump/*.mjs', + "py.typed", + "bin/*", + "test/*", + "*.pyi", + "_C/*.pyi", + "cuda/*.pyi", + "fx/*.pyi", + "optim/*.pyi", + "autograd/*.pyi", + "nn/*.pyi", + "nn/modules/*.pyi", + "nn/parallel/*.pyi", + "utils/data/*.pyi", + "utils/data/datapipes/*.pyi", + "lib/*.so*", + "lib/*.dylib*", + "lib/*.dll", + "lib/*.lib", + "lib/*.pdb", + "lib/torch_shm_manager", + "lib/*.h", + "include/*.h", + "include/ATen/*.h", + "include/ATen/cpu/*.h", + "include/ATen/cpu/vec/vec256/*.h", + "include/ATen/cpu/vec/vec256/vsx/*.h", + "include/ATen/cpu/vec/vec512/*.h", + "include/ATen/cpu/vec/*.h", + "include/ATen/core/*.h", + "include/ATen/cuda/*.cuh", + "include/ATen/cuda/*.h", + "include/ATen/cuda/detail/*.cuh", + "include/ATen/cuda/detail/*.h", + "include/ATen/cudnn/*.h", + "include/ATen/functorch/*.h", + "include/ATen/ops/*.h", + "include/ATen/hip/*.cuh", + "include/ATen/hip/*.h", + "include/ATen/hip/detail/*.cuh", + "include/ATen/hip/detail/*.h", + "include/ATen/hip/impl/*.h", + "include/ATen/mps/*.h", + "include/ATen/miopen/*.h", + "include/ATen/detail/*.h", + "include/ATen/native/*.h", + "include/ATen/native/cpu/*.h", + "include/ATen/native/cuda/*.h", + "include/ATen/native/cuda/*.cuh", + "include/ATen/native/hip/*.h", + "include/ATen/native/hip/*.cuh", + "include/ATen/native/mps/*.h", + "include/ATen/native/quantized/*.h", + "include/ATen/native/quantized/cpu/*.h", + "include/ATen/quantized/*.h", + "include/caffe2/serialize/*.h", + "include/c10/*.h", + "include/c10/macros/*.h", + "include/c10/core/*.h", + "include/ATen/core/boxing/*.h", + "include/ATen/core/boxing/impl/*.h", + "include/ATen/core/dispatch/*.h", + "include/ATen/core/op_registration/*.h", + "include/c10/core/impl/*.h", + "include/c10/core/impl/cow/*.h", + "include/c10/util/*.h", + "include/c10/cuda/*.h", + "include/c10/cuda/impl/*.h", + "include/c10/hip/*.h", + "include/c10/hip/impl/*.h", + "include/torch/*.h", + "include/torch/csrc/*.h", + "include/torch/csrc/api/include/torch/*.h", + "include/torch/csrc/api/include/torch/data/*.h", + "include/torch/csrc/api/include/torch/data/dataloader/*.h", + "include/torch/csrc/api/include/torch/data/datasets/*.h", + "include/torch/csrc/api/include/torch/data/detail/*.h", + "include/torch/csrc/api/include/torch/data/samplers/*.h", + "include/torch/csrc/api/include/torch/data/transforms/*.h", + "include/torch/csrc/api/include/torch/detail/*.h", + "include/torch/csrc/api/include/torch/detail/ordered_dict.h", + "include/torch/csrc/api/include/torch/nn/*.h", + "include/torch/csrc/api/include/torch/nn/functional/*.h", + "include/torch/csrc/api/include/torch/nn/options/*.h", + "include/torch/csrc/api/include/torch/nn/modules/*.h", + "include/torch/csrc/api/include/torch/nn/modules/container/*.h", + "include/torch/csrc/api/include/torch/nn/parallel/*.h", + "include/torch/csrc/api/include/torch/nn/utils/*.h", + "include/torch/csrc/api/include/torch/optim/*.h", + "include/torch/csrc/api/include/torch/optim/schedulers/*.h", + "include/torch/csrc/api/include/torch/serialize/*.h", + "include/torch/csrc/autograd/*.h", + "include/torch/csrc/autograd/functions/*.h", + "include/torch/csrc/autograd/generated/*.h", + "include/torch/csrc/autograd/utils/*.h", + "include/torch/csrc/cuda/*.h", + "include/torch/csrc/distributed/c10d/*.h", + "include/torch/csrc/distributed/c10d/*.hpp", + "include/torch/csrc/distributed/rpc/*.h", + "include/torch/csrc/distributed/autograd/context/*.h", + "include/torch/csrc/distributed/autograd/functions/*.h", + "include/torch/csrc/distributed/autograd/rpc_messages/*.h", + "include/torch/csrc/dynamo/eval_frame.h", + "include/torch/csrc/inductor/*.h", + "include/torch/csrc/jit/*.h", + "include/torch/csrc/jit/backends/*.h", + "include/torch/csrc/jit/generated/*.h", + "include/torch/csrc/jit/passes/*.h", + "include/torch/csrc/jit/passes/quantization/*.h", + "include/torch/csrc/jit/passes/utils/*.h", + "include/torch/csrc/jit/runtime/*.h", + "include/torch/csrc/jit/ir/*.h", + "include/torch/csrc/jit/frontend/*.h", + "include/torch/csrc/jit/api/*.h", + "include/torch/csrc/jit/serialization/*.h", + "include/torch/csrc/jit/python/*.h", + "include/torch/csrc/jit/mobile/*.h", + "include/torch/csrc/jit/testing/*.h", + "include/torch/csrc/jit/tensorexpr/*.h", + "include/torch/csrc/jit/tensorexpr/operators/*.h", + "include/torch/csrc/jit/codegen/cuda/*.h", + "include/torch/csrc/jit/codegen/cuda/ops/*.h", + "include/torch/csrc/jit/codegen/cuda/scheduler/*.h", + "include/torch/csrc/onnx/*.h", + "include/torch/csrc/profiler/*.h", + "include/torch/csrc/profiler/orchestration/*.h", + "include/torch/csrc/profiler/stubs/*.h", + "include/torch/csrc/utils/*.h", + "include/torch/csrc/tensor/*.h", + "include/torch/csrc/lazy/backend/*.h", + "include/torch/csrc/lazy/core/*.h", + "include/torch/csrc/lazy/core/internal_ops/*.h", + "include/torch/csrc/lazy/core/ops/*.h", + "include/torch/csrc/lazy/python/python_util.h", + "include/torch/csrc/lazy/ts_backend/*.h", + "include/pybind11/*.h", + "include/pybind11/detail/*.h", + "include/TH/*.h*", + "include/TH/generic/*.h*", + "include/THC/*.cuh", + "include/THC/*.h*", + "include/THC/generic/*.h", + "include/THH/*.cuh", + "include/THH/*.h*", + "include/THH/generic/*.h", + "include/sleef.h", + "_inductor/codegen/*.cpp", + "_inductor/codegen/*.h", + "share/cmake/ATen/*.cmake", + "share/cmake/Caffe2/*.cmake", + "share/cmake/Caffe2/public/*.cmake", + "share/cmake/Caffe2/Modules_CUDA_fix/*.cmake", + "share/cmake/Caffe2/Modules_CUDA_fix/upstream/*.cmake", + "share/cmake/Caffe2/Modules_CUDA_fix/upstream/FindCUDA/*.cmake", + "share/cmake/Gloo/*.cmake", + "share/cmake/Tensorpipe/*.cmake", + "share/cmake/Torch/*.cmake", + "utils/benchmark/utils/*.cpp", + "utils/benchmark/utils/valgrind_wrapper/*.cpp", + "utils/benchmark/utils/valgrind_wrapper/*.h", + "utils/model_dump/skeleton.html", + "utils/model_dump/code.js", + "utils/model_dump/*.mjs", ] - if get_cmake_cache_vars()['BUILD_NVFUSER']: - torch_package_data.extend([ - 'share/cmake/nvfuser/*.cmake', - 'include/nvfuser/*.h', - 'include/nvfuser/kernel_db/*.h', - 'include/nvfuser/multidevice/*.h', - 'include/nvfuser/ops/*.h', - 'include/nvfuser/python_frontend/*.h', - 'include/nvfuser/scheduler/*.h', - ]) + if get_cmake_cache_vars()["BUILD_NVFUSER"]: + torch_package_data.extend( + [ + "share/cmake/nvfuser/*.cmake", + "include/nvfuser/*.h", + "include/nvfuser/kernel_db/*.h", + "include/nvfuser/multidevice/*.h", + "include/nvfuser/ops/*.h", + "include/nvfuser/python_frontend/*.h", + "include/nvfuser/scheduler/*.h", + ] + ) - if get_cmake_cache_vars()['BUILD_CAFFE2']: - torch_package_data.extend([ - 'include/caffe2/**/*.h', - 'include/caffe2/utils/*.h', - 'include/caffe2/utils/**/*.h', - ]) - if get_cmake_cache_vars()['USE_TENSORPIPE']: - torch_package_data.extend([ - 'include/tensorpipe/*.h', - 'include/tensorpipe/channel/*.h', - 'include/tensorpipe/channel/basic/*.h', - 'include/tensorpipe/channel/cma/*.h', - 'include/tensorpipe/channel/mpt/*.h', - 'include/tensorpipe/channel/xth/*.h', - 'include/tensorpipe/common/*.h', - 'include/tensorpipe/core/*.h', - 'include/tensorpipe/transport/*.h', - 'include/tensorpipe/transport/ibv/*.h', - 'include/tensorpipe/transport/shm/*.h', - 'include/tensorpipe/transport/uv/*.h', - ]) + if get_cmake_cache_vars()["BUILD_CAFFE2"]: + torch_package_data.extend( + [ + "include/caffe2/**/*.h", + "include/caffe2/utils/*.h", + "include/caffe2/utils/**/*.h", + ] + ) + if get_cmake_cache_vars()["USE_TENSORPIPE"]: + torch_package_data.extend( + [ + "include/tensorpipe/*.h", + "include/tensorpipe/channel/*.h", + "include/tensorpipe/channel/basic/*.h", + "include/tensorpipe/channel/cma/*.h", + "include/tensorpipe/channel/mpt/*.h", + "include/tensorpipe/channel/xth/*.h", + "include/tensorpipe/common/*.h", + "include/tensorpipe/core/*.h", + "include/tensorpipe/transport/*.h", + "include/tensorpipe/transport/ibv/*.h", + "include/tensorpipe/transport/shm/*.h", + "include/tensorpipe/transport/uv/*.h", + ] + ) torchgen_package_data = [ # Recursive glob doesn't work in setup.py, # https://github.com/pypa/setuptools/issues/1806 # To make this robust we should replace it with some code that # returns a list of everything under packaged/ - 'packaged/ATen/*', - 'packaged/ATen/native/*', - 'packaged/ATen/templates/*', + "packaged/ATen/*", + "packaged/ATen/native/*", + "packaged/ATen/templates/*", ] setup( name=package_name, version=version, - description=("Tensors and Dynamic neural networks in " - "Python with strong GPU acceleration"), + description=( + "Tensors and Dynamic neural networks in " + "Python with strong GPU acceleration" + ), long_description=long_description, long_description_content_type="text/markdown", ext_modules=extensions, @@ -1276,39 +1377,43 @@ def main(): install_requires=install_requires, extras_require=extras_require, package_data={ - 'torch': torch_package_data, - 'torchgen': torchgen_package_data, - 'caffe2': [ - 'python/serialized_test/data/operator_test/*.zip', + "torch": torch_package_data, + "torchgen": torchgen_package_data, + "caffe2": [ + "python/serialized_test/data/operator_test/*.zip", ], }, - url='https://pytorch.org/', - download_url='https://github.com/pytorch/pytorch/tags', - author='PyTorch Team', - author_email='packages@pytorch.org', - python_requires=f'>={python_min_version_str}', + url="https://pytorch.org/", + download_url="https://github.com/pytorch/pytorch/tags", + author="PyTorch Team", + author_email="packages@pytorch.org", + python_requires=f">={python_min_version_str}", # PyPI package information. classifiers=[ - 'Development Status :: 5 - Production/Stable', - 'Intended Audience :: Developers', - 'Intended Audience :: Education', - 'Intended Audience :: Science/Research', - 'License :: OSI Approved :: BSD License', - 'Topic :: Scientific/Engineering', - 'Topic :: Scientific/Engineering :: Mathematics', - 'Topic :: Scientific/Engineering :: Artificial Intelligence', - 'Topic :: Software Development', - 'Topic :: Software Development :: Libraries', - 'Topic :: Software Development :: Libraries :: Python Modules', - 'Programming Language :: C++', - 'Programming Language :: Python :: 3', - ] + [f'Programming Language :: Python :: 3.{i}' for i in range(python_min_version[1], version_range_max)], - license='BSD-3', - keywords='pytorch, machine learning', + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Intended Audience :: Education", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: BSD License", + "Topic :: Scientific/Engineering", + "Topic :: Scientific/Engineering :: Mathematics", + "Topic :: Scientific/Engineering :: Artificial Intelligence", + "Topic :: Software Development", + "Topic :: Software Development :: Libraries", + "Topic :: Software Development :: Libraries :: Python Modules", + "Programming Language :: C++", + "Programming Language :: Python :: 3", + ] + + [ + f"Programming Language :: Python :: 3.{i}" + for i in range(python_min_version[1], version_range_max) + ], + license="BSD-3", + keywords="pytorch, machine learning", ) if EMIT_BUILD_WARNING: print_box(build_update_message) -if __name__ == '__main__': +if __name__ == "__main__": main()