OSSForks/tensorflow
1
0
Fork 0
mirror of https://github.com/zebrajr/tensorflow.git synced 2026-01-15 12:15:41 +00:00
Code Issues Actions 5 Packages Projects Releases Wiki Activity

Create a new ML Build container for ARM64. The new container will leverage the hermetic Python and hermetic cuda requirements to reduce its size by removing unneeded softwares.

Browse Source 
PiperOrigin-RevId: 695410924
This commit is contained in:
Quoc Truong
2024-11-11 11:02:14 -08:00
committed by TensorFlower Gardener
parent adc16d6119
commit 4c8b0d6025
15 changed files with 2078 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

65
ci/official/containers/ml_build_arm64/Dockerfile Normal file
View File

@@ -0,0 +1,65 @@
################################################################################
FROM ubuntu:20.04@sha256:8e5c4f0285ecbb4ead070431d29b576a530d3166df73ec44affc1cd27555141b as devel
################################################################################
# Install devtoolset build dependencies
RUN echo "Installing devtoolset build dependencies"
COPY setup.sources.sh /setup.sources.sh
COPY setup.packages.sh /setup.packages.sh
COPY builder.packages.txt /builder.packages.txt
RUN /setup.sources.sh && /setup.packages.sh /builder.packages.txt
RUN update-ca-certificates
# Install devtoolset-9 in /dt10 with glibc 2.17 and libstdc++ 4.8, for building
# manylinux2014-compatible packages.
COPY builder.devtoolset/fixlinks_aarch64.sh /fixlinks.sh
COPY builder.devtoolset/rpm-patch.sh /rpm-patch.sh
COPY builder.devtoolset/build_devtoolset.sh /build_devtoolset.sh
COPY builder.devtoolset/gcc9-fixups.patch /gcc9-fixups.patch
COPY builder.devtoolset/stringop_trunc.patch /stringop_trunc.patch
RUN /build_devtoolset.sh devtoolset-10 /dt10
# Build later version of patchelf that is not so buggy
RUN wget https://github.com/NixOS/patchelf/releases/download/0.18.0/patchelf-0.18.0-aarch64.tar.gz && tar -zxvf patchelf-0.18.0-aarch64.tar.gz -C /usr && rm -rf patchelf-0.18.0-aarch64.tar.gz
# Install various tools.
# - bats: bash unit testing framework
# NOTE: v1.6.0 seems to have a bug that made "git" in setup_file break
# - bazelisk: always use the correct bazel version
# - buildifier: clean bazel build depshttps://github.com/bazelbuild/buildtools/releases/download/v7.3.1/buildifier-linux-arm64
# - buildozer: clean bazel build deps
RUN git clone --branch v1.11.0 https://github.com/bats-core/bats-core.git && bats-core/install.sh /usr/local && rm -rf bats-core
RUN wget https://github.com/bazelbuild/bazelisk/releases/download/v1.21.0/bazelisk-linux-arm64 -O /usr/local/bin/bazel && chmod +x /usr/local/bin/bazel
RUN wget https://github.com/bazelbuild/buildtools/releases/download/v7.3.1/buildifier-linux-arm64 -O /usr/local/bin/buildifier && chmod +x /usr/local/bin/buildifier
RUN wget https://github.com/bazelbuild/buildtools/releases/download/v7.3.1/buildozer-linux-arm64 -O /usr/local/bin/buildozer && chmod +x /usr/local/bin/buildozer
RUN groupadd -g 1001 buildslave && useradd -m -u 1001 -g buildslave buildslave
RUN mkdir -p /tf/venv
RUN chown -R buildslave:buildslave /tf
# All lines past this point are reset when $CACHEBUSTER is set. We need this
# for Python specifically because we install some nightly packages which are
# likely to change daily.
ARG CACHEBUSTER=0
RUN echo $CACHEBUSTER
# Setup build and environment
COPY devel.bashrc /root/.bashrc
COPY ld.so.conf /dt10/etc/
# Make sure clang is on the path
RUN ln -s /usr/lib/llvm-18/bin/clang /usr/bin/clang
# Setup JAX Python environment.
COPY requirements.txt /requirements.txt
COPY setup.python.sh /setup.python.sh
RUN /setup.python.sh python3.9 requirements.txt
RUN /setup.python.sh python3.10 requirements.txt
RUN /setup.python.sh python3.11 requirements.txt
RUN /setup.python.sh python3.12 requirements.txt
RUN /setup.python.sh python3.13 requirements.txt
# "python3" commands by default run under 3.10
RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.10 1

15
ci/official/containers/ml_build_arm64/apt.conf Normal file
View File

@@ -0,0 +1,15 @@
/* Copyright 2023 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
APT::Default-Release "focal";

76
ci/official/containers/ml_build_arm64/build.sh Executable file
View File

@@ -0,0 +1,76 @@
#!/bin/bash
# Copyright 2023 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
# Builds the following Docker images for Linux ARM64. See the accompanying
# Dockerfile for more details:
# - us-central1-docker.pkg.dev/tensorflow-sigs/build-arm64:jax-latest-multi-python
# - us-central1-docker.pkg.dev/tensorflow-sigs/build-arm64:tf-latest-multi-python
set -exo pipefail
function is_continuous_or_release() {
[[ "$KOKORO_JOB_TYPE" == "CONTINUOUS_INTEGRATION" ]] || [[ "$KOKORO_JOB_TYPE" == "RELEASE" ]]
}
# Move into the directory of the script
cd "$(dirname "$0")"
if is_continuous_or_release || [[ -z "$KOKORO_BUILD_ID" ]]; then
# A continuous job is the only one to publish to latest
TAG="latest-multi-python"
else
# If it is a change, grab a good tag for iterative builds
if [[ -z "${KOKORO_GITHUB_PULL_REQUEST_NUMBER}" ]]; then
TAG=$(head -n 1 "$KOKORO_PIPER_DIR/presubmit_request.txt" | cut -d" " -f2)
else
TAG="pr-${KOKORO_GITHUB_PULL_REQUEST_NUMBER}"
fi
fi
AR_IMAGE_PATH="us-central1-docker.pkg.dev/tensorflow-sigs/tensorflow/ml-build-arm64"
# Build for both JAX and TF usage. We do these in one place because they share
# almost all of the same cache layers
export DOCKER_BUILDKIT=1
for target in jax tf; do
AR_IMAGE="$AR_IMAGE_PATH:$target-$TAG"
docker pull "$AR_IMAGE" || true
# Due to some flakiness of resources pulled in the build, allow the docker
# command to reattempt build a few times in the case of failure (b/302558736)
set +e
for i in $(seq 1 5)
do
docker build \
--build-arg REQUIREMENTS_FILE=jax.requirements.txt \
--target=$target \
--cache-from "$AR_IMAGE" \
-t "$AR_IMAGE" . && break
done
final=$?
if [ $final -ne 0 ]; then
exit $final
fi
set -e
INFRA_PUBLIC_TAG=infrastructure-public-image-$(docker images "$AR_IMAGE" --quiet)
AR_IMAGE_INFRA_PUBLIC="$AR_IMAGE_PATH:$INFRA_PUBLIC_TAG"
docker image tag "$AR_IMAGE" "$AR_IMAGE_INFRA_PUBLIC"
if [[ -n "$KOKORO_BUILD_ID" ]]; then
gcloud auth configure-docker us-central1-docker.pkg.dev
docker push "$AR_IMAGE_PATH" --all-tags
fi
done

157
ci/official/containers/ml_build_arm64/builder.devtoolset/build_devtoolset.sh Executable file
View File

@@ -0,0 +1,157 @@
#!/bin/bash -eu
# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
#
# Builds a devtoolset cross-compiler targeting manylinux2014 (glibc 2.17 / libstdc++ 4.8).
VERSION="$1"
TARGET="$2"
case "${VERSION}" in
devtoolset-9)
LIBSTDCXX_VERSION="6.0.28"
LIBSTDCXX_ABI="new"
;;
devtoolset-10)
LIBSTDCXX_VERSION="6.0.28"
LIBSTDCXX_ABI="new"
;;
*)
echo "Usage: $0 {devtoolset-9|devtoolset-10} <target-directory> <arch>"
echo "Use 'devtoolset-9' to build a manylinux2014 compatible toolchain"
exit 1
;;
esac
mkdir -p "${TARGET}"
mkdir -p ${TARGET}/usr/include
# Put the current kernel headers from ubuntu in place.
ln -s "/usr/include/linux" "${TARGET}/usr/include/linux"
ln -s "/usr/include/asm-generic" "${TARGET}/usr/include/asm-generic"
ln -s "/usr/include/aarch64-linux-gnu/asm" "${TARGET}/usr/include/asm"
# Download glibc's shared and development libraries based on the value of the
# `VERSION` parameter.
# Note: 'Templatizing' this and the other conditional branches would require
# defining several variables (version, os, path) making it difficult to maintain
# and extend for future modifications.
mkdir -p glibc-src
mkdir -p glibc-build
cd glibc-src
wget --retry-connrefused --waitretry=1 --read-timeout=20 --timeout=15 --tries=5 "https://vault.centos.org/centos/7/os/Source/SPackages/glibc-2.17-317.el7.src.rpm"
rpm2cpio "glibc-2.17-317.el7.src.rpm" |cpio -idmv
tar -xvzf "glibc-2.17-c758a686.tar.gz" --strip 1
tar -xvzf "glibc-2.17-c758a686-releng.tar.gz" --strip 1
sed -i '/patch0060/d' glibc.spec
/rpm-patch.sh "glibc.spec"
rm -f "glibc-2.17-317.el7.src.rpm" "glibc-2.17-c758a686.tar.gz" "glibc-2.17-c758a686-releng.tar.gz"
patch -p1 < /gcc9-fixups.patch
patch -p1 < /stringop_trunc.patch
cd ../glibc-build
../glibc-src/configure --prefix=/usr --disable-werror --enable-obsolete-rpc --disable-profile
make -j$(nproc)
make install DESTDIR=${TARGET}
cd ..
# Symlinks in the binary distribution are set up for installation in /usr, we
# need to fix up all the links to stay within /${TARGET}.
/fixlinks.sh "/${TARGET}"
# Patch to allow non-glibc 2.12 compatible builds to work.
sed -i '54i#define TCP_USER_TIMEOUT 18' "/${TARGET}/usr/include/netinet/tcp.h"
# Download specific version of libstdc++ shared library based on the value of
# the `VERSION` parameter
# Download binary libstdc++ 4.8 shared library release
wget --retry-connrefused --waitretry=1 --read-timeout=20 --timeout=15 --tries=5 "http://old-releases.ubuntu.com/ubuntu/pool/main/g/gcc-4.8/libstdc++6_4.8.1-10ubuntu8_arm64.deb" && \
unar "libstdc++6_4.8.1-10ubuntu8_arm64.deb" && \
tar -C "${TARGET}" -xvzf "libstdc++6_4.8.1-10ubuntu8_arm64/data.tar.gz" "./usr/lib/aarch64-linux-gnu/libstdc++.so.6.0.18" && \
rm -rf "libstdc++6_4.8.1-10ubuntu8_arm64.deb" "libstdc++6_4.8.1-10ubuntu8_arm64"
mkdir -p "${TARGET}-src"
cd "${TARGET}-src"
# Build a devtoolset cross-compiler based on our glibc 2.12/glibc 2.17 sysroot setup.
case "${VERSION}" in
devtoolset-9)
wget --retry-connrefused --waitretry=1 --read-timeout=20 --timeout=15 --tries=5 "https://vault.centos.org/centos/7/sclo/Source/rh/devtoolset-9-gcc-9.3.1-2.2.el7.src.rpm"
rpm2cpio "devtoolset-9-gcc-9.3.1-2.2.el7.src.rpm" |cpio -idmv
tar -xvf "gcc-9.3.1-20200408.tar.xz" --strip 1
;;
devtoolset-10)
wget --retry-connrefused --waitretry=1 --read-timeout=20 --timeout=15 --tries=5 "https://vault.centos.org/centos/7/sclo/Source/rh/devtoolset-10-gcc-10.2.1-11.2.el7.src.rpm"
rpm2cpio "devtoolset-10-gcc-10.2.1-11.2.el7.src.rpm" |cpio -idmv
tar -xvf "gcc-10.2.1-20210130.tar.xz" --strip 1
;;
esac
# Apply the devtoolset patches to gcc.
/rpm-patch.sh "gcc.spec"
./contrib/download_prerequisites
mkdir -p "${TARGET}-build"
cd "${TARGET}-build"
"${TARGET}-src/configure" \
--prefix="${TARGET}/usr" \
--with-sysroot="/${TARGET}" \
--disable-bootstrap \
--disable-libmpx \
--enable-libsanitizer \
--disable-libunwind-exceptions \
--disable-libunwind-exceptions \
--disable-lto \
--disable-multilib \
--enable-__cxa_atexit \
--enable-gnu-indirect-function \
--enable-gnu-unique-object \
--enable-initfini-array \
--enable-languages="c,c++" \
--enable-linker-build-id \
--enable-plugin \
--enable-shared \
--enable-threads=posix \
--with-default-libstdcxx-abi=${LIBSTDCXX_ABI} \
--with-gcc-major-version-only \
--with-linker-hash-style="gnu" \
&& \
make -j$(nproc) && \
make install
# Create the devtoolset libstdc++ linkerscript that links dynamically against
# the system libstdc++ 4.4 and provides all other symbols statically.
# Note that the installation path for libstdc++ here is ${TARGET}/usr/lib64/
mv "${TARGET}/usr/lib64/libstdc++.so.${LIBSTDCXX_VERSION}" \
"${TARGET}/usr/lib64/libstdc++.so.${LIBSTDCXX_VERSION}.backup"
echo -e "OUTPUT_FORMAT(elf64-littleaarch64)\nINPUT ( libstdc++.so.6.0.18 -lstdc++_nonshared44 )" \
> "${TARGET}/usr/lib64/libstdc++.so.${LIBSTDCXX_VERSION}"
cp "./aarch64-unknown-linux-gnu/libstdc++-v3/src/.libs/libstdc++_nonshared44.a" \
"${TARGET}/usr/lib64"
# Link in architecture specific includes from the system; note that we cannot
# link in the whole aarch64-linux-gnu folder, as otherwise we're overlaying
# system gcc paths that we do not want to find.
# TODO(klimek): Automate linking in all non-gcc / non-kernel include
# directories.
mkdir -p "${TARGET}/usr/include/aarch64-linux-gnu"
PYTHON_VERSIONS=("python3.9" "python3.10" "python3.11" "python3.12")
for v in "${PYTHON_VERSIONS[@]}"; do
ln -s "/usr/local/include/${v}" "${TARGET}/usr/include/aarch64-linux-gnu/${v}"
done

28
ci/official/containers/ml_build_arm64/builder.devtoolset/fixlinks_aarch64.sh Executable file
View File

@@ -0,0 +1,28 @@
#!/bin/bash
# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
#
# Re-direct all links in $1 that are relative to be canonical
BASE="$1"
find "${BASE}" -type l | \
while read l ; do
if [[ "$(readlink "$l")" == \.\./* ]]; then
CANONICAL="$(readlink "$l")";
rm "$l";
ln -s "${CANONICAL}" "$l"
fi
done

270
ci/official/containers/ml_build_arm64/builder.devtoolset/gcc9-fixups.patch Normal file
View File

@@ -0,0 +1,270 @@
diff --git a/iconv/gconv.h b/iconv/gconv.h
index 3f9112e..8e60197 100644
--- a/iconv/gconv.h
+++ b/iconv/gconv.h
@@ -174,7 +174,7 @@ typedef struct __gconv_info
{
size_t __nsteps;
struct __gconv_step *__steps;
- __extension__ struct __gconv_step_data __data __flexarr;
+ __extension__ struct __gconv_step_data __data[0];
} *__gconv_t;
#endif /* gconv.h */
diff --git a/include/libc-symbols.h b/include/libc-symbols.h
index c555bf2..143b26d 100644
--- a/include/libc-symbols.h
+++ b/include/libc-symbols.h
@@ -107,6 +107,11 @@
# endif
#endif
+#ifndef __attribute_copy__
+/* Provide an empty definition when cdefs.h is not included. */
+# define __attribute_copy__(arg)
+#endif
+
#ifndef __ASSEMBLER__
/* GCC understands weak symbols and aliases; use its interface where
possible, instead of embedded assembly language. */
@@ -114,7 +119,8 @@
/* Define ALIASNAME as a strong alias for NAME. */
# define strong_alias(name, aliasname) _strong_alias(name, aliasname)
# define _strong_alias(name, aliasname) \
- extern __typeof (name) aliasname __attribute__ ((alias (#name)));
+ extern __typeof (name) aliasname __attribute__ ((alias (#name))) \
+ __attribute_copy__ (name);
/* This comes between the return type and function name in
a function definition to make that definition weak. */
@@ -125,14 +131,16 @@
If weak aliases are not available, this defines a strong alias. */
# define weak_alias(name, aliasname) _weak_alias (name, aliasname)
# define _weak_alias(name, aliasname) \
- extern __typeof (name) aliasname __attribute__ ((weak, alias (#name)));
+ extern __typeof (name) aliasname __attribute__ ((weak, alias (#name))) \
+ __attribute_copy__ (name);
/* Same as WEAK_ALIAS, but mark symbol as hidden. */
# define weak_hidden_alias(name, aliasname) \
_weak_hidden_alias (name, aliasname)
# define _weak_hidden_alias(name, aliasname) \
extern __typeof (name) aliasname \
- __attribute__ ((weak, alias (#name), __visibility__ ("hidden")));
+ __attribute__ ((weak, alias (#name), __visibility__ ("hidden"))) \
+ __attribute_copy__ (name);
/* Declare SYMBOL as weak undefined symbol (resolved to 0 if not defined). */
# define weak_extern(symbol) _weak_extern (weak symbol)
@@ -528,7 +536,8 @@ for linking")
# define __hidden_ver1(local, internal, name) \
extern __typeof (name) __EI_##name __asm__(__hidden_asmname (#internal)); \
extern __typeof (name) __EI_##name \
- __attribute__((alias (__hidden_asmname (#local))))
+ __attribute__((alias (__hidden_asmname (#local)))) \
+ __attribute_copy__ (name)
# define hidden_ver(local, name) __hidden_ver1(local, __GI_##name, name);
# define hidden_data_ver(local, name) hidden_ver(local, name)
# define hidden_def(name) __hidden_ver1(__GI_##name, name, name);
@@ -541,7 +550,8 @@ for linking")
# define __hidden_nolink1(local, internal, name, version) \
__hidden_nolink2 (local, internal, name, version)
# define __hidden_nolink2(local, internal, name, version) \
- extern __typeof (name) internal __attribute__ ((alias (#local))); \
+ extern __typeof (name) internal __attribute__ ((alias (#local))) \
+ __attribute_copy__ (name); \
__hidden_nolink3 (local, internal, #name "@" #version)
# define __hidden_nolink3(local, internal, vername) \
__asm__ (".symver " #internal ", " vername);
diff --git a/locale/weightwc.h b/locale/weightwc.h
index e966c03..22ab790 100644
--- a/locale/weightwc.h
+++ b/locale/weightwc.h
@@ -79,19 +79,19 @@ findidx (const wint_t **cpp, size_t len)
if (cp[cnt] != usrc[cnt])
break;
- if (cnt < nhere - 1)
+ if (cnt < nhere - 1 || cnt == len)
{
cp += 2 * nhere;
continue;
}
- if (cp[nhere - 1] > usrc[nhere -1])
+ if (cp[nhere - 1] > usrc[nhere - 1])
{
cp += 2 * nhere;
continue;
}
- if (cp[2 * nhere - 1] < usrc[nhere -1])
+ if (cp[2 * nhere - 1] < usrc[nhere - 1])
{
cp += 2 * nhere;
continue;
diff --git a/locale/xlocale.h b/locale/xlocale.h
index 98c080b..843bd45 100644
--- a/locale/xlocale.h
+++ b/locale/xlocale.h
@@ -20,6 +20,9 @@
#ifndef _XLOCALE_H
#define _XLOCALE_H 1
+#ifndef _BITS_TYPES___LOCALE_T_H
+#define _BITS_TYPES___LOCALE_T_H 1
+
/* Structure for reentrant locale using functions. This is an
(almost) opaque type for the user level programs. The file and
this data structure is not standardized. Don't rely on it. It can
@@ -41,4 +44,6 @@ typedef struct __locale_struct
/* POSIX 2008 makes locale_t official. */
typedef __locale_t locale_t;
+#endif /* bits/types/__locale_t.h */
+
#endif /* xlocale.h */
diff --git a/misc/sys/cdefs.h b/misc/sys/cdefs.h
index d1cb3dd..30482a1 100644
--- a/misc/sys/cdefs.h
+++ b/misc/sys/cdefs.h
@@ -423,4 +423,14 @@
# endif
#endif
+/* Undefine (also defined in libc-symbols.h). */
+#undef __attribute_copy__
+#if __GNUC_PREREQ (9, 0)
+/* Copies attributes from the declaration or type referenced by
+ the argument. */
+# define __attribute_copy__(arg) __attribute__ ((__copy__ (arg)))
+#else
+# define __attribute_copy__(arg)
+#endif
+
#endif /* sys/cdefs.h */
diff --git a/stdlib/setenv.c b/stdlib/setenv.c
index 45efe2e..06bfab0 100644
--- a/stdlib/setenv.c
+++ b/stdlib/setenv.c
@@ -319,6 +319,7 @@ unsetenv (const char *name)
ep = __environ;
if (ep != NULL)
+ {
while (*ep != NULL)
if (!strncmp (*ep, name, len) && (*ep)[len] == '=')
{
@@ -332,6 +333,7 @@ unsetenv (const char *name)
}
else
++ep;
+ }
UNLOCK;
diff --git a/support/Makefile b/support/Makefile
index a253698..2f4e2a9 100644
--- a/support/Makefile
+++ b/support/Makefile
@@ -167,13 +167,6 @@ CFLAGS-support_paths.c = \
-DINSTDIR_PATH=\"$(prefix)\" \
-DLIBDIR_PATH=\"$(libdir)\"
-ifeq (,$(CXX))
-LINKS_DSO_PROGRAM = links-dso-program-c
-else
-LINKS_DSO_PROGRAM = links-dso-program
-LDLIBS-links-dso-program = -lstdc++ -lgcc -lgcc_s $(libunwind)
-endif
-
LDLIBS-test-container = $(libsupport)
others += test-container
@@ -182,9 +175,6 @@ others-noinstall += test-container
others += shell-container echo-container true-container
others-noinstall += shell-container echo-container true-container
-others += $(LINKS_DSO_PROGRAM)
-others-noinstall += $(LINKS_DSO_PROGRAM)
-
$(objpfx)test-container : $(libsupport)
$(objpfx)shell-container : $(libsupport)
$(objpfx)echo-container : $(libsupport)
diff --git a/support/links-dso-program.cc b/support/links-dso-program.cc
index 8ff3155..f9d2b77 100644
--- a/support/links-dso-program.cc
+++ b/support/links-dso-program.cc
@@ -3,6 +3,11 @@
backported. */
#define _ISOMAC 1
+#define __GLIBC_USE(F) __GLIBC_USE_ ## F
+
+# define __attribute_alloc_size__(params) \
+ __attribute__ ((__alloc_size__ params))
+
#include <iostream>
using namespace std;
diff --git a/sysdeps/aarch64/dl-machine.h b/sysdeps/aarch64/dl-machine.h
index 185402f..bbdeae0 100644
--- a/sysdeps/aarch64/dl-machine.h
+++ b/sysdeps/aarch64/dl-machine.h
@@ -49,23 +49,11 @@ elf_machine_load_address (void)
/* To figure out the load address we use the definition that for any symbol:
dynamic_addr(symbol) = static_addr(symbol) + load_addr
- The choice of symbol is arbitrary. The static address we obtain
- by constructing a non GOT reference to the symbol, the dynamic
- address of the symbol we compute using adrp/add to compute the
- symbol's address relative to the PC. */
-
- ElfW(Addr) static_addr;
- ElfW(Addr) dynamic_addr;
-
- asm (" \n\
- adrp %1, _dl_start; \n\
- add %1, %1, #:lo12:_dl_start \n\
- ldr %w0, 1f \n\
- b 2f \n\
-1: .word _dl_start \n\
-2: \n\
- " : "=r" (static_addr), "=r" (dynamic_addr));
- return dynamic_addr - static_addr;
+ _DYNAMIC sysmbol is used here as its link-time address stored in
+ the special unrelocated first GOT entry. */
+
+ extern ElfW(Dyn) _DYNAMIC[] attribute_hidden;
+ return (ElfW(Addr)) &_DYNAMIC - elf_machine_dynamic ();
}
/* Set up the loaded object described by L so its unrelocated PLT
diff --git a/sysdeps/ieee754/dbl-64/k_rem_pio2.c b/sysdeps/ieee754/dbl-64/k_rem_pio2.c
index fcf956a..e2c5d29 100644
--- a/sysdeps/ieee754/dbl-64/k_rem_pio2.c
+++ b/sysdeps/ieee754/dbl-64/k_rem_pio2.c
@@ -172,7 +172,8 @@ int __kernel_rem_pio2(double *x, double *y, int e0, int nx, int prec, const int3
/* compute q[0],q[1],...q[jk] */
for (i=0;i<=jk;i++) {
- for(j=0,fw=0.0;j<=jx;j++) fw += x[j]*f[jx+i-j]; q[i] = fw;
+ for(j=0,fw=0.0;j<=jx;j++) fw += x[j]*f[jx+i-j];
+ q[i] = fw;
}
jz = jk;
diff --git a/sysdeps/ieee754/flt-32/k_rem_pio2f.c b/sysdeps/ieee754/flt-32/k_rem_pio2f.c
index e54a067..215b0e0 100644
--- a/sysdeps/ieee754/flt-32/k_rem_pio2f.c
+++ b/sysdeps/ieee754/flt-32/k_rem_pio2f.c
@@ -65,7 +65,8 @@ int __kernel_rem_pio2f(float *x, float *y, int e0, int nx, int prec, const int32
/* compute q[0],q[1],...q[jk] */
for (i=0;i<=jk;i++) {
- for(j=0,fw=0.0;j<=jx;j++) fw += x[j]*f[jx+i-j]; q[i] = fw;
+ for(j=0,fw=0.0;j<=jx;j++) fw += x[j]*f[jx+i-j];
+ q[i] = fw;
}
jz = jk;

28
ci/official/containers/ml_build_arm64/builder.devtoolset/rpm-patch.sh Executable file
View File

@@ -0,0 +1,28 @@
#!/bin/bash -eu
# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
#
# Given an RPM spec file $1, apply its patches.
SPEC="$1"
grep '%patch' "${SPEC}" |while read cmd ; do
N=$(echo "${cmd}" |sed 's,%patch\([0-9]\+\).*,\1,')
file=$(grep "Patch$N:" "${SPEC}" |sed 's,.*: ,,')
parg=$(echo "${cmd}" |sed 's,.*\(-p[0-9]\).*,\1,')
if [[ ! "${file}" =~ doxygen && "${cmd}" != \#* ]]; then
echo "patch ${parg} -s < ${file}"
patch ${parg} -s < "${file}"
fi
done

1204
ci/official/containers/ml_build_arm64/builder.devtoolset/stringop_trunc.patch Normal file
View File

File diff suppressed because it is too large Load Diff

47
ci/official/containers/ml_build_arm64/builder.packages.txt Normal file
View File

@@ -0,0 +1,47 @@
# Packages needed to build devtoolset
file
flex
make
build-essential
patch
rpm2cpio
unar
wget
xz-utils
cpio
gawk
texinfo
gettext
# Other build-related tools
software-properties-common
apt-transport-https
autoconf
automake
ca-certificates
pkg-config
libcurl3-dev
libcurl4-openssl-dev
libfreetype6-dev
libhdf5-serial-dev
libomp-18-dev
libssl-dev
libtool
libssl-dev
libxml2-dev
libxslt1-dev
libzmq3-dev
llvm-18
clang-18
clang-tidy-18
lld-18
clang-format-12
curl
git
parallel
sudo
swig
unzip
zip
openjdk-21-jdk
vim

26
ci/official/containers/ml_build_arm64/devel.bashrc Normal file
View File

@@ -0,0 +1,26 @@
# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# ==============================================================================
# Do not print anything if this is not being used interactively
[ -z "$PS1" ] && return
# Set up attractive prompt
export PS1="\[\e[31m\]tf-docker\[\e[m\] \[\e[33m\]\w\[\e[m\] > "
export TERM=xterm-256color
alias grep="grep --color=auto"
alias ls="ls --color=auto"
# Fix nvidia-docker
ldconfig

18
ci/official/containers/ml_build_arm64/ld.so.conf Normal file
View File

@@ -0,0 +1,18 @@
# Copyright 2023 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
#
# Builds a devtoolset cross-compiler targeting manylinux2014 (glibc 2.17 / libstdc++ 4.8).
/lib64

4
ci/official/containers/ml_build_arm64/requirements.txt Normal file
View File

@@ -0,0 +1,4 @@
portpicker==1.6.0
# For wheel verification, and uploading
auditwheel ~= 6.1.0
twine ~= 5.1.1

28
ci/official/containers/ml_build_arm64/setup.packages.sh Executable file
View File

@@ -0,0 +1,28 @@
#!/usr/bin/env bash
#
# Copyright 2022 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
#
# setup.packages.sh: Given a list of Ubuntu packages, install them and clean up.
# Usage: setup.packages.sh <package_list.txt>
set -e
# Prevent apt install tzinfo from asking our location (assumes UTC)
export DEBIAN_FRONTEND=noninteractive
apt-get update
# Remove commented lines and blank lines
apt-get install -y --no-install-recommends $(sed -e '/^\s*#.*$/d' -e '/^\s*$/d' "$1" | sort -u)
rm -rf /var/lib/apt/lists/*

67
ci/official/containers/ml_build_arm64/setup.python.sh Executable file
View File

@@ -0,0 +1,67 @@
#!/usr/bin/env bash
#
# Copyright 2022 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
#
# setup.python.sh: Install a specific Python version and packages for it.
# Usage: setup.python.sh <pyversion> <requirements.txt>
set -xe
source ~/.bashrc
VERSION=$1
REQUIREMENTS=$2
add-apt-repository ppa:deadsnakes/ppa
# Install Python packages for this container's version
if [[ ${VERSION} == "python3.13" ]]; then
cat >pythons.txt <<EOF
$VERSION
$VERSION-dev
$VERSION-venv
EOF
else
cat >pythons.txt <<EOF
$VERSION
$VERSION-dev
$VERSION-venv
EOF
fi
/setup.packages.sh pythons.txt
# Re-link pyconfig.h from aarch64-linux-gnu into the devtoolset directory
# for any Python version present
pushd /usr/include/aarch64-linux-gnu
for f in $(ls | grep python); do
# set up symlink for devtoolset-10
rm -f /dt10/usr/include/aarch64-linux-gnu/$f
ln -s /usr/include/aarch64-linux-gnu/$f /dt10/usr/include/aarch64-linux-gnu/$f
done
popd
# Python 3.10 include headers fix:
# sysconfig.get_path('include') incorrectly points to /usr/local/include/python
# map /usr/include/python3.10 to /usr/local/include/python3.10
if [[ ! -f "/usr/local/include/$VERSION" ]]; then
ln -sf /usr/include/$VERSION /usr/local/include/$VERSION
fi
# Install pip
wget --retry-connrefused --waitretry=1 --read-timeout=20 --timeout=15 --tries=5 https://bootstrap.pypa.io/get-pip.py
/usr/bin/$VERSION get-pip.py
/usr/bin/$VERSION -m pip install --no-cache-dir --upgrade pip
# Disable the cache dir to save image space, and install packages
/usr/bin/$VERSION -m pip install --no-cache-dir -r $REQUIREMENTS -U

45
ci/official/containers/ml_build_arm64/setup.sources.sh Executable file
View File

@@ -0,0 +1,45 @@
#!/usr/bin/env bash
#
# Copyright 2022 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
#
# setup.python.sh: Install a specific Python version and packages for it.
# Usage: setup.python.sh <pyversion> <requirements.txt>
# Sets up custom apt sources for our TF images.
# Prevent apt install tzinfo from asking our location (assumes UTC)
export DEBIAN_FRONTEND=noninteractive
# Set up shared custom sources
apt-get update
apt-get install -y gnupg ca-certificates
# Deadsnakes: https://launchpad.net/~deadsnakes/+archive/ubuntu/ppa
apt-key adv --keyserver keyserver.ubuntu.com --recv-keys F23C5A6CF475977595C89F51BA6932366A755776
# LLVM/Clang: https://apt.llvm.org/
apt-key adv --fetch-keys https://apt.llvm.org/llvm-snapshot.gpg.key
# Set up custom sources
cat >/etc/apt/sources.list.d/custom.list <<SOURCES
# More Python versions: Deadsnakes
deb http://ppa.launchpad.net/deadsnakes/ppa/ubuntu focal main
deb-src http://ppa.launchpad.net/deadsnakes/ppa/ubuntu focal main
# LLVM/Clang 18 repository
deb http://apt.llvm.org/focal/ llvm-toolchain-focal-18 main
deb-src http://apt.llvm.org/focal/ llvm-toolchain-focal-18 main
SOURCES