ci: Add anaconda pruning to CI pipeline (#44651)

Summary: Pull Request resolved: https://github.com/pytorch/pytorch/pull/44651 Adds pruning for our anaconda channels (pytorch-nightly, pytorch-test) into our CI pipeline so that it gets run on a more consistent basis. Signed-off-by: Eli Uriegas <eliuriegas@fb.com> Test Plan: Imported from OSS Reviewed By: walterddr Differential Revision: D23692851 Pulled By: seemethere fbshipit-source-id: fa69b506b73805bf2ffbde75d221aef1ee3f753e
2026-01-15 12:15:51 +00:00 · 2020-09-15 10:29:03 -07:00
parent 1d733d660d
commit d62994a94d
6 changed files with 172 additions and 0 deletions
--- a/.circleci/cimodel/data/simple/anaconda_prune_defintions.py
+++ b/.circleci/cimodel/data/simple/anaconda_prune_defintions.py
@@ -0,0 +1,28 @@
+from collections import OrderedDict
+
+from cimodel.data.simple.util.branch_filters import gen_filter_dict
+from cimodel.lib.miniutils import quote
+
+
+CHANNELS_TO_PRUNE = ["pytorch-nightly", "pytorch-test"]
+PACKAGES_TO_PRUNE = "pytorch torchvision torchaudio torchtext ignite torchcsprng"
+
+
+def gen_workflow_job(channel: str):
+    return OrderedDict(
+        {
+            "anaconda_prune": OrderedDict(
+                {
+                    "name": f"anaconda-prune-{channel}",
+                    "context": quote("org-member"),
+                    "packages": quote(PACKAGES_TO_PRUNE),
+                    "channel": channel,
+                    "filters": gen_filter_dict(branches_list=["postnightly"]),
+                }
+            )
+        }
+    )
+
+
+def get_workflow_jobs():
+    return [gen_workflow_job(channel) for channel in CHANNELS_TO_PRUNE]
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -1113,6 +1113,35 @@ jobs:
          cat "$script"
          source "$script"

+  anaconda_prune:
+    parameters:
+      packages:
+        type: string
+        description: "What packages are we pruning? (quoted, space-separated string. eg. 'pytorch', 'torchvision torchaudio', etc.)"
+        default: "pytorch"
+      channel:
+        type: string
+        description: "What channel are we pruning? (eq. pytorch-nightly)"
+        default: "pytorch-nightly"
+    docker:
+      - image: continuumio/miniconda3
+    environment:
+      - PACKAGES: "<< parameters.packages >>"
+      - CHANNEL: "<< parameters.channel >>"
+    steps:
+      - checkout
+      - run:
+          name: Install dependencies
+          no_output_timeout: "1h"
+          command: |
+            conda install -yq anaconda-client
+      - run:
+          name: Prune packages
+          no_output_timeout: "1h"
+          command: |
+              ANACONDA_API_TOKEN="${CONDA_PYTORCHBOT_TOKEN}" \
+              scripts/release/anaconda-prune/run.sh
+
  pytorch_doc_push:
    resource_class: medium
    machine:
@@ -7055,6 +7084,24 @@ workflows:
          name: nightly_pytorch_linux_xenial_py3_clang5_android_ndk_r19c_x86_32_android_publish_snapshot
          requires:
            - nightly_pytorch_linux_xenial_py3_clang5_android_ndk_r19c_android_gradle_build
+      - anaconda_prune:
+          name: anaconda-prune-pytorch-nightly
+          context: "org-member"
+          packages: "pytorch torchvision torchaudio torchtext ignite torchcsprng"
+          channel: pytorch-nightly
+          filters:
+            branches:
+              only:
+                - postnightly
+      - anaconda_prune:
+          name: anaconda-prune-pytorch-test
+          context: "org-member"
+          packages: "pytorch torchvision torchaudio torchtext ignite torchcsprng"
+          channel: pytorch-test
+          filters:
+            branches:
+              only:
+                - postnightly
      - pytorch_windows_build:
          build_environment: pytorch-win-vs2019-cuda10-cudnn7-py3
          cuda_version: "10"
--- a/.circleci/generate_config_yml.py
+++ b/.circleci/generate_config_yml.py
@@ -22,6 +22,7 @@ import cimodel.data.simple.macos_definitions
 import cimodel.data.simple.mobile_definitions
 import cimodel.data.simple.nightly_android
 import cimodel.data.simple.nightly_ios
+import cimodel.data.simple.anaconda_prune_defintions
 import cimodel.data.windows_build_definitions as windows_build_definitions
 import cimodel.lib.miniutils as miniutils
 import cimodel.lib.miniyaml as miniyaml
@@ -93,6 +94,7 @@ def gen_build_workflows_tree():
        cimodel.data.simple.binary_smoketest.get_workflow_jobs,
        cimodel.data.simple.nightly_ios.get_workflow_jobs,
        cimodel.data.simple.nightly_android.get_workflow_jobs,
+        cimodel.data.simple.anaconda_prune_defintions.get_workflow_jobs,
        windows_build_definitions.get_windows_workflows,
        binary_build_definitions.get_post_upload_jobs,
        binary_build_definitions.get_binary_smoke_test_jobs,
--- a/.circleci/verbatim-sources/job-specs/binary-job-specs.yml
+++ b/.circleci/verbatim-sources/job-specs/binary-job-specs.yml
@@ -324,3 +324,32 @@
          cat "$script"
          source "$script"

+  anaconda_prune:
+    parameters:
+      packages:
+        type: string
+        description: "What packages are we pruning? (quoted, space-separated string. eg. 'pytorch', 'torchvision torchaudio', etc.)"
+        default: "pytorch"
+      channel:
+        type: string
+        description: "What channel are we pruning? (eq. pytorch-nightly)"
+        default: "pytorch-nightly"
+    docker:
+      - image: continuumio/miniconda3
+    environment:
+      - PACKAGES: "<< parameters.packages >>"
+      - CHANNEL: "<< parameters.channel >>"
+    steps:
+      - checkout
+      - run:
+          name: Install dependencies
+          no_output_timeout: "1h"
+          command: |
+            conda install -yq anaconda-client
+      - run:
+          name: Prune packages
+          no_output_timeout: "1h"
+          command: |
+              ANACONDA_API_TOKEN="${CONDA_PYTORCHBOT_TOKEN}" \
+              scripts/release/anaconda-prune/run.sh
+
--- a/scripts/release/anaconda-prune/prune.sh
+++ b/scripts/release/anaconda-prune/prune.sh
@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+
+grab_prune_version() {
+    conda search -c "${CHANNEL}" --platform "${platform}" "${PKG}" 2>/dev/null | \
+        grep "${CHANNEL}" | \
+        awk -F '  *' '{print $2}' | \
+        uniq | \
+        head -n -1 | \
+        xargs
+}
+
+grab_latest_version() {
+    conda search -c "${CHANNEL}" --platform "${platform}" "${PKG}" 2>/dev/null | \
+        grep "${CHANNEL}" | \
+        awk -F '  *' '{print $2}' | \
+        uniq | \
+        tail -n 1 | \
+        xargs
+}
+
+grab_specs_for_version() {
+    conda search -c "${CHANNEL}" --platform "${platform}" "${PKG}" 2>/dev/null | \
+        grep "${CHANNEL}" | \
+        grep "$1" | \
+        awk -F '  *' '{print $3}' | \
+        uniq | \
+        xargs
+}
+
+set -eou pipefail
+
+CHANNEL=${CHANNEL:-pytorch-nightly}
+PKG=${PKG:-pytorch}
+PLATFORMS=${PLATFORMS:-noarch osx-64 linux-64 win-64}
+
+for platform in ${PLATFORMS}; do
+    latest_version="$(grab_latest_version || true)"
+    specs_in_latest_version="$(grab_specs_for_version "${latest_version}" || true)"
+    versions_to_prune="$(grab_prune_version || true)"
+    for version in ${versions_to_prune}; do
+        specs_in_prune_version="$(grab_specs_for_version "${version}" || true)"
+        for spec in ${specs_in_prune_version}; do
+        # If this spec is included in specs_in_latest_version, then remove it.
+        if [[ "${specs_in_latest_version}" =~ ${spec} ]];then
+            (
+                set -x
+                anaconda remove --force "${CHANNEL}/${PKG}/${version}/${platform}/${PKG}-${version}-${spec}.tar.bz2"
+            )
+        fi
+        done
+    done
+done
--- a/scripts/release/anaconda-prune/run.sh
+++ b/scripts/release/anaconda-prune/run.sh
@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
+CHANNEL=${CHANNEL:-pytorch-nightly}
+PACKAGES=${PACKAGES:-pytorch}
+
+for pkg in ${PACKAGES}; do
+    echo "+ Attempting to prune: ${CHANNEL}/${pkg}"
+    CHANNEL="${CHANNEL}" PKG="${pkg}" "${DIR}/prune.sh"
+    echo
+done