From d62994a94de08a9be3c1b2b487302a683b0d07bb Mon Sep 17 00:00:00 2001 From: Eli Uriegas Date: Tue, 15 Sep 2020 10:29:03 -0700 Subject: [PATCH] ci: Add anaconda pruning to CI pipeline (#44651) Summary: Pull Request resolved: https://github.com/pytorch/pytorch/pull/44651 Adds pruning for our anaconda channels (pytorch-nightly, pytorch-test) into our CI pipeline so that it gets run on a more consistent basis. Signed-off-by: Eli Uriegas Test Plan: Imported from OSS Reviewed By: walterddr Differential Revision: D23692851 Pulled By: seemethere fbshipit-source-id: fa69b506b73805bf2ffbde75d221aef1ee3f753e --- .../data/simple/anaconda_prune_defintions.py | 28 ++++++++++ .circleci/config.yml | 47 +++++++++++++++++ .circleci/generate_config_yml.py | 2 + .../job-specs/binary-job-specs.yml | 29 +++++++++++ scripts/release/anaconda-prune/prune.sh | 52 +++++++++++++++++++ scripts/release/anaconda-prune/run.sh | 14 +++++ 6 files changed, 172 insertions(+) create mode 100644 .circleci/cimodel/data/simple/anaconda_prune_defintions.py create mode 100755 scripts/release/anaconda-prune/prune.sh create mode 100755 scripts/release/anaconda-prune/run.sh diff --git a/.circleci/cimodel/data/simple/anaconda_prune_defintions.py b/.circleci/cimodel/data/simple/anaconda_prune_defintions.py new file mode 100644 index 000000000000..81af5f42c097 --- /dev/null +++ b/.circleci/cimodel/data/simple/anaconda_prune_defintions.py @@ -0,0 +1,28 @@ +from collections import OrderedDict + +from cimodel.data.simple.util.branch_filters import gen_filter_dict +from cimodel.lib.miniutils import quote + + +CHANNELS_TO_PRUNE = ["pytorch-nightly", "pytorch-test"] +PACKAGES_TO_PRUNE = "pytorch torchvision torchaudio torchtext ignite torchcsprng" + + +def gen_workflow_job(channel: str): + return OrderedDict( + { + "anaconda_prune": OrderedDict( + { + "name": f"anaconda-prune-{channel}", + "context": quote("org-member"), + "packages": quote(PACKAGES_TO_PRUNE), + "channel": channel, + "filters": gen_filter_dict(branches_list=["postnightly"]), + } + ) + } + ) + + +def get_workflow_jobs(): + return [gen_workflow_job(channel) for channel in CHANNELS_TO_PRUNE] diff --git a/.circleci/config.yml b/.circleci/config.yml index e113edc50937..f2aa001e47fa 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1113,6 +1113,35 @@ jobs: cat "$script" source "$script" + anaconda_prune: + parameters: + packages: + type: string + description: "What packages are we pruning? (quoted, space-separated string. eg. 'pytorch', 'torchvision torchaudio', etc.)" + default: "pytorch" + channel: + type: string + description: "What channel are we pruning? (eq. pytorch-nightly)" + default: "pytorch-nightly" + docker: + - image: continuumio/miniconda3 + environment: + - PACKAGES: "<< parameters.packages >>" + - CHANNEL: "<< parameters.channel >>" + steps: + - checkout + - run: + name: Install dependencies + no_output_timeout: "1h" + command: | + conda install -yq anaconda-client + - run: + name: Prune packages + no_output_timeout: "1h" + command: | + ANACONDA_API_TOKEN="${CONDA_PYTORCHBOT_TOKEN}" \ + scripts/release/anaconda-prune/run.sh + pytorch_doc_push: resource_class: medium machine: @@ -7055,6 +7084,24 @@ workflows: name: nightly_pytorch_linux_xenial_py3_clang5_android_ndk_r19c_x86_32_android_publish_snapshot requires: - nightly_pytorch_linux_xenial_py3_clang5_android_ndk_r19c_android_gradle_build + - anaconda_prune: + name: anaconda-prune-pytorch-nightly + context: "org-member" + packages: "pytorch torchvision torchaudio torchtext ignite torchcsprng" + channel: pytorch-nightly + filters: + branches: + only: + - postnightly + - anaconda_prune: + name: anaconda-prune-pytorch-test + context: "org-member" + packages: "pytorch torchvision torchaudio torchtext ignite torchcsprng" + channel: pytorch-test + filters: + branches: + only: + - postnightly - pytorch_windows_build: build_environment: pytorch-win-vs2019-cuda10-cudnn7-py3 cuda_version: "10" diff --git a/.circleci/generate_config_yml.py b/.circleci/generate_config_yml.py index fda4e05154a9..f1af924bd3e2 100755 --- a/.circleci/generate_config_yml.py +++ b/.circleci/generate_config_yml.py @@ -22,6 +22,7 @@ import cimodel.data.simple.macos_definitions import cimodel.data.simple.mobile_definitions import cimodel.data.simple.nightly_android import cimodel.data.simple.nightly_ios +import cimodel.data.simple.anaconda_prune_defintions import cimodel.data.windows_build_definitions as windows_build_definitions import cimodel.lib.miniutils as miniutils import cimodel.lib.miniyaml as miniyaml @@ -93,6 +94,7 @@ def gen_build_workflows_tree(): cimodel.data.simple.binary_smoketest.get_workflow_jobs, cimodel.data.simple.nightly_ios.get_workflow_jobs, cimodel.data.simple.nightly_android.get_workflow_jobs, + cimodel.data.simple.anaconda_prune_defintions.get_workflow_jobs, windows_build_definitions.get_windows_workflows, binary_build_definitions.get_post_upload_jobs, binary_build_definitions.get_binary_smoke_test_jobs, diff --git a/.circleci/verbatim-sources/job-specs/binary-job-specs.yml b/.circleci/verbatim-sources/job-specs/binary-job-specs.yml index b73290b3c7bf..bd26e8b2b373 100644 --- a/.circleci/verbatim-sources/job-specs/binary-job-specs.yml +++ b/.circleci/verbatim-sources/job-specs/binary-job-specs.yml @@ -324,3 +324,32 @@ cat "$script" source "$script" + anaconda_prune: + parameters: + packages: + type: string + description: "What packages are we pruning? (quoted, space-separated string. eg. 'pytorch', 'torchvision torchaudio', etc.)" + default: "pytorch" + channel: + type: string + description: "What channel are we pruning? (eq. pytorch-nightly)" + default: "pytorch-nightly" + docker: + - image: continuumio/miniconda3 + environment: + - PACKAGES: "<< parameters.packages >>" + - CHANNEL: "<< parameters.channel >>" + steps: + - checkout + - run: + name: Install dependencies + no_output_timeout: "1h" + command: | + conda install -yq anaconda-client + - run: + name: Prune packages + no_output_timeout: "1h" + command: | + ANACONDA_API_TOKEN="${CONDA_PYTORCHBOT_TOKEN}" \ + scripts/release/anaconda-prune/run.sh + diff --git a/scripts/release/anaconda-prune/prune.sh b/scripts/release/anaconda-prune/prune.sh new file mode 100755 index 000000000000..f3b377dd94fa --- /dev/null +++ b/scripts/release/anaconda-prune/prune.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash + +grab_prune_version() { + conda search -c "${CHANNEL}" --platform "${platform}" "${PKG}" 2>/dev/null | \ + grep "${CHANNEL}" | \ + awk -F ' *' '{print $2}' | \ + uniq | \ + head -n -1 | \ + xargs +} + +grab_latest_version() { + conda search -c "${CHANNEL}" --platform "${platform}" "${PKG}" 2>/dev/null | \ + grep "${CHANNEL}" | \ + awk -F ' *' '{print $2}' | \ + uniq | \ + tail -n 1 | \ + xargs +} + +grab_specs_for_version() { + conda search -c "${CHANNEL}" --platform "${platform}" "${PKG}" 2>/dev/null | \ + grep "${CHANNEL}" | \ + grep "$1" | \ + awk -F ' *' '{print $3}' | \ + uniq | \ + xargs +} + +set -eou pipefail + +CHANNEL=${CHANNEL:-pytorch-nightly} +PKG=${PKG:-pytorch} +PLATFORMS=${PLATFORMS:-noarch osx-64 linux-64 win-64} + +for platform in ${PLATFORMS}; do + latest_version="$(grab_latest_version || true)" + specs_in_latest_version="$(grab_specs_for_version "${latest_version}" || true)" + versions_to_prune="$(grab_prune_version || true)" + for version in ${versions_to_prune}; do + specs_in_prune_version="$(grab_specs_for_version "${version}" || true)" + for spec in ${specs_in_prune_version}; do + # If this spec is included in specs_in_latest_version, then remove it. + if [[ "${specs_in_latest_version}" =~ ${spec} ]];then + ( + set -x + anaconda remove --force "${CHANNEL}/${PKG}/${version}/${platform}/${PKG}-${version}-${spec}.tar.bz2" + ) + fi + done + done +done diff --git a/scripts/release/anaconda-prune/run.sh b/scripts/release/anaconda-prune/run.sh new file mode 100755 index 000000000000..284c88214739 --- /dev/null +++ b/scripts/release/anaconda-prune/run.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +set -euo pipefail + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + +CHANNEL=${CHANNEL:-pytorch-nightly} +PACKAGES=${PACKAGES:-pytorch} + +for pkg in ${PACKAGES}; do + echo "+ Attempting to prune: ${CHANNEL}/${pkg}" + CHANNEL="${CHANNEL}" PKG="${pkg}" "${DIR}/prune.sh" + echo +done