summaryrefslogtreecommitdiff
path: root/.circleci
diff options
context:
space:
mode:
Diffstat (limited to '.circleci')
-rw-r--r--.circleci/config.yml49
1 files changed, 29 insertions, 20 deletions
diff --git a/.circleci/config.yml b/.circleci/config.yml
index 3a27c53876..0b229b411f 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -37,7 +37,8 @@ setup_ci_environment: &setup_ci_environment
linux-headers-$(uname -r) \
linux-image-generic \
moreutils \
- nvidia-docker2
+ nvidia-docker2 \
+ expect-dev
sudo pkill -SIGHUP dockerd
@@ -98,7 +99,8 @@ pytorch_linux_build_defaults: &pytorch_linux_build_defaults
docker cp /home/circleci/project/. $id:/var/lib/jenkins/workspace
- ((echo "export JOB_BASE_NAME=${JOB_BASE_NAME}" && echo "source ./workspace/env" && echo 'sudo chown -R jenkins workspace && cd workspace && .jenkins/pytorch/build.sh') | docker exec -u jenkins -i "$id" bash) 2>&1 | ts
+ export COMMAND='((echo "export JOB_BASE_NAME=${JOB_BASE_NAME}" && echo "source ./workspace/env" && echo "sudo chown -R jenkins workspace && cd workspace && .jenkins/pytorch/build.sh") | docker exec -u jenkins -i "$id" bash) 2>&1'
+ echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts
# Push intermediate Docker image for next phase to use
if [ -z "${BUILD_ONLY}" ]; then
@@ -122,15 +124,16 @@ pytorch_linux_test_defaults: &pytorch_linux_test_defaults
echo "DOCKER_IMAGE: "${COMMIT_DOCKER_IMAGE}
docker pull ${COMMIT_DOCKER_IMAGE} >/dev/null
if [ -n "${CUDA_VERSION}" ]; then
- id=$(docker run --runtime=nvidia -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE})
+ export id=$(docker run --runtime=nvidia -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE})
else
- id=$(docker run -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE})
+ export id=$(docker run -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE})
fi
if [ -n "${MULTI_GPU}" ]; then
- ((echo "export JOB_BASE_NAME=${JOB_BASE_NAME}" && echo "source ./workspace/env" && echo 'sudo chown -R jenkins workspace && cd workspace && .jenkins/pytorch/multigpu-test.sh') | docker exec -u jenkins -i "$id" bash) 2>&1 | ts
+ export COMMAND='((echo "export JOB_BASE_NAME=${JOB_BASE_NAME}" && echo "source ./workspace/env" && echo "sudo chown -R jenkins workspace && cd workspace && .jenkins/pytorch/multigpu-test.sh") | docker exec -u jenkins -i "$id" bash) 2>&1'
else
- ((echo "export JOB_BASE_NAME=${JOB_BASE_NAME}" && echo "source ./workspace/env" && echo 'sudo chown -R jenkins workspace && cd workspace && .jenkins/pytorch/test.sh') | docker exec -u jenkins -i "$id" bash) 2>&1 | ts
+ export COMMAND='((echo "export JOB_BASE_NAME=${JOB_BASE_NAME}" && echo "source ./workspace/env" && echo "sudo chown -R jenkins workspace && cd workspace && .jenkins/pytorch/test.sh") | docker exec -u jenkins -i "$id" bash) 2>&1'
fi
+ echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts
caffe2_linux_build_defaults: &caffe2_linux_build_defaults
resource_class: large
@@ -197,7 +200,8 @@ caffe2_linux_build_defaults: &caffe2_linux_build_defaults
export id=$(docker run -t -d -w /var/lib/jenkins ${DOCKER_IMAGE})
docker cp /home/circleci/project/. $id:/var/lib/jenkins/workspace
- ((echo "source ./workspace/env" && echo 'sudo chown -R jenkins workspace && cd workspace && ./ci_build_script.sh') | docker exec -u jenkins -i "$id" bash) 2>&1 | ts
+ export COMMAND='((echo "source ./workspace/env" && echo "sudo chown -R jenkins workspace && cd workspace && ./ci_build_script.sh") | docker exec -u jenkins -i "$id" bash) 2>&1'
+ echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts
# Push intermediate Docker image for next phase to use
if [ -z "${BUILD_ONLY}" ]; then
@@ -271,13 +275,14 @@ caffe2_linux_test_defaults: &caffe2_linux_test_defaults
echo "DOCKER_IMAGE: "${COMMIT_DOCKER_IMAGE}
docker pull ${COMMIT_DOCKER_IMAGE} >/dev/null
if [ -n "${CUDA_VERSION}" ]; then
- id=$(docker run --runtime=nvidia -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE})
+ export id=$(docker run --runtime=nvidia -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE})
else
- id=$(docker run -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE})
+ export id=$(docker run -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE})
fi
docker cp /home/circleci/project/. "$id:/var/lib/jenkins/workspace"
- ((echo "source ./workspace/env" && echo 'sudo chown -R jenkins workspace && cd workspace && ./ci_test_script.sh') | docker exec -u jenkins -i "$id" bash) 2>&1 | ts
+ export COMMAND='((echo "source ./workspace/env" && echo "sudo chown -R jenkins workspace && cd workspace && ./ci_test_script.sh") | docker exec -u jenkins -i "$id" bash) 2>&1'
+ echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts
caffe2_macos_build_defaults: &caffe2_macos_build_defaults
macos:
@@ -296,6 +301,8 @@ caffe2_macos_build_defaults: &caffe2_macos_build_defaults
brew install moreutils --without-parallel
brew install cmake
+ brew install expect
+
# Reinitialize submodules
git submodule sync && git submodule update -q --init --recursive
@@ -342,12 +349,12 @@ caffe2_macos_build_defaults: &caffe2_macos_build_defaults
# Build
if [ "${BUILD_IOS:-0}" -eq 1 ]; then
- scripts/build_ios.sh 2>&1 | ts
+ unbuffer scripts/build_ios.sh 2>&1 | ts
elif [ -n "${CAFFE2_USE_ANACONDA}" ]; then
# All conda build logic should be in scripts/build_anaconda.sh
- scripts/build_anaconda.sh 2>&1 | ts
+ unbuffer scripts/build_anaconda.sh 2>&1 | ts
else
- scripts/build_local.sh 2>&1 | ts
+ unbuffer scripts/build_local.sh 2>&1 | ts
fi
# Show sccache stats if it is running
@@ -579,7 +586,7 @@ jobs:
export COMMIT_DOCKER_IMAGE=${DOCKER_IMAGE}-${CIRCLE_SHA1}
echo "DOCKER_IMAGE: "${COMMIT_DOCKER_IMAGE}
docker pull ${COMMIT_DOCKER_IMAGE} >/dev/null
- id=$(docker run --runtime=nvidia -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE})
+ export id=$(docker run --runtime=nvidia -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE})
docker cp $id:/var/lib/jenkins/workspace/env /home/circleci/project/env
# This IAM user allows write access to S3 bucket for perf test numbers
@@ -587,7 +594,8 @@ jobs:
echo "declare -x AWS_SECRET_ACCESS_KEY=${CIRCLECI_AWS_SECRET_KEY_FOR_PERF_TEST_S3_BUCKET_V2}" >> /home/circleci/project/env
docker cp /home/circleci/project/env $id:/var/lib/jenkins/workspace/env
- ((echo "export JOB_BASE_NAME=${JOB_BASE_NAME}" && echo "source ./workspace/env" && echo 'sudo chown -R jenkins workspace && cd workspace && .jenkins/pytorch/short-perf-test-gpu.sh') | docker exec -u jenkins -i "$id" bash) 2>&1 | ts
+ export COMMAND='((echo "export JOB_BASE_NAME=${JOB_BASE_NAME}" && echo "source ./workspace/env" && echo "sudo chown -R jenkins workspace && cd workspace && .jenkins/pytorch/short-perf-test-gpu.sh") | docker exec -u jenkins -i "$id" bash) 2>&1'
+ echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts
pytorch_doc_push:
environment:
@@ -611,7 +619,7 @@ jobs:
export COMMIT_DOCKER_IMAGE=${DOCKER_IMAGE}-${CIRCLE_SHA1}
echo "DOCKER_IMAGE: "${COMMIT_DOCKER_IMAGE}
docker pull ${COMMIT_DOCKER_IMAGE} >/dev/null
- id=$(docker run -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE})
+ export id=$(docker run -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE})
cat >/home/circleci/project/doc_push_script.sh <<EOL
# =================== The following code will be executed inside Docker container ===================
@@ -663,7 +671,8 @@ jobs:
chmod +x /home/circleci/project/doc_push_script.sh
docker cp /home/circleci/project/doc_push_script.sh $id:/var/lib/jenkins/workspace/doc_push_script.sh
- ((echo "export JOB_BASE_NAME=${JOB_BASE_NAME}" && echo "source ./workspace/env" && echo 'sudo chown -R jenkins workspace && cd workspace && ./doc_push_script.sh') | docker exec -u jenkins -i "$id" bash) 2>&1 | ts
+ export COMMAND='((echo "export JOB_BASE_NAME=${JOB_BASE_NAME}" && echo "source ./workspace/env" && echo "sudo chown -R jenkins workspace && cd workspace && ./doc_push_script.sh") | docker exec -u jenkins -i "$id" bash) 2>&1'
+ echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts
pytorch_macos_10_13_py3_build:
macos:
@@ -694,7 +703,7 @@ jobs:
git submodule sync && git submodule update -q --init
chmod a+x .jenkins/pytorch/macos-build.sh
- .jenkins/pytorch/macos-build.sh 2>&1 | ts
+ unbuffer .jenkins/pytorch/macos-build.sh 2>&1 | ts
mkdir -p /Users/distiller/pytorch-ci-env/workspace
cp -r /Users/distiller/project/. /Users/distiller/pytorch-ci-env/workspace
@@ -729,7 +738,7 @@ jobs:
cp -r /Users/distiller/pytorch-ci-env/workspace/. /Users/distiller/project
chmod a+x .jenkins/pytorch/macos-test.sh
- .jenkins/pytorch/macos-test.sh 2>&1 | ts
+ unbuffer .jenkins/pytorch/macos-test.sh 2>&1 | ts
pytorch_macos_10_13_cuda9_2_cudnn7_py3_build:
macos:
@@ -776,7 +785,7 @@ jobs:
git submodule sync && git submodule update -q --init
chmod a+x .jenkins/pytorch/macos-build.sh
- .jenkins/pytorch/macos-build.sh 2>&1 | ts
+ unbuffer .jenkins/pytorch/macos-build.sh 2>&1 | ts
caffe2_py2_cuda8_0_cudnn6_ubuntu16_04_build:
environment:
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-22 20:52:06 +0000