CircleCI: disable output buffering to better locate test timeout (#13516)

Summary:
ASAN test timeout such as https://circleci.com/gh/pytorch/pytorch/165649?utm_campaign=vcs-integration-link&utm_medium=referral&utm_source=github-build-link doesn't actually show where the timeout happened because of the bash output buffering. This PR turns off the buffering to better surface the error.
Pull Request resolved: https://github.com/pytorch/pytorch/pull/13516

Differential Revision: D12952513

Pulled By: yf225

fbshipit-source-id: 48058c021470e5aa7a2246e1fcd974cfabf5df54

1 files changed, 29 insertions, 20 deletions

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 3a27c53876..0b229b411f 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -37,7 +37,8 @@ setup_ci_environment: &setup_ci_environment
       linux-headers-$(uname -r) \
       linux-image-generic \
       moreutils \
-      nvidia-docker2
+      nvidia-docker2 \
+      expect-dev
 
     sudo pkill -SIGHUP dockerd
 
@@ -98,7 +99,8 @@ pytorch_linux_build_defaults: &pytorch_linux_build_defaults
 
         docker cp /home/circleci/project/. $id:/var/lib/jenkins/workspace
 
-        ((echo "export JOB_BASE_NAME=${JOB_BASE_NAME}" && echo "source ./workspace/env" && echo 'sudo chown -R jenkins workspace && cd workspace && .jenkins/pytorch/build.sh') | docker exec -u jenkins -i "$id" bash) 2>&1 | ts
+        export COMMAND='((echo "export JOB_BASE_NAME=${JOB_BASE_NAME}" && echo "source ./workspace/env" && echo "sudo chown -R jenkins workspace && cd workspace && .jenkins/pytorch/build.sh") | docker exec -u jenkins -i "$id" bash) 2>&1'
+        echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts
 
         # Push intermediate Docker image for next phase to use
         if [ -z "${BUILD_ONLY}" ]; then
@@ -122,15 +124,16 @@ pytorch_linux_test_defaults: &pytorch_linux_test_defaults
         echo "DOCKER_IMAGE: "${COMMIT_DOCKER_IMAGE}
         docker pull ${COMMIT_DOCKER_IMAGE} >/dev/null
         if [ -n "${CUDA_VERSION}" ]; then
-          id=$(docker run --runtime=nvidia -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE})
+          export id=$(docker run --runtime=nvidia -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE})
         else
-          id=$(docker run -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE})
+          export id=$(docker run -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE})
         fi
         if [ -n "${MULTI_GPU}" ]; then
-          ((echo "export JOB_BASE_NAME=${JOB_BASE_NAME}" && echo "source ./workspace/env" && echo 'sudo chown -R jenkins workspace && cd workspace && .jenkins/pytorch/multigpu-test.sh') | docker exec -u jenkins -i "$id" bash) 2>&1 | ts
+          export COMMAND='((echo "export JOB_BASE_NAME=${JOB_BASE_NAME}" && echo "source ./workspace/env" && echo "sudo chown -R jenkins workspace && cd workspace && .jenkins/pytorch/multigpu-test.sh") | docker exec -u jenkins -i "$id" bash) 2>&1'
         else
-          ((echo "export JOB_BASE_NAME=${JOB_BASE_NAME}" && echo "source ./workspace/env" && echo 'sudo chown -R jenkins workspace && cd workspace && .jenkins/pytorch/test.sh') | docker exec -u jenkins -i "$id" bash) 2>&1 | ts
+          export COMMAND='((echo "export JOB_BASE_NAME=${JOB_BASE_NAME}" && echo "source ./workspace/env" && echo "sudo chown -R jenkins workspace && cd workspace && .jenkins/pytorch/test.sh") | docker exec -u jenkins -i "$id" bash) 2>&1'
         fi
+        echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts
 
 caffe2_linux_build_defaults: &caffe2_linux_build_defaults
   resource_class: large
@@ -197,7 +200,8 @@ caffe2_linux_build_defaults: &caffe2_linux_build_defaults
         export id=$(docker run -t -d -w /var/lib/jenkins ${DOCKER_IMAGE})
         docker cp /home/circleci/project/. $id:/var/lib/jenkins/workspace
 
-        ((echo "source ./workspace/env" && echo 'sudo chown -R jenkins workspace && cd workspace && ./ci_build_script.sh') | docker exec -u jenkins -i "$id" bash) 2>&1 | ts
+        export COMMAND='((echo "source ./workspace/env" && echo "sudo chown -R jenkins workspace && cd workspace && ./ci_build_script.sh") | docker exec -u jenkins -i "$id" bash) 2>&1'
+        echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts
 
         # Push intermediate Docker image for next phase to use
         if [ -z "${BUILD_ONLY}" ]; then
@@ -271,13 +275,14 @@ caffe2_linux_test_defaults: &caffe2_linux_test_defaults
         echo "DOCKER_IMAGE: "${COMMIT_DOCKER_IMAGE}
         docker pull ${COMMIT_DOCKER_IMAGE} >/dev/null
         if [ -n "${CUDA_VERSION}" ]; then
-          id=$(docker run --runtime=nvidia -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE})
+          export id=$(docker run --runtime=nvidia -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE})
         else
-          id=$(docker run -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE})
+          export id=$(docker run -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE})
         fi
         docker cp /home/circleci/project/. "$id:/var/lib/jenkins/workspace"
 
-        ((echo "source ./workspace/env" && echo 'sudo chown -R jenkins workspace && cd workspace && ./ci_test_script.sh') | docker exec -u jenkins -i "$id" bash) 2>&1 | ts
+        export COMMAND='((echo "source ./workspace/env" && echo "sudo chown -R jenkins workspace && cd workspace && ./ci_test_script.sh") | docker exec -u jenkins -i "$id" bash) 2>&1'
+        echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts
 
 caffe2_macos_build_defaults: &caffe2_macos_build_defaults
   macos:
@@ -296,6 +301,8 @@ caffe2_macos_build_defaults: &caffe2_macos_build_defaults
           brew install moreutils --without-parallel
           brew install cmake
 
+          brew install expect
+
           # Reinitialize submodules
           git submodule sync && git submodule update -q --init --recursive
 
@@ -342,12 +349,12 @@ caffe2_macos_build_defaults: &caffe2_macos_build_defaults
 
           # Build
           if [ "${BUILD_IOS:-0}" -eq 1 ]; then
-            scripts/build_ios.sh 2>&1 | ts
+            unbuffer scripts/build_ios.sh 2>&1 | ts
           elif [ -n "${CAFFE2_USE_ANACONDA}" ]; then
             # All conda build logic should be in scripts/build_anaconda.sh
-            scripts/build_anaconda.sh 2>&1 | ts
+            unbuffer scripts/build_anaconda.sh 2>&1 | ts
           else
-            scripts/build_local.sh 2>&1 | ts
+            unbuffer scripts/build_local.sh 2>&1 | ts
           fi
 
           # Show sccache stats if it is running
@@ -579,7 +586,7 @@ jobs:
           export COMMIT_DOCKER_IMAGE=${DOCKER_IMAGE}-${CIRCLE_SHA1}
           echo "DOCKER_IMAGE: "${COMMIT_DOCKER_IMAGE}
           docker pull ${COMMIT_DOCKER_IMAGE} >/dev/null
-          id=$(docker run --runtime=nvidia -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE})
+          export id=$(docker run --runtime=nvidia -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE})
 
           docker cp $id:/var/lib/jenkins/workspace/env /home/circleci/project/env
           # This IAM user allows write access to S3 bucket for perf test numbers
@@ -587,7 +594,8 @@ jobs:
           echo "declare -x AWS_SECRET_ACCESS_KEY=${CIRCLECI_AWS_SECRET_KEY_FOR_PERF_TEST_S3_BUCKET_V2}" >> /home/circleci/project/env
           docker cp /home/circleci/project/env $id:/var/lib/jenkins/workspace/env
 
-          ((echo "export JOB_BASE_NAME=${JOB_BASE_NAME}" && echo "source ./workspace/env" && echo 'sudo chown -R jenkins workspace && cd workspace && .jenkins/pytorch/short-perf-test-gpu.sh') | docker exec -u jenkins -i "$id" bash) 2>&1 | ts
+          export COMMAND='((echo "export JOB_BASE_NAME=${JOB_BASE_NAME}" && echo "source ./workspace/env" && echo "sudo chown -R jenkins workspace && cd workspace && .jenkins/pytorch/short-perf-test-gpu.sh") | docker exec -u jenkins -i "$id" bash) 2>&1'
+          echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts
 
   pytorch_doc_push:
     environment:
@@ -611,7 +619,7 @@ jobs:
           export COMMIT_DOCKER_IMAGE=${DOCKER_IMAGE}-${CIRCLE_SHA1}
           echo "DOCKER_IMAGE: "${COMMIT_DOCKER_IMAGE}
           docker pull ${COMMIT_DOCKER_IMAGE} >/dev/null
-          id=$(docker run -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE})
+          export id=$(docker run -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE})
 
           cat >/home/circleci/project/doc_push_script.sh <<EOL
           # =================== The following code will be executed inside Docker container ===================
@@ -663,7 +671,8 @@ jobs:
           chmod +x /home/circleci/project/doc_push_script.sh
           docker cp /home/circleci/project/doc_push_script.sh $id:/var/lib/jenkins/workspace/doc_push_script.sh
 
-          ((echo "export JOB_BASE_NAME=${JOB_BASE_NAME}" && echo "source ./workspace/env" && echo 'sudo chown -R jenkins workspace && cd workspace && ./doc_push_script.sh') | docker exec -u jenkins -i "$id" bash) 2>&1 | ts
+          export COMMAND='((echo "export JOB_BASE_NAME=${JOB_BASE_NAME}" && echo "source ./workspace/env" && echo "sudo chown -R jenkins workspace && cd workspace && ./doc_push_script.sh") | docker exec -u jenkins -i "$id" bash) 2>&1'
+          echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts
 
   pytorch_macos_10_13_py3_build:
     macos:
@@ -694,7 +703,7 @@ jobs:
 
             git submodule sync && git submodule update -q --init
             chmod a+x .jenkins/pytorch/macos-build.sh
-            .jenkins/pytorch/macos-build.sh 2>&1 | ts
+            unbuffer .jenkins/pytorch/macos-build.sh 2>&1 | ts
 
             mkdir -p /Users/distiller/pytorch-ci-env/workspace
             cp -r /Users/distiller/project/. /Users/distiller/pytorch-ci-env/workspace
@@ -729,7 +738,7 @@ jobs:
             cp -r /Users/distiller/pytorch-ci-env/workspace/. /Users/distiller/project
 
             chmod a+x .jenkins/pytorch/macos-test.sh
-            .jenkins/pytorch/macos-test.sh 2>&1 | ts
+            unbuffer .jenkins/pytorch/macos-test.sh 2>&1 | ts
 
   pytorch_macos_10_13_cuda9_2_cudnn7_py3_build:
     macos:
@@ -776,7 +785,7 @@ jobs:
 
             git submodule sync && git submodule update -q --init
             chmod a+x .jenkins/pytorch/macos-build.sh
-            .jenkins/pytorch/macos-build.sh 2>&1 | ts
+            unbuffer .jenkins/pytorch/macos-build.sh 2>&1 | ts
 
   caffe2_py2_cuda8_0_cudnn6_ubuntu16_04_build:
     environment: