Rollup merge of #50606 - kennytm:retry-docker-cache, r=alexcrichton

Retry when downloading the Docker cache.

As a safety measure, prevent spuriously needing to rebuild the docker image in case the network was reset while downloading.

Also, adjusted the retry function to insert a sleep between retries, because retrying immediately will often just hit the same issue.

src/ci/docker/run.sh

@@ -36,8 +36,10 @@ if [ -f "$docker_dir/$image/Dockerfile" ]; then
       s3url="s3://$SCCACHE_BUCKET/docker/$cksum"
       url="https://s3-us-west-1.amazonaws.com/$SCCACHE_BUCKET/docker/$cksum"
       echo "Attempting to download $s3url"
+      rm -f /tmp/rustci_docker_cache
       set +e
-      loaded_images=$(curl $url | docker load | sed 's/.* sha/sha/')
+      retry curl -f -L -C - -o /tmp/rustci_docker_cache "$url"
+      loaded_images=$(docker load -i /tmp/rustci_docker_cache | sed 's/.* sha/sha/')
       set -e
       echo "Downloaded containers:\n$loaded_images"
     fi

src/ci/shared.sh

@@ -21,11 +21,12 @@ function retry {
   while true; do
     "$@" && break || {
       if [[ $n -lt $max ]]; then
+        sleep $n  # don't retry immediately
         ((n++))
         echo "Command failed. Attempt $n/$max:"
       else
         echo "The command has failed after $n attempts."
-        exit 1
+        return 1
       fi
     }
   done