Rollup merge of #40780 - aidanhs:aphs-cache-git-modules, r=alexcrichton

Attempt to cache git modules

Partial resolution of #40772, appveyor remains to be done once travis looks like it's working ok.

The approach in this PR is based on the `--reference` flag to `git-clone`/`git-submodule --update` and is a compromise based on the current limitations of the tools we're using.

The ideal would be:
1. have a cached pristine copy of rust-lang/rust master in `$HOME/rustsrc` with all submodules initialised
2. clone the PR branch with `git clone --recurse-submodules --reference $HOME/rustsrc git@github.com:rust-lang/rust.git`

This would (in the nonexistent ideal world) use the pristine copy as an object cache for the top level repo and all submodules, transferring over the network only the changes on the branch. Unfortunately, a) there is no way to manually control the initial clone with travis and b) even if there was, cloned submodules don't use the submodules of the reference as an object cache. So the steps we end up with are:

1. have a cached pristine copy of rust-lang/rust master in `$HOME/rustsrc` with all submodules initialised
2. have a cloned PR branch
3. extract the path of each submodule, and explicitly `git submodule update --init --reference $HOME/rustsrc/$module $module` (i.e. point directly to the location of the pristine submodule repo) for each one

I've also taken some care to make this forward compatible, both for adding and removing submodules.

r? @alexcrichton

.travis.yml

@@ -133,13 +133,14 @@ before_script:
 script:
   - >
       if [ "$ALLOW_PR" = "" ] && [ "$TRAVIS_BRANCH" != "auto" ]; then
-          echo skipping, not a full build;
-      elif [ "$TRAVIS_OS_NAME" = "osx" ]; then
-          travis_retry stamp sh -c 'git submodule deinit -f . && git submodule update --init' &&
-          stamp src/ci/run.sh;
+          echo skipping, not a full build
       else
-          travis_retry stamp sh -c 'git submodule deinit -f . && git submodule update --init' &&
-          stamp src/ci/docker/run.sh $IMAGE;
+          stamp src/ci/init_repo.sh . "$HOME/rustsrc" &&
+          if [ "$TRAVIS_OS_NAME" = "osx" ]; then
+              stamp src/ci/run.sh;
+          else
+              stamp src/ci/docker/run.sh $IMAGE;
+          fi
       fi
 
 after_success:
@@ -169,13 +170,21 @@ after_failure:
   - dmesg | grep -i kill
 
 # Save tagged docker images we created and load them if they're available
+# Travis saves caches whether the build failed or not, nuke rustsrc if
+# the failure was while updating it (as it may be in an bad state)
+# https://github.com/travis-ci/travis-ci/issues/4472
 before_cache:
   - docker history -q rust-ci |
     grep -v missing |
     xargs docker save |
     gzip > $HOME/docker/rust-ci.tar.gz
+  - if [ ! -f $HOME/rustsrc/cache_valid1 ]; then
+        echo "WARNING rustsrc cache was invalid when saving";
+        rm -rf $HOME/rustsrc && mkdir $HOME/rustsrc;
+    fi
 before_install:
   - zcat $HOME/docker/rust-ci.tar.gz | docker load || true
+  - mkdir -p $HOME/rustsrc
 
 notifications:
   email: false
@@ -183,6 +192,7 @@ notifications:
 cache:
   directories:
     - $HOME/docker
+    - $HOME/rustsrc
 
 before_deploy:
   - mkdir -p deploy/$TRAVIS_COMMIT

appveyor.yml

@@ -141,7 +141,8 @@ install:
   - set SCCACHE_ERROR_LOG=%CD%/sccache.log
 
 test_script:
-  - appveyor-retry sh -c 'git submodule deinit -f . && git submodule update --init'
+  - mkdir C:\cache\rustsrc
+  - sh src/ci/init_repo.sh . /c/cache/rustsrc
   - set SRC=.
   - set NO_CCACHE=1
   - sh src/ci/run.sh
@@ -150,6 +151,7 @@ on_failure:
   - cat %CD%/sccache.log
 
 cache:
+  - C:\cache\rustsrc
   - "build/i686-pc-windows-msvc/llvm -> src/rustllvm/llvm-rebuild-trigger"
   - "build/x86_64-pc-windows-msvc/llvm -> src/rustllvm/llvm-rebuild-trigger"
   - "i686-pc-windows-msvc/llvm -> src/rustllvm/llvm-rebuild-trigger"

src/ci/docker/run.sh

@@ -57,6 +57,7 @@ exec docker \
   --env DEPLOY_ALT=$DEPLOY_ALT \
   --env LOCAL_USER_ID=`id -u` \
   --volume "$HOME/.cargo:/cargo" \
+  --volume "$HOME/rustsrc:$HOME/rustsrc" \
   --privileged \
   --rm \
   rust-ci \

src/ci/init_repo.sh

@@ -0,0 +1,71 @@
+#!/bin/bash
+# Copyright 2016 The Rust Project Developers. See the COPYRIGHT
+# file at the top-level directory of this distribution and at
+# http://rust-lang.org/COPYRIGHT.
+#
+# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+# option. This file may not be copied, modified, or distributed
+# except according to those terms.
+
+set -o errexit
+set -o pipefail
+set -o nounset
+
+set -o xtrace
+
+ci_dir=$(cd $(dirname $0) && pwd)
+. "$ci_dir/shared.sh"
+
+REPO_DIR="$1"
+CACHE_DIR="$2"
+
+cache_src_dir="$CACHE_DIR/src"
+# If the layout of the cache directory changes, bump the number here
+# (and anywhere else this file is referenced) so the cache is wiped
+cache_valid_file="$CACHE_DIR/cache_valid1"
+
+if [ ! -d "$REPO_DIR" -o ! -d "$REPO_DIR/.git" ]; then
+    echo "Error: $REPO_DIR does not exist or is not a git repo"
+    exit 1
+fi
+cd $REPO_DIR
+if [ ! -d "$CACHE_DIR" ]; then
+    echo "Error: $CACHE_DIR does not exist or is not an absolute path"
+    exit 1
+fi
+
+# Wipe the cache if it's not valid, or mark it as invalid while we update it
+if [ ! -f "$cache_valid_file" ]; then
+    rm -rf "$CACHE_DIR" && mkdir "$CACHE_DIR"
+else
+    rm "$cache_valid_file"
+fi
+
+# Update the cache (a pristine copy of the rust source master)
+if [ ! -d "$cache_src_dir/.git" ]; then
+    retry sh -c "rm -rf $cache_src_dir && mkdir -p $cache_src_dir && \
+        git clone https://github.com/rust-lang/rust.git $cache_src_dir"
+fi
+retry sh -c "cd $cache_src_dir && git reset --hard && git pull"
+retry sh -c "cd $cache_src_dir && \
+    git submodule deinit -f . && git submodule sync && git submodule update --init"
+
+# Cache was updated without errors, mark it as valid
+touch "$cache_valid_file"
+
+# Update the submodules of the repo we're in, using the pristine repo as
+# a cache for any object files
+# No, `git submodule foreach` won't work:
+# http://stackoverflow.com/questions/12641469/list-submodules-in-a-git-repository
+modules="$(git config --file .gitmodules --get-regexp '\.path$' | cut -d' ' -f2)"
+for module in $modules; do
+    if [ ! -d "$cache_src_dir/$module" ]; then
+        echo "WARNING: $module not found in pristine repo"
+        retry sh -c "git submodule deinit -f $module && git submodule update --init $module"
+        continue
+    fi
+    retry sh -c "git submodule deinit -f $module && \
+        git submodule update --init --reference $cache_src_dir/$module $module"
+done

src/ci/shared.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/bin/false
 # Copyright 2016 The Rust Project Developers. See the COPYRIGHT
 # file at the top-level directory of this distribution and at
 # http://rust-lang.org/COPYRIGHT.
@@ -9,13 +9,16 @@
 # option. This file may not be copied, modified, or distributed
 # except according to those terms.
 
+# This file is intended to be sourced with `. shared.sh` or
+# `source shared.sh`, hence the invalid shebang and not being
+# marked as an executable file in git.
+
 # See http://unix.stackexchange.com/questions/82598
 function retry {
+  echo "Attempting with retry:" "$@"
   local n=1
   local max=5
-  local delay=15
   while true; do
-    echo "Attempting:" "$@"
     "$@" && break || {
       if [[ $n -lt $max ]]; then
         ((n++))