diff --git a/.github/deploy.sh b/.github/deploy.sh
index aa76f8f4e410..1d206e61167e 100755
--- a/.github/deploy.sh
+++ b/.github/deploy.sh
@@ -1,8 +1,7 @@
 #!/bin/bash
 # Automatically deploy on gh-pages
 
-set -e
-set -x
+set -ex
 
 SOURCE_BRANCH="master"
 TARGET_BRANCH="gh-pages"
diff --git a/.travis.yml b/.travis.yml
index 077a17e0bace..22dc2f572ae0 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -29,28 +29,27 @@ install:
   - nvm use stable
   - npm install remark-cli remark-lint
 
+matrix:
+  include:
+    - env: BASE_TESTS=true # runs the base tests
+    - env: INTEGRATION=rust-lang/cargo
+    - env: INTEGRATION=rust-lang-nursery/rand
+  allow_failures:
+    - env: INTEGRATION=rust-lang/cargo
+    - env: INTEGRATION=rust-lang-nursery/rand
+
 script:
- - PATH=$PATH:./node_modules/.bin
- - remark -f *.md > /dev/null
- - set -e
- - cargo build --features debugging
- - cargo test --features debugging
- - mkdir -p ~/rust/cargo/bin
- - cp target/debug/cargo-clippy ~/rust/cargo/bin/cargo-clippy
- - cp target/debug/clippy-driver ~/rust/cargo/bin/clippy-driver
- - PATH=$PATH:~/rust/cargo/bin cargo clippy --all -- -D clippy
- - cd clippy_workspace_tests && PATH=$PATH:~/rust/cargo/bin cargo clippy -- -D clippy && cd ..
- - cd clippy_workspace_tests/src && PATH=$PATH:~/rust/cargo/bin cargo clippy -- -D clippy && cd ../..
- - cd clippy_workspace_tests/subcrate && PATH=$PATH:~/rust/cargo/bin cargo clippy -- -D clippy && cd ../..
- - cd clippy_workspace_tests/subcrate/src && PATH=$PATH:~/rust/cargo/bin cargo clippy -- -D clippy && cd ../../..
- - PATH=$PATH:~/rust/cargo/bin cargo clippy --manifest-path=clippy_workspace_tests/Cargo.toml -- -D clippy
- - cd clippy_workspace_tests/subcrate && PATH=$PATH:~/rust/cargo/bin cargo clippy --manifest-path=../Cargo.toml -- -D clippy && cd ../..
- - set +e
+  - |
+    if [ -z ${INTEGRATION} ]; then
+      ./ci/base-tests.sh
+    else
+      ./ci/integration-tests.sh
+    fi
 
 after_success: |
   #!/bin/bash
   if [ $(uname) == Linux ]; then
-    set -e
+    set -ex
     ./.github/deploy.sh
     # trigger rebuild of the clippy-service, to keep it up to date with clippy itself
     if [ "$TRAVIS_PULL_REQUEST" == "false" ] &&
diff --git a/ci/base-tests.sh b/ci/base-tests.sh
new file mode 100755
index 000000000000..daec740212db
--- /dev/null
+++ b/ci/base-tests.sh
@@ -0,0 +1,15 @@
+set -ex
+PATH=$PATH:./node_modules/.bin
+remark -f *.md > /dev/null
+cargo build --features debugging
+cargo test --features debugging
+mkdir -p ~/rust/cargo/bin
+cp target/debug/cargo-clippy ~/rust/cargo/bin/cargo-clippy
+cp target/debug/clippy-driver ~/rust/cargo/bin/clippy-driver
+PATH=$PATH:~/rust/cargo/bin cargo clippy --all -- -D clippy
+cd clippy_workspace_tests && PATH=$PATH:~/rust/cargo/bin cargo clippy -- -D clippy && cd ..
+cd clippy_workspace_tests/src && PATH=$PATH:~/rust/cargo/bin cargo clippy -- -D clippy && cd ../..
+cd clippy_workspace_tests/subcrate && PATH=$PATH:~/rust/cargo/bin cargo clippy -- -D clippy && cd ../..
+cd clippy_workspace_tests/subcrate/src && PATH=$PATH:~/rust/cargo/bin cargo clippy -- -D clippy && cd ../../..
+PATH=$PATH:~/rust/cargo/bin cargo clippy --manifest-path=clippy_workspace_tests/Cargo.toml -- -D clippy
+cd clippy_workspace_tests/subcrate && PATH=$PATH:~/rust/cargo/bin cargo clippy --manifest-path=../Cargo.toml -- -D clippy && cd ../..
diff --git a/ci/integration-tests.sh b/ci/integration-tests.sh
new file mode 100755
index 000000000000..2fe15179b5f4
--- /dev/null
+++ b/ci/integration-tests.sh
@@ -0,0 +1,25 @@
+set -x
+cargo install --force
+
+echo "Running integration test for crate ${INTEGRATION}"
+
+git clone --depth=1 https://github.com/${INTEGRATION}.git checkout
+cd checkout
+
+function check() {
+  cargo clippy --all &> clippy_output
+  cat clippy_output
+  ! cat clippy_output | grep -q "internal compiler error"
+  if [[ $? != 0 ]]; then
+    return 1
+  fi
+}
+
+case ${INTEGRATION} in
+  rust-lang/cargo)
+    check
+    ;;
+  *)
+    check
+    ;;
+esac