diff --git a/.travis.yml b/.travis.yml
index 817d1ace7f87..075d3756b416 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -2,7 +2,12 @@ sudo: true
 
 language: cpp
 
-cache: ccache
+cache:
+  directories:
+    - $HOME/.ccache
+    - $HOME/.cache/pip
+    - $HOME/.mxnet
+    - $HOME/Library/Caches/Homebrew
 
 os:
   - osx
@@ -17,7 +22,7 @@ before_install:
   - export PYTHONPATH=${PYTHONPATH}:${PWD}/python
 
 install:
-  - brew install ccache
+  - HOMEBREW_NO_AUTO_UPDATE=1 brew install ccache
   - export PATH="/usr/local/opt/ccache/libexec:$PATH"
   - source ci/travis/install.sh
 
@@ -29,4 +34,7 @@ script:
   - export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0
   - mv make/osx.mk config.mk
   - make -j 2
-  - python -m nose --verbose tests/python/unittest/
+  # We ignore several tests to avoid possible timeouts on large PRs.
+  # This lowers our test coverage, but is required for consistent Travis runs.
+  # These tests will be tested in a variety of environments in Jenkins based tests.
+  - python -m nose --with-timer --timer-top-n 20 --exclude-test=test_sparse_operator.test_elemwise_binary_ops --exclude-test=test_gluon_model_zoo.test_models --exclude-test=test_random.test_shuffle --exclude-test=test_operator.test_broadcast_binary_op --exclude-test=test_operator.test_pick --exclude-test=test_profiler.test_continuous_profile_and_instant_marker --exclude-test=test_metric_perf.test_metric_performance --exclude-test=test_operator.test_order --verbose tests/python/unittest/
diff --git a/ci/travis/install.sh b/ci/travis/install.sh
index 16db601211a5..ae959767133a 100644
--- a/ci/travis/install.sh
+++ b/ci/travis/install.sh
@@ -17,14 +17,10 @@
 # specific language governing permissions and limitations
 # under the License.
 
+# Disable brew auto-update to avoid long running updates while running tests in CI.
+export HOMEBREW_NO_AUTO_UPDATE=1
+
 if [ ${TRAVIS_OS_NAME} == "osx" ]; then
-    brew update
     brew install opencv
-    brew install python3
-    brew install fftw
-    brew install libpng
-    brew install ImageMagick
-    brew install swig
-    python -m pip install --user nose numpy cython scipy requests mock
-    python3 -m pip install --user nose numpy cython scipy requests mock
+    python -m pip install --user nose numpy cython scipy requests mock nose-timer nose-exclude
 fi