diff --git a/.travis.yml b/.travis.yml
index beaa3b53e..c1c397967 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -16,7 +16,7 @@ env:
   - MODULE=openml
   matrix:
   - DISTRIB="conda" PYTHON_VERSION="3.5" SKLEARN_VERSION="0.21.2"
-  - DISTRIB="conda" PYTHON_VERSION="3.6" SKLEARN_VERSION="0.21.2"
+  - DISTRIB="conda" PYTHON_VERSION="3.6" SKLEARN_VERSION="0.21.2" TEST_DIST="true"
   - DISTRIB="conda" PYTHON_VERSION="3.7" SKLEARN_VERSION="0.21.2" RUN_FLAKE8="true" SKIP_TESTS="true"
   - DISTRIB="conda" PYTHON_VERSION="3.7" SKLEARN_VERSION="0.21.2" COVERAGE="true" DOCPUSH="true"
   - DISTRIB="conda" PYTHON_VERSION="3.7" SKLEARN_VERSION="0.20.2"
diff --git a/Makefile b/Makefile
index c36acbe9f..165bcea80 100644
--- a/Makefile
+++ b/Makefile
@@ -9,7 +9,7 @@ all: clean inplace test
 
 clean:
 	$(PYTHON) setup.py clean
-	rm -rf dist
+	rm -rf dist openml.egg-info
 
 in: inplace # just a shortcut
 inplace:
diff --git a/ci_scripts/install.sh b/ci_scripts/install.sh
index 5c338fe5e..15cb84bca 100644
--- a/ci_scripts/install.sh
+++ b/ci_scripts/install.sh
@@ -32,15 +32,24 @@ source activate testenv
 if [[ -v SCIPY_VERSION ]]; then
     conda install --yes scipy=$SCIPY_VERSION
 fi
-
 python --version
-pip install -e '.[test]'
+
+if [[ "$TEST_DIST" == "true" ]]; then
+    pip install twine nbconvert jupyter_client matplotlib pytest pytest-xdist pytest-timeout \
+        nbformat oslo.concurrency flaky
+    python setup.py sdist
+    # Find file which was modified last as done in https://stackoverflow.com/a/4561987
+    dist=`find dist -type f -printf '%T@ %p\n' | sort -n | tail -1 | cut -f2- -d" "`
+    echo "Installing $dist"
+    pip install "$dist"
+    twine check "$dist"
+else
+    pip install -e '.[test]'
+fi
+
 python -c "import numpy; print('numpy %s' % numpy.__version__)"
 python -c "import scipy; print('scipy %s' % scipy.__version__)"
 
-if [[ "$DOCTEST" == "true" ]]; then
-    pip install sphinx_bootstrap_theme
-fi
 if [[ "$DOCPUSH" == "true" ]]; then
     conda install --yes gxx_linux-64 gcc_linux-64 swig
     pip install -e '.[examples,examples_unix]'
diff --git a/ci_scripts/test.sh b/ci_scripts/test.sh
index 8659a105b..5ffced544 100644
--- a/ci_scripts/test.sh
+++ b/ci_scripts/test.sh
@@ -15,14 +15,8 @@ run_tests() {
 
     cwd=`pwd`
     test_dir=$cwd/tests
-    doctest_dir=$cwd/doc
 
     cd $TEST_DIR
-    if [[ "$EXAMPLES" == "true" ]]; then
-        pytest -sv $test_dir/test_examples/
-    elif [[ "$DOCTEST" == "true" ]]; then
-        python -m doctest $doctest_dir/usage.rst
-    fi
 
     if [[ "$COVERAGE" == "true" ]]; then
         PYTEST_ARGS='--cov=openml'
diff --git a/doc/progress.rst b/doc/progress.rst
index b7d4b4992..52fdf283d 100644
--- a/doc/progress.rst
+++ b/doc/progress.rst
@@ -11,6 +11,8 @@ Changelog
 
 * FIX #873: Fixes an issue which resulted in incorrect URLs when printing OpenML objects after
   switching the server
+* MAINT #767: Source distribution installation is now unit-tested.
+* MAINT #865: OpenML no longer bundles test files in the source distribution.
 
 0.10.2
 ~~~~~~
diff --git a/setup.py b/setup.py
index 9c9766636..46e4ae8b2 100644
--- a/setup.py
+++ b/setup.py
@@ -35,7 +35,12 @@
                      "Source Code": "https://github.com/openml/openml-python"
                  },
                  version=version,
-                 packages=setuptools.find_packages(),
+                 # Make sure to remove stale files such as the egg-info before updating this:
+                 # https://stackoverflow.com/a/26547314
+                 packages=setuptools.find_packages(
+                     include=['openml.*', 'openml'],
+                     exclude=["*.tests", "*.tests.*", "tests.*", "tests"],
+                 ),
                  package_data={'': ['*.txt', '*.md']},
                  python_requires=">=3.5",
                  install_requires=[