diff --git a/images/docker-stacks-foundation/start.sh b/images/docker-stacks-foundation/start.sh
index 688c4586..76419b60 100755
--- a/images/docker-stacks-foundation/start.sh
+++ b/images/docker-stacks-foundation/start.sh
@@ -121,9 +121,6 @@ if [ "$(id -u)" == 0 ]; then
         done
     fi
 
-    # Update potentially outdated environment variables since the image build
-    export XDG_CACHE_HOME="/home/${NB_USER}/.cache"
-
     # Prepend ${CONDA_DIR}/bin to sudo secure_path
     sed -r "s#Defaults\s+secure_path\s*=\s*\"?([^\"]+)\"?#Defaults secure_path=\"${CONDA_DIR}/bin:\1\"#" /etc/sudoers | grep secure_path > /etc/sudoers.d/path
 
diff --git a/images/scipy-notebook/Dockerfile b/images/scipy-notebook/Dockerfile
index 8b687eb7..55bf7eb5 100644
--- a/images/scipy-notebook/Dockerfile
+++ b/images/scipy-notebook/Dockerfile
@@ -74,9 +74,6 @@ RUN git clone https://github.com/PAIR-code/facets && \
     fix-permissions "${CONDA_DIR}" && \
     fix-permissions "/home/${NB_USER}"
 
-# Import matplotlib the first time to build the font cache.
-ENV XDG_CACHE_HOME="/home/${NB_USER}/.cache/"
-
 RUN MPLBACKEND=Agg python -c "import matplotlib.pyplot" && \
     fix-permissions "/home/${NB_USER}"