Bump to spark 2.4.5 + minor improvements

pyspark-notebook/test/test_spark.py

@@ -0,0 +1,19 @@
+# Copyright (c) Jupyter Development Team.
+# Distributed under the terms of the Modified BSD License.
+import time
+import logging
+
+import pytest
+
+LOGGER = logging.getLogger(__name__)
+
+def test_spark_shell(container):
+    """Checking if Spark (spark-shell) is running properly"""
+    c = container.run(
+        tty=True,
+        command=['start.sh', 'bash', '-c', 'spark-shell <<< "1+1"']
+    )
+    c.wait(timeout=30)
+    logs = c.logs(stdout=True).decode('utf-8')
+    LOGGER.debug(logs)
+    assert 'res0: Int = 2' in logs