hazza/docker-stacks
1
0
Fork 0
mirror of https://github.com/jupyter/docker-stacks.git synced 2025-10-10 19:42:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix docs: we're not installing stable version of spark anymore (#2165)

Browse Source
This commit is contained in:
Ayaz Salikhov
2024-10-29 10:11:31 +00:00
committed by GitHub
parent f74a764584
commit 03e5fe572d
3 changed files with 6 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
images/pyspark-notebook/setup_spark.py
View File

@@ -29,11 +29,11 @@ def get_all_refs(url: str) -> list[str]:
def get_latest_spark_version() -> str:
"""
Returns the last stable version of Spark using spark archive
Returns the last version of Spark using spark archive
"""
LOGGER.info("Downloading Spark versions information")
all_refs = get_all_refs("https://archive.apache.org/dist/spark/")
stable_versions = [
versions = [
ref.removeprefix("spark-").removesuffix("/")
for ref in all_refs
if ref.startswith("spark-") and "incubating" not in ref
@@ -49,7 +49,7 @@ def get_latest_spark_version() -> str:
patch, _, preview = arr[2].partition("-")
return (major, minor, int(patch), preview)
latest_version = max(stable_versions, key=lambda ver: version_array(ver))
latest_version = max(versions, key=lambda ver: version_array(ver))
LOGGER.info(f"Latest version: {latest_version}")
return latest_version