hazza/docker-stacks
1
0
Fork 0
mirror of https://github.com/jupyter/docker-stacks.git synced 2025-10-13 21:12:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

Make Spark scripts more robust: support preview versions and Spark 4 output

Browse Source
This commit is contained in:
Ayaz Salikhov
2024-10-19 19:08:04 +01:00
parent b6b22e1546
commit c4cb04ec37
2 changed files with 17 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
images/pyspark-notebook/setup_spark.py
View File

@@ -38,10 +38,18 @@ def get_latest_spark_version() -> str:
for ref in all_refs for ref in all_refs
if ref.startswith("spark-") and "incubating" not in ref and "preview" not in ref if ref.startswith("spark-") and "incubating" not in ref and "preview" not in ref
] ]
# Compare versions semantically # Compare versions semantically
latest_version = max( def version_array(ver: str) -> tuple[int, int, int, str]:
stable_versions, key=lambda ver: [int(sub_ver) for sub_ver in ver.split(".")] # 3.5.3 -> [3, 5, 3, ""]
) # 4.0.0-preview2 -> [4, 0, 0, "preview2"]
arr = ver.split(".")
assert len(arr) == 3, arr
major, minor = int(arr[0]), int(arr[1])
patch, _, preview = arr[2].partition("-")
return (major, minor, int(patch), preview)
latest_version = max(stable_versions, key=lambda ver: version_array(ver))
LOGGER.info(f"Latest version: {latest_version}") LOGGER.info(f"Latest version: {latest_version}")
return latest_version return latest_version

8
tagging/taggers.py
View File

@@ -122,8 +122,12 @@ class SparkVersionTagger(TaggerInterface):
SPARK_VERSION_LINE_PREFIX = r" /___/ .__/\_,_/_/ /_/\_\ version" SPARK_VERSION_LINE_PREFIX = r" /___/ .__/\_,_/_/ /_/\_\ version"
spark_version = _get_program_version(container, "spark-submit") spark_version = _get_program_version(container, "spark-submit")
version_line = spark_version.split("\n")[4] version_line = next(
assert version_line.startswith(SPARK_VERSION_LINE_PREFIX) filter(
lambda line: line.startswith(SPARK_VERSION_LINE_PREFIX),
spark_version.split("\n"),
)
)
return "spark-" + version_line.split(" ")[-1] return "spark-" + version_line.split(" ")[-1]