hazza/docker-stacks
1
0
Fork 0
mirror of https://github.com/jupyter/docker-stacks.git synced 2025-10-17 15:02:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

Make Spark scripts more robust: support preview versions and Spark 4 output

Browse Source
This commit is contained in:
Ayaz Salikhov
2024-10-19 19:08:04 +01:00
parent b6b22e1546
commit c4cb04ec37
2 changed files with 17 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
tagging/taggers.py
View File

@@ -122,8 +122,12 @@ class SparkVersionTagger(TaggerInterface):
SPARK_VERSION_LINE_PREFIX = r" /___/ .__/\_,_/_/ /_/\_\ version"
spark_version = _get_program_version(container, "spark-submit")
version_line = spark_version.split("\n")[4]
assert version_line.startswith(SPARK_VERSION_LINE_PREFIX)
version_line = next(
filter(
lambda line: line.startswith(SPARK_VERSION_LINE_PREFIX),
spark_version.split("\n"),
)
)
return "spark-" + version_line.split(" ")[-1]