hazza/docker-stacks
1
0
Fork 0
mirror of https://github.com/jupyter/docker-stacks.git synced 2025-10-12 04:22:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

Only consider r'^spark-\d' as possible spark version

Browse Source
This commit is contained in:
Ayaz Salikhov
2025-05-12 09:14:29 +01:00
parent bd63eaa408
commit 388066466d
1 changed files with 4 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
images/pyspark-notebook/setup_spark.py
View File

@@ -9,6 +9,7 @@
import argparse import argparse
import logging import logging
import os import os
import re
import subprocess import subprocess
from pathlib import Path from pathlib import Path
@@ -33,11 +34,13 @@ def get_latest_spark_version() -> str:
""" """
LOGGER.info("Downloading Spark versions information") LOGGER.info("Downloading Spark versions information")
all_refs = get_all_refs("https://archive.apache.org/dist/spark/") all_refs = get_all_refs("https://archive.apache.org/dist/spark/")
LOGGER.info(f"All refs: {all_refs}")
versions = [ versions = [
ref.removeprefix("spark-").removesuffix("/") ref.removeprefix("spark-").removesuffix("/")
for ref in all_refs for ref in all_refs
if ref.startswith("spark-") and "incubating" not in ref if re.match(r"^spark-\d", ref) is not None and "incubating" not in ref
] ]
LOGGER.info(f"Available versions: {versions}")
# Compare versions semantically # Compare versions semantically
def version_array(ver: str) -> tuple[int, int, int, str]: def version_array(ver: str) -> tuple[int, int, int, str]: