| #!/usr/bin/env python |
| |
| """ |
| This script is shared between SDL2, SDL2_image, SDL2_mixer and SDL2_ttf. |
| Don't specialize this script for doing project-specific modifications. |
| Rather, modify release-info.json. |
| """ |
| |
| import argparse |
| import collections |
| from collections.abc import Callable |
| import contextlib |
| import datetime |
| import fnmatch |
| import glob |
| import io |
| import json |
| import logging |
| import multiprocessing |
| import os |
| from pathlib import Path |
| import platform |
| import re |
| import shutil |
| import subprocess |
| import sys |
| import tarfile |
| import tempfile |
| import textwrap |
| import typing |
| import zipfile |
| |
| logger = logging.getLogger(__name__) |
| |
| |
| GIT_HASH_FILENAME = ".git-hash" |
| |
| |
| def safe_isotime_to_datetime(str_isotime: str) -> datetime.datetime: |
| try: |
| return datetime.datetime.fromisoformat(str_isotime) |
| except ValueError: |
| pass |
| logger.warning("Invalid iso time: %s", str_isotime) |
| if str_isotime[-6:-5] in ("+", "-"): |
| # Commits can have isotime with invalid timezone offset (e.g. "2021-07-04T20:01:40+32:00") |
| modified_str_isotime = str_isotime[:-6] + "+00:00" |
| try: |
| return datetime.datetime.fromisoformat(modified_str_isotime) |
| except ValueError: |
| pass |
| raise ValueError(f"Invalid isotime: {str_isotime}") |
| |
| |
| class VsArchPlatformConfig: |
| def __init__(self, arch: str, platform: str, configuration: str): |
| self.arch = arch |
| self.platform = platform |
| self.configuration = configuration |
| |
| def configure(self, s: str) -> str: |
| return s.replace("@ARCH@", self.arch).replace("@PLATFORM@", self.platform).replace("@CONFIGURATION@", self.configuration) |
| |
| |
| @contextlib.contextmanager |
| def chdir(path): |
| original_cwd = os.getcwd() |
| try: |
| os.chdir(path) |
| yield |
| finally: |
| os.chdir(original_cwd) |
| |
| |
| class Executer: |
| def __init__(self, root: Path, dry: bool=False): |
| self.root = root |
| self.dry = dry |
| |
| def run(self, cmd, cwd=None, env=None): |
| logger.info("Executing args=%r", cmd) |
| sys.stdout.flush() |
| if not self.dry: |
| subprocess.run(cmd, check=True, cwd=cwd or self.root, env=env, text=True) |
| |
| def check_output(self, cmd, cwd=None, dry_out=None, env=None, text=True): |
| logger.info("Executing args=%r", cmd) |
| sys.stdout.flush() |
| if self.dry: |
| return dry_out |
| return subprocess.check_output(cmd, cwd=cwd or self.root, env=env, text=text) |
| |
| |
| class SectionPrinter: |
| @contextlib.contextmanager |
| def group(self, title: str): |
| print(f"{title}:") |
| yield |
| |
| |
| class GitHubSectionPrinter(SectionPrinter): |
| def __init__(self): |
| super().__init__() |
| self.in_group = False |
| |
| @contextlib.contextmanager |
| def group(self, title: str): |
| print(f"::group::{title}") |
| assert not self.in_group, "Can enter a group only once" |
| self.in_group = True |
| yield |
| self.in_group = False |
| print("::endgroup::") |
| |
| |
| class VisualStudio: |
| def __init__(self, executer: Executer, year: typing.Optional[str]=None): |
| self.executer = executer |
| self.vsdevcmd = self.find_vsdevcmd(year) |
| self.msbuild = self.find_msbuild() |
| |
| @property |
| def dry(self) -> bool: |
| return self.executer.dry |
| |
| VS_YEAR_TO_VERSION = { |
| "2022": 17, |
| "2019": 16, |
| "2017": 15, |
| "2015": 14, |
| "2013": 12, |
| } |
| |
| def find_vsdevcmd(self, year: typing.Optional[str]=None) -> typing.Optional[Path]: |
| vswhere_spec = ["-latest"] |
| if year is not None: |
| try: |
| version = self.VS_YEAR_TO_VERSION[year] |
| except KeyError: |
| logger.error("Invalid Visual Studio year") |
| return None |
| vswhere_spec.extend(["-version", f"[{version},{version+1})"]) |
| vswhere_cmd = ["vswhere"] + vswhere_spec + ["-property", "installationPath"] |
| vs_install_path = Path(self.executer.check_output(vswhere_cmd, dry_out="/tmp").strip()) |
| logger.info("VS install_path = %s", vs_install_path) |
| assert vs_install_path.is_dir(), "VS installation path does not exist" |
| vsdevcmd_path = vs_install_path / "Common7/Tools/vsdevcmd.bat" |
| logger.info("vsdevcmd path = %s", vsdevcmd_path) |
| if self.dry: |
| vsdevcmd_path.parent.mkdir(parents=True, exist_ok=True) |
| vsdevcmd_path.touch(exist_ok=True) |
| assert vsdevcmd_path.is_file(), "vsdevcmd.bat batch file does not exist" |
| return vsdevcmd_path |
| |
| def find_msbuild(self) -> typing.Optional[Path]: |
| vswhere_cmd = ["vswhere", "-latest", "-requires", "Microsoft.Component.MSBuild", "-find", r"MSBuild\**\Bin\MSBuild.exe"] |
| msbuild_path = Path(self.executer.check_output(vswhere_cmd, dry_out="/tmp/MSBuild.exe").strip()) |
| logger.info("MSBuild path = %s", msbuild_path) |
| if self.dry: |
| msbuild_path.parent.mkdir(parents=True, exist_ok=True) |
| msbuild_path.touch(exist_ok=True) |
| assert msbuild_path.is_file(), "MSBuild.exe does not exist" |
| return msbuild_path |
| |
| def build(self, arch_platform: VsArchPlatformConfig, projects: list[Path]): |
| assert projects, "Need at least one project to build" |
| |
| vsdev_cmd_str = f"\"{self.vsdevcmd}\" -arch={arch_platform.arch}" |
| msbuild_cmd_str = " && ".join([f"\"{self.msbuild}\" \"{project}\" /m /p:BuildInParallel=true /p:Platform={arch_platform.platform} /p:Configuration={arch_platform.configuration}" for project in projects]) |
| bat_contents = f"{vsdev_cmd_str} && {msbuild_cmd_str}\n" |
| bat_path = Path(tempfile.gettempdir()) / "cmd.bat" |
| with bat_path.open("w") as f: |
| f.write(bat_contents) |
| |
| logger.info("Running cmd.exe script (%s): %s", bat_path, bat_contents) |
| cmd = ["cmd.exe", "/D", "/E:ON", "/V:OFF", "/S", "/C", f"CALL {str(bat_path)}"] |
| self.executer.run(cmd) |
| |
| |
| class Archiver: |
| def __init__(self, zip_path: typing.Optional[Path]=None, tgz_path: typing.Optional[Path]=None, txz_path: typing.Optional[Path]=None): |
| self._zip_files = [] |
| self._tar_files = [] |
| self._added_files = set() |
| if zip_path: |
| self._zip_files.append(zipfile.ZipFile(zip_path, "w", compression=zipfile.ZIP_DEFLATED)) |
| if tgz_path: |
| self._tar_files.append(tarfile.open(tgz_path, "w:gz")) |
| if txz_path: |
| self._tar_files.append(tarfile.open(txz_path, "w:xz")) |
| |
| @property |
| def added_files(self) -> set[str]: |
| return self._added_files |
| |
| def add_file_data(self, arcpath: str, data: bytes, mode: int, time: datetime.datetime): |
| for zf in self._zip_files: |
| file_data_time = (time.year, time.month, time.day, time.hour, time.minute, time.second) |
| zip_info = zipfile.ZipInfo(filename=arcpath, date_time=file_data_time) |
| zip_info.external_attr = mode << 16 |
| zip_info.compress_type = zipfile.ZIP_DEFLATED |
| zf.writestr(zip_info, data=data) |
| for tf in self._tar_files: |
| tar_info = tarfile.TarInfo(arcpath) |
| tar_info.type = tarfile.REGTYPE |
| tar_info.mode = mode |
| tar_info.size = len(data) |
| tar_info.mtime = int(time.timestamp()) |
| tf.addfile(tar_info, fileobj=io.BytesIO(data)) |
| |
| self._added_files.add(arcpath) |
| |
| def add_symlink(self, arcpath: str, target: str, time: datetime.datetime, files_for_zip): |
| for zf in self._zip_files: |
| file_data_time = (time.year, time.month, time.day, time.hour, time.minute, time.second) |
| for f in files_for_zip: |
| zip_info = zipfile.ZipInfo(filename=f["arcpath"], date_time=file_data_time) |
| zip_info.external_attr = f["mode"] << 16 |
| zip_info.compress_type = zipfile.ZIP_DEFLATED |
| zf.writestr(zip_info, data=f["data"]) |
| for tf in self._tar_files: |
| tar_info = tarfile.TarInfo(arcpath) |
| tar_info.type = tarfile.SYMTYPE |
| tar_info.mode = 0o777 |
| tar_info.mtime = int(time.timestamp()) |
| tar_info.linkname = target |
| tf.addfile(tar_info) |
| |
| self._added_files.update(f["arcpath"] for f in files_for_zip) |
| |
| def add_git_hash(self, commit: str, arcdir: typing.Optional[str]=None, time: typing.Optional[datetime.datetime]=None): |
| arcpath = GIT_HASH_FILENAME |
| if arcdir and arcdir[-1:] != "/": |
| arcpath = f"{arcdir}/{arcpath}" |
| if not time: |
| time = datetime.datetime(year=2024, month=4, day=1) |
| data = f"{commit}\n".encode() |
| self.add_file_data(arcpath=arcpath, data=data, mode=0o100644, time=time) |
| |
| def add_file_path(self, arcpath: str, path: Path): |
| assert path.is_file(), f"{path} should be a file" |
| for zf in self._zip_files: |
| zf.write(path, arcname=arcpath) |
| for tf in self._tar_files: |
| tf.add(path, arcname=arcpath) |
| |
| def add_file_directory(self, arcdirpath: str, dirpath: Path): |
| assert dirpath.is_dir() |
| if arcdirpath and arcdirpath[-1:] != "/": |
| arcdirpath += "/" |
| for f in dirpath.iterdir(): |
| if f.is_file(): |
| arcpath = f"{arcdirpath}{f.name}" |
| logger.debug("Adding %s to %s", f, arcpath) |
| self.add_file_path(arcpath=arcpath, path=f) |
| |
| def close(self): |
| # Archiver is intentionally made invalid after this function |
| del self._zip_files |
| self._zip_files = None |
| del self._tar_files |
| self._tar_files = None |
| |
| def __enter__(self): |
| return self |
| |
| def __exit__(self, type, value, traceback): |
| self.close() |
| |
| |
| class SourceCollector: |
| TreeItem = collections.namedtuple("TreeItem", ("path", "mode", "data", "symtarget", "directory", "time")) |
| def __init__(self, root: Path, commit: str, filter: typing.Optional[Callable[[str], bool]], executer: Executer): |
| self.root = root |
| self.commit = commit |
| self.filter = filter |
| self.executer = executer |
| self._git_contents: typing.Optional[dict[str, SourceCollector.TreeItem]] = None |
| |
| def _get_git_contents(self) -> dict[str, TreeItem]: |
| contents_tgz = subprocess.check_output(["git", "archive", "--format=tar.gz", self.commit, "-o", "/dev/stdout"], cwd=self.root, text=False) |
| tar_archive = tarfile.open(fileobj=io.BytesIO(contents_tgz), mode="r:gz") |
| filenames = tuple(m.name for m in tar_archive if (m.isfile() or m.issym())) |
| |
| file_times = self._get_file_times(paths=filenames) |
| git_contents = {} |
| for ti in tar_archive: |
| if self.filter and not self.filter(ti.name): |
| continue |
| data = None |
| symtarget = None |
| directory = False |
| file_time = None |
| if ti.isfile(): |
| contents_file = tar_archive.extractfile(ti.name) |
| data = contents_file.read() |
| file_time = file_times[ti.name] |
| elif ti.issym(): |
| symtarget = ti.linkname |
| file_time = file_times[ti.name] |
| elif ti.isdir(): |
| directory = True |
| else: |
| raise ValueError(f"{ti.name}: unknown type") |
| git_contents[ti.name] = self.TreeItem(path=ti.name, mode=ti.mode, data=data, symtarget=symtarget, directory=directory, time=file_time) |
| return git_contents |
| |
| @property |
| def git_contents(self) -> dict[str, TreeItem]: |
| if self._git_contents is None: |
| self._git_contents = self._get_git_contents() |
| return self._git_contents |
| |
| def _get_file_times(self, paths: tuple[str, ...]) -> dict[str, datetime.datetime]: |
| dry_out = textwrap.dedent("""\ |
| time=2024-03-14T15:40:25-07:00 |
| |
| M\tCMakeLists.txt |
| """) |
| git_log_out = self.executer.check_output(["git", "log", "--name-status", '--pretty=time=%cI', self.commit], dry_out=dry_out, cwd=self.root).splitlines(keepends=False) |
| current_time = None |
| set_paths = set(paths) |
| path_times: dict[str, datetime.datetime] = {} |
| for line in git_log_out: |
| if not line: |
| continue |
| if line.startswith("time="): |
| current_time = safe_isotime_to_datetime(line.removeprefix("time=")) |
| continue |
| mod_type, file_paths = line.split(maxsplit=1) |
| assert current_time is not None |
| for file_path in file_paths.split("\t"): |
| if file_path in set_paths and file_path not in path_times: |
| path_times[file_path] = current_time |
| |
| # FIXME: find out why some files are not shown in "git log" |
| # assert set(path_times.keys()) == set_paths |
| if set(path_times.keys()) != set_paths: |
| found_times = set(path_times.keys()) |
| paths_without_times = set_paths.difference(found_times) |
| logger.warning("No times found for these paths: %s", paths_without_times) |
| max_time = max(time for time in path_times.values()) |
| for path in paths_without_times: |
| path_times[path] = max_time |
| |
| return path_times |
| |
| def add_to_archiver(self, archive_base: str, archiver: Archiver): |
| remaining_symlinks = set() |
| added_files = dict() |
| |
| def calculate_symlink_target(s: SourceCollector.TreeItem) -> str: |
| dest_dir = os.path.dirname(s.path) |
| if dest_dir: |
| dest_dir += "/" |
| target = dest_dir + s.symtarget |
| while True: |
| new_target, n = re.subn(r"([^/]+/+[.]{2}/)", "", target) |
| print(f"{target=} {new_target=}") |
| target = new_target |
| if not n: |
| break |
| return target |
| |
| # Add files in first pass |
| for git_file in self.git_contents.values(): |
| if git_file.data is not None: |
| archiver.add_file_data(arcpath=f"{archive_base}/{git_file.path}", data=git_file.data, time=git_file.time, mode=git_file.mode) |
| added_files[git_file.path] = git_file |
| elif git_file.symtarget is not None: |
| remaining_symlinks.add(git_file) |
| |
| # Resolve symlinks in second pass: zipfile does not support symlinks, so add files to zip archive |
| while True: |
| if not remaining_symlinks: |
| break |
| symlinks_this_time = set() |
| extra_added_files = {} |
| for symlink in remaining_symlinks: |
| symlink_files_for_zip = {} |
| symlink_target_path = calculate_symlink_target(symlink) |
| if symlink_target_path in added_files: |
| symlink_files_for_zip[symlink.path] = added_files[symlink_target_path] |
| else: |
| symlink_target_path_slash = symlink_target_path + "/" |
| for added_file in added_files: |
| if added_file.startswith(symlink_target_path_slash): |
| path_in_symlink = symlink.path + "/" + added_file.removeprefix(symlink_target_path_slash) |
| symlink_files_for_zip[path_in_symlink] = added_files[added_file] |
| if symlink_files_for_zip: |
| symlinks_this_time.add(symlink) |
| extra_added_files.update(symlink_files_for_zip) |
| files_for_zip = [{"arcpath": f"{archive_base}/{sym_path}", "data": sym_info.data, "mode": sym_info.mode} for sym_path, sym_info in symlink_files_for_zip.items()] |
| archiver.add_symlink(arcpath=f"{archive_base}/{symlink.path}", target=symlink.symtarget, time=symlink.time, files_for_zip=files_for_zip) |
| # if not symlinks_this_time: |
| # logger.info("files added: %r", set(path for path in added_files.keys())) |
| assert symlinks_this_time, f"No targets found for symlinks: {remaining_symlinks}" |
| remaining_symlinks.difference_update(symlinks_this_time) |
| added_files.update(extra_added_files) |
| |
| |
| class Releaser: |
| def __init__(self, release_info: dict, commit: str, root: Path, dist_path: Path, section_printer: SectionPrinter, executer: Executer, cmake_generator: str, deps_path: Path, overwrite: bool, github: bool, fast: bool): |
| self.release_info = release_info |
| self.project = release_info["name"] |
| self.version = self.extract_sdl_version(root=root, release_info=release_info) |
| self.root = root |
| self.commit = commit |
| self.dist_path = dist_path |
| self.section_printer = section_printer |
| self.executer = executer |
| self.cmake_generator = cmake_generator |
| self.cpu_count = multiprocessing.cpu_count() |
| self.deps_path = deps_path |
| self.overwrite = overwrite |
| self.github = github |
| self.fast = fast |
| |
| self.artifacts: dict[str, Path] = {} |
| |
| @property |
| def dry(self) -> bool: |
| return self.executer.dry |
| |
| def prepare(self): |
| logger.debug("Creating dist folder") |
| self.dist_path.mkdir(parents=True, exist_ok=True) |
| |
| @classmethod |
| def _path_filter(cls, path: str) -> bool: |
| if ".gitmodules" in path: |
| return True |
| if path.startswith(".git"): |
| return False |
| return True |
| |
| @classmethod |
| def _external_repo_path_filter(cls, path: str) -> bool: |
| if not cls._path_filter(path): |
| return False |
| if path.startswith("test/") or path.startswith("tests/"): |
| return False |
| return True |
| |
| def create_source_archives(self) -> None: |
| archive_base = f"{self.project}-{self.version}" |
| |
| project_souce_collector = SourceCollector(root=self.root, commit=self.commit, executer=self.executer, filter=self._path_filter) |
| |
| latest_mod_time = max(item.time for item in project_souce_collector.git_contents.values() if item.time) |
| |
| zip_path = self.dist_path / f"{archive_base}.zip" |
| tgz_path = self.dist_path / f"{archive_base}.tar.gz" |
| txz_path = self.dist_path / f"{archive_base}.tar.xz" |
| |
| logger.info("Creating zip/tgz/txz source archives ...") |
| if self.dry: |
| zip_path.touch() |
| tgz_path.touch() |
| txz_path.touch() |
| else: |
| with Archiver(zip_path=zip_path, tgz_path=tgz_path, txz_path=txz_path) as archiver: |
| archiver.add_file_data(arcpath=f"{archive_base}/VERSION.txt", data=f"{self.version}\n".encode(), mode=0o100644, time=latest_mod_time) |
| archiver.add_file_data(arcpath=f"{archive_base}/{GIT_HASH_FILENAME}", data=f"{self.commit}\n".encode(), mode=0o100644, time=latest_mod_time) |
| |
| print(f"Adding source files of main project ...") |
| project_souce_collector.add_to_archiver(archive_base=archive_base, archiver=archiver) |
| |
| for extra_repo in self.release_info["source"].get("extra-repos", []): |
| extra_repo_root = self.root / extra_repo |
| assert (extra_repo_root / ".git").exists(), f"{extra_repo_root} must be a git repo" |
| extra_repo_commit = self.executer.check_output(["git", "rev-parse", "HEAD"], dry_out=f"gitsha-extra-repo-{extra_repo}", cwd=extra_repo_root).strip() |
| extra_repo_source_collector = SourceCollector(root=extra_repo_root, commit=extra_repo_commit, executer=self.executer, filter=self._external_repo_path_filter) |
| print(f"Adding source files of {extra_repo} ...") |
| extra_repo_source_collector.add_to_archiver(archive_base=f"{archive_base}/{extra_repo}", archiver=archiver) |
| |
| for file in self.release_info["source"]["checks"]: |
| assert f"{archive_base}/{file}" in archiver.added_files, f"'{archive_base}/{file}' must exist" |
| |
| logger.info("... done") |
| |
| self.artifacts["src-zip"] = zip_path |
| self.artifacts["src-tar-gz"] = tgz_path |
| self.artifacts["src-tar-xz"] = txz_path |
| |
| if not self.dry: |
| with tgz_path.open("r+b") as f: |
| # Zero the embedded timestamp in the gzip'ed tarball |
| f.seek(4, 0) |
| f.write(b"\x00\x00\x00\x00") |
| |
| def create_dmg(self, configuration: str="Release") -> None: |
| dmg_in = self.root / self.release_info["dmg"]["path"] |
| xcode_project = self.root / self.release_info["dmg"]["project"] |
| assert xcode_project.is_dir(), f"{xcode_project} must be a directory" |
| assert (xcode_project / "project.pbxproj").is_file, f"{xcode_project} must contain project.pbxproj" |
| dmg_in.unlink(missing_ok=True) |
| build_xcconfig = self.release_info["dmg"].get("build-xcconfig") |
| if build_xcconfig: |
| shutil.copy(self.root / build_xcconfig, xcode_project.parent / "build.xcconfig") |
| |
| xcode_scheme = self.release_info["dmg"].get("scheme") |
| xcode_target = self.release_info["dmg"].get("target") |
| assert xcode_scheme or xcode_target, "dmg needs scheme or target" |
| assert not (xcode_scheme and xcode_target), "dmg cannot have both scheme and target set" |
| if xcode_scheme: |
| scheme_or_target = "-scheme" |
| target_like = xcode_scheme |
| else: |
| scheme_or_target = "-target" |
| target_like = xcode_target |
| self.executer.run(["xcodebuild", "ONLY_ACTIVE_ARCH=NO", "-project", xcode_project, scheme_or_target, target_like, "-configuration", configuration]) |
| if self.dry: |
| dmg_in.parent.mkdir(parents=True, exist_ok=True) |
| dmg_in.touch() |
| |
| assert dmg_in.is_file(), f"{self.project}.dmg was not created by xcodebuild" |
| |
| dmg_out = self.dist_path / f"{self.project}-{self.version}.dmg" |
| shutil.copy(dmg_in, dmg_out) |
| self.artifacts["dmg"] = dmg_out |
| |
| @property |
| def git_hash_data(self) -> bytes: |
| return f"{self.commit}\n".encode() |
| |
| def _tar_add_git_hash(self, tar_object: tarfile.TarFile, root: typing.Optional[str]=None, time: typing.Optional[datetime.datetime]=None): |
| if not time: |
| time = datetime.datetime(year=2024, month=4, day=1) |
| path = GIT_HASH_FILENAME |
| if root: |
| path = f"{root}/{path}" |
| |
| tar_info = tarfile.TarInfo(path) |
| tar_info.mode = 0o100644 |
| tar_info.size = len(self.git_hash_data) |
| tar_info.mtime = int(time.timestamp()) |
| tar_object.addfile(tar_info, fileobj=io.BytesIO(self.git_hash_data)) |
| |
| def create_mingw_archives(self) -> None: |
| build_type = "Release" |
| build_parent_dir = self.root / "build-mingw" |
| assert "autotools" in self.release_info["mingw"] |
| assert "cmake" not in self.release_info["mingw"] |
| mingw_archs = self.release_info["mingw"]["autotools"]["archs"] |
| ARCH_TO_TRIPLET = { |
| "x86": "i686-w64-mingw32", |
| "x64": "x86_64-w64-mingw32", |
| } |
| |
| new_env = dict(os.environ) |
| |
| if "dependencies" in self.release_info["mingw"]: |
| mingw_deps_path = self.deps_path / "mingw-deps" |
| shutil.rmtree(mingw_deps_path, ignore_errors=True) |
| mingw_deps_path.mkdir() |
| |
| for triplet in ARCH_TO_TRIPLET.values(): |
| (mingw_deps_path / triplet).mkdir() |
| |
| def extract_filter(member: tarfile.TarInfo, path: str, /): |
| if member.name.startswith("SDL"): |
| member.name = "/".join(Path(member.name).parts[1:]) |
| return member |
| for dep in self.release_info["dependencies"].keys(): |
| extract_dir = mingw_deps_path / f"extract-{dep}" |
| extract_dir.mkdir() |
| with chdir(extract_dir): |
| tar_path = glob.glob(self.release_info["mingw"]["dependencies"][dep]["artifact"], root_dir=self.deps_path)[0] |
| logger.info("Extracting %s to %s", tar_path, mingw_deps_path) |
| with tarfile.open(self.deps_path / tar_path, mode="r:gz") as tarf: |
| tarf.extractall(filter=extract_filter) |
| for triplet in ARCH_TO_TRIPLET.values(): |
| self.executer.run(["make", f"-j{os.cpu_count()}", "-C", str(extract_dir), "install-package", f"arch={triplet}", f"prefix={str(mingw_deps_path / triplet)}"]) |
| |
| dep_binpath = mingw_deps_path / triplet / "bin" |
| assert dep_binpath.is_dir(), f"{dep_binpath} for PATH should exist" |
| dep_pkgconfig = mingw_deps_path / triplet / "lib/pkgconfig" |
| assert dep_pkgconfig.is_dir(), f"{dep_pkgconfig} for PKG_CONFIG_PATH should exist" |
| |
| new_env["PATH"] = os.pathsep.join([str(dep_binpath), new_env["PATH"]]) |
| new_env["PKG_CONFIG_PATH"] = str(dep_pkgconfig) |
| |
| new_env["CFLAGS"] = f"-O2 -ffile-prefix-map={self.root}=/src/{self.project}" |
| new_env["CXXFLAGS"] = f"-O2 -ffile-prefix-map={self.root}=/src/{self.project}" |
| |
| arch_install_paths = {} |
| arch_files = {} |
| for arch in mingw_archs: |
| triplet = ARCH_TO_TRIPLET[arch] |
| new_env["CC"] = f"{triplet}-gcc" |
| new_env["CXX"] = f"{triplet}-g++" |
| new_env["RC"] = f"{triplet}-windres" |
| |
| build_path = build_parent_dir / f"build-{triplet}" |
| install_path = build_parent_dir / f"install-{triplet}" |
| arch_install_paths[arch] = install_path |
| shutil.rmtree(install_path, ignore_errors=True) |
| build_path.mkdir(parents=True, exist_ok=True) |
| with self.section_printer.group(f"Configuring MinGW {triplet}"): |
| extra_args = [arg.replace("@DEP_PREFIX@", str(mingw_deps_path / triplet)) for arg in self.release_info["mingw"]["autotools"]["args"]] |
| assert "@" not in " ".join(extra_args), f"@ should not be present in extra arguments ({extra_args})" |
| self.executer.run([ |
| self.root / "configure", |
| f"--prefix={install_path}", |
| f"--includedir={install_path}/include", |
| f"--libdir={install_path}/lib", |
| f"--bindir={install_path}/bin", |
| f"--host={triplet}", |
| f"--build=x86_64-none-linux-gnu", |
| ] + extra_args, cwd=build_path, env=new_env) |
| with self.section_printer.group(f"Build MinGW {triplet}"): |
| self.executer.run(["make", f"-j{self.cpu_count}"], cwd=build_path, env=new_env) |
| with self.section_printer.group(f"Install MinGW {triplet}"): |
| self.executer.run(["make", "install"], cwd=build_path, env=new_env) |
| arch_files[arch] = list(Path(r) / f for r, _, files in os.walk(install_path) for f in files) |
| |
| print("Collecting files for MinGW development archive ...") |
| archived_files = {} |
| arc_root = f"{self.project}-{self.version}" |
| for arch in mingw_archs: |
| triplet = ARCH_TO_TRIPLET[arch] |
| install_path = arch_install_paths[arch] |
| arcname_parent = f"{arc_root}/{triplet}" |
| for file in arch_files[arch]: |
| arcname = os.path.join(arcname_parent, file.relative_to(install_path)) |
| logger.debug("Adding %s as %s", file, arcname) |
| archived_files[arcname] = file |
| for meta_destdir, file_globs in self.release_info["mingw"]["files"].items(): |
| assert meta_destdir[0] == "/" and meta_destdir[-1] == "/", f"'{meta_destdir}' must begin and end with '/'" |
| if "@" in meta_destdir: |
| destdirs = list(meta_destdir.replace("@TRIPLET@", triplet) for triplet in ARCH_TO_TRIPLET.values()) |
| assert not any("A" in d for d in destdirs) |
| else: |
| destdirs = [meta_destdir] |
| |
| assert isinstance(file_globs, list), f"'{file_globs}' in release_info.json must be a list of globs instead" |
| for file_glob in file_globs: |
| file_paths = glob.glob(file_glob, root_dir=self.root) |
| assert file_paths, f"glob '{file_glob}' does not match any file" |
| for file_path in file_paths: |
| file_path = self.root / file_path |
| for destdir in destdirs: |
| arcname = f"{arc_root}{destdir}{file_path.name}" |
| logger.debug("Adding %s as %s", file_path, arcname) |
| archived_files[arcname] = file_path |
| print("... done") |
| |
| print("Creating zip/tgz/txz development archives ...") |
| zip_path = self.dist_path / f"{self.project}-devel-{self.version}-mingw.zip" |
| tgz_path = self.dist_path / f"{self.project}-devel-{self.version}-mingw.tar.gz" |
| txz_path = self.dist_path / f"{self.project}-devel-{self.version}-mingw.tar.xz" |
| with Archiver(zip_path=zip_path, tgz_path=tgz_path, txz_path=txz_path) as archiver: |
| for arcpath, path in archived_files.items(): |
| archiver.add_file_path(arcpath=arcpath, path=path) |
| print("... done") |
| |
| self.artifacts["mingw-devel-zip"] = zip_path |
| self.artifacts["mingw-devel-tar-gz"] = tgz_path |
| self.artifacts["mingw-devel-tar-xz"] = txz_path |
| |
| def download_dependencies(self): |
| shutil.rmtree(self.deps_path, ignore_errors=True) |
| self.deps_path.mkdir(parents=True) |
| |
| if self.github: |
| with open(os.environ["GITHUB_OUTPUT"], "a") as f: |
| f.write(f"dep-path={self.deps_path.absolute()}\n") |
| |
| for dep, depinfo in self.release_info["dependencies"].items(): |
| startswith = depinfo["startswith"] |
| dep_repo = depinfo["repo"] |
| dep_string_data = self.executer.check_output(["gh", "-R", dep_repo, "release", "list", "--exclude-drafts", "--exclude-pre-releases", "--json", "name,createdAt,tagName", "--jq", f'[.[]|select(.name|startswith("{startswith}"))]|max_by(.createdAt)']).strip() |
| dep_data = json.loads(dep_string_data) |
| dep_tag = dep_data["tagName"] |
| dep_version = dep_data["name"] |
| logger.info("Download dependency %s version %s (tag=%s) ", dep, dep_version, dep_tag) |
| self.executer.run(["gh", "-R", dep_repo, "release", "download", dep_tag], cwd=self.deps_path) |
| if self.github: |
| with open(os.environ["GITHUB_OUTPUT"], "a") as f: |
| f.write(f"dep-{dep.lower()}-version={dep_version}\n") |
| |
| def verify_dependencies(self): |
| for dep, depinfo in self.release_info.get("dependencies", {}).items(): |
| mingw_matches = glob.glob(self.release_info["mingw"]["dependencies"][dep]["artifact"], root_dir=self.deps_path) |
| assert len(mingw_matches) == 1, f"Exactly one archive matches mingw {dep} dependency: {mingw_matches}" |
| dmg_matches = glob.glob(self.release_info["dmg"]["dependencies"][dep]["artifact"], root_dir=self.deps_path) |
| assert len(dmg_matches) == 1, f"Exactly one archive matches dmg {dep} dependency: {dmg_matches}" |
| msvc_matches = glob.glob(self.release_info["msvc"]["dependencies"][dep]["artifact"], root_dir=self.deps_path) |
| assert len(msvc_matches) == 1, f"Exactly one archive matches msvc {dep} dependency: {msvc_matches}" |
| |
| def build_vs(self, arch_platform: VsArchPlatformConfig, vs: VisualStudio): |
| msvc_deps_path = self.deps_path / "msvc-deps" |
| shutil.rmtree(msvc_deps_path, ignore_errors=True) |
| if "dependencies" in self.release_info["msvc"]: |
| for dep, depinfo in self.release_info["msvc"]["dependencies"].items(): |
| msvc_zip = self.deps_path / glob.glob(depinfo["artifact"], root_dir=self.deps_path)[0] |
| |
| src_globs = [arch_platform.configure(instr["src"]) for instr in depinfo["copy"]] |
| with zipfile.ZipFile(msvc_zip, "r") as zf: |
| for member in zf.namelist(): |
| member_path = "/".join(Path(member).parts[1:]) |
| for src_i, src_glob in enumerate(src_globs): |
| if fnmatch.fnmatch(member_path, src_glob): |
| dst = (self.root / arch_platform.configure(depinfo["copy"][src_i]["dst"])).resolve() / Path(member_path).name |
| zip_data = zf.read(member) |
| if dst.exists(): |
| identical = False |
| if dst.is_file(): |
| orig_bytes = dst.read_bytes() |
| if orig_bytes == zip_data: |
| identical = True |
| if not identical: |
| logger.warning("Extracting dependency %s, will cause %s to be overwritten", dep, dst) |
| if not self.overwrite: |
| raise RuntimeError("Run with --overwrite to allow overwriting") |
| logger.debug("Extracting %s -> %s", member, dst) |
| |
| dst.parent.mkdir(exist_ok=True, parents=True) |
| dst.write_bytes(zip_data) |
| |
| assert "msbuild" in self.release_info["msvc"] |
| assert "cmake" not in self.release_info["msvc"] |
| built_paths = [ |
| self.root / arch_platform.configure(f) for msbuild_files in self.release_info["msvc"]["msbuild"]["files"] for f in msbuild_files["paths"] |
| ] |
| |
| for b in built_paths: |
| b.unlink(missing_ok=True) |
| |
| projects = self.release_info["msvc"]["msbuild"]["projects"] |
| |
| with self.section_printer.group(f"Build {arch_platform.arch} VS binary"): |
| vs.build(arch_platform=arch_platform, projects=projects) |
| |
| if self.dry: |
| for b in built_paths: |
| b.parent.mkdir(parents=True, exist_ok=True) |
| b.touch() |
| |
| for b in built_paths: |
| assert b.is_file(), f"{b} has not been created" |
| b.parent.mkdir(parents=True, exist_ok=True) |
| b.touch() |
| |
| zip_path = self.dist_path / f"{self.project}-{self.version}-win32-{arch_platform.arch}.zip" |
| zip_path.unlink(missing_ok=True) |
| logger.info("Creating %s", zip_path) |
| |
| with Archiver(zip_path=zip_path) as archiver: |
| for msbuild_files in self.release_info["msvc"]["msbuild"]["files"]: |
| if "lib" in msbuild_files: |
| arcdir = arch_platform.configure(msbuild_files["lib"]) |
| for p in msbuild_files["paths"]: |
| p = arch_platform.configure(p) |
| archiver.add_file_path(path=self.root / p, arcpath=f"{arcdir}/{Path(p).name}") |
| for extra_files in self.release_info["msvc"]["files"]: |
| if "lib" in extra_files: |
| arcdir = arch_platform.configure(extra_files["lib"]) |
| for p in extra_files["paths"]: |
| p = arch_platform.configure(p) |
| archiver.add_file_path(path=self.root / p, arcpath=f"{arcdir}/{Path(p).name}") |
| |
| archiver.add_git_hash(commit=self.commit) |
| self.artifacts[f"VC-{arch_platform.arch}"] = zip_path |
| |
| for p in built_paths: |
| assert p.is_file(), f"{p} should exist" |
| |
| def build_vs_devel(self, arch_platforms: list[VsArchPlatformConfig]) -> None: |
| zip_path = self.dist_path / f"{self.project}-devel-{self.version}-VC.zip" |
| archive_prefix = f"{self.project}-{self.version}" |
| |
| with Archiver(zip_path=zip_path) as archiver: |
| for msbuild_files in self.release_info["msvc"]["msbuild"]["files"]: |
| if "devel" in msbuild_files: |
| for meta_glob_path in msbuild_files["paths"]: |
| if "@" in meta_glob_path or "@" in msbuild_files["devel"]: |
| for arch_platform in arch_platforms: |
| glob_path = arch_platform.configure(meta_glob_path) |
| paths = glob.glob(glob_path, root_dir=self.root) |
| dst_subdirpath = arch_platform.configure(msbuild_files['devel']) |
| for path in paths: |
| path = self.root / path |
| arcpath = f"{archive_prefix}/{dst_subdirpath}/{Path(path).name}" |
| archiver.add_file_path(path=path, arcpath=arcpath) |
| else: |
| paths = glob.glob(meta_glob_path, root_dir=self.root) |
| for path in paths: |
| path = self.root / path |
| arcpath = f"{archive_prefix}/{msbuild_files['devel']}/{Path(path).name}" |
| archiver.add_file_path(path=path, arcpath=arcpath) |
| for extra_files in self.release_info["msvc"]["files"]: |
| if "devel" in extra_files: |
| for meta_glob_path in extra_files["paths"]: |
| if "@" in meta_glob_path or "@" in extra_files["devel"]: |
| for arch_platform in arch_platforms: |
| glob_path = arch_platform.configure(meta_glob_path) |
| paths = glob.glob(glob_path, root_dir=self.root) |
| dst_subdirpath = arch_platform.configure(extra_files['devel']) |
| for path in paths: |
| path = self.root / path |
| arcpath = f"{archive_prefix}/{dst_subdirpath}/{Path(path).name}" |
| archiver.add_file_path(path=path, arcpath=arcpath) |
| else: |
| paths = glob.glob(meta_glob_path, root_dir=self.root) |
| for path in paths: |
| path = self.root / path |
| arcpath = f"{archive_prefix}/{extra_files['devel']}/{Path(path).name}" |
| archiver.add_file_path(path=path, arcpath=arcpath) |
| |
| archiver.add_git_hash(commit=self.commit, arcdir=archive_prefix) |
| self.artifacts["VC-devel"] = zip_path |
| |
| @classmethod |
| def extract_sdl_version(cls, root: Path, release_info: dict) -> str: |
| with open(root / release_info["version"]["file"], "r") as f: |
| text = f.read() |
| major = next(re.finditer(release_info["version"]["re_major"], text, flags=re.M)).group(1) |
| minor = next(re.finditer(release_info["version"]["re_minor"], text, flags=re.M)).group(1) |
| micro = next(re.finditer(release_info["version"]["re_micro"], text, flags=re.M)).group(1) |
| return f"{major}.{minor}.{micro}" |
| |
| |
| def main(argv=None) -> int: |
| if sys.version_info < (3, 11): |
| logger.error("This script needs at least python 3.11") |
| return 1 |
| |
| parser = argparse.ArgumentParser(allow_abbrev=False, description="Create SDL release artifacts") |
| parser.add_argument("--root", metavar="DIR", type=Path, default=Path(__file__).absolute().parents[1], help="Root of project") |
| parser.add_argument("--release-info", metavar="JSON", dest="path_release_info", type=Path, default=Path(__file__).absolute().parent / "release-info.json", help="Path of release-info.json") |
| parser.add_argument("--dependency-folder", metavar="FOLDER", dest="deps_path", type=Path, default="deps", help="Directory containing pre-built archives of dependencies (will be removed when downloading archives)") |
| parser.add_argument("--out", "-o", metavar="DIR", dest="dist_path", type=Path, default="dist", help="Output directory") |
| parser.add_argument("--github", action="store_true", help="Script is running on a GitHub runner") |
| parser.add_argument("--commit", default="HEAD", help="Git commit/tag of which a release should be created") |
| parser.add_argument("--actions", choices=["download", "source", "mingw", "msvc", "dmg"], required=True, nargs="+", dest="actions", help="What to do?") |
| parser.set_defaults(loglevel=logging.INFO) |
| parser.add_argument('--vs-year', dest="vs_year", help="Visual Studio year") |
| parser.add_argument('--cmake-generator', dest="cmake_generator", default="Ninja", help="CMake Generator") |
| parser.add_argument('--debug', action='store_const', const=logging.DEBUG, dest="loglevel", help="Print script debug information") |
| parser.add_argument('--dry-run', action='store_true', dest="dry", help="Don't execute anything") |
| parser.add_argument('--force', action='store_true', dest="force", help="Ignore a non-clean git tree") |
| parser.add_argument('--overwrite', action='store_true', dest="overwrite", help="Allow potentially overwriting other projects") |
| parser.add_argument('--fast', action='store_true', dest="fast", help="Don't do a rebuild") |
| |
| args = parser.parse_args(argv) |
| logging.basicConfig(level=args.loglevel, format='[%(levelname)s] %(message)s') |
| args.deps_path = args.deps_path.absolute() |
| args.dist_path = args.dist_path.absolute() |
| args.root = args.root.absolute() |
| args.dist_path = args.dist_path.absolute() |
| if args.dry: |
| args.dist_path = args.dist_path / "dry" |
| |
| if args.github: |
| section_printer: SectionPrinter = GitHubSectionPrinter() |
| else: |
| section_printer = SectionPrinter() |
| |
| if args.github and "GITHUB_OUTPUT" not in os.environ: |
| os.environ["GITHUB_OUTPUT"] = "/tmp/github_output.txt" |
| |
| executer = Executer(root=args.root, dry=args.dry) |
| |
| root_git_hash_path = args.root / GIT_HASH_FILENAME |
| root_is_maybe_archive = root_git_hash_path.is_file() |
| if root_is_maybe_archive: |
| logger.warning("%s detected: Building from archive", GIT_HASH_FILENAME) |
| archive_commit = root_git_hash_path.read_text().strip() |
| if args.commit != archive_commit: |
| logger.warning("Commit argument is %s, but archive commit is %s. Using %s.", args.commit, archive_commit, archive_commit) |
| args.commit = archive_commit |
| else: |
| args.commit = executer.check_output(["git", "rev-parse", args.commit], dry_out="e5812a9fd2cda317b503325a702ba3c1c37861d9").strip() |
| logger.info("Using commit %s", args.commit) |
| |
| try: |
| with args.path_release_info.open() as f: |
| release_info = json.load(f) |
| except FileNotFoundError: |
| logger.error(f"Could not find {args.path_release_info}") |
| |
| releaser = Releaser( |
| release_info=release_info, |
| commit=args.commit, |
| root=args.root, |
| dist_path=args.dist_path, |
| executer=executer, |
| section_printer=section_printer, |
| cmake_generator=args.cmake_generator, |
| deps_path=args.deps_path, |
| overwrite=args.overwrite, |
| github=args.github, |
| fast=args.fast, |
| ) |
| |
| if root_is_maybe_archive: |
| logger.warning("Building from archive. Skipping clean git tree check.") |
| else: |
| porcelain_status = executer.check_output(["git", "status", "--ignored", "--porcelain"], dry_out="\n").strip() |
| if porcelain_status: |
| print(porcelain_status) |
| logger.warning("The tree is dirty! Do not publish any generated artifacts!") |
| if not args.force: |
| raise Exception("The git repo contains modified and/or non-committed files. Run with --force to ignore.") |
| |
| if args.fast: |
| logger.warning("Doing fast build! Do not publish generated artifacts!") |
| |
| with section_printer.group("Arguments"): |
| print(f"project = {releaser.project}") |
| print(f"version = {releaser.version}") |
| print(f"commit = {args.commit}") |
| print(f"out = {args.dist_path}") |
| print(f"actions = {args.actions}") |
| print(f"dry = {args.dry}") |
| print(f"force = {args.force}") |
| print(f"overwrite = {args.overwrite}") |
| print(f"cmake_generator = {args.cmake_generator}") |
| |
| releaser.prepare() |
| |
| if "download" in args.actions: |
| releaser.download_dependencies() |
| |
| if set(args.actions).intersection({"msvc", "mingw"}): |
| print("Verifying presence of dependencies (run 'download' action to download) ...") |
| releaser.verify_dependencies() |
| print("... done") |
| |
| if "source" in args.actions: |
| if root_is_maybe_archive: |
| raise Exception("Cannot build source archive from source archive") |
| with section_printer.group("Create source archives"): |
| releaser.create_source_archives() |
| |
| if "dmg" in args.actions: |
| if platform.system() != "Darwin" and not args.dry: |
| parser.error("framework artifact(s) can only be built on Darwin") |
| |
| releaser.create_dmg() |
| |
| if "msvc" in args.actions: |
| if platform.system() != "Windows" and not args.dry: |
| parser.error("msvc artifact(s) can only be built on Windows") |
| with section_printer.group("Find Visual Studio"): |
| vs = VisualStudio(executer=executer) |
| |
| arch_platforms = [ |
| VsArchPlatformConfig(arch="x86", platform="Win32", configuration="Release"), |
| VsArchPlatformConfig(arch="x64", platform="x64", configuration="Release"), |
| ] |
| for arch_platform in arch_platforms: |
| releaser.build_vs(arch_platform=arch_platform, vs=vs) |
| with section_printer.group("Create SDL VC development zip"): |
| releaser.build_vs_devel(arch_platforms) |
| |
| if "mingw" in args.actions: |
| releaser.create_mingw_archives() |
| |
| with section_printer.group("Summary"): |
| print(f"artifacts = {releaser.artifacts}") |
| |
| if args.github: |
| with open(os.environ["GITHUB_OUTPUT"], "a") as f: |
| f.write(f"project={releaser.project}\n") |
| f.write(f"version={releaser.version}\n") |
| for k, v in releaser.artifacts.items(): |
| f.write(f"{k}={v.name}\n") |
| return 0 |
| |
| |
| if __name__ == "__main__": |
| raise SystemExit(main()) |
