From 38e846cff4f0e1229dab784168a67c5e75c6ede3 Mon Sep 17 00:00:00 2001 From: LLM Date: Mon, 6 Apr 2026 12:25:12 +0000 Subject: [PATCH] [+] improve cli_bootstrap: bootstrap args, overrides, whl cache python version 1. add argv_extract_t for targeted argument extraction from argv; 2. add --bootstrap-help and --bootstrap-override cli args; 3. apply_overrides_to_constraints patches constraint file per override; 4. fix whl_cache_download to use target python_version, not host; 5. fix whl cache check to verify python_tag matches target version; 6. parse_whl_name_version now extracts python_tag from wheel filename; 7. add parse_req_name for extracting package name from spec; 8. use contextlib.ExitStack for temp file cleanup in compile; --- python/m.py | 228 +++++++++++++++--- .../pr34/commands_typed/cli_bootstrap.py | 228 +++++++++++++++--- 2 files changed, 392 insertions(+), 64 deletions(-) diff --git a/python/m.py b/python/m.py index 85d28dd..9a73599 100755 --- a/python/m.py +++ b/python/m.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +import contextlib import glob import importlib import json @@ -478,6 +479,7 @@ class packaging_t: class pkg_id_t: name: str version: str + python_tag: Optional[str] = None @staticmethod def canonicalize_name(name: str) -> str: @@ -486,6 +488,12 @@ class packaging_t: @staticmethod def parse_whl_name_version(filename: str) -> Optional['packaging_t.pkg_id_t']: parts = filename.split('-') + if len(parts) >= 5 and filename.endswith('.whl'): + return packaging_t.pkg_id_t( + name=packaging_t.canonicalize_name(parts[0]), + version=parts[1], + python_tag=parts[2], + ) if len(parts) >= 3 and filename.endswith('.whl'): return packaging_t.pkg_id_t( name=packaging_t.canonicalize_name(parts[0]), @@ -503,20 +511,93 @@ class packaging_t: ) return None + @staticmethod + def parse_req_name(spec: str) -> Optional[str]: + """Extract canonical package name from a requirement spec like 'pip>=23' or 'librt>=0.8'.""" + m = re.match(r'^([a-zA-Z0-9._-]+)', spec.strip()) + if m: + return packaging_t.canonicalize_name(m.group(1)) + return None + + @staticmethod + def apply_overrides_to_constraints( + requirements_path: pathlib.Path, + overrides: list[str], + output: 'typing.IO[str]', + ) -> None: + """Copy requirements file to output, replacing blocks for overridden packages. + + Handles multi-line entries (continuations with \\ and --hash lines). + For each overridden package, its entire block is replaced with the override spec. + """ + override_map: dict[str, str] = {} + for ov in overrides: + name = packaging_t.parse_req_name(ov) + if name is not None: + override_map[name] = ov + + with io.open(requirements_path, 'r') as f: + skip_block = False + current_override: Optional[str] = None + + for line in f: + stripped = line.strip() + + if stripped.startswith('#') or stripped.startswith('--hash'): + if skip_block: + continue + output.write(line) + continue + + if stripped == '' or stripped == '\\': + if skip_block: + continue + output.write(line) + continue + + if stripped.endswith('\\'): + spec_part = stripped.rstrip('\\').strip() + else: + spec_part = stripped + + parsed = packaging_t.parse_req_spec(spec_part) + if parsed is not None and parsed.name in override_map: + if not skip_block: + skip_block = True + current_override = override_map.pop(parsed.name) + output.write(current_override + '\n') + continue + + if skip_block and (stripped.startswith('--hash') or stripped.endswith('\\')): + continue + + skip_block = False + current_override = None + output.write(line) + + for name, ov in override_map.items(): + output.write(ov + '\n') + def whl_cache_download( whl_cache_path: pathlib.Path, requirements_path: pathlib.Path, - uv_python_version: list[str], + python_version: Optional[str], pip_find_links_args: list[str], ) -> None: whl_cache_path.mkdir(parents=True, exist_ok=True) + py_tag_prefix = 'cp' + python_version.replace('.', '') if python_version else None + cached_pkgs: set[tuple[str, str]] = set() for whl in whl_cache_path.glob('*.whl'): parsed = packaging_t.parse_whl_name_version(whl.name) - if parsed is not None: - cached_pkgs.add((parsed.name, parsed.version)) + if parsed is None: + continue + if py_tag_prefix is not None and parsed.python_tag is not None: + if not parsed.python_tag.startswith(py_tag_prefix) and parsed.python_tag not in ('py3', 'py2.py3'): + continue + cached_pkgs.add((parsed.name, parsed.version)) missing_reqs: list[str] = [] with io.open(requirements_path, 'r') as f: @@ -546,6 +627,10 @@ def whl_cache_download( f.flush() missing_req_path = f.name + pip_python_version_args: list[str] = [] + if python_version is not None: + pip_python_version_args = ['--python-version', python_version.replace('.', '')] + try: cmd = [ sys.executable, @@ -553,7 +638,7 @@ def whl_cache_download( 'pip', 'download', '--only-binary=:all:', - *uv_python_version, + *pip_python_version_args, *pip_find_links_args, '-r', missing_req_path, @@ -584,6 +669,7 @@ def check_host_prerequisites() -> None: def env_bootstrap( bootstrap_settings: BootstrapSettings, pyproject: PyProject, + overrides: Optional[list[str]] = None, ) -> None: check_host_prerequisites() @@ -707,38 +793,51 @@ def env_bootstrap( if len(constraint_args) > 0: uv_compile_args = [o for o in uv_compile_args if o not in ('-U', '--upgrade')] - cmd = [ - 'uv', - '--cache-dir', - bootstrap_settings.uv_cache_dir, - 'pip', - 'compile', - *uv_python_version, - '--generate-hashes', - '--no-annotate', - '--no-header', - *pip_find_links_args, - *cache_find_links_args, - *constraint_args, - *uv_compile_args, - '-o', - f_out.name, - f_in.name, - ] - logger.info(dict(cmd=cmd)) + with contextlib.ExitStack() as stack: + if overrides and len(constraint_args) > 0: + patched = stack.enter_context( + tempfile.NamedTemporaryFile( + mode='w', prefix='constraints_', suffix='.txt' + ) + ) + packaging_t.apply_overrides_to_constraints( + requirements_path, overrides, patched + ) + patched.flush() + constraint_args = ['-c', patched.name] - try: - subprocess.check_call(cmd) - os.replace(f_out.name, str(requirements_path)) - except subprocess.CalledProcessError: - os.unlink(f_out.name) - raise + cmd = [ + 'uv', + '--cache-dir', + bootstrap_settings.uv_cache_dir, + 'pip', + 'compile', + *uv_python_version, + '--generate-hashes', + '--no-annotate', + '--no-header', + *pip_find_links_args, + *cache_find_links_args, + *constraint_args, + *uv_compile_args, + '-o', + f_out.name, + f_in.name, + ] + logger.info(dict(cmd=cmd)) + + try: + subprocess.check_call(cmd) + os.replace(f_out.name, str(requirements_path)) + except subprocess.CalledProcessError: + os.unlink(f_out.name) + raise if not bootstrap_settings.whl_cache_path.exists() or bootstrap_settings.whl_cache_update: whl_cache_download( whl_cache_path=bootstrap_settings.whl_cache_path, requirements_path=requirements_path, - uv_python_version=uv_python_version, + python_version=bootstrap_settings.python_version, pip_find_links_args=pip_find_links_args, ) if bootstrap_settings.whl_cache_path.exists(): @@ -797,6 +896,53 @@ def paths_equal(a: pathlib.Path | str, b: pathlib.Path | str) -> bool: return os.path.abspath(str(a)) == os.path.abspath(str(b)) +import argparse as _argparse + + +class argv_extract_t: + """Extract known arguments from argv by scanning parser action definitions.""" + + @dataclasses.dataclass + class res_t: + namespace: _argparse.Namespace + rest: list[str] + + @staticmethod + def extract( + parser: _argparse.ArgumentParser, + argv: list[str], + ) -> 'argv_extract_t.res_t': + flag_map: dict[str, _argparse.Action] = {} + for action in parser._actions: + for opt in action.option_strings: + flag_map[opt] = action + + matched_argv: list[str] = [] + rest: list[str] = [] + i = 0 + while i < len(argv): + action = flag_map.get(argv[i]) + if action is not None: + matched_argv.append(argv[i]) + i += 1 + if action.nargs in (None, 1) and action.const is None and not isinstance( + action, (_argparse._StoreTrueAction, _argparse._StoreFalseAction, _argparse._CountAction, _argparse._HelpAction) + ): + if i < len(argv): + matched_argv.append(argv[i]) + i += 1 + else: + rest.append(argv[i]) + i += 1 + + namespace = parser.parse_args(matched_argv) + + return argv_extract_t.res_t( + namespace=namespace, + rest=rest, + ) + + def run( d: Optional[pathlib.Path] = None, cli_path: Optional[pathlib.Path] = None, @@ -807,6 +953,22 @@ def run( if d is None: d = pathlib.Path(__file__).parent / 'pyproject.toml' + bootstrap_parser = _argparse.ArgumentParser(add_help=False) + bootstrap_parser.add_argument( + '--bootstrap-help', + action='help', + help='show bootstrap help and exit', + ) + bootstrap_parser.add_argument( + '--bootstrap-override', + dest='overrides', + action='append', + default=[], + help='override for uv pip compile (e.g. "librt>=0.8")', + ) + + bootstrap_args = argv_extract_t.extract(bootstrap_parser, sys.argv[1:]) + bootstrap_settings = BootstrapSettings.get() pyproject: PyProject = pyproject_load(d) @@ -820,6 +982,7 @@ def run( env_bootstrap( bootstrap_settings=bootstrap_settings, pyproject=pyproject, + overrides=bootstrap_args.namespace.overrides or None, ) logger.info([sys.executable, sys.argv, bootstrap_settings.python_path]) @@ -829,7 +992,8 @@ def run( str(bootstrap_settings.python_path), [ str(bootstrap_settings.python_path), - *sys.argv, + sys.argv[0], + *bootstrap_args.rest, ], ) @@ -838,7 +1002,7 @@ def run( [ str(bootstrap_settings.python_path), str(cli_path), - *sys.argv[1:], + *bootstrap_args.rest, ], ) diff --git a/python/online/fxreader/pr34/commands_typed/cli_bootstrap.py b/python/online/fxreader/pr34/commands_typed/cli_bootstrap.py index 576d29d..f18c4f7 100644 --- a/python/online/fxreader/pr34/commands_typed/cli_bootstrap.py +++ b/python/online/fxreader/pr34/commands_typed/cli_bootstrap.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +import contextlib import glob import importlib import json @@ -478,6 +479,7 @@ class packaging_t: class pkg_id_t: name: str version: str + python_tag: Optional[str] = None @staticmethod def canonicalize_name(name: str) -> str: @@ -486,6 +488,12 @@ class packaging_t: @staticmethod def parse_whl_name_version(filename: str) -> Optional['packaging_t.pkg_id_t']: parts = filename.split('-') + if len(parts) >= 5 and filename.endswith('.whl'): + return packaging_t.pkg_id_t( + name=packaging_t.canonicalize_name(parts[0]), + version=parts[1], + python_tag=parts[2], + ) if len(parts) >= 3 and filename.endswith('.whl'): return packaging_t.pkg_id_t( name=packaging_t.canonicalize_name(parts[0]), @@ -503,20 +511,93 @@ class packaging_t: ) return None + @staticmethod + def parse_req_name(spec: str) -> Optional[str]: + """Extract canonical package name from a requirement spec like 'pip>=23' or 'librt>=0.8'.""" + m = re.match(r'^([a-zA-Z0-9._-]+)', spec.strip()) + if m: + return packaging_t.canonicalize_name(m.group(1)) + return None + + @staticmethod + def apply_overrides_to_constraints( + requirements_path: pathlib.Path, + overrides: list[str], + output: 'typing.IO[str]', + ) -> None: + """Copy requirements file to output, replacing blocks for overridden packages. + + Handles multi-line entries (continuations with \\ and --hash lines). + For each overridden package, its entire block is replaced with the override spec. + """ + override_map: dict[str, str] = {} + for ov in overrides: + name = packaging_t.parse_req_name(ov) + if name is not None: + override_map[name] = ov + + with io.open(requirements_path, 'r') as f: + skip_block = False + current_override: Optional[str] = None + + for line in f: + stripped = line.strip() + + if stripped.startswith('#') or stripped.startswith('--hash'): + if skip_block: + continue + output.write(line) + continue + + if stripped == '' or stripped == '\\': + if skip_block: + continue + output.write(line) + continue + + if stripped.endswith('\\'): + spec_part = stripped.rstrip('\\').strip() + else: + spec_part = stripped + + parsed = packaging_t.parse_req_spec(spec_part) + if parsed is not None and parsed.name in override_map: + if not skip_block: + skip_block = True + current_override = override_map.pop(parsed.name) + output.write(current_override + '\n') + continue + + if skip_block and (stripped.startswith('--hash') or stripped.endswith('\\')): + continue + + skip_block = False + current_override = None + output.write(line) + + for name, ov in override_map.items(): + output.write(ov + '\n') + def whl_cache_download( whl_cache_path: pathlib.Path, requirements_path: pathlib.Path, - uv_python_version: list[str], + python_version: Optional[str], pip_find_links_args: list[str], ) -> None: whl_cache_path.mkdir(parents=True, exist_ok=True) + py_tag_prefix = 'cp' + python_version.replace('.', '') if python_version else None + cached_pkgs: set[tuple[str, str]] = set() for whl in whl_cache_path.glob('*.whl'): parsed = packaging_t.parse_whl_name_version(whl.name) - if parsed is not None: - cached_pkgs.add((parsed.name, parsed.version)) + if parsed is None: + continue + if py_tag_prefix is not None and parsed.python_tag is not None: + if not parsed.python_tag.startswith(py_tag_prefix) and parsed.python_tag not in ('py3', 'py2.py3'): + continue + cached_pkgs.add((parsed.name, parsed.version)) missing_reqs: list[str] = [] with io.open(requirements_path, 'r') as f: @@ -546,6 +627,10 @@ def whl_cache_download( f.flush() missing_req_path = f.name + pip_python_version_args: list[str] = [] + if python_version is not None: + pip_python_version_args = ['--python-version', python_version.replace('.', '')] + try: cmd = [ sys.executable, @@ -553,7 +638,7 @@ def whl_cache_download( 'pip', 'download', '--only-binary=:all:', - *uv_python_version, + *pip_python_version_args, *pip_find_links_args, '-r', missing_req_path, @@ -584,6 +669,7 @@ def check_host_prerequisites() -> None: def env_bootstrap( bootstrap_settings: BootstrapSettings, pyproject: PyProject, + overrides: Optional[list[str]] = None, ) -> None: check_host_prerequisites() @@ -707,38 +793,51 @@ def env_bootstrap( if len(constraint_args) > 0: uv_compile_args = [o for o in uv_compile_args if o not in ('-U', '--upgrade')] - cmd = [ - 'uv', - '--cache-dir', - bootstrap_settings.uv_cache_dir, - 'pip', - 'compile', - *uv_python_version, - '--generate-hashes', - '--no-annotate', - '--no-header', - *pip_find_links_args, - *cache_find_links_args, - *constraint_args, - *uv_compile_args, - '-o', - f_out.name, - f_in.name, - ] - logger.info(dict(cmd=cmd)) + with contextlib.ExitStack() as stack: + if overrides and len(constraint_args) > 0: + patched = stack.enter_context( + tempfile.NamedTemporaryFile( + mode='w', prefix='constraints_', suffix='.txt' + ) + ) + packaging_t.apply_overrides_to_constraints( + requirements_path, overrides, patched + ) + patched.flush() + constraint_args = ['-c', patched.name] - try: - subprocess.check_call(cmd) - os.replace(f_out.name, str(requirements_path)) - except subprocess.CalledProcessError: - os.unlink(f_out.name) - raise + cmd = [ + 'uv', + '--cache-dir', + bootstrap_settings.uv_cache_dir, + 'pip', + 'compile', + *uv_python_version, + '--generate-hashes', + '--no-annotate', + '--no-header', + *pip_find_links_args, + *cache_find_links_args, + *constraint_args, + *uv_compile_args, + '-o', + f_out.name, + f_in.name, + ] + logger.info(dict(cmd=cmd)) + + try: + subprocess.check_call(cmd) + os.replace(f_out.name, str(requirements_path)) + except subprocess.CalledProcessError: + os.unlink(f_out.name) + raise if not bootstrap_settings.whl_cache_path.exists() or bootstrap_settings.whl_cache_update: whl_cache_download( whl_cache_path=bootstrap_settings.whl_cache_path, requirements_path=requirements_path, - uv_python_version=uv_python_version, + python_version=bootstrap_settings.python_version, pip_find_links_args=pip_find_links_args, ) if bootstrap_settings.whl_cache_path.exists(): @@ -797,6 +896,53 @@ def paths_equal(a: pathlib.Path | str, b: pathlib.Path | str) -> bool: return os.path.abspath(str(a)) == os.path.abspath(str(b)) +import argparse as _argparse + + +class argv_extract_t: + """Extract known arguments from argv by scanning parser action definitions.""" + + @dataclasses.dataclass + class res_t: + namespace: _argparse.Namespace + rest: list[str] + + @staticmethod + def extract( + parser: _argparse.ArgumentParser, + argv: list[str], + ) -> 'argv_extract_t.res_t': + flag_map: dict[str, _argparse.Action] = {} + for action in parser._actions: + for opt in action.option_strings: + flag_map[opt] = action + + matched_argv: list[str] = [] + rest: list[str] = [] + i = 0 + while i < len(argv): + action = flag_map.get(argv[i]) + if action is not None: + matched_argv.append(argv[i]) + i += 1 + if action.nargs in (None, 1) and action.const is None and not isinstance( + action, (_argparse._StoreTrueAction, _argparse._StoreFalseAction, _argparse._CountAction, _argparse._HelpAction) + ): + if i < len(argv): + matched_argv.append(argv[i]) + i += 1 + else: + rest.append(argv[i]) + i += 1 + + namespace = parser.parse_args(matched_argv) + + return argv_extract_t.res_t( + namespace=namespace, + rest=rest, + ) + + def run( d: Optional[pathlib.Path] = None, cli_path: Optional[pathlib.Path] = None, @@ -807,6 +953,22 @@ def run( if d is None: d = pathlib.Path(__file__).parent / 'pyproject.toml' + bootstrap_parser = _argparse.ArgumentParser(add_help=False) + bootstrap_parser.add_argument( + '--bootstrap-help', + action='help', + help='show bootstrap help and exit', + ) + bootstrap_parser.add_argument( + '--bootstrap-override', + dest='overrides', + action='append', + default=[], + help='override for uv pip compile (e.g. "librt>=0.8")', + ) + + bootstrap_args = argv_extract_t.extract(bootstrap_parser, sys.argv[1:]) + bootstrap_settings = BootstrapSettings.get() pyproject: PyProject = pyproject_load(d) @@ -820,6 +982,7 @@ def run( env_bootstrap( bootstrap_settings=bootstrap_settings, pyproject=pyproject, + overrides=bootstrap_args.namespace.overrides or None, ) logger.info([sys.executable, sys.argv, bootstrap_settings.python_path]) @@ -829,7 +992,8 @@ def run( str(bootstrap_settings.python_path), [ str(bootstrap_settings.python_path), - *sys.argv, + sys.argv[0], + *bootstrap_args.rest, ], ) @@ -838,7 +1002,7 @@ def run( [ str(bootstrap_settings.python_path), str(cli_path), - *sys.argv[1:], + *bootstrap_args.rest, ], )