From 9b5fff93c0b5a3e563c1ced4d0fac46703e8853a Mon Sep 17 00:00:00 2001 From: Siarhei Siniak Date: Sat, 18 Jan 2025 19:34:46 +0300 Subject: [PATCH] [+] add pip_resolve 1. add freezing packages based on pip resolver; 1.1. the command should resolve final packages with versions, and provide constraints with file hashes; to act alike go mod tool; --- m.py | 8 +- python/online/fxreader/pr34/commands.py | 26 +++ .../fxreader/pr34/commands_typed/pip.py | 203 +++++++++++++++++- 3 files changed, 230 insertions(+), 7 deletions(-) diff --git a/m.py b/m.py index bb5a4c8..fed3334 100755 --- a/m.py +++ b/m.py @@ -93,7 +93,7 @@ def env_bootstrap( pyproject: PyProject, ) -> None: subprocess.check_call([ - 'uv', 'venv', '--seed', '--offline', + 'uv', 'venv', '--seed', str(bootstrap_settings.env_path) ]) @@ -103,7 +103,6 @@ def env_bootstrap( 'install', '-p', bootstrap_settings.python_path, - '--offline', 'uv', ]) @@ -111,7 +110,6 @@ def env_bootstrap( bootstrap_settings.python_path, '-m', 'uv', 'pip', 'install', - '--offline', 'build', 'setuptools', 'meson-python', 'pybind11', ]) @@ -126,7 +124,6 @@ def env_bootstrap( bootstrap_settings.python_path, '-m', 'uv', 'pip', 'install', - '--offline', *early_wheels, ]) @@ -144,7 +141,6 @@ def env_bootstrap( bootstrap_settings.python_path, '-m', 'uv', 'pip', 'install', - '--offline', *early_dependencies, ]) @@ -207,4 +203,4 @@ if __name__ == '__main__': run( d=pathlib.Path(__file__).parent / 'python' / 'pyproject.toml', cli_path=pathlib.Path(__file__).parent / 'python' / 'cli.py', - ) \ No newline at end of file + ) diff --git a/python/online/fxreader/pr34/commands.py b/python/online/fxreader/pr34/commands.py index 54d248c..e03afc7 100644 --- a/python/online/fxreader/pr34/commands.py +++ b/python/online/fxreader/pr34/commands.py @@ -3878,6 +3878,30 @@ class Command(enum.StrEnum): scrap_yt_music = 'scrap-yt-music' vpn = 'vpn' backup = 'backup' + pip_resolve = 'pip_resolve' + +def pip_resolve(args: list[str]) -> None: + parser = argparse.ArgumentParser() + parser.add_argument( + '-m', '--mode', + choices=['copy_paste', 'monkey_patch'], + required=True, + ) + + options, argv = parser.parse_known_args(args) + + from online.fxreader.pr34.commands_typed.pip import pip_resolve + sys.stdout.write('\n'.join([ + '%s %s' % ( + o.constraint, + '--hash:sha256=%s' % o.sha256, + ) + for o in pip_resolve( + argv, + mode=options.mode, + ) + ])) + sys.stdout.flush() def commands_cli( argv: Optional[list[str]] = None @@ -3959,6 +3983,8 @@ def commands_cli( suspend_timer(args) elif options.command is Command.desktop_services: desktop_services(args) + elif options.command is Command.pip_resolve: + pip_resolve(args) elif options.command is Command.pm_service: pm_service(args) elif options.command is Command.backup: diff --git a/python/online/fxreader/pr34/commands_typed/pip.py b/python/online/fxreader/pr34/commands_typed/pip.py index 2af5e32..ce63dfe 100644 --- a/python/online/fxreader/pr34/commands_typed/pip.py +++ b/python/online/fxreader/pr34/commands_typed/pip.py @@ -1,4 +1,22 @@ +import contextlib +import pathlib +import dataclasses import pip._internal.commands.show +import pip._internal.commands.download +import pip._internal.cli.main_parser +import pip._internal.models.index +import pip._internal.utils.temp_dir +import pip._internal.cli.main +import pip._internal.network.download +import pip._internal.resolution.base +import pip._internal.resolution.resolvelib.resolver +import unittest.mock +import logging + +from typing import (Literal,) + +logger = logging.getLogger(__name__) + def pip_show( argv: list[str], @@ -7,4 +25,187 @@ def pip_show( pip._internal.commands.show.search_packages_info( argv, ) - ) \ No newline at end of file + ) + +class pip_resolve_t: + class res_t: + @dataclasses.dataclass + class download_info_t: + url: str + sha256: str + constraint: str + +def pip_resolve( + argv: list[str], + mode: Literal['copy_paste', 'monkey_patch'], +) -> list[ + pip_resolve_t.res_t.download_info_t +]: + if mode == 'copy_paste': + with contextlib.ExitStack() as stack: + stack.enter_context( + pip._internal.utils.temp_dir.global_tempdir_manager() + ) + + t2 = pip._internal.cli.main_parser.create_main_parser() + + t3 = t2.parse_args(['download']) + t1 = pip._internal.commands.download.DownloadCommand( + 'blah', + 'shit' + ) + + stack.enter_context(t1.main_context()) + + #options = pip._internal.commands.download.Values() + options = t3[0] + options.python_version = None + options.platforms = [] + options.abis = [] + options.implementation = [] + options.format_control = None + options.ignore_dependencies = None + options.index_url = pip._internal.models.index.PyPI.simple_url + options.extra_index_urls = [] + options.no_index = None + options.find_links = [] + options.pre = None + options.prefer_binary = True + options.only_binary = True + options.constraints = [] + options.use_pep517 = None + options.editables = [] + options.requirements = [] + options.src_dir = str(pathlib.Path(__file__).parent) + options.build_isolation = None + options.check_build_deps = None + options.progress_bar = True + options.require_hashes = None + options.ignore_requires_python = None + #options.cache_dir + pip._internal.commands.download.cmdoptions.check_dist_restriction(options) + # t1._in_main_context = True + session = t1.get_default_session(options) + target_python = pip._internal.commands.download.make_target_python(options) + finder = t1._build_package_finder( + options=options, + session=session, + target_python=target_python, + ignore_requires_python=options.ignore_requires_python, + ) + build_tracker = t1.enter_context(pip._internal.commands.download.get_build_tracker()) + reqs = t1.get_requirements([ + #'pip', 'uv', 'ipython', + *argv, + ], options, finder, session) + pip._internal.commands.download.check_legacy_setup_py_options(options, reqs) + directory = pip._internal.commands.download.TempDirectory( + delete=True, + kind='download', + globally_managed=True + ) + preparer = t1.make_requirement_preparer( + temp_build_dir=directory, + options=options, + build_tracker=build_tracker, + session=session, + finder=finder, + download_dir=None, + use_user_site=False, + verbosity=False, + ) + resolver = t1.make_resolver( + preparer=preparer, + finder=finder, + options=options, + ignore_requires_python=options.ignore_requires_python, + use_pep517=options.use_pep517, + py_version_info=options.python_version, + ) + t1.trace_basic_info(finder) + requirement_set = resolver.resolve(reqs, check_supported_wheels=True) + + return [ + pip_resolve_t.res_t.download_info_t( + constraint=k, + sha256=v.download_info.info.hashes['sha256'], + url=v.download_info.url, + ) + for k, v in requirement_set.requirements.items() + ] + elif mode == 'monkey_patch': + downloader_call_def = pip._internal.network.download.Downloader.__call__ + + def downloader_call(*args): + # import ipdb + # ipdb.set_trace() + # print(args) + + logger.warn(dict( + url=args[1].url, + )) + + return downloader_call_def(*args) + + #base_resolver_resolve_def = pip._internal.resolution.base.BaseResolver.resolve + base_resolver_resolve_def = pip._internal.resolution.resolvelib.resolver.Resolver.resolve + + reqs = [] + + def base_resolver_resolve(*args, **kwargs): + # print(args, kwargs) + + res = base_resolver_resolve_def( + *args, + **kwargs + ) + + reqs.append(res) + return res + + patches = [] + patches.append( + unittest.mock.patch.object( + pip._internal.network.download.Downloader, + '__call__', + downloader_call + ) + ) + #patches.append( + # unittest.mock.patch.object( + # pip._internal.resolution.base.BaseResolver, 'resolve', base_resolver_resolve)) + + patches.append( + unittest.mock.patch.object( + pip._internal.resolution.resolvelib.resolver.Resolver, + 'resolve', + base_resolver_resolve + ) + ) + + with contextlib.ExitStack() as stack: + for p in patches: + stack.enter_context(p) + + pip._internal.cli.main.main([ + 'download', + '--no-cache', + '-d', + '/dev/null', + *argv, + # 'numpy', + ]) + + return sum([ + [ + pip_resolve_t.res_t.download_info_t( + constraint=k, + sha256=v.download_info.info.hashes['sha256'], + url=v.download_info.url, + ) + for k, v in o.requirements.items() + ] + for o in reqs + ], []) + else: + raise NotImplementedError