From d643e8f97bfc23957f1ed3afc80a3308c0729ea3 Mon Sep 17 00:00:00 2001 From: Siarhei Siniak Date: Sat, 18 Jan 2025 20:56:03 +0300 Subject: [PATCH] [+] improve pip_resolve 1. do not download anything; 2. provide hashes per dependency; 3. test that works on numpy --- python/online/fxreader/pr34/commands.py | 18 ++- .../fxreader/pr34/commands_typed/pip.py | 128 +++++++++++++++--- 2 files changed, 122 insertions(+), 24 deletions(-) diff --git a/python/online/fxreader/pr34/commands.py b/python/online/fxreader/pr34/commands.py index e03afc7..314e9d4 100644 --- a/python/online/fxreader/pr34/commands.py +++ b/python/online/fxreader/pr34/commands.py @@ -3892,9 +3892,13 @@ def pip_resolve(args: list[str]) -> None: from online.fxreader.pr34.commands_typed.pip import pip_resolve sys.stdout.write('\n'.join([ - '%s %s' % ( + '#%s\n%s %s' % ( + o.url, o.constraint, - '--hash:sha256=%s' % o.sha256, + ' '.join([ + '--hash:sha256=%s' % o2 + for o2 in o.sha256 + ]) ) for o in pip_resolve( argv, @@ -3909,10 +3913,12 @@ def commands_cli( if argv is None: argv = sys.argv[1:] - logging.getLogger().setLevel(logging.INFO) - logger.setLevel(logging.INFO) - handler = logging.StreamHandler(sys.stderr) - logging.getLogger().addHandler(handler) + from online.fxreader.pr34.commands_typed.logging import setup as logging_setup + logging_setup() + #logging.getLogger().setLevel(logging.INFO) + #logger.setLevel(logging.INFO) + #handler = logging.StreamHandler(sys.stderr) + #logging.getLogger().addHandler(handler) msg : Optional[str] = None diff --git a/python/online/fxreader/pr34/commands_typed/pip.py b/python/online/fxreader/pr34/commands_typed/pip.py index ce63dfe..e48cf67 100644 --- a/python/online/fxreader/pr34/commands_typed/pip.py +++ b/python/online/fxreader/pr34/commands_typed/pip.py @@ -10,6 +10,7 @@ import pip._internal.cli.main import pip._internal.network.download import pip._internal.resolution.base import pip._internal.resolution.resolvelib.resolver +import pip._internal.operations.prepare import unittest.mock import logging @@ -137,20 +138,30 @@ def pip_resolve( downloader_call_def = pip._internal.network.download.Downloader.__call__ def downloader_call(*args): - # import ipdb - # ipdb.set_trace() - # print(args) - - logger.warn(dict( + logger.info(dict( url=args[1].url, )) return downloader_call_def(*args) + batch_downloader_call_def = pip._internal.network.download.BatchDownloader.__call__ + + def batch_downloader_call(*args): + # print(args) + + logger.info(dict( + args=args, + )) + + return [ + (o, ('/dev/null', None)) + for o in args[1] + ] + #base_resolver_resolve_def = pip._internal.resolution.base.BaseResolver.resolve base_resolver_resolve_def = pip._internal.resolution.resolvelib.resolver.Resolver.resolve - reqs = [] + result_requirements = [] def base_resolver_resolve(*args, **kwargs): # print(args, kwargs) @@ -160,9 +171,52 @@ def pip_resolve( **kwargs ) - reqs.append(res) + result_requirements.append(res) + raise NotImplementedError return res + get_http_url_def = pip._internal.operations.prepare.get_http_url + + def get_http_url(link, download, download_dir=None, hashes=None): + + logger.info(dict( + url=link.url, + hashes=hashes, + )) + + if link.url.endswith('.whl'): + print('blah') + hashes = None + + return '/dev/null', None + else: + return get_http_url_def( + link, + download, + download_dir, + hashes + ) + + prepare_linked_requirements_more_def = \ + pip._internal.operations.prepare.RequirementPreparer\ + .prepare_linked_requirements_more + + def prepare_linked_requirements_more( + _self, reqs, *args, **kwargs + ): + result_requirements.append(reqs) + raise NotImplementedError + + _complete_partial_requirements_def = \ + pip._internal.operations.prepare.RequirementPreparer\ + ._complete_partial_requirements + + def _complete_partial_requirements( + _self, reqs, *args, **kwargs + ): + result_requirements.append(reqs) + raise NotImplementedError + patches = [] patches.append( unittest.mock.patch.object( @@ -173,6 +227,13 @@ def pip_resolve( ) #patches.append( # unittest.mock.patch.object( + # pip._internal.network.download.BatchDownloader, + # '__call__', + # batch_downloader_call + # ) + #) + #patches.append( + # unittest.mock.patch.object( # pip._internal.resolution.base.BaseResolver, 'resolve', base_resolver_resolve)) patches.append( @@ -182,6 +243,27 @@ def pip_resolve( base_resolver_resolve ) ) + patches.append( + unittest.mock.patch.object( + pip._internal.operations.prepare, + 'get_http_url', + get_http_url, + ) + ) + patches.append( + unittest.mock.patch.object( + pip._internal.operations.prepare.RequirementPreparer, + 'prepare_linked_requirements_more', + prepare_linked_requirements_more + ) + ) + #patches.append( + # unittest.mock.patch.object( + # pip._internal.operations.prepare.RequirementPreparer, + # '_complete_partial_requirements', + # _complete_partial_requirements + # ) + #) with contextlib.ExitStack() as stack: for p in patches: @@ -189,6 +271,7 @@ def pip_resolve( pip._internal.cli.main.main([ 'download', + '-q', '--no-cache', '-d', '/dev/null', @@ -196,16 +279,25 @@ def pip_resolve( # 'numpy', ]) - return sum([ - [ - pip_resolve_t.res_t.download_info_t( - constraint=k, - sha256=v.download_info.info.hashes['sha256'], - url=v.download_info.url, - ) - for k, v in o.requirements.items() - ] - for o in reqs - ], []) + #return sum([ + # [ + # pip_resolve_t.res_t.download_info_t( + # constraint=k, + # sha256=v.download_info.info.hashes['sha256'], + # url=v.download_info.url, + # ) + # for k, v in o.requirements.items() + # ] + # for o in result_requirements + #], []) + logger.warn(result_requirements) + return [ + pip_resolve_t.res_t.download_info_t( + constraint=str(o.req), + sha256=o.hashes()._allowed['sha256'], + url=o.link.url, + ) + for o in sum(result_requirements, []) + ] else: raise NotImplementedError