fxreader.online/freelance-project-34-marketing-blog
2
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

[+] reformat with ruff

Browse Source
This commit is contained in:
Siarhei Siniak 2025-05-20 11:13:17 +03:00
parent 8510d49015
commit 0f17070c62
20 changed files with 7886 additions and 7960 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

390
python/_m.py
View File

@ -1,5 +1,5 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
#vim: set filetype=python # vim: set filetype=python
import logging import logging
import json import json
@ -7,158 +7,184 @@ import enum
import pathlib import pathlib
import sys import sys
import argparse import argparse
#import optparse
# import optparse
import dataclasses import dataclasses
import subprocess import subprocess
import os import os
from typing import ( from typing import (
Optional, Any, TypeAlias, Literal, cast, BinaryIO, Generator, Optional,
ClassVar, Self, Any,
TypeAlias,
Literal,
cast,
BinaryIO,
Generator,
ClassVar,
Self,
) )
logger = logging.getLogger() logger = logging.getLogger()
@dataclasses.dataclass @dataclasses.dataclass
class Settings: class Settings:
project_root : pathlib.Path = pathlib.Path.cwd() project_root: pathlib.Path = pathlib.Path.cwd()
env_path : pathlib.Path = project_root / 'tmp' / 'env3' env_path: pathlib.Path = project_root / 'tmp' / 'env3'
_settings : ClassVar[Optional['Settings']] = None _settings: ClassVar[Optional['Settings']] = None
@classmethod @classmethod
def settings(cls) -> Self: def settings(cls) -> Self:
if cls._settings is None: if cls._settings is None:
cls._settings = cls() cls._settings = cls()
return cls._settings
return cls._settings
def js(argv: list[str]) -> int: def js(argv: list[str]) -> int:
return subprocess.check_call([ return subprocess.check_call(
'sudo', [
'docker-compose', 'sudo',
'--project-directory', 'docker-compose',
Settings.settings().project_root, '--project-directory',
'-f', Settings.settings().project_root,
Settings.settings().project_root / 'docker' / 'js' / 'docker-compose.yml', '-f',
*argv, Settings.settings().project_root / 'docker' / 'js' / 'docker-compose.yml',
]) *argv,
]
)
def env( def env(
argv: Optional[list[str]] = None, argv: Optional[list[str]] = None,
mode: Literal['exec', 'subprocess'] = 'subprocess', mode: Literal['exec', 'subprocess'] = 'subprocess',
**kwargs: Any, **kwargs: Any,
) -> Optional[subprocess.CompletedProcess[bytes]]: ) -> Optional[subprocess.CompletedProcess[bytes]]:
env_path = Settings.settings().env_path env_path = Settings.settings().env_path
if not env_path.exists(): if not env_path.exists():
subprocess.check_call([ subprocess.check_call([sys.executable, '-m', 'venv', '--system-site-packages', str(env_path)])
sys.executable, '-m', 'venv',
'--system-site-packages',
str(env_path)
])
subprocess.check_call([ subprocess.check_call(
env_path / 'bin' / 'python3', [
'-m', 'pip', env_path / 'bin' / 'python3',
'install', '-r', 'requirements.txt', '-m',
]) 'pip',
'install',
'-r',
'requirements.txt',
]
)
if not argv is None: if not argv is None:
python_path = str(env_path / 'bin' / 'python3') python_path = str(env_path / 'bin' / 'python3')
if mode == 'exec': if mode == 'exec':
os.execv( os.execv(
python_path, python_path,
[ [
python_path, python_path,
*argv, *argv,
], ],
) )
return None return None
elif mode == 'subprocess': elif mode == 'subprocess':
return subprocess.run([ return subprocess.run(
python_path, [
*argv, python_path,
], **kwargs) *argv,
else: ],
raise NotImplementedError **kwargs,
)
else:
raise NotImplementedError
return None
return None
def ruff(argv: list[str]) -> None: def ruff(argv: list[str]) -> None:
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
parser.add_argument( parser.add_argument(
'-i', '-i',
dest='paths', dest='paths',
help='specify paths to check', help='specify paths to check',
default=[], default=[],
action='append', action='append',
) )
parser.add_argument( parser.add_argument(
'-e', '-e',
dest='exclude', dest='exclude',
help='rules to ignore', help='rules to ignore',
default=[], default=[],
action='append', action='append',
) )
options, args = parser.parse_known_args(argv) options, args = parser.parse_known_args(argv)
if len(options.paths) == 0: if len(options.paths) == 0:
options.paths.extend([ options.paths.extend(
'.', [
'dotfiles/.local/bin/commands', '.',
]) 'dotfiles/.local/bin/commands',
]
)
if len(options.exclude) == 0: if len(options.exclude) == 0:
options.exclude.extend([ options.exclude.extend(
'E731', [
'E713', 'E731',
'E714', 'E713',
'E703', 'E714',
]) 'E703',
]
)
res = env([ res = env(
'-m', [
'ruff', '-m',
'check', 'ruff',
*args, 'check',
'--output-format', 'json', *args,
'--ignore', ','.join(options.exclude), '--output-format',
*options.paths, 'json',
], stdout=subprocess.PIPE, stderr=subprocess.PIPE) '--ignore',
','.join(options.exclude),
*options.paths,
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
assert not res is None assert not res is None
errors = json.loads(res.stdout.decode('utf-8')) errors = json.loads(res.stdout.decode('utf-8'))
g: dict[str, Any] = dict() g: dict[str, Any] = dict()
for o in errors: for o in errors:
if not o['filename'] in g: if not o['filename'] in g:
g[o['filename']] = [] g[o['filename']] = []
g[o['filename']].append(o) g[o['filename']].append(o)
h = { h = {k: len(v) for k, v in g.items()}
k : len(v)
for k, v in g.items()
}
logger.info(json.dumps(errors, indent=4)) logger.info(json.dumps(errors, indent=4))
logger.info(json.dumps(h, indent=4)) logger.info(json.dumps(h, indent=4))
def inside_env() -> bool: def inside_env() -> bool:
try: try:
import numpy import numpy
return True
except Exception:
return False
#class Commands(enum.StrEnum): return True
except Exception:
return False
# class Commands(enum.StrEnum):
# js = 'js' # js = 'js'
# mypy = 'mypy' # mypy = 'mypy'
# env = 'env' # env = 'env'
@ -172,83 +198,97 @@ def inside_env() -> bool:
# argv, # argv,
# ) # )
def host_deps(argv: list[str]) -> None: def host_deps(argv: list[str]) -> None:
if sys.platform in ['linux']: if sys.platform in ['linux']:
subprocess.check_call(r''' subprocess.check_call(
r"""
exec yay -S $(cat requirements-archlinux.txt) exec yay -S $(cat requirements-archlinux.txt)
''', shell=True,) """,
else: shell=True,
raise NotImplementedError )
else:
raise NotImplementedError
Command_args = ['js', 'mypy', 'env', 'ruff', 'm2', 'host_deps',]
Command : TypeAlias = Literal['js', 'mypy', 'env', 'ruff', 'm2', 'host_deps',] Command_args = [
'js',
'mypy',
'env',
'ruff',
'm2',
'host_deps',
]
Command: TypeAlias = Literal[
'js',
'mypy',
'env',
'ruff',
'm2',
'host_deps',
]
def run(argv: Optional[list[str]] = None) -> None: def run(argv: Optional[list[str]] = None) -> None:
logging.basicConfig( logging.basicConfig(
level=logging.INFO, level=logging.INFO,
format=( format=('%(levelname)s:%(name)s:%(message)s:%(process)d:%(asctime)s:%(pathname)s:%(funcName)s:%(lineno)s'),
'%(levelname)s:%(name)s:%(message)s' )
':%(process)d'
':%(asctime)s'
':%(pathname)s:%(funcName)s:%(lineno)s'
),
)
if argv is None: if argv is None:
argv = sys.argv[:] argv = sys.argv[:]
parser = argparse.ArgumentParser()
parser.add_argument(
'command',
#'_command',
choices=[o for o in Command_args],
# required=True,
)
parser = argparse.ArgumentParser() options, args = parser.parse_known_args(argv[1:])
parser.add_argument(
'command',
#'_command',
choices=[
o
for o in Command_args
],
#required=True,
)
options, args = parser.parse_known_args(argv[1:]) assert options.command in Command_args
assert options.command in Command_args if len(args) > 0 and args[0] == '--':
del args[0]
if len(args) > 0 and args[0] == '--': # options.command = Commands(options._command)
del args[0]
#options.command = Commands(options._command) if options.command == 'js':
js(args)
elif options.command == 'host_deps':
host_deps(args)
elif options.command == 'env':
env(
args,
mode='exec',
)
# elif options.command == 'mypy':
# if not inside_env():
# env(
# [
# pathlib.Path(__file__).parent / 'm.py',
# *argv[1:],
# ],
# mode='exec'
# )
# else:
# mypy(args)
elif options.command == 'ruff':
ruff(args)
elif options.command == 'm2':
if not inside_env():
env(['--', '_m.py', 'm2', *args])
return
if options.command == 'js': import python.tasks.cython
js(args)
elif options.command == 'host_deps': python.tasks.cython.mypyc_build(pathlib.Path('_m.py'))
host_deps(args) else:
elif options.command == 'env': raise NotImplementedError
env(args, mode='exec',)
# elif options.command == 'mypy':
# if not inside_env():
# env(
# [
# pathlib.Path(__file__).parent / 'm.py',
# *argv[1:],
# ],
# mode='exec'
# )
# else:
# mypy(args)
elif options.command == 'ruff':
ruff(args)
elif options.command == 'm2':
if not inside_env():
env(['--', '_m.py', 'm2', *args])
return
import python.tasks.cython
python.tasks.cython.mypyc_build(
pathlib.Path('_m.py')
)
else:
raise NotImplementedError
if __name__ == '__main__': if __name__ == '__main__':
run() run()

228
python/cli.py
View File

@ -10,7 +10,10 @@ import enum
import argparse import argparse
import dataclasses import dataclasses
from typing import (Optional, override,) from typing import (
Optional,
override,
)
from online.fxreader.pr34.commands_typed.logging import setup as logging_setup from online.fxreader.pr34.commands_typed.logging import setup as logging_setup
@ -24,139 +27,134 @@ logger = logging.getLogger(__name__)
class Command(enum.StrEnum): class Command(enum.StrEnum):
mypy = 'mypy' mypy = 'mypy'
deploy_wheel = 'deploy:wheel' deploy_wheel = 'deploy:wheel'
tests = 'tests' tests = 'tests'
@dataclasses.dataclass @dataclasses.dataclass
class Settings( class Settings(
_cli.DistSettings, _cli.DistSettings,
): ):
base_dir: pathlib.Path = pathlib.Path(__file__).parent.parent base_dir: pathlib.Path = pathlib.Path(__file__).parent.parent
build_dir: pathlib.Path = base_dir / 'tmp' / 'build' build_dir: pathlib.Path = base_dir / 'tmp' / 'build'
wheel_dir: pathlib.Path = base_dir / 'deps' / 'dist' wheel_dir: pathlib.Path = base_dir / 'deps' / 'dist'
env_path: pathlib.Path = cli_bootstrap.BootstrapSettings.get(base_dir).env_path env_path: pathlib.Path = cli_bootstrap.BootstrapSettings.get(base_dir).env_path
python_path: pathlib.Path = cli_bootstrap.BootstrapSettings.get(base_dir).python_path python_path: pathlib.Path = cli_bootstrap.BootstrapSettings.get(base_dir).python_path
class CLI(_cli.CLI): class CLI(_cli.CLI):
def __init__(self) -> None: def __init__(self) -> None:
self.settings = Settings() self.settings = Settings()
self._projects: dict[str, _cli.Project] = { self._projects: dict[str, _cli.Project] = {
'online.fxreader.pr34': _cli.Project( 'online.fxreader.pr34': _cli.Project(
source_dir=self.settings.base_dir / 'python', source_dir=self.settings.base_dir / 'python',
build_dir=self.settings.base_dir / 'tmp' / 'online' / 'fxreader' / 'pr34' / 'build', build_dir=self.settings.base_dir / 'tmp' / 'online' / 'fxreader' / 'pr34' / 'build',
dest_dir=self.settings.base_dir / 'tmp' / 'online' / 'fxreader' / 'pr34' / 'install', dest_dir=self.settings.base_dir / 'tmp' / 'online' / 'fxreader' / 'pr34' / 'install',
) )
} }
self._dependencies : dict[str, _cli.Dependency] = dict() self._dependencies: dict[str, _cli.Dependency] = dict()
@override @override
@property @property
def dist_settings(self) -> _cli.DistSettings: def dist_settings(self) -> _cli.DistSettings:
return self.settings return self.settings
@override @override
@property @property
def projects(self) -> dict[str, _cli.Project]: def projects(self) -> dict[str, _cli.Project]:
return self._projects return self._projects
def mypy( def mypy(
self, self,
argv: list[str], argv: list[str],
) -> None: ) -> None:
import online.fxreader.pr34.commands_typed.mypy as _mypy import online.fxreader.pr34.commands_typed.mypy as _mypy
project = self._projects['online.fxreader.pr34'] project = self._projects['online.fxreader.pr34']
_mypy.run( _mypy.run(
argv, argv,
settings=_mypy.MypySettings( settings=_mypy.MypySettings(
paths=[ paths=[
#Settings.settings().project_root / 'dotfiles/.local/bin/commands', # Settings.settings().project_root / 'dotfiles/.local/bin/commands',
# project.source_dir / 'm.py', # project.source_dir / 'm.py',
project.source_dir / '_m.py', project.source_dir / '_m.py',
project.source_dir / 'online', project.source_dir / 'online',
project.source_dir / 'cli.py', project.source_dir / 'cli.py',
self.settings.base_dir / 'm.py', self.settings.base_dir / 'm.py',
# Settings.settings().project_root / 'deps/com.github.aiortc.aiortc/src', # Settings.settings().project_root / 'deps/com.github.aiortc.aiortc/src',
#Settings.settings().project_root / 'm.py', # Settings.settings().project_root / 'm.py',
], ],
max_errors={ max_errors={
'python/online/fxreader/pr34/commands_typed': 0, 'python/online/fxreader/pr34/commands_typed': 0,
'python/cli.py': 0, 'python/cli.py': 0,
'm.py': 0, 'm.py': 0,
'deps/com.github.aiortc.aiortc/src/online_fxreader': 0, 'deps/com.github.aiortc.aiortc/src/online_fxreader': 0,
'deps/com.github.aiortc.aiortc/src/aiortc/contrib/signaling': 0 'deps/com.github.aiortc.aiortc/src/aiortc/contrib/signaling': 0,
} },
), ),
) )
@override @override
@property @property
def dependencies(self) -> dict[str, _cli.Dependency]: def dependencies(self) -> dict[str, _cli.Dependency]:
return self._dependencies return self._dependencies
def run(self, argv: Optional[list[str]] = None) -> None: def run(self, argv: Optional[list[str]] = None) -> None:
if argv is None: if argv is None:
argv = copy.deepcopy(sys.argv) argv = copy.deepcopy(sys.argv)
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
parser.add_argument( parser.add_argument('command', choices=[o.value for o in Command])
'command', parser.add_argument('-p', '--project', choices=[o for o in self.projects])
choices=[ parser.add_argument(
o.value '-o',
for o in Command '--output_dir',
] default=None,
) help='wheel output dir for deploy:wheel',
parser.add_argument( )
'-p', '--project', parser.add_argument(
choices=[ '-f',
o '--force',
for o in self.projects default=False,
] action='store_true',
) help='remove install dir, before installing, default = false',
parser.add_argument( )
'-o', '--output_dir',
default=None,
help='wheel output dir for deploy:wheel',
)
parser.add_argument(
'-f', '--force',
default=False,
action='store_true',
help='remove install dir, before installing, default = false',
)
options, args = parser.parse_known_args(argv[1:]) options, args = parser.parse_known_args(argv[1:])
options.command = Command(options.command) options.command = Command(options.command)
if options.command is Command.deploy_wheel: if options.command is Command.deploy_wheel:
assert not options.project is None assert not options.project is None
self.deploy_wheel(
project_name=options.project,
argv=args,
output_dir=options.output_dir,
mypy=True,
)
elif options.command is Command.mypy:
self.mypy(
argv=args,
)
elif options.command is Command.tests:
for k, v in self.projects.items():
subprocess.check_call(
[
sys.executable,
'-m',
'unittest',
'online.fxreader.pr34.tests.test_crypto',
*args,
],
cwd=str(v.source_dir),
)
else:
raise NotImplementedError
self.deploy_wheel(
project_name=options.project,
argv=args,
output_dir=options.output_dir,
mypy=True,
)
elif options.command is Command.mypy:
self.mypy(
argv=args,
)
elif options.command is Command.tests:
for k, v in self.projects.items():
subprocess.check_call([
sys.executable,
'-m',
'unittest',
'online.fxreader.pr34.tests.test_crypto',
*args,
], cwd=str(v.source_dir))
else:
raise NotImplementedError
if __name__ == '__main__': if __name__ == '__main__':
CLI().run() CLI().run()

6925
python/online/fxreader/pr34/commands.py
View File

File diff suppressed because it is too large Load Diff

31
python/online/fxreader/pr34/commands_typed/argparse.py
View File

@ -1,27 +1,28 @@
__all__ = ( __all__ = ('parse_args',)
'parse_args',
)
import sys import sys
import argparse import argparse
from typing import (Optional,) from typing import (
Optional,
)
def parse_args( def parse_args(
parser: argparse.ArgumentParser, parser: argparse.ArgumentParser,
args: Optional[list[str]] = None, args: Optional[list[str]] = None,
) -> tuple[argparse.Namespace, list[str]]: ) -> tuple[argparse.Namespace, list[str]]:
if args is None: if args is None:
args = sys.argv[1:] args = sys.argv[1:]
argv : list[str] = [] argv: list[str] = []
for i, o in enumerate(args): for i, o in enumerate(args):
if o == '--': if o == '--':
argv.extend(args[i + 1:]) argv.extend(args[i + 1 :])
del args[i:] del args[i:]
break break
return parser.parse_args(args), argv return parser.parse_args(args), argv

23
python/online/fxreader/pr34/commands_typed/asyncio.py
View File

@ -1,14 +1,23 @@
import logging import logging
import asyncio import asyncio
from typing import (Any,) from typing import (
Any,
)
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
def handle_task_result(fut: asyncio.Future[Any]) -> None:
try:
fut.result()
logger.debug(dict(fut=fut, msg='done'), stacklevel=2,) def handle_task_result(fut: asyncio.Future[Any]) -> None:
except: try:
logger.exception('', stacklevel=2,) fut.result()
logger.debug(
dict(fut=fut, msg='done'),
stacklevel=2,
)
except:
logger.exception(
'',
stacklevel=2,
)

923
python/online/fxreader/pr34/commands_typed/cli.py
View File

@ -12,467 +12,490 @@ import abc
from .os import shutil_which from .os import shutil_which
from typing import ( from typing import (
Optional, Optional,
Literal, Literal,
Any, Any,
) )
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@dataclasses.dataclass @dataclasses.dataclass
class Project: class Project:
source_dir : pathlib.Path source_dir: pathlib.Path
build_dir : pathlib.Path build_dir: pathlib.Path
dest_dir : pathlib.Path dest_dir: pathlib.Path
meson_path: Optional[pathlib.Path] = None meson_path: Optional[pathlib.Path] = None
@dataclasses.dataclass @dataclasses.dataclass
class Dependency: class Dependency:
name: str name: str
mode : Literal['pyproject', 'meson', 'meson-python', 'm'] mode: Literal['pyproject', 'meson', 'meson-python', 'm']
source_path : pathlib.Path source_path: pathlib.Path
args: Optional[list[str]] = None args: Optional[list[str]] = None
@dataclasses.dataclass @dataclasses.dataclass
class DistSettings: class DistSettings:
wheel_dir : pathlib.Path wheel_dir: pathlib.Path
python_path: pathlib.Path python_path: pathlib.Path
env_path: pathlib.Path env_path: pathlib.Path
class CLI(abc.ABC): class CLI(abc.ABC):
@property @property
@abc.abstractmethod @abc.abstractmethod
def dist_settings(self) -> DistSettings: def dist_settings(self) -> DistSettings:
raise NotImplementedError raise NotImplementedError
@property @property
@abc.abstractmethod @abc.abstractmethod
def projects(self) -> dict[str, Project]: def projects(self) -> dict[str, Project]:
raise NotImplementedError raise NotImplementedError
@property @property
@abc.abstractmethod @abc.abstractmethod
def dependencies(self) -> dict[str, Dependency]: def dependencies(self) -> dict[str, Dependency]:
raise NotImplementedError raise NotImplementedError
def mypy( def mypy(self, argv: list[str]) -> None:
self, from . import mypy as _mypy
argv: list[str]
) -> None: _mypy.run(
from . import mypy as _mypy argv,
)
_mypy.run(
argv, def ruff(
) self,
project_name: str,
def ruff( argv: list[str],
self, ) -> None:
project_name: str, project = self.projects[project_name]
argv: list[str],
) -> None: if len(argv) == 0:
project = self.projects[project_name] argv = [
'check',
if len(argv) == 0: '.',
argv = ['check', '.',] ]
subprocess.check_call([ subprocess.check_call(
self.dist_settings.python_path, [
'-m', self.dist_settings.python_path,
'ruff', '-m',
'--config', str(project.source_dir / 'pyproject.toml'), 'ruff',
*argv, '--config',
]) str(project.source_dir / 'pyproject.toml'),
*argv,
def pyright( ]
self, )
project_name: str,
argv: list[str], def pyright(
) -> None: self,
project = self.projects[project_name] project_name: str,
argv: list[str],
if len(argv) == 0: ) -> None:
argv = ['--threads', '3'] project = self.projects[project_name]
cmd = [ if len(argv) == 0:
str(self.dist_settings.python_path), argv = ['--threads', '3']
'-m',
'pyright', cmd = [
'--pythonpath', str(self.dist_settings.python_path), str(self.dist_settings.python_path),
'-p', str(project.source_dir / 'pyproject.toml'), '-m',
*argv, 'pyright',
] '--pythonpath',
str(self.dist_settings.python_path),
logger.info(cmd) '-p',
str(project.source_dir / 'pyproject.toml'),
subprocess.check_call(cmd) *argv,
]
def pip_sync(
self, logger.info(cmd)
project: str,
features: list[str], subprocess.check_call(cmd)
) -> None:
from . import cli_bootstrap def pip_sync(
self,
pyproject = cli_bootstrap.pyproject_load( project: str,
self.projects[project].source_dir / 'pyproject.toml' features: list[str],
) ) -> None:
from . import cli_bootstrap
dependencies = sum([
pyproject.dependencies[o] pyproject = cli_bootstrap.pyproject_load(self.projects[project].source_dir / 'pyproject.toml')
for o in features
], []) dependencies = sum([pyproject.dependencies[o] for o in features], [])
pip_find_links : list[pathlib.Path] = [] pip_find_links: list[pathlib.Path] = []
if not pyproject.pip_find_links is None: if not pyproject.pip_find_links is None:
pip_find_links.extend(pyproject.pip_find_links) pip_find_links.extend(pyproject.pip_find_links)
logger.info(
logger.info(dict( dict(
dependencies=dependencies, dependencies=dependencies,
)) )
)
if len(dependencies) > 0:
subprocess.check_call([ if len(dependencies) > 0:
self.dist_settings.python_path, subprocess.check_call(
'-m', [
'uv', 'pip', 'install', self.dist_settings.python_path,
*sum([ '-m',
['-f', str(o),] 'uv',
for o in pip_find_links 'pip',
], []), 'install',
# '-f', str(pathlib.Path(__file__).parent / 'deps' / 'dist'), *sum(
'--offline', [
*dependencies, [
]) '-f',
str(o),
def deploy_fetch_dist( ]
self, for o in pip_find_links
force: bool, ],
) -> None: [],
for k, d in self.dependencies.items(): ),
whl_glob = self.dist_settings.wheel_dir / ('*%s*.whl' % d.name.replace('.', '_')) # '-f', str(pathlib.Path(__file__).parent / 'deps' / 'dist'),
if len(glob.glob( '--offline',
str(whl_glob) *dependencies,
)) == 0 or force: ]
if d.source_path.exists(): )
def whl_files_get() -> list[dict[str, Any]]:
return [ def deploy_fetch_dist(
dict( self,
path=o, force: bool,
stat=os.stat(o).st_mtime, ) -> None:
) for k, d in self.dependencies.items():
for o in glob.glob( whl_glob = self.dist_settings.wheel_dir / ('*%s*.whl' % d.name.replace('.', '_'))
str(whl_glob) if len(glob.glob(str(whl_glob))) == 0 or force:
) if d.source_path.exists():
]
def whl_files_get() -> list[dict[str, Any]]:
present_files = whl_files_get() return [
dict(
if d.mode == 'm': path=o,
if (d.source_path / 'm.py').exists(): stat=os.stat(o).st_mtime,
cmd = [ )
sys.executable, for o in glob.glob(str(whl_glob))
str(d.source_path / 'm.py'), ]
'deploy:wheel',
'-o', present_files = whl_files_get()
str(self.dist_settings.wheel_dir),
] if d.mode == 'm':
if (d.source_path / 'm.py').exists():
if not d.args is None: cmd = [
cmd.extend(d.args) sys.executable,
str(d.source_path / 'm.py'),
subprocess.check_call( 'deploy:wheel',
cmd, '-o',
cwd=d.source_path, str(self.dist_settings.wheel_dir),
) ]
else:
raise NotImplementedError if not d.args is None:
cmd.extend(d.args)
updated_files = whl_files_get()
subprocess.check_call(
def index_get(o: dict[str, Any]) -> tuple[Any, ...]: cmd,
return (o['path'], o['stat']) cwd=d.source_path,
)
present_files_index = { else:
index_get(o) : o raise NotImplementedError
for o in present_files
} updated_files = whl_files_get()
new_files : list[dict[str, Any]] = [] def index_get(o: dict[str, Any]) -> tuple[Any, ...]:
return (o['path'], o['stat'])
for o in updated_files:
entry_index = index_get(o) present_files_index = {index_get(o): o for o in present_files}
if not entry_index in present_files_index: new_files: list[dict[str, Any]] = []
new_files.append(o)
for o in updated_files:
if len(new_files) == 0: entry_index = index_get(o)
raise NotImplementedError
if not entry_index in present_files_index:
latest_file = sorted( new_files.append(o)
new_files,
key=lambda x: x['stat'] if len(new_files) == 0:
)[-1] raise NotImplementedError
subprocess.check_call([ latest_file = sorted(new_files, key=lambda x: x['stat'])[-1]
self.dist_settings.python_path,
'-m', 'pip', subprocess.check_call(
'install', [
latest_file['path'], self.dist_settings.python_path,
]) '-m',
'pip',
@property 'install',
def pkg_config_path(self,) -> set[pathlib.Path]: latest_file['path'],
return { ]
pathlib.Path(o) )
for o in glob.glob(
str(self.dist_settings.env_path / 'lib' / 'python*' / '**' / 'pkgconfig'), @property
recursive=True, def pkg_config_path(
) self,
} ) -> set[pathlib.Path]:
return {
def deploy_wheel( pathlib.Path(o)
self, for o in glob.glob(
project_name: str, str(self.dist_settings.env_path / 'lib' / 'python*' / '**' / 'pkgconfig'),
argv: Optional[list[str]] = None, recursive=True,
output_dir: Optional[pathlib.Path] = None, )
force: Optional[bool] = None, }
env: Optional[dict[str, str]] = None,
mypy: bool = False, def deploy_wheel(
tests: bool = False, self,
) -> None: project_name: str,
project = self.projects[project_name] argv: Optional[list[str]] = None,
output_dir: Optional[pathlib.Path] = None,
# subprocess.check_call([ force: Optional[bool] = None,
# sys.argv[0], env: Optional[dict[str, str]] = None,
# # sys.executable, mypy: bool = False,
# '-p', options.project, tests: bool = False,
# Command.meson_setup.value, ) -> None:
# ]) project = self.projects[project_name]
if argv is None: # subprocess.check_call([
argv = [] # sys.argv[0],
# # sys.executable,
# assert argv is None or len(argv) == 0 # '-p', options.project,
# Command.meson_setup.value,
if not project.meson_path is None: # ])
if tests:
self.meson_test( if argv is None:
project_name=project_name, argv = []
)
# assert argv is None or len(argv) == 0
self.meson_install(
project_name=project_name, if not project.meson_path is None:
force=force, if tests:
) self.meson_test(
project_name=project_name,
if mypy: )
self.mypy([])
self.meson_install(
if env is None: project_name=project_name,
env = dict() force=force,
)
extra_args: list[str] = []
if mypy:
if len(self.third_party_roots) > 0: self.mypy([])
extra_args.extend([
'-Csetup-args=%s' % ( if env is None:
'-Dthird_party_roots=%s' % str(o.absolute()) env = dict()
)
for o in self.third_party_roots extra_args: list[str] = []
])
if len(self.third_party_roots) > 0:
cmd = [ extra_args.extend(['-Csetup-args=%s' % ('-Dthird_party_roots=%s' % str(o.absolute())) for o in self.third_party_roots])
sys.executable,
'-m', cmd = [
'build', sys.executable,
'-w', '-n', '-m',
*extra_args, 'build',
'-Csetup-args=-Dmodes=pyproject', '-w',
'-Cbuild-dir=%s' % str(project.build_dir / 'pyproject'), '-n',
'-Csetup-args=-Dinstall_path=%s' % str(project.dest_dir), *extra_args,
# '-Cbuild-dir=%s' % str(project.build_dir), '-Csetup-args=-Dmodes=pyproject',
str(project.source_dir), '-Cbuild-dir=%s' % str(project.build_dir / 'pyproject'),
*argv, '-Csetup-args=-Dinstall_path=%s' % str(project.dest_dir),
] # '-Cbuild-dir=%s' % str(project.build_dir),
str(project.source_dir),
if not output_dir is None: *argv,
cmd.extend(['-o', str(output_dir)]) ]
logger.info(dict(env=env)) if not output_dir is None:
cmd.extend(['-o', str(output_dir)])
subprocess.check_call(
cmd, logger.info(dict(env=env))
env=dict(list(os.environ.items())) | env,
) subprocess.check_call(
cmd,
if not project.meson_path is None: env=dict(list(os.environ.items())) | env,
if tests: )
subprocess.check_call(
[ if not project.meson_path is None:
'ninja', if tests:
'-C', subprocess.check_call(
str(project.build_dir / 'pyproject'), [
'test', 'ninja',
] '-C',
) str(project.build_dir / 'pyproject'),
'test',
def meson_install( ]
self, )
project_name: str,
force: Optional[bool] = None, def meson_install(
argv: Optional[list[str]] = None, self,
) -> None: project_name: str,
project = self.projects[project_name] force: Optional[bool] = None,
argv: Optional[list[str]] = None,
if force is None: ) -> None:
force = False project = self.projects[project_name]
if argv is None: if force is None:
argv = [] force = False
if force and project.dest_dir.exists(): if argv is None:
shutil.rmtree(project.dest_dir) argv = []
subprocess.check_call([ if force and project.dest_dir.exists():
shutil_which('meson', True,), shutil.rmtree(project.dest_dir)
'install',
'-C', subprocess.check_call(
project.build_dir / 'meson', [
'--destdir', project.dest_dir, shutil_which(
*argv, 'meson',
]) True,
),
for o in glob.glob( 'install',
str(project.dest_dir / 'lib' / 'pkgconfig' / '*.pc'), '-C',
recursive=True, project.build_dir / 'meson',
): '--destdir',
logger.info(dict( project.dest_dir,
path=o, *argv,
action='patch prefix', ]
)) )
with io.open(o, 'r') as f: for o in glob.glob(
content = f.read() str(project.dest_dir / 'lib' / 'pkgconfig' / '*.pc'),
recursive=True,
with io.open(o, 'w') as f: ):
f.write( logger.info(
content.replace('prefix=/', 'prefix=${pcfiledir}/../../') dict(
) path=o,
def ninja( action='patch prefix',
self, )
project_name: str, )
argv: Optional[list[str]] = None,
env: Optional[dict[str, str]] = None, with io.open(o, 'r') as f:
) -> None: content = f.read()
project = self.projects[project_name]
with io.open(o, 'w') as f:
if argv is None: f.write(content.replace('prefix=/', 'prefix=${pcfiledir}/../../'))
argv = []
def ninja(
if env is None: self,
env = dict() project_name: str,
argv: Optional[list[str]] = None,
logger.info(dict(env=env)) env: Optional[dict[str, str]] = None,
) -> None:
subprocess.check_call( project = self.projects[project_name]
[
shutil_which('ninja', True), if argv is None:
'-C', argv = []
str(project.build_dir / 'meson'),
*argv, if env is None:
], env = dict()
env=dict(list(os.environ.items())) | env,
) logger.info(dict(env=env))
def meson_test( subprocess.check_call(
self, [
project_name: str, shutil_which('ninja', True),
argv: Optional[list[str]] = None, '-C',
) -> None: str(project.build_dir / 'meson'),
project = self.projects[project_name] *argv,
],
if argv is None: env=dict(list(os.environ.items())) | env,
argv = [] )
subprocess.check_call([ def meson_test(
shutil_which('meson', True,), self,
'test', project_name: str,
'-C', argv: Optional[list[str]] = None,
project.build_dir / 'meson', ) -> None:
*argv, project = self.projects[project_name]
])
if argv is None:
argv = []
def meson_compile(
self, subprocess.check_call(
project_name: str, [
argv: Optional[list[str]] = None, shutil_which(
) -> None: 'meson',
project = self.projects[project_name] True,
),
if argv is None: 'test',
argv = [] '-C',
project.build_dir / 'meson',
subprocess.check_call([ *argv,
shutil_which('meson', True,), ]
'compile', )
'-C',
project.build_dir / 'meson', def meson_compile(
*argv, self,
]) project_name: str,
argv: Optional[list[str]] = None,
@property ) -> None:
def third_party_roots(self) -> list[pathlib.Path]: project = self.projects[project_name]
return []
if argv is None:
def meson_setup( argv = []
self,
project_name: str, subprocess.check_call(
force: bool, [
argv: Optional[list[str]] = None, shutil_which(
env: Optional[dict[str, str]] = None, 'meson',
# third_party_roots: Optional[list[pathlib.Path]] = None, True,
) -> None: ),
project = self.projects[project_name] 'compile',
'-C',
if argv is None: project.build_dir / 'meson',
argv = [] *argv,
]
if env is None: )
env = dict()
@property
logger.info(dict(env=env)) def third_party_roots(self) -> list[pathlib.Path]:
return []
if force:
if (project.build_dir / 'meson').exists(): def meson_setup(
logger.info(dict(action='removing build dir', path=project.build_dir / 'meson')) self,
shutil.rmtree(project.build_dir / 'meson') project_name: str,
force: bool,
extra_args : list[str] = [] argv: Optional[list[str]] = None,
env: Optional[dict[str, str]] = None,
if len(self.third_party_roots) > 0: # third_party_roots: Optional[list[pathlib.Path]] = None,
extra_args.extend([ ) -> None:
'-Dthird_party_roots=%s' % str(o.absolute()) project = self.projects[project_name]
for o in self.third_party_roots
]) if argv is None:
argv = []
cmd = [
shutil_which('meson', True,), if env is None:
'setup', env = dict()
str(project.source_dir),
str(project.build_dir / 'meson'), logger.info(dict(env=env))
'-Dmodes=["meson"]',
*extra_args, if force:
# '-Dpkgconfig.relocatable=true', if (project.build_dir / 'meson').exists():
'-Dprefix=/', logger.info(dict(action='removing build dir', path=project.build_dir / 'meson'))
*argv, shutil.rmtree(project.build_dir / 'meson')
]
extra_args: list[str] = []
logger.info(dict(cmd=cmd))
if len(self.third_party_roots) > 0:
subprocess.check_call( extra_args.extend(['-Dthird_party_roots=%s' % str(o.absolute()) for o in self.third_party_roots])
cmd,
env=dict(list(os.environ.items())) | env, cmd = [
) shutil_which(
'meson',
True,
),
'setup',
str(project.source_dir),
str(project.build_dir / 'meson'),
'-Dmodes=["meson"]',
*extra_args,
# '-Dpkgconfig.relocatable=true',
'-Dprefix=/',
*argv,
]
logger.info(dict(cmd=cmd))
subprocess.check_call(
cmd,
env=dict(list(os.environ.items())) | env,
)

525
python/online/fxreader/pr34/commands_typed/cli_bootstrap.py
View File

@ -10,327 +10,324 @@ import os
import logging import logging
from typing import (Optional, Any,) from typing import (
Optional,
Any,
)
from typing_extensions import ( from typing_extensions import (
Self, BinaryIO, Self,
BinaryIO,
) )
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
def toml_load(f: BinaryIO) -> Any: def toml_load(f: BinaryIO) -> Any:
try: try:
import tomllib import tomllib
return tomllib.load(f)
except:
pass
try: return tomllib.load(f)
import tomli except:
return tomli.load(f) pass
except:
pass try:
import tomli
return tomli.load(f)
except:
pass
raise NotImplementedError
raise NotImplementedError
@dataclasses.dataclass @dataclasses.dataclass
class PyProject: class PyProject:
path: pathlib.Path path: pathlib.Path
dependencies: dict[str, list[str]] dependencies: dict[str, list[str]]
early_features: Optional[list[str]] = None early_features: Optional[list[str]] = None
pip_find_links: Optional[list[pathlib.Path]] = None pip_find_links: Optional[list[pathlib.Path]] = None
runtime_libdirs: Optional[list[pathlib.Path]] = None runtime_libdirs: Optional[list[pathlib.Path]] = None
runtime_preload: Optional[list[pathlib.Path]] = None runtime_preload: Optional[list[pathlib.Path]] = None
requirements: dict[str, pathlib.Path] = dataclasses.field(default_factory=lambda : dict()) requirements: dict[str, pathlib.Path] = dataclasses.field(default_factory=lambda: dict())
def pyproject_load( def pyproject_load(
d: pathlib.Path, d: pathlib.Path,
) -> PyProject: ) -> PyProject:
with io.open(d, 'rb') as f: with io.open(d, 'rb') as f:
content = toml_load(f) content = toml_load(f)
assert isinstance(content, dict) assert isinstance(content, dict)
dependencies : dict[str, list[str]] = dict() dependencies: dict[str, list[str]] = dict()
dependencies['default'] = content['project']['dependencies'] dependencies['default'] = content['project']['dependencies']
if ( if 'optional-dependencies' in content['project']:
'optional-dependencies' in content['project'] assert isinstance(content['project']['optional-dependencies'], dict)
):
assert isinstance(
content['project']['optional-dependencies'],
dict
)
for k, v in content['project']['optional-dependencies'].items(): for k, v in content['project']['optional-dependencies'].items():
assert isinstance(v, list) assert isinstance(v, list)
assert isinstance(k, str) assert isinstance(k, str)
dependencies[k] = v dependencies[k] = v
res = PyProject(
path=d,
dependencies=dependencies,
)
res = PyProject( tool_name = 'online.fxreader.pr34'.replace('.', '-')
path=d,
dependencies=dependencies,
)
tool_name = 'online.fxreader.pr34'.replace('.', '-') if 'tool' in content and isinstance(content['tool'], dict) and tool_name in content['tool'] and isinstance(content['tool'][tool_name], dict):
if 'early_features' in content['tool'][tool_name]:
res.early_features = content['tool'][tool_name]['early_features']
if ( if 'pip_find_links' in content['tool'][tool_name]:
'tool' in content and res.pip_find_links = [d.parent / pathlib.Path(o) for o in content['tool'][tool_name]['pip_find_links']]
isinstance(
content['tool'], dict
) and
tool_name in content['tool'] and
isinstance(
content['tool'][tool_name],
dict
)
):
if 'early_features' in content['tool'][tool_name]:
res.early_features = content['tool'][tool_name]['early_features']
if 'pip_find_links' in content['tool'][tool_name]: if 'runtime_libdirs' in content['tool'][tool_name]:
res.pip_find_links = [ res.runtime_libdirs = [
d.parent / pathlib.Path(o) d.parent / pathlib.Path(o)
for o in content['tool'][tool_name]['pip_find_links'] # pathlib.Path(o)
] for o in content['tool'][tool_name]['runtime_libdirs']
]
if 'runtime_libdirs' in content['tool'][tool_name]: if 'runtime_preload' in content['tool'][tool_name]:
res.runtime_libdirs = [ res.runtime_preload = [
d.parent / pathlib.Path(o) d.parent / pathlib.Path(o)
# pathlib.Path(o) # pathlib.Path(o)
for o in content['tool'][tool_name]['runtime_libdirs'] for o in content['tool'][tool_name]['runtime_preload']
] ]
if 'runtime_preload' in content['tool'][tool_name]: if 'requirements' in content['tool'][tool_name]:
res.runtime_preload = [ assert isinstance(content['tool'][tool_name]['requirements'], dict)
d.parent / pathlib.Path(o)
# pathlib.Path(o)
for o in content['tool'][tool_name]['runtime_preload']
]
if 'requirements' in content['tool'][tool_name]: res.requirements = {
assert isinstance(content['tool'][tool_name]['requirements'], dict) k: d.parent / pathlib.Path(v)
# pathlib.Path(o)
for k, v in content['tool'][tool_name]['requirements'].items()
}
res.requirements = { return res
k : d.parent / pathlib.Path(v)
# pathlib.Path(o)
for k, v in content['tool'][tool_name]['requirements'].items()
}
return res
@dataclasses.dataclass @dataclasses.dataclass
class BootstrapSettings: class BootstrapSettings:
env_path: pathlib.Path env_path: pathlib.Path
python_path: pathlib.Path python_path: pathlib.Path
base_dir: pathlib.Path base_dir: pathlib.Path
python_version: Optional[str] = dataclasses.field( python_version: Optional[str] = dataclasses.field(
default_factory=lambda : os.environ.get( default_factory=lambda: os.environ.get(
'PYTHON_VERSION', 'PYTHON_VERSION',
'%d.%d' % ( '%d.%d'
sys.version_info.major, % (
sys.version_info.minor, sys.version_info.major,
), sys.version_info.minor,
).strip() ),
) ).strip()
uv_args: list[str] = dataclasses.field( )
default_factory=lambda : os.environ.get( uv_args: list[str] = dataclasses.field(
'UV_ARGS', default_factory=lambda: os.environ.get(
'--offline', 'UV_ARGS',
).split(), '--offline',
) ).split(),
)
@classmethod @classmethod
def get( def get(
cls, cls,
base_dir: Optional[pathlib.Path] = None, base_dir: Optional[pathlib.Path] = None,
) -> Self: ) -> Self:
if base_dir is None: if base_dir is None:
base_dir = pathlib.Path.cwd() base_dir = pathlib.Path.cwd()
env_path = base_dir / '.venv' env_path = base_dir / '.venv'
python_path = env_path / 'bin' / 'python3' python_path = env_path / 'bin' / 'python3'
return cls(
base_dir=base_dir,
env_path=env_path,
python_path=python_path,
)
return cls(
base_dir=base_dir,
env_path=env_path,
python_path=python_path,
)
def env_bootstrap( def env_bootstrap(
bootstrap_settings: BootstrapSettings, bootstrap_settings: BootstrapSettings,
pyproject: PyProject, pyproject: PyProject,
) -> None: ) -> None:
pip_find_links : list[pathlib.Path] = [] pip_find_links: list[pathlib.Path] = []
if not pyproject.pip_find_links is None: if not pyproject.pip_find_links is None:
pip_find_links.extend(pyproject.pip_find_links) pip_find_links.extend(pyproject.pip_find_links)
pip_find_links_args = sum([ pip_find_links_args = sum(
['-f', str(o),] [
for o in pip_find_links [
], []) '-f',
str(o),
]
for o in pip_find_links
],
[],
)
features : list[str] = [] features: list[str] = []
if pyproject.early_features: if pyproject.early_features:
features.extend(pyproject.early_features) features.extend(pyproject.early_features)
requirements_python_version: Optional[str] = None requirements_python_version: Optional[str] = None
if not bootstrap_settings.python_version is None: if not bootstrap_settings.python_version is None:
requirements_python_version = bootstrap_settings.python_version.replace('.', '_') requirements_python_version = bootstrap_settings.python_version.replace('.', '_')
requirements_name = '_'.join(sorted(features))
if requirements_python_version:
requirements_name += '_' + requirements_python_version
requirements_path: Optional[pathlib.Path] = None
if requirements_name in pyproject.requirements:
requirements_path = pyproject.requirements[requirements_name]
else:
requirements_path = pyproject.path.parent / 'requirements.txt'
requirements_in: list[str] = []
requirements_in.extend(['uv', 'pip', 'build', 'setuptools', 'meson-python', 'pybind11'])
if pyproject.early_features:
early_dependencies = sum([pyproject.dependencies[o] for o in pyproject.early_features], [])
logger.info(
dict(
requirements_name=requirements_name,
early_dependencies=early_dependencies,
)
)
requirements_in.extend(early_dependencies)
# if len(early_dependencies) > 0:
# subprocess.check_call([
# bootstrap_settings.python_path,
# '-m',
# 'uv', 'pip', 'install',
# *pip_find_links_args,
# # '-f', str(pathlib.Path(__file__).parent / 'deps' / 'dist'),
# *bootstrap_settings.uv_args,
# *early_dependencies,
# ])
if not requirements_path.exists():
with tempfile.NamedTemporaryFile(
mode='w',
prefix='requirements',
suffix='.in',
) as f:
f.write('\n'.join(requirements_in))
f.flush()
subprocess.check_call(
[
'uv',
'pip',
'compile',
'--generate-hashes',
*pip_find_links_args,
# '-p',
# bootstrap_settings.python_path,
*bootstrap_settings.uv_args,
'-o',
str(requirements_path),
f.name,
]
)
uv_python_version: list[str] = []
if not bootstrap_settings.python_version is None:
uv_python_version.extend(
[
'-p',
bootstrap_settings.python_version,
]
)
subprocess.check_call(
[
'uv',
'venv',
*uv_python_version,
*pip_find_links_args,
# '--seed',
*bootstrap_settings.uv_args,
str(bootstrap_settings.env_path),
]
)
subprocess.check_call(
[
'uv',
'pip',
'install',
*pip_find_links_args,
'-p',
bootstrap_settings.python_path,
'--require-hashes',
*bootstrap_settings.uv_args,
'-r',
str(requirements_path),
]
)
requirements_name = '_'.join(sorted(features)) def paths_equal(a: pathlib.Path | str, b: pathlib.Path | str) -> bool:
return os.path.abspath(str(a)) == os.path.abspath(str(b))
if requirements_python_version:
requirements_name += '_' + requirements_python_version
requirements_path : Optional[pathlib.Path] = None
if requirements_name in pyproject.requirements:
requirements_path = pyproject.requirements[requirements_name]
else:
requirements_path = pyproject.path.parent / 'requirements.txt'
requirements_in : list[str] = []
requirements_in.extend([
'uv', 'pip', 'build', 'setuptools', 'meson-python', 'pybind11'
])
if pyproject.early_features:
early_dependencies = sum([
pyproject.dependencies[o]
for o in pyproject.early_features
], [])
logger.info(dict(
requirements_name=requirements_name,
early_dependencies=early_dependencies,
))
requirements_in.extend(early_dependencies)
# if len(early_dependencies) > 0:
# subprocess.check_call([
# bootstrap_settings.python_path,
# '-m',
# 'uv', 'pip', 'install',
# *pip_find_links_args,
# # '-f', str(pathlib.Path(__file__).parent / 'deps' / 'dist'),
# *bootstrap_settings.uv_args,
# *early_dependencies,
# ])
if not requirements_path.exists():
with tempfile.NamedTemporaryFile(
mode='w',
prefix='requirements',
suffix='.in',
) as f:
f.write(
'\n'.join(requirements_in)
)
f.flush()
subprocess.check_call([
'uv',
'pip',
'compile',
'--generate-hashes',
*pip_find_links_args,
# '-p',
# bootstrap_settings.python_path,
*bootstrap_settings.uv_args,
'-o', str(requirements_path),
f.name,
])
uv_python_version: list[str] = []
if not bootstrap_settings.python_version is None:
uv_python_version.extend([
'-p', bootstrap_settings.python_version,
])
subprocess.check_call([
'uv', 'venv',
*uv_python_version,
*pip_find_links_args,
# '--seed',
*bootstrap_settings.uv_args,
str(bootstrap_settings.env_path)
])
subprocess.check_call([
'uv',
'pip',
'install',
*pip_find_links_args,
'-p',
bootstrap_settings.python_path,
'--require-hashes',
*bootstrap_settings.uv_args,
'-r', str(requirements_path),
])
def paths_equal(
a: pathlib.Path | str,
b: pathlib.Path | str
) -> bool:
return (
os.path.abspath(str(a)) ==
os.path.abspath(str(b))
)
def run( def run(
d: Optional[pathlib.Path] = None, d: Optional[pathlib.Path] = None,
cli_path: Optional[pathlib.Path] = None, cli_path: Optional[pathlib.Path] = None,
) -> None: ) -> None:
if cli_path is None: if cli_path is None:
cli_path = pathlib.Path(__file__).parent / 'cli.py' cli_path = pathlib.Path(__file__).parent / 'cli.py'
if d is None: if d is None:
d = pathlib.Path(__file__).parent / 'pyproject.toml' d = pathlib.Path(__file__).parent / 'pyproject.toml'
bootstrap_settings = BootstrapSettings.get() bootstrap_settings = BootstrapSettings.get()
pyproject : PyProject = pyproject_load( pyproject: PyProject = pyproject_load(d)
d
)
logging.basicConfig(level=logging.INFO) logging.basicConfig(level=logging.INFO)
if not bootstrap_settings.env_path.exists(): if not bootstrap_settings.env_path.exists():
env_bootstrap( env_bootstrap(
bootstrap_settings=bootstrap_settings, bootstrap_settings=bootstrap_settings,
pyproject=pyproject, pyproject=pyproject,
) )
logger.info([sys.executable, sys.argv, bootstrap_settings.python_path]) logger.info([sys.executable, sys.argv, bootstrap_settings.python_path])
if not paths_equal(sys.executable, bootstrap_settings.python_path): if not paths_equal(sys.executable, bootstrap_settings.python_path):
os.execv( os.execv(
str(bootstrap_settings.python_path), str(bootstrap_settings.python_path),
[ [
str(bootstrap_settings.python_path), str(bootstrap_settings.python_path),
*sys.argv, *sys.argv,
] ],
) )
os.execv(
str(bootstrap_settings.python_path),
[
str(bootstrap_settings.python_path),
str(cli_path),
*sys.argv[1:],
],
)
os.execv(
str(bootstrap_settings.python_path),
[
str(bootstrap_settings.python_path),
str(
cli_path
),
*sys.argv[1:],
]
)
if __name__ == '__main__': if __name__ == '__main__':
run() run()

151
python/online/fxreader/pr34/commands_typed/crypto.py
View File

@ -3,88 +3,95 @@ import os
import cryptography.hazmat.primitives.kdf.scrypt import cryptography.hazmat.primitives.kdf.scrypt
from typing import (Literal, overload, Optional,) from typing import (
Literal,
overload,
Optional,
)
class PasswordUtils: class PasswordUtils:
@overload @overload
@classmethod @classmethod
def secret_hash( def secret_hash(
cls, cls,
secret: str | bytes, secret: str | bytes,
mode: Literal['base64'], mode: Literal['base64'],
salt: Optional[bytes] = None, salt: Optional[bytes] = None,
) -> tuple[str, str]: ... ) -> tuple[str, str]: ...
@overload @overload
@classmethod @classmethod
def secret_hash( def secret_hash(
cls, cls,
secret: str | bytes, secret: str | bytes,
mode: Literal['bytes'], mode: Literal['bytes'],
salt: Optional[bytes] = None, salt: Optional[bytes] = None,
) -> tuple[bytes, bytes]: ... ) -> tuple[bytes, bytes]: ...
@classmethod @classmethod
def secret_hash( def secret_hash(
cls, cls,
secret: str | bytes, secret: str | bytes,
mode: Literal['bytes', 'base64'], mode: Literal['bytes', 'base64'],
salt: Optional[bytes] = None, salt: Optional[bytes] = None,
) -> tuple[str, str] | tuple[bytes, bytes]: ) -> tuple[str, str] | tuple[bytes, bytes]:
if salt is None: if salt is None:
salt = os.urandom(16) salt = os.urandom(16)
if isinstance(secret, str): if isinstance(secret, str):
secret = secret.encode('utf-8') secret = secret.encode('utf-8')
# derive # derive
kdf = cls._scrypt_init(salt=salt) kdf = cls._scrypt_init(salt=salt)
hashed_secret = kdf.derive(secret) hashed_secret = kdf.derive(secret)
if mode == 'bytes': if mode == 'bytes':
return (salt, hashed_secret) return (salt, hashed_secret)
elif mode == 'base64': elif mode == 'base64':
res_tuple = tuple(( res_tuple = tuple(
base64.b64encode(o).decode('utf-8') (
for o in (salt, hashed_secret,) base64.b64encode(o).decode('utf-8')
)) for o in (
return (res_tuple[0], res_tuple[1]) salt,
else: hashed_secret,
raise NotImplementedError )
)
)
return (res_tuple[0], res_tuple[1])
else:
raise NotImplementedError
@classmethod @classmethod
def _scrypt_init( def _scrypt_init(cls, salt: bytes) -> cryptography.hazmat.primitives.kdf.scrypt.Scrypt:
cls, return cryptography.hazmat.primitives.kdf.scrypt.Scrypt(
salt: bytes salt=salt,
) -> cryptography.hazmat.primitives.kdf.scrypt.Scrypt: length=32,
return cryptography.hazmat.primitives.kdf.scrypt.Scrypt( n=2**14,
salt=salt, r=8,
length=32, p=1,
n=2**14, )
r=8,
p=1,
)
@classmethod @classmethod
def secret_check( def secret_check(
cls, cls,
secret: str | bytes, secret: str | bytes,
salt: str | bytes, salt: str | bytes,
hashed_secret: str | bytes, hashed_secret: str | bytes,
) -> bool: ) -> bool:
if isinstance(salt, str): if isinstance(salt, str):
salt = base64.b64decode(salt) salt = base64.b64decode(salt)
if isinstance(secret, str): if isinstance(secret, str):
secret = secret.encode('utf-8') secret = secret.encode('utf-8')
if isinstance(hashed_secret, str): if isinstance(hashed_secret, str):
hashed_secret = base64.b64decode(hashed_secret) hashed_secret = base64.b64decode(hashed_secret)
kdf = cls._scrypt_init(salt=salt) kdf = cls._scrypt_init(salt=salt)
try: try:
kdf.verify(secret, hashed_secret) kdf.verify(secret, hashed_secret)
return True return True
except cryptography.exceptions.InvalidKey: except cryptography.exceptions.InvalidKey:
return False return False

52
python/online/fxreader/pr34/commands_typed/debug.py
View File

@ -1,35 +1,39 @@
import os import os
import logging import logging
from typing import (Optional,) from typing import (
Optional,
)
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
class DebugPy: class DebugPy:
@classmethod @classmethod
def set_trace( def set_trace(
cls, cls,
host: Optional[str] = None, host: Optional[str] = None,
port: Optional[int] = None, port: Optional[int] = None,
wait: Optional[bool] = None, wait: Optional[bool] = None,
) -> None: ) -> None:
if host is None: if host is None:
host = '127.0.0.1' host = '127.0.0.1'
if port is None: if port is None:
port = 4444 port = 4444
if wait is None: if wait is None:
wait = True wait = True
import debugpy import debugpy
if os.environ.get('DEBUGPY_RUNNING') != 'true': if os.environ.get('DEBUGPY_RUNNING') != 'true':
logger.info('debugpy init') logger.info('debugpy init')
import debugpy import debugpy
debugpy.listen((host, port))
os.environ['DEBUGPY_RUNNING'] = 'true'
if wait: debugpy.listen((host, port))
debugpy.wait_for_client() os.environ['DEBUGPY_RUNNING'] = 'true'
debugpy.breakpoint()
logger.info('debugpy done') if wait:
debugpy.wait_for_client()
debugpy.breakpoint()
logger.info('debugpy done')

22
python/online/fxreader/pr34/commands_typed/logging.py
View File

@ -1,16 +1,14 @@
import logging import logging
from typing import (Optional,) from typing import (
Optional,
)
def setup(level: Optional[int] = None) -> None: def setup(level: Optional[int] = None) -> None:
if level is None: if level is None:
level = logging.INFO level = logging.INFO
logging.basicConfig( logging.basicConfig(
level=level, level=level,
format=( format=('%(levelname)s:%(name)s:%(message)s:%(process)d:%(asctime)s:%(pathname)s:%(funcName)s:%(lineno)s'),
'%(levelname)s:%(name)s:%(message)s' )
':%(process)d'
':%(asctime)s'
':%(pathname)s:%(funcName)s:%(lineno)s'
),
)

340
python/online/fxreader/pr34/commands_typed/mypy.py
View File

@ -9,208 +9,232 @@ import logging
import sys import sys
import argparse import argparse
from pydantic import (Field,) from pydantic import (
Field,
)
from typing import (ClassVar, Generator, Annotated, Optional, Any,) from typing import (
ClassVar,
Generator,
Annotated,
Optional,
Any,
)
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@pydantic.dataclasses.dataclass @pydantic.dataclasses.dataclass
class MypyFormatEntry: class MypyFormatEntry:
name : str name: str
value : str value: str
def __eq__(self, other: object) -> bool: def __eq__(self, other: object) -> bool:
if not isinstance(other, type(self)): if not isinstance(other, type(self)):
raise NotImplementedError raise NotImplementedError
return self.value == other.value
return self.value == other.value
class MypyFormat: class MypyFormat:
vscode : ClassVar[MypyFormatEntry] = MypyFormatEntry(name='vscode', value='vscode') vscode: ClassVar[MypyFormatEntry] = MypyFormatEntry(name='vscode', value='vscode')
json : ClassVar[MypyFormatEntry] = MypyFormatEntry(name='json', value='json') json: ClassVar[MypyFormatEntry] = MypyFormatEntry(name='json', value='json')
@classmethod
def from_value(cls, value: str) -> MypyFormatEntry:
for e in cls.entries():
if value == e.value:
return e
@classmethod raise NotImplementedError
def from_value(cls, value: str) -> MypyFormatEntry:
for e in cls.entries():
if value == e.value:
return e
raise NotImplementedError @classmethod
def entries(
cls,
) -> Generator[
MypyFormatEntry,
None,
None,
]:
for o in dir(cls):
e = getattr(cls, o)
if not isinstance(e, MypyFormatEntry):
continue
@classmethod yield e
def entries(cls) -> Generator[MypyFormatEntry, None, None,]:
for o in dir(cls):
e = getattr(cls, o)
if not isinstance(e, MypyFormatEntry):
continue
yield e
class MypySettings(pydantic_settings.BaseSettings): class MypySettings(pydantic_settings.BaseSettings):
model_config = pydantic_settings.SettingsConfigDict( model_config = pydantic_settings.SettingsConfigDict(
env_prefix='online_fxreader_pr34_mypy_', env_prefix='online_fxreader_pr34_mypy_',
case_sensitive=False, case_sensitive=False,
) )
config_path: pathlib.Path = pathlib.Path.cwd() / '.mypy.ini'
max_errors: dict[str, int] = dict()
paths: Annotated[list[pathlib.Path], Field(default_factory=lambda: ['.'])]
config_path : pathlib.Path = pathlib.Path.cwd() / '.mypy.ini'
max_errors : dict[str, int] = dict()
paths : Annotated[list[pathlib.Path], Field(default_factory=lambda : ['.'])]
def run( def run(
argv: Optional[list[str]] = None, argv: Optional[list[str]] = None,
settings: Optional[MypySettings] = None, settings: Optional[MypySettings] = None,
) -> None: ) -> None:
if argv is None: if argv is None:
argv = [] argv = []
if settings is None: if settings is None:
settings = MypySettings() settings = MypySettings()
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
parser.add_argument( parser.add_argument(
'-q', '--quiet', '-q',
dest='quiet', '--quiet',
action='store_true', dest='quiet',
help='do not print anything if the program is correct according to max_errors limits', action='store_true',
default=False, help='do not print anything if the program is correct according to max_errors limits',
) default=False,
parser.add_argument( )
'-i', parser.add_argument(
dest='paths', '-i',
help='specify paths to check', dest='paths',
default=[], help='specify paths to check',
action='append', default=[],
) action='append',
parser.add_argument( )
'-f', '--format', parser.add_argument(
dest='_format', '-f',
help='output format of errors', '--format',
default=MypyFormat.json.value, dest='_format',
choices=[ help='output format of errors',
o.value default=MypyFormat.json.value,
for o in MypyFormat.entries() choices=[o.value for o in MypyFormat.entries()],
], )
) options, args = parser.parse_known_args(argv)
options, args = parser.parse_known_args(argv)
if len(args) > 0 and args[0] == '--': if len(args) > 0 and args[0] == '--':
del args[0] del args[0]
options.format = MypyFormat.from_value(options._format) options.format = MypyFormat.from_value(options._format)
if len(options.paths) == 0: if len(options.paths) == 0:
options.paths.extend(settings.paths) options.paths.extend(settings.paths)
started_at = datetime.datetime.now() started_at = datetime.datetime.now()
mypy_cmd = [ mypy_cmd = [
sys.executable, sys.executable,
'-m', '-m',
'mypy', 'mypy',
'--config-file', str(settings.config_path), '--config-file',
'--strict', str(settings.config_path),
'-O', '--strict',
'json', '-O',
*args, 'json',
*options.paths, *args,
] *options.paths,
]
logger.info(dict(cmd=mypy_cmd))
logger.info(dict(cmd=mypy_cmd)) res = subprocess.run(
mypy_cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
res = subprocess.run( done_at = datetime.datetime.now()
mypy_cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
done_at = datetime.datetime.now() try:
assert not res.returncode is None
try: errors = sorted(
assert not res.returncode is None [json.loads(o) for o in res.stdout.decode('utf-8').splitlines() if not o.strip() == ''],
key=lambda x: (
x.get('file', ''),
x.get('line', 0),
),
)
errors = sorted([ if not options.quiet:
json.loads(o) if (len(res.stderr)) > 0:
for o in res.stdout.decode('utf-8').splitlines() logger.error(res.stderr.decode('utf-8'))
if not o.strip() == '' except:
], key=lambda x: ( logger.exception('')
x.get('file', ''), logger.error(res.stdout.decode('utf-8'))
x.get('line', 0), logger.error(res.stderr.decode('utf-8'))
)) sys.exit(res.returncode)
if not options.quiet: g: dict[str, Any] = dict()
if (len(res.stderr)) > 0: for o in errors:
logger.error(res.stderr.decode('utf-8')) if not o['file'] in g:
except: g[o['file']] = []
logger.exception('') g[o['file']].append(o)
logger.error(res.stdout.decode('utf-8'))
logger.error(res.stderr.decode('utf-8'))
sys.exit(res.returncode)
h = {
k: len(v)
for k, v in sorted(
list(g.items()),
key=lambda x: x[0],
)
}
g : dict[str, Any] = dict() mentioned_paths = marisa_trie.Trie(list(h))
for o in errors:
if not o['file'] in g:
g[o['file']] = []
g[o['file']].append(o)
h = { violated_limits: dict[str, str] = dict()
k : len(v)
for k, v in sorted(
list(g.items()),
key=lambda x: x[0],
)
}
mentioned_paths = marisa_trie.Trie(list(h)) for k, v in settings.max_errors.items():
matching_paths = mentioned_paths.keys(k)
total_errors = sum([h[o] for o in matching_paths], 0)
violated_limits : dict[str, str] = dict() if total_errors > v:
violated_limits[k] = '%s - [%s]: has %d errors > %d' % (
k,
', '.join(matching_paths),
total_errors,
v,
)
for k, v in settings.max_errors.items(): if len(violated_limits) > 0 or not options.quiet:
matching_paths = mentioned_paths.keys(k) if options.format == MypyFormat.vscode:
total_errors = sum([ for o in errors:
h[o] sys.stdout.write(
for o in matching_paths '[%s] %s:%d,%d %s - %s - %s\n'
], 0) % (
o['severity'],
o['file'],
o['line'],
o['column'],
o['message'],
o['hint'],
o['code'],
)
)
sys.stdout.flush()
# logger.info(json.dumps(errors, indent=4))
else:
logger.info(json.dumps(errors, indent=4))
if total_errors > v: # if len(violated_limits) > 0:
violated_limits[k] = '%s - [%s]: has %d errors > %d' % ( # logger.info(json.dumps(violated_limits, indent=4))
k, ', '.join(matching_paths), total_errors, v, logger.info(
) json.dumps(
dict(
max_errors=settings.max_errors,
violated_limits=violated_limits,
histogram=h,
elapsed=(done_at - started_at).total_seconds(),
),
indent=4,
)
)
if len(violated_limits) > 0 or not options.quiet: if len(violated_limits) > 0:
if options.format == MypyFormat.vscode: sys.exit(1)
for o in errors:
sys.stdout.write('[%s] %s:%d,%d %s - %s - %s\n' % (
o['severity'],
o['file'],
o['line'],
o['column'],
o['message'],
o['hint'],
o['code'],
))
sys.stdout.flush()
#logger.info(json.dumps(errors, indent=4))
else:
logger.info(json.dumps(errors, indent=4))
#if len(violated_limits) > 0:
# logger.info(json.dumps(violated_limits, indent=4))
logger.info(json.dumps(dict(
max_errors=settings.max_errors,
violated_limits=violated_limits,
histogram=h,
elapsed=(done_at - started_at).total_seconds(),
), indent=4))
if len(violated_limits) > 0:
sys.exit(1)
if __name__ == '__main__': if __name__ == '__main__':
from . import logging as _logging from . import logging as _logging
_logging.setup()
run(sys.argv[1:]) _logging.setup()
run(sys.argv[1:])

165
python/online/fxreader/pr34/commands_typed/os.py
View File

@ -11,112 +11,115 @@ import dataclasses
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
from typing import (overload, Optional, Literal, Any, Annotated,) from typing import (
overload,
Optional,
Literal,
Any,
Annotated,
)
from .cli_bootstrap import PyProject from .cli_bootstrap import PyProject
@overload @overload
def shutil_which( def shutil_which(
name: str, name: str,
raise_on_failure: Literal[True], raise_on_failure: Literal[True],
) -> str: ... ) -> str: ...
@overload @overload
def shutil_which( def shutil_which(
name: str, name: str,
raise_on_failure: bool, raise_on_failure: bool,
) -> Optional[str]: ... ) -> Optional[str]: ...
def shutil_which( def shutil_which(
name: str, name: str,
raise_on_failure: bool, raise_on_failure: bool,
) -> Optional[str]: ) -> Optional[str]:
res = shutil.which(name) res = shutil.which(name)
if res is None and raise_on_failure: if res is None and raise_on_failure:
raise NotImplementedError raise NotImplementedError
else: else:
return res return res
def runtime_libdirs_init( def runtime_libdirs_init(
project: PyProject, project: PyProject,
) -> None: ) -> None:
if sys.platform == 'linux': if sys.platform == 'linux':
ld_library_path : list[pathlib.Path] = [ ld_library_path: list[pathlib.Path] = [
o o
for o in [ for o in [
*[ *[o.absolute() for o in (project.runtime_libdirs if project.runtime_libdirs else [])],
o.absolute() *[pathlib.Path(o) for o in os.environ.get('LD_LIBRARY_PATH', '').split(os.path.pathsep) if o != ''],
for o in ( ]
project.runtime_libdirs ]
if project.runtime_libdirs
else []
)
],
*[
pathlib.Path(o)
for o in os.environ.get(
'LD_LIBRARY_PATH',
''
).split(os.path.pathsep)
if o != ''
]
]
]
ld_library_path_present : list[pathlib.Path] = [] ld_library_path_present: list[pathlib.Path] = []
for o in ld_library_path: for o in ld_library_path:
if not o.exists(): if not o.exists():
logger.warning(dict( logger.warning(
ld_library_path=o, dict(
msg='not found', ld_library_path=o,
)) msg='not found',
)
)
ld_library_path_present.append(o) ld_library_path_present.append(o)
os.environ.update( os.environ.update(LD_LIBRARY_PATH=os.path.pathsep.join([str(o) for o in ld_library_path_present]))
LD_LIBRARY_PATH=os.path.pathsep.join([
str(o) for o in ld_library_path_present
])
)
for preload_path in (project.runtime_preload or []): for preload_path in project.runtime_preload or []:
for preload_found in glob.glob(str( for preload_found in glob.glob(str(preload_path.parent / ('lib%s.so' % preload_path.name))):
preload_path.parent / ('lib%s.so' % preload_path.name) logger.info(
)): dict(
logger.info(dict( preload_path=preload_path,
preload_path=preload_path, preload_found=preload_found, preload_found=preload_found,
# lib_path=o, # lib_path=o,
msg='load_library', msg='load_library',
)) )
)
ctypes.cdll.LoadLibrary(preload_found)
else:
raise NotImplementedError
ctypes.cdll.LoadLibrary(preload_found)
else:
raise NotImplementedError
class interfaces_index_t: class interfaces_index_t:
@dataclasses.dataclass @dataclasses.dataclass
class Interface: class Interface:
@dataclasses.dataclass @dataclasses.dataclass
class AddrInfo: class AddrInfo:
family: str family: str
local: str local: str
name: Annotated[
str,
pydantic.Field(
alias='ifname',
),
]
addr_info: list[AddrInfo]
name: Annotated[
str,
pydantic.Field(
alias='ifname',
)
]
addr_info: list[AddrInfo]
def interfaces_index() -> list[interfaces_index_t.Interface]: def interfaces_index() -> list[interfaces_index_t.Interface]:
res = pydantic.RootModel[ res = (
list[interfaces_index_t.Interface] pydantic.RootModel[list[interfaces_index_t.Interface]]
].model_validate_json( .model_validate_json(
subprocess.check_output([ subprocess.check_output(
'ip', '-j', 'addr', [
]).decode('utf-8') 'ip',
).root '-j',
'addr',
]
).decode('utf-8')
)
.root
)
return res return res

838
python/online/fxreader/pr34/commands_typed/pip.py
View File

@ -11,514 +11,470 @@ import logging
import typing import typing
if typing.TYPE_CHECKING: if typing.TYPE_CHECKING:
import pip._internal.commands.show import pip._internal.commands.show
import pip._internal.commands.download import pip._internal.commands.download
import pip._internal.cli.main_parser import pip._internal.cli.main_parser
import pip._internal.models.index import pip._internal.models.index
import pip._internal.utils.temp_dir import pip._internal.utils.temp_dir
import pip._internal.cli.main import pip._internal.cli.main
import pip._internal.network.download import pip._internal.network.download
import pip._internal.resolution.base import pip._internal.resolution.base
import pip._internal.resolution.resolvelib.resolver import pip._internal.resolution.resolvelib.resolver
import pip._internal.operations.prepare import pip._internal.operations.prepare
from typing import ( from typing import (
Literal, Optional, Iterable, Any, Literal,
Optional,
Iterable,
Any,
) )
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
def pip_show( def pip_show(
argv: list[str], argv: list[str],
) -> list['pip._internal.commands.show._PackageInfo']: ) -> list['pip._internal.commands.show._PackageInfo']:
import pip._internal.commands.show import pip._internal.commands.show
return list(
pip._internal.commands.show.search_packages_info( return list(
argv, pip._internal.commands.show.search_packages_info(
) argv,
) )
)
class pip_resolve_t: class pip_resolve_t:
class kwargs_t: class kwargs_t:
class mode_t(enum.StrEnum): class mode_t(enum.StrEnum):
copy_paste = "copy_paste" copy_paste = 'copy_paste'
monkey_patch = "monkey_patch" monkey_patch = 'monkey_patch'
uv_pip_freeze = "uv_pip_freeze" uv_pip_freeze = 'uv_pip_freeze'
uv_pip_compile = "uv_pip_compile" uv_pip_compile = 'uv_pip_compile'
@dataclasses.dataclass @dataclasses.dataclass
class res_t: class res_t:
@dataclasses.dataclass @dataclasses.dataclass
class download_info_t: class download_info_t:
url: str url: str
sha256: str sha256: str
constraint: str constraint: str
txt: Optional[str] = None txt: Optional[str] = None
entries: Optional[list[download_info_t]] = None entries: Optional[list[download_info_t]] = None
def pip_resolve_entries_to_txt( def pip_resolve_entries_to_txt(entries: list[pip_resolve_t.res_t.download_info_t]) -> str:
entries: list[pip_resolve_t.res_t.download_info_t] return '\n'.join(['#%s\n%s %s' % (o.url, o.constraint, ' '.join(['--hash=sha256:%s' % o2 for o2 in o.sha256])) for o in entries])
) -> str:
return '\n'.join([
'#%s\n%s %s' % (
o.url,
o.constraint,
' '.join([
'--hash=sha256:%s' % o2
for o2 in o.sha256
])
)
for o in entries
])
def pip_resolve( def pip_resolve(
argv: list[str], argv: list[str],
mode: pip_resolve_t.kwargs_t.mode_t, mode: pip_resolve_t.kwargs_t.mode_t,
requirements: Optional[list[str]] = None, requirements: Optional[list[str]] = None,
) -> pip_resolve_t.res_t: ) -> pip_resolve_t.res_t:
if mode is pip_resolve_t.kwargs_t.mode_t.copy_paste: if mode is pip_resolve_t.kwargs_t.mode_t.copy_paste:
import pip._internal.commands.show import pip._internal.commands.show
import pip._internal.commands.download import pip._internal.commands.download
import pip._internal.cli.cmdoptions import pip._internal.cli.cmdoptions
import pip._internal.cli.main_parser import pip._internal.cli.main_parser
import pip._internal.models.index import pip._internal.models.index
import pip._internal.utils.temp_dir import pip._internal.utils.temp_dir
import pip._internal.cli.main import pip._internal.cli.main
import pip._internal.network.download import pip._internal.network.download
import pip._internal.resolution.base import pip._internal.resolution.base
import pip._internal.req.req_install import pip._internal.req.req_install
import pip._internal.resolution.resolvelib.resolver import pip._internal.resolution.resolvelib.resolver
import pip._internal.operations.prepare import pip._internal.operations.prepare
import pip._internal.utils.temp_dir import pip._internal.utils.temp_dir
import pip._internal.operations.build.build_tracker import pip._internal.operations.build.build_tracker
import pip._internal.models.direct_url import pip._internal.models.direct_url
with contextlib.ExitStack() as stack: with contextlib.ExitStack() as stack:
stack.enter_context(pip._internal.utils.temp_dir.global_tempdir_manager()) stack.enter_context(pip._internal.utils.temp_dir.global_tempdir_manager())
t2 = pip._internal.cli.main_parser.create_main_parser() t2 = pip._internal.cli.main_parser.create_main_parser()
t3 = t2.parse_args(["download"]) t3 = t2.parse_args(['download'])
t1 = pip._internal.commands.download.DownloadCommand("blah", "shit") t1 = pip._internal.commands.download.DownloadCommand('blah', 'shit')
stack.enter_context(t1.main_context()) stack.enter_context(t1.main_context())
# options = pip._internal.commands.download.Values() # options = pip._internal.commands.download.Values()
options = t3[0] options = t3[0]
options.python_version = None options.python_version = None
options.platforms = [] options.platforms = []
options.abis = [] options.abis = []
options.implementation = [] options.implementation = []
options.format_control = None options.format_control = None
options.ignore_dependencies = None options.ignore_dependencies = None
options.index_url = pip._internal.models.index.PyPI.simple_url options.index_url = pip._internal.models.index.PyPI.simple_url
options.extra_index_urls = [] options.extra_index_urls = []
options.no_index = None options.no_index = None
options.find_links = [] options.find_links = []
options.pre = None options.pre = None
options.prefer_binary = True options.prefer_binary = True
options.only_binary = True options.only_binary = True
options.constraints = [] options.constraints = []
options.use_pep517 = None options.use_pep517 = None
options.editables = [] options.editables = []
options.requirements = [] options.requirements = []
options.src_dir = str(pathlib.Path(__file__).parent) options.src_dir = str(pathlib.Path(__file__).parent)
options.build_isolation = None options.build_isolation = None
options.check_build_deps = None options.check_build_deps = None
options.progress_bar = True options.progress_bar = True
options.require_hashes = None options.require_hashes = None
options.ignore_requires_python = False options.ignore_requires_python = False
# options.cache_dir # options.cache_dir
pip._internal.cli.cmdoptions.check_dist_restriction(options) pip._internal.cli.cmdoptions.check_dist_restriction(options)
# t1._in_main_context = True # t1._in_main_context = True
session = t1.get_default_session(options) session = t1.get_default_session(options)
target_python = pip._internal.cli.cmdoptions.make_target_python(options) target_python = pip._internal.cli.cmdoptions.make_target_python(options)
finder = t1._build_package_finder( finder = t1._build_package_finder(
options=options, options=options,
session=session, session=session,
target_python=target_python, target_python=target_python,
ignore_requires_python=options.ignore_requires_python, ignore_requires_python=options.ignore_requires_python,
) )
build_tracker = t1.enter_context( build_tracker = t1.enter_context(pip._internal.operations.build.build_tracker.get_build_tracker())
pip._internal.operations.build.build_tracker.get_build_tracker() reqs = t1.get_requirements(
) [
reqs = t1.get_requirements( #'pip', 'uv', 'ipython',
[ *argv,
#'pip', 'uv', 'ipython', ],
*argv, options,
], finder,
options, session,
finder, )
session, pip._internal.req.req_install.check_legacy_setup_py_options(options, reqs)
) directory = pip._internal.utils.temp_dir.TempDirectory(delete=True, kind='download', globally_managed=True)
pip._internal.req.req_install.check_legacy_setup_py_options(options, reqs) preparer = t1.make_requirement_preparer(
directory = pip._internal.utils.temp_dir.TempDirectory( temp_build_dir=directory,
delete=True, kind="download", globally_managed=True options=options,
) build_tracker=build_tracker,
preparer = t1.make_requirement_preparer( session=session,
temp_build_dir=directory, finder=finder,
options=options, download_dir=None,
build_tracker=build_tracker, use_user_site=False,
session=session, verbosity=False,
finder=finder, )
download_dir=None, resolver = t1.make_resolver(
use_user_site=False, preparer=preparer,
verbosity=False, finder=finder,
) options=options,
resolver = t1.make_resolver( ignore_requires_python=options.ignore_requires_python,
preparer=preparer, use_pep517=options.use_pep517,
finder=finder, py_version_info=options.python_version,
options=options, )
ignore_requires_python=options.ignore_requires_python, t1.trace_basic_info(finder)
use_pep517=options.use_pep517, requirement_set = resolver.resolve(reqs, check_supported_wheels=True)
py_version_info=options.python_version,
)
t1.trace_basic_info(finder)
requirement_set = resolver.resolve(reqs, check_supported_wheels=True)
res = pip_resolve_t.res_t() res = pip_resolve_t.res_t()
res.entries = [] res.entries = []
for k, v in requirement_set.requirements.items(): for k, v in requirement_set.requirements.items():
assert not v.download_info is None assert not v.download_info is None
assert isinstance( assert isinstance(
v.download_info.info, v.download_info.info,
pip._internal.models.direct_url.ArchiveInfo, pip._internal.models.direct_url.ArchiveInfo,
) )
assert not v.download_info.info.hashes is None assert not v.download_info.info.hashes is None
res.entries.append( res.entries.append(
pip_resolve_t.res_t.download_info_t( pip_resolve_t.res_t.download_info_t(
constraint=k, constraint=k,
sha256=v.download_info.info.hashes["sha256"], sha256=v.download_info.info.hashes['sha256'],
url=v.download_info.url, url=v.download_info.url,
) )
) )
res.txt = pip_resolve_entries_to_txt( res.txt = pip_resolve_entries_to_txt(res.entries)
res.entries
)
return res return res
elif mode is pip_resolve_t.kwargs_t.mode_t.monkey_patch: elif mode is pip_resolve_t.kwargs_t.mode_t.monkey_patch:
import pip._internal.commands.show import pip._internal.commands.show
import pip._internal.commands.download import pip._internal.commands.download
import pip._internal.cli.main_parser import pip._internal.cli.main_parser
import pip._internal.models.index import pip._internal.models.index
import pip._internal.models.link import pip._internal.models.link
from pip._internal.models.link import ( from pip._internal.models.link import (
Link, Link,
) )
import pip._internal.utils.temp_dir import pip._internal.utils.temp_dir
from pip._internal.metadata.base import ( from pip._internal.metadata.base import (
BaseDistribution, BaseDistribution,
) )
import pip._internal.cli.main import pip._internal.cli.main
import pip._internal.network.download import pip._internal.network.download
import pip._internal.resolution.base import pip._internal.resolution.base
import pip._internal.resolution.resolvelib.resolver import pip._internal.resolution.resolvelib.resolver
import pip._internal.operations.prepare import pip._internal.operations.prepare
from pip._internal.network.download import ( from pip._internal.network.download import (
Downloader, Downloader,
) )
from pip._internal.operations.prepare import ( from pip._internal.operations.prepare import (
File, File,
) )
from pip._internal.req.req_set import RequirementSet from pip._internal.req.req_set import RequirementSet
from pip._internal.utils.hashes import Hashes from pip._internal.utils.hashes import Hashes
from pip._internal.req.req_install import InstallRequirement from pip._internal.req.req_install import InstallRequirement
downloader_call_def = pip._internal.network.download.Downloader.__call__ downloader_call_def = pip._internal.network.download.Downloader.__call__
def downloader_call( def downloader_call(
_self: pip._internal.network.download.Downloader, _self: pip._internal.network.download.Downloader,
link: pip._internal.models.link.Link, link: pip._internal.models.link.Link,
location: str, location: str,
) -> tuple[str, str]: ) -> tuple[str, str]:
logger.info( logger.info(
dict( dict(
url=link.url, url=link.url,
) )
) )
return downloader_call_def( return downloader_call_def(
_self, _self,
link, location, link,
) location,
)
batch_downloader_call_def = ( batch_downloader_call_def = pip._internal.network.download.BatchDownloader.__call__
pip._internal.network.download.BatchDownloader.__call__
)
def batch_downloader_call( def batch_downloader_call(
_self: pip._internal.network.download.BatchDownloader, _self: pip._internal.network.download.BatchDownloader,
links: Iterable[pip._internal.models.link.Link], links: Iterable[pip._internal.models.link.Link],
location: str, location: str,
) -> Iterable[ ) -> Iterable[tuple[pip._internal.models.link.Link, tuple[str, str]]]:
tuple[ # print(args)
pip._internal.models.link.Link,
tuple[str, str]
]
]:
# print(args)
logger.info( logger.info(
dict( dict(
links=links, links=links,
location=location, location=location,
) )
) )
return [ return [(o, ('/dev/null', '')) for o in links]
(o, ("/dev/null", ''))
for o in links
]
# base_resolver_resolve_def = pip._internal.resolution.base.BaseResolver.resolve # base_resolver_resolve_def = pip._internal.resolution.base.BaseResolver.resolve
base_resolver_resolve_def = ( base_resolver_resolve_def = pip._internal.resolution.resolvelib.resolver.Resolver.resolve
pip._internal.resolution.resolvelib.resolver.Resolver.resolve
)
result_requirements : list[ result_requirements: list[RequirementSet | InstallRequirement] = []
RequirementSet | InstallRequirement
] = []
def base_resolver_resolve( def base_resolver_resolve(
_self: pip._internal.resolution.resolvelib.resolver.Resolver, _self: pip._internal.resolution.resolvelib.resolver.Resolver,
root_reqs: list[ root_reqs: list[InstallRequirement,],
InstallRequirement, check_supported_wheels: bool,
], ) -> RequirementSet:
check_supported_wheels: bool, # print(args, kwargs)
) -> RequirementSet:
# print(args, kwargs)
res = base_resolver_resolve_def( res = base_resolver_resolve_def(_self, root_reqs, check_supported_wheels)
_self,
root_reqs,
check_supported_wheels
)
result_requirements.append(res) result_requirements.append(res)
raise NotImplementedError raise NotImplementedError
return res return res
get_http_url_def = pip._internal.operations.prepare.get_http_url get_http_url_def = pip._internal.operations.prepare.get_http_url
def get_http_url( def get_http_url(
link: Link, link: Link,
download: Downloader, download: Downloader,
download_dir: Optional[str] = None, download_dir: Optional[str] = None,
hashes: Optional[Hashes] = None, hashes: Optional[Hashes] = None,
) -> File: ) -> File:
logger.info( logger.info(
dict( dict(
url=link.url, url=link.url,
hashes=hashes, hashes=hashes,
) )
) )
if link.url.endswith(".whl"): if link.url.endswith('.whl'):
print("blah") print('blah')
hashes = None hashes = None
return File( return File(
"/dev/null", '/dev/null',
'', '',
) )
else: else:
return get_http_url_def( return get_http_url_def(link, download, download_dir, hashes)
link,
download,
download_dir,
hashes
)
prepare_linked_requirements_more_def = pip._internal.operations.prepare.RequirementPreparer.prepare_linked_requirements_more prepare_linked_requirements_more_def = pip._internal.operations.prepare.RequirementPreparer.prepare_linked_requirements_more
def prepare_linked_requirements_more( def prepare_linked_requirements_more(
_self: pip._internal.resolution.resolvelib.resolver.Resolver, _self: pip._internal.resolution.resolvelib.resolver.Resolver,
reqs: Iterable[InstallRequirement], reqs: Iterable[InstallRequirement],
parallel_builds: bool = False, parallel_builds: bool = False,
) -> None: ) -> None:
result_requirements.extend( result_requirements.extend(reqs)
reqs raise NotImplementedError
)
raise NotImplementedError
_complete_partial_requirements_def = pip._internal.operations.prepare.RequirementPreparer._complete_partial_requirements _complete_partial_requirements_def = pip._internal.operations.prepare.RequirementPreparer._complete_partial_requirements
def _complete_partial_requirements( def _complete_partial_requirements(
_self: pip._internal.resolution.resolvelib.resolver.Resolver, _self: pip._internal.resolution.resolvelib.resolver.Resolver,
partially_downloaded_reqs: Iterable[InstallRequirement], partially_downloaded_reqs: Iterable[InstallRequirement],
parallel_builds: bool = False, parallel_builds: bool = False,
) -> None: ) -> None:
result_requirements.extend( result_requirements.extend(partially_downloaded_reqs)
partially_downloaded_reqs raise NotImplementedError
)
raise NotImplementedError
patches : list[Any] = [] patches: list[Any] = []
patches.append( patches.append(unittest.mock.patch.object(pip._internal.network.download.Downloader, '__call__', downloader_call))
unittest.mock.patch.object( # patches.append(
pip._internal.network.download.Downloader, "__call__", downloader_call # unittest.mock.patch.object(
) # pip._internal.network.download.BatchDownloader,
) # '__call__',
# patches.append( # batch_downloader_call
# unittest.mock.patch.object( # )
# pip._internal.network.download.BatchDownloader, # )
# '__call__', # patches.append(
# batch_downloader_call # unittest.mock.patch.object(
# ) # pip._internal.resolution.base.BaseResolver, 'resolve', base_resolver_resolve))
# )
# patches.append(
# unittest.mock.patch.object(
# pip._internal.resolution.base.BaseResolver, 'resolve', base_resolver_resolve))
patches.append( patches.append(
unittest.mock.patch.object( unittest.mock.patch.object(
pip._internal.resolution.resolvelib.resolver.Resolver, pip._internal.resolution.resolvelib.resolver.Resolver,
"resolve", 'resolve',
base_resolver_resolve, base_resolver_resolve,
) )
) )
patches.append( patches.append(
unittest.mock.patch.object( unittest.mock.patch.object(
pip._internal.operations.prepare, pip._internal.operations.prepare,
"get_http_url", 'get_http_url',
get_http_url, get_http_url,
) )
) )
patches.append( patches.append(
unittest.mock.patch.object( unittest.mock.patch.object(
pip._internal.operations.prepare.RequirementPreparer, pip._internal.operations.prepare.RequirementPreparer,
"prepare_linked_requirements_more", 'prepare_linked_requirements_more',
prepare_linked_requirements_more, prepare_linked_requirements_more,
) )
) )
# patches.append( # patches.append(
# unittest.mock.patch.object( # unittest.mock.patch.object(
# pip._internal.operations.prepare.RequirementPreparer, # pip._internal.operations.prepare.RequirementPreparer,
# '_complete_partial_requirements', # '_complete_partial_requirements',
# _complete_partial_requirements # _complete_partial_requirements
# ) # )
# ) # )
with contextlib.ExitStack() as stack: with contextlib.ExitStack() as stack:
for p in patches: for p in patches:
stack.enter_context(p) stack.enter_context(p)
pip._internal.cli.main.main( pip._internal.cli.main.main(
[ [
"download", 'download',
"-q", '-q',
"--no-cache", '--no-cache',
"-d", '-d',
"/dev/null", '/dev/null',
*argv, *argv,
# 'numpy', # 'numpy',
] ]
) )
# return sum([ # return sum([
# [ # [
# pip_resolve_t.res_t.download_info_t( # pip_resolve_t.res_t.download_info_t(
# constraint=k, # constraint=k,
# sha256=v.download_info.info.hashes['sha256'], # sha256=v.download_info.info.hashes['sha256'],
# url=v.download_info.url, # url=v.download_info.url,
# ) # )
# for k, v in o.requirements.items() # for k, v in o.requirements.items()
# ] # ]
# for o in result_requirements # for o in result_requirements
# ], []) # ], [])
logger.warn(result_requirements) logger.warn(result_requirements)
res = pip_resolve_t.res_t() res = pip_resolve_t.res_t()
res.entries = [] res.entries = []
for o in result_requirements: for o in result_requirements:
assert isinstance(o, InstallRequirement) assert isinstance(o, InstallRequirement)
sha256_hashes = o.hashes()._allowed["sha256"] sha256_hashes = o.hashes()._allowed['sha256']
assert len(sha256_hashes) == 1 assert len(sha256_hashes) == 1
assert not o.link is None assert not o.link is None
res.entries.append( res.entries.append(
pip_resolve_t.res_t.download_info_t( pip_resolve_t.res_t.download_info_t(
constraint=str(o.req), constraint=str(o.req),
sha256=sha256_hashes[0], sha256=sha256_hashes[0],
url=o.link.url, url=o.link.url,
) )
) )
res.txt = pip_resolve_entries_to_txt( res.txt = pip_resolve_entries_to_txt(res.entries)
res.entries
)
return res return res
elif mode is pip_resolve_t.kwargs_t.mode_t.uv_pip_freeze: elif mode is pip_resolve_t.kwargs_t.mode_t.uv_pip_freeze:
assert len(argv) == 0 assert len(argv) == 0
pip_freeze = subprocess.check_output( pip_freeze = subprocess.check_output(
[ [
sys.executable, sys.executable,
"-m", '-m',
"uv", 'uv',
"pip", 'pip',
"freeze", 'freeze',
], ],
).decode('utf-8') ).decode('utf-8')
pip_compile = subprocess.check_output( pip_compile = subprocess.check_output(
[ [
sys.executable, '-m', sys.executable,
'uv', 'pip', 'compile', '-m',
'--generate-hashes', 'uv',
'-', 'pip',
'compile',
'--generate-hashes',
'-',
],
input=pip_freeze.encode('utf-8'),
).decode('utf-8')
], return pip_resolve_t.res_t(
input=pip_freeze.encode('utf-8') txt=pip_compile,
).decode('utf-8') )
elif mode is pip_resolve_t.kwargs_t.mode_t.uv_pip_compile:
with contextlib.ExitStack() as stack:
if not requirements is None:
# assert len(argv) == 0
return pip_resolve_t.res_t( f = stack.enter_context(
txt=pip_compile, tempfile.NamedTemporaryFile(
) suffix='.txt',
elif mode is pip_resolve_t.kwargs_t.mode_t.uv_pip_compile: )
with contextlib.ExitStack() as stack: )
if not requirements is None: f.write(('\n'.join(requirements)).encode('utf-8'))
# assert len(argv) == 0 f.flush()
f = stack.enter_context( argv.append(f.name)
tempfile.NamedTemporaryFile(
suffix='.txt',
)
)
f.write(
('\n'.join(requirements)).encode('utf-8')
)
f.flush()
argv.append(f.name) if argv[0] == '--':
del argv[0]
if argv[0] == '--': pip_compile = subprocess.check_output(
del argv[0] [
sys.executable,
'-m',
'uv',
'pip',
'compile',
'--generate-hashes',
*argv,
],
).decode('utf-8')
pip_compile = subprocess.check_output( return pip_resolve_t.res_t(
[ txt=pip_compile,
sys.executable, '-m', )
'uv', 'pip', 'compile', else:
'--generate-hashes', raise NotImplementedError
*argv,
],
).decode('utf-8')
return pip_resolve_t.res_t(
txt=pip_compile,
)
else:
raise NotImplementedError

27
python/online/fxreader/pr34/commands_typed/typing.py
View File

@ -6,22 +6,23 @@ from typing import Any
from typing_extensions import Protocol from typing_extensions import Protocol
from abc import abstractmethod from abc import abstractmethod
C = typing.TypeVar("C", bound="Comparable") C = typing.TypeVar('C', bound='Comparable')
class Comparable(Protocol): class Comparable(Protocol):
@abstractmethod @abstractmethod
def __eq__(self, other: Any) -> bool: def __eq__(self, other: Any) -> bool:
pass pass
@abstractmethod @abstractmethod
def __lt__(self: C, other: C) -> bool: def __lt__(self: C, other: C) -> bool:
pass pass
def __gt__(self: C, other: C) -> bool: def __gt__(self: C, other: C) -> bool:
return (not self < other) and self != other return (not self < other) and self != other
def __le__(self: C, other: C) -> bool: def __le__(self: C, other: C) -> bool:
return self < other or self == other return self < other or self == other
def __ge__(self: C, other: C) -> bool: def __ge__(self: C, other: C) -> bool:
return (not self < other) return not self < other

170
python/online/fxreader/pr34/tasks/ble.py
View File

@ -5,121 +5,107 @@ import pprint
async def f1(): async def f1():
devices = await bleak.BleakScanner.discover() devices = await bleak.BleakScanner.discover()
return devices return devices
async def f2(device, timeout=None): async def f2(device, timeout=None):
if timeout is None: if timeout is None:
timeout = 1.0 timeout = 1.0
assert isinstance(timeout, float) and timeout >= 1e-8 assert isinstance(timeout, float) and timeout >= 1e-8
p = await bleak.BleakClient(
device,
timeout=timeout,
).__aenter__()
return p
p = await bleak.BleakClient(
device,
timeout=timeout,
).__aenter__()
return p
async def f3(client): async def f3(client):
t1 = [ t1 = [dict(service=o.__dict__, characteristics=[o2.__dict__ for o2 in o.characteristics]) for o in client.services]
dict( return t1
service=o.__dict__,
characteristics=[
o2.__dict__
for o2 in o.characteristics
]
)
for o in client.services
]
return t1
async def f5( async def f5(
name_check=None, name_check=None,
): ):
t2 = [] t2 = []
attempt = 0 attempt = 0
while True: while True:
t1 = await f1() t1 = await f1()
pprint.pprint([o.__dict__ for o in t1]) pprint.pprint([o.__dict__ for o in t1])
if not name_check is None: if not name_check is None:
assert inspect.isfunction(name_check) assert inspect.isfunction(name_check)
t5 = { t5 = {i: o.details[0].name() for i, o in enumerate(t1)}
i : o.details[0].name()
for i, o in enumerate(t1)
}
t2.extend( t2.extend([t1[k] for k, v in t5.items() if isinstance(v, str) and name_check(v)])
[ else:
t1[k] t2.extend(t1)
for k, v in t5.items()
if isinstance(v, str) and name_check(v)
]
)
else:
t2.extend(t1)
if len(t2) > 0: if len(t2) > 0:
break break
attempt += 1 attempt += 1
print('\rattempt #%d' % attempt, end='') print('\rattempt #%d' % attempt, end='')
return t2
return t2
async def f4( async def f4(
timeout=None, timeout=None,
characteristics=None, characteristics=None,
operations=None, operations=None,
name_check=None, name_check=None,
): ):
if isinstance(name_check, str): if isinstance(name_check, str):
assert name_check in [ assert name_check in [
'watch fit', 'watch fit',
] ]
name_check2 = lambda current_name: name_check.lower() in current_name.lower() name_check2 = lambda current_name: name_check.lower() in current_name.lower()
else: else:
name_check2 = name_check name_check2 = name_check
assert not name_check2 is None assert not name_check2 is None
if characteristics is None: if characteristics is None:
characteristics = [ characteristics = [
'0000ffd1-0000-1000-8000-00805f9b34fb', '0000ffd1-0000-1000-8000-00805f9b34fb',
] ]
t2 = await f5( t2 = await f5(
name_check=name_check2, name_check=name_check2,
) )
if len(t2) == 0: if len(t2) == 0:
print('not found') print('not found')
return return
t3 = None t3 = None
try: try:
t3 = await f2(t2[0], timeout=timeout) t3 = await f2(t2[0], timeout=timeout)
t4 = await f3(t3) t4 = await f3(t3)
pprint.pprint(t4) pprint.pprint(t4)
if not operations is None and inspect.isfunction(operations): if not operations is None and inspect.isfunction(operations):
await operations( await operations(
client=t3, client=t3,
t4=t4, t4=t4,
) )
else: else:
t6 = {} t6 = {}
for o in characteristics: for o in characteristics:
try: try:
t7 = await t3.read_gatt_char(o) t7 = await t3.read_gatt_char(o)
except Exception as exception: except Exception as exception:
print(traceback.format_exc()) print(traceback.format_exc())
t7 = None t7 = None
t6[o] = t7 t6[o] = t7
pprint.pprint(t6) pprint.pprint(t6)
finally: finally:
if not t3 is None: if not t3 is None:
await t3.disconnect() await t3.disconnect()

307
python/online/fxreader/pr34/tasks/cython.py
View File

@ -10,162 +10,149 @@ import threading
import cython import cython
import datetime import datetime
from typing import (Any, Optional, TypeVar, Type, cast) from typing import Any, Optional, TypeVar, Type, cast
# from scoping import scoping as s # from scoping import scoping as s
def test(
_id: int,
T: float,
a: numpy.ndarray[Any, numpy.dtype[numpy.int32]],
) -> None:
with cython.nogil:
#if True:
started_at = datetime.datetime.now()
print('started')
def elapsed() -> float:
return (datetime.datetime.now() - started_at).total_seconds()
#a = 0
while elapsed() < T:
#a += 1
for k in range(1024 * 1024):
a[_id] += 1
print(['done', started_at, elapsed(), a[_id]]) def test(
_id: int,
T: float,
a: numpy.ndarray[Any, numpy.dtype[numpy.int32]],
) -> None:
with cython.nogil:
# if True:
started_at = datetime.datetime.now()
print('started')
def elapsed() -> float:
return (datetime.datetime.now() - started_at).total_seconds()
# a = 0
while elapsed() < T:
# a += 1
for k in range(1024 * 1024):
a[_id] += 1
print(['done', started_at, elapsed(), a[_id]])
M = TypeVar('M', bound=Type[Any]) M = TypeVar('M', bound=Type[Any])
def build(content: str, module: M) -> M: def build(content: str, module: M) -> M:
import pathlib import pathlib
import tempfile import tempfile
import hashlib import hashlib
import Cython.Build.Inline import Cython.Build.Inline
sha256sum = hashlib.sha256(content.encode('utf-8')).digest().hex() sha256sum = hashlib.sha256(content.encode('utf-8')).digest().hex()
output_dir = (pathlib.Path('.') / 'tmp' / 'cython' / sha256sum).absolute() output_dir = (pathlib.Path('.') / 'tmp' / 'cython' / sha256sum).absolute()
if not output_dir.exists() or True:
os.makedirs(str(output_dir), exist_ok=True)
if not output_dir.exists() or True: source_path = output_dir / ('_%s.pyx' % sha256sum)
os.makedirs(str(output_dir), exist_ok=True) if not source_path.exists():
with io.open(str(source_path), 'w') as f:
f.write(content)
source_path = output_dir / ('_%s.pyx' % sha256sum) t1 = Cython.Build.Inline._get_build_extension()
if not source_path.exists(): t1.extensions = Cython.Build.cythonize(str(source_path))
with io.open(str(source_path), 'w') as f: t1.build_temp = str(pathlib.Path('/'))
f.write(content) t1.build_lib = str(output_dir)
# t2 = Cython.Build.Inline.Extension(
# name=sha256sum,
# )
t1.run()
t1 = Cython.Build.Inline._get_build_extension() return cast(M, Cython.Build.Inline.load_dynamic('_%s' % sha256sum, glob.glob(str(output_dir / ('_%s*.so' % sha256sum)))[0]))
t1.extensions = Cython.Build.cythonize(str(source_path))
t1.build_temp = str(pathlib.Path('/'))
t1.build_lib = str(output_dir)
#t2 = Cython.Build.Inline.Extension(
# name=sha256sum,
#)
t1.run()
return cast( raise NotImplementedError
M,
Cython.Build.Inline.load_dynamic(
'_%s' % sha256sum,
glob.glob(
str(output_dir / ('_%s*.so' % sha256sum))
)[0]
)
)
raise NotImplementedError
def mypyc_build(file_path: pathlib.Path) -> Any: def mypyc_build(file_path: pathlib.Path) -> Any:
import pathlib import pathlib
import tempfile import tempfile
import hashlib import hashlib
import mypyc.build import mypyc.build
import Cython.Build.Inline import Cython.Build.Inline
assert isinstance(file_path, pathlib.Path) assert isinstance(file_path, pathlib.Path)
#sha256sum = hashlib.sha256(content.encode('utf-8')).digest().hex() # sha256sum = hashlib.sha256(content.encode('utf-8')).digest().hex()
#output_dir = (pathlib.Path('.') / 'tmp' / 'cython' / sha256sum).absolute() # output_dir = (pathlib.Path('.') / 'tmp' / 'cython' / sha256sum).absolute()
output_dir = pathlib.Path('.') / 'tmp' / 'mypyc' output_dir = pathlib.Path('.') / 'tmp' / 'mypyc'
sha256sum = file_path.stem sha256sum = file_path.stem
lib_pattern = file_path.parent / ('%s.cpython*.so' % sha256sum) lib_pattern = file_path.parent / ('%s.cpython*.so' % sha256sum)
lib_dir = pathlib.Path('.') lib_dir = pathlib.Path('.')
def lib_path_glob(path: str | pathlib.Path) -> Optional[pathlib.Path]:
res: list[str] = glob.glob(str(path))
def lib_path_glob(path: str | pathlib.Path) -> Optional[pathlib.Path]: if len(res) == 0:
res : list[str] = glob.glob(str(path)) return None
else:
return pathlib.Path(res[0])
if len(res) == 0: need_build: bool = False
return None
else:
return pathlib.Path(res[0])
need_build : bool = False lib_path: Optional[pathlib.Path] = None
lib_path : Optional[pathlib.Path] = None lib_path = lib_path_glob(lib_pattern)
lib_path = lib_path_glob(lib_pattern) if not lib_path is None:
t2 = file_path.stat()
t3 = lib_path.stat()
if t3.st_mtime < t2.st_mtime:
need_build = True
if not lib_path is None: del t2
t2 = file_path.stat() del t3
t3 = lib_path.stat() else:
if t3.st_mtime < t2.st_mtime: need_build = True
need_build = True
del t2 if need_build:
del t3 for o in [
else: output_dir,
need_build = True output_dir / 'build' / file_path.parent,
]:
os.makedirs(str(o), exist_ok=True)
# source_path = output_dir / ('_%s.py' % sha256sum)
source_path = file_path
# with io.open(str(source_path), 'w') as f:
# f.write(content)
t1 = Cython.Build.Inline._get_build_extension()
t1.extensions = mypyc.build.mypycify([str(source_path)], target_dir=str(output_dir / 'build'))
t1.build_temp = str(output_dir)
t1.build_lib = str(lib_dir)
# t2 = Cython.Build.Inline.Extension(
# name=sha256sum,
# )
t1.run()
if need_build: lib_path = lib_path_glob(lib_pattern)
for o in [
output_dir,
output_dir / 'build' / file_path.parent,
]:
os.makedirs(
str(o),
exist_ok=True
)
#source_path = output_dir / ('_%s.py' % sha256sum)
source_path = file_path
#with io.open(str(source_path), 'w') as f:
# f.write(content)
t1 = Cython.Build.Inline._get_build_extension() return Cython.Build.Inline.load_dynamic(
t1.extensions = mypyc.build.mypycify( #'_%s' % sha256sum,
[str(source_path)], # t1.extensions[0].name,
target_dir=str(output_dir / 'build') file_path.stem,
) str(lib_path),
t1.build_temp = str(output_dir) )
t1.build_lib = str(lib_dir)
#t2 = Cython.Build.Inline.Extension(
# name=sha256sum,
#)
t1.run()
lib_path = lib_path_glob(lib_pattern) raise NotImplementedError
return Cython.Build.Inline.load_dynamic(
#'_%s' % sha256sum,
#t1.extensions[0].name,
file_path.stem,
str(lib_path),
)
raise NotImplementedError
class Source: class Source:
@staticmethod @staticmethod
def test2( def test2(_a: numpy.ndarray[Any, numpy.dtype[numpy.int64]], _id: numpy.dtype[numpy.int32] | int, T: float = 16) -> int:
_a : numpy.ndarray[Any, numpy.dtype[numpy.int64]], raise NotImplementedError
_id : numpy.dtype[numpy.int32] | int,
T : float=16
) -> int:
raise NotImplementedError
source = build(r''' source = build(
r"""
cimport cython cimport cython
@cython.boundscheck(False) @cython.boundscheck(False)
@ -226,52 +213,52 @@ def test2(long long [:] _a, int _id, double T=16) -> int:
return _a[_id] return _a[_id]
''', Source) """,
Source,
)
def test_cython(N: int=4, T:int=16) -> None:
#a = [0] * N
a = numpy.zeros((N,), dtype=numpy.int64)
t = [ def test_cython(N: int = 4, T: int = 16) -> None:
threading.Thread( # a = [0] * N
target=functools.partial( a = numpy.zeros((N,), dtype=numpy.int64)
source.test2,
a,
k,
T,
)
)
for k in range(N)
]
for o in t: t = [
o.start() threading.Thread(
for o in t: target=functools.partial(
o.join() source.test2,
a,
k,
T,
)
)
for k in range(N)
]
#cython_module['test2'](a, 0) for o in t:
o.start()
for o in t:
o.join()
def test_mypyc(N: int=4, W:int=35) -> None: # cython_module['test2'](a, 0)
cython2 = mypyc_build(
(pathlib.Path(__file__).parent / 'cython2.py').relative_to(
pathlib.Path.cwd()
)
)
# from .cython2 import fib
#a = [0] * N def test_mypyc(N: int = 4, W: int = 35) -> None:
t = [ cython2 = mypyc_build((pathlib.Path(__file__).parent / 'cython2.py').relative_to(pathlib.Path.cwd()))
threading.Thread(
target=functools.partial(
cython2.fib,
W,
)
)
for k in range(N)
]
for o in t: # from .cython2 import fib
o.start()
for o in t: # a = [0] * N
o.join() t = [
threading.Thread(
target=functools.partial(
cython2.fib,
W,
)
)
for k in range(N)
]
for o in t:
o.start()
for o in t:
o.join()

10
python/online/fxreader/pr34/tasks/cython2.py
View File

@ -1,10 +1,12 @@
import time import time
def fib(n: int) -> int: def fib(n: int) -> int:
if n <= 1: if n <= 1:
return n return n
else: else:
return fib(n - 2) + fib(n - 1) return fib(n - 2) + fib(n - 1)
t0 = time.time() t0 = time.time()
fib(32) fib(32)

590
python/online/fxreader/pr34/tasks/jigsaw_toxic.py
View File

@ -5,378 +5,334 @@ import os
def kernel_1_sample_scrap( def kernel_1_sample_scrap(
max_articles=None, max_articles=None,
): ):
if max_articles is None: if max_articles is None:
max_articles = 1 max_articles = 1
with requests.get( with requests.get(
'https://dev.to', 'https://dev.to',
) as p: ) as p:
t10 = p.content.decode('utf-8') t10 = p.content.decode('utf-8')
t11 = pyquery.PyQuery(t10) t11 = pyquery.PyQuery(t10)
t13 = t11('.crayons-story__title > a') t13 = t11('.crayons-story__title > a')
t12 = [ t12 = [pyquery.PyQuery(o).attr('href') for o in t13]
pyquery.PyQuery(o).attr('href') pprint.pprint(t12)
for o in t13 t14 = ['https://dev.to/%s' % o for o in t12]
]
pprint.pprint(t12)
t14 = [
'https://dev.to/%s' % o
for o in t12
]
t8 = [] t8 = []
for t7 in t14[:max_articles]: for t7 in t14[:max_articles]:
with requests.get( with requests.get(
t7, t7,
) as p: ) as p:
t1 = p.content.decode('utf-8') t1 = p.content.decode('utf-8')
t2 = pyquery.PyQuery(t1) t2 = pyquery.PyQuery(t1)
t3 = t2('.comment__content') t3 = t2('.comment__content')
t6 = [] t6 = []
for o in t3: for o in t3:
t4 = pyquery.PyQuery(o) t4 = pyquery.PyQuery(o)
t5 = t4('.comment__header > a').attr['href'] t5 = t4('.comment__header > a').attr['href']
t9 = t4('.comment__body').text() t9 = t4('.comment__body').text()
t6.append( t6.append(
dict( dict(
author=t5, author=t5,
text=t9, text=t9,
) )
) )
#pprint.pprint(t3) # pprint.pprint(t3)
pprint.pprint(t6) pprint.pprint(t6)
t8.append( t8.append(
dict( dict(
article=t7, article=t7,
comments=t6, comments=t6,
) )
) )
pprint.pprint(t8) pprint.pprint(t8)
return dict(
t1=t1,
t2=t2,
t3=t3,
t6=t6,
t8=t8,
t12=t12,
)
return dict(
t1=t1,
t2=t2,
t3=t3,
t6=t6,
t8=t8,
t12=t12,
)
def kernel_2(): def kernel_2():
import numpy as np # linear algebra import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv) import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
from tqdm import tqdm from tqdm import tqdm
from sklearn.model_selection import train_test_split from sklearn.model_selection import train_test_split
import tensorflow as tf import tensorflow as tf
from keras.models import Sequential from keras.models import Sequential
from keras.layers.recurrent import LSTM, GRU,SimpleRNN from keras.layers.recurrent import LSTM, GRU, SimpleRNN
from keras.layers.core import Dense, Activation, Dropout from keras.layers.core import Dense, Activation, Dropout
from keras.layers.embeddings import Embedding from keras.layers.embeddings import Embedding
from keras.layers.normalization import BatchNormalization from keras.layers.normalization import BatchNormalization
from keras.utils import np_utils from keras.utils import np_utils
from sklearn import preprocessing, decomposition, model_selection, metrics, pipeline from sklearn import preprocessing, decomposition, model_selection, metrics, pipeline
from keras.layers import GlobalMaxPooling1D, Conv1D, MaxPooling1D, Flatten, Bidirectional, SpatialDropout1D from keras.layers import GlobalMaxPooling1D, Conv1D, MaxPooling1D, Flatten, Bidirectional, SpatialDropout1D
from keras.preprocessing import sequence, text from keras.preprocessing import sequence, text
from keras.callbacks import EarlyStopping from keras.callbacks import EarlyStopping
import matplotlib.pyplot as plt
import seaborn as sns
import matplotlib.pyplot as plt # %matplotlib inline
import seaborn as sns from plotly import graph_objs as go
#%matplotlib inline import plotly.express as px
from plotly import graph_objs as go import plotly.figure_factory as ff
import plotly.express as px
import plotly.figure_factory as ff
# %% [markdown] # %% [markdown]
# # Configuring TPU's # # Configuring TPU's
# #
# For this version of Notebook we will be using TPU's as we have to built a BERT Model # For this version of Notebook we will be using TPU's as we have to built a BERT Model
# %% [code] # %% [code]
# Detect hardware, return appropriate distribution strategy # Detect hardware, return appropriate distribution strategy
try: try:
# TPU detection. No parameters necessary if TPU_NAME environment variable is # TPU detection. No parameters necessary if TPU_NAME environment variable is
# set: this is always the case on Kaggle. # set: this is always the case on Kaggle.
tpu = tf.distribute.cluster_resolver.TPUClusterResolver() tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master()) print('Running on TPU ', tpu.master())
except ValueError: except ValueError:
tpu = None tpu = None
if tpu: if tpu:
tf.config.experimental_connect_to_cluster(tpu) tf.config.experimental_connect_to_cluster(tpu)
tf.tpu.experimental.initialize_tpu_system(tpu) tf.tpu.experimental.initialize_tpu_system(tpu)
strategy = tf.distribute.experimental.TPUStrategy(tpu) strategy = tf.distribute.experimental.TPUStrategy(tpu)
else: else:
# Default distribution strategy in Tensorflow. Works on CPU and single GPU. # Default distribution strategy in Tensorflow. Works on CPU and single GPU.
strategy = tf.distribute.get_strategy() strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync) print('REPLICAS: ', strategy.num_replicas_in_sync)
# %% [code] # %% [code]
train = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv') train = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv')
validation = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv') validation = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv')
test = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv') test = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv')
# %% [markdown] # %% [markdown]
# We will drop the other columns and approach this problem as a Binary Classification Problem and also we will have our exercise done on a smaller subsection of the dataset(only 12000 data points) to make it easier to train the models # We will drop the other columns and approach this problem as a Binary Classification Problem and also we will have our exercise done on a smaller subsection of the dataset(only 12000 data points) to make it easier to train the models
# %% [code] # %% [code]
train.drop(['severe_toxic','obscene','threat','insult','identity_hate'],axis=1,inplace=True) train.drop(['severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate'], axis=1, inplace=True)
# %% [code] # %% [code]
train = train.loc[:12000,:] train = train.loc[:12000, :]
train.shape train.shape
# %% [markdown] # %% [markdown]
# We will check the maximum number of words that can be present in a comment , this will help us in padding later # We will check the maximum number of words that can be present in a comment , this will help us in padding later
# %% [code] # %% [code]
train['comment_text'].apply(lambda x:len(str(x).split())).max() train['comment_text'].apply(lambda x: len(str(x).split())).max()
# %% [markdown]
# ### Data Preparation
# %% [markdown] # %% [code]
# ### Data Preparation xtrain, xvalid, ytrain, yvalid = train_test_split(
train.comment_text.values, train.toxic.values, stratify=train.toxic.values, random_state=42, test_size=0.2, shuffle=True
)
# %% [code] # %% [markdown]
xtrain, xvalid, ytrain, yvalid = train_test_split(train.comment_text.values, train.toxic.values, # # Before We Begin
stratify=train.toxic.values, #
random_state=42, # Before we Begin If you are a complete starter with NLP and never worked with text data, I am attaching a few kernels that will serve as a starting point of your journey
test_size=0.2, shuffle=True) # * https://www.kaggle.com/arthurtok/spooky-nlp-and-topic-modelling-tutorial
# * https://www.kaggle.com/abhishek/approaching-almost-any-nlp-problem-on-kaggle
#
# If you want a more basic dataset to practice with here is another kernel which I wrote:
# * https://www.kaggle.com/tanulsingh077/what-s-cooking
#
# Below are some Resources to get started with basic level Neural Networks, It will help us to easily understand the upcoming parts
# * https://www.youtube.com/watch?v=aircAruvnKk&list=PL_h2yd2CGtBHEKwEH5iqTZH85wLS-eUzv
# * https://www.youtube.com/watch?v=IHZwWFHWa-w&list=PL_h2yd2CGtBHEKwEH5iqTZH85wLS-eUzv&index=2
# * https://www.youtube.com/watch?v=Ilg3gGewQ5U&list=PL_h2yd2CGtBHEKwEH5iqTZH85wLS-eUzv&index=3
# * https://www.youtube.com/watch?v=tIeHLnjs5U8&list=PL_h2yd2CGtBHEKwEH5iqTZH85wLS-eUzv&index=4
#
# For Learning how to visualize test data and what to use view:
# * https://www.kaggle.com/tanulsingh077/twitter-sentiment-extaction-analysis-eda-and-model
# * https://www.kaggle.com/jagangupta/stop-the-s-toxic-comments-eda
# %% [markdown] # %% [markdown]
# # Before We Begin # # Simple RNN
# #
# Before we Begin If you are a complete starter with NLP and never worked with text data, I am attaching a few kernels that will serve as a starting point of your journey # ## Basic Overview
# * https://www.kaggle.com/arthurtok/spooky-nlp-and-topic-modelling-tutorial #
# * https://www.kaggle.com/abhishek/approaching-almost-any-nlp-problem-on-kaggle # What is a RNN?
# #
# If you want a more basic dataset to practice with here is another kernel which I wrote: # Recurrent Neural Network(RNN) are a type of Neural Network where the output from previous step are fed as input to the current step. In traditional neural networks, all the inputs and outputs are independent of each other, but in cases like when it is required to predict the next word of a sentence, the previous words are required and hence there is a need to remember the previous words. Thus RNN came into existence, which solved this issue with the help of a Hidden Layer.
# * https://www.kaggle.com/tanulsingh077/what-s-cooking #
# # Why RNN's?
# Below are some Resources to get started with basic level Neural Networks, It will help us to easily understand the upcoming parts #
# * https://www.youtube.com/watch?v=aircAruvnKk&list=PL_h2yd2CGtBHEKwEH5iqTZH85wLS-eUzv # https://www.quora.com/Why-do-we-use-an-RNN-instead-of-a-simple-neural-network
# * https://www.youtube.com/watch?v=IHZwWFHWa-w&list=PL_h2yd2CGtBHEKwEH5iqTZH85wLS-eUzv&index=2 #
# * https://www.youtube.com/watch?v=Ilg3gGewQ5U&list=PL_h2yd2CGtBHEKwEH5iqTZH85wLS-eUzv&index=3 # ## In-Depth Understanding
# * https://www.youtube.com/watch?v=tIeHLnjs5U8&list=PL_h2yd2CGtBHEKwEH5iqTZH85wLS-eUzv&index=4 #
# # * https://medium.com/mindorks/understanding-the-recurrent-neural-network-44d593f112a2
# For Learning how to visualize test data and what to use view: # * https://www.youtube.com/watch?v=2E65LDnM2cA&list=PL1F3ABbhcqa3BBWo170U4Ev2wfsF7FN8l
# * https://www.kaggle.com/tanulsingh077/twitter-sentiment-extaction-analysis-eda-and-model # * https://www.d2l.ai/chapter_recurrent-neural-networks/rnn.html
# * https://www.kaggle.com/jagangupta/stop-the-s-toxic-comments-eda #
# ## Code Implementation
#
# So first I will implement the and then I will explain the code step by step
# %% [markdown] # %% [code]
# # Simple RNN # using keras tokenizer here
# token = text.Tokenizer(num_words=None)
# ## Basic Overview max_len = 1500
#
# What is a RNN?
#
# Recurrent Neural Network(RNN) are a type of Neural Network where the output from previous step are fed as input to the current step. In traditional neural networks, all the inputs and outputs are independent of each other, but in cases like when it is required to predict the next word of a sentence, the previous words are required and hence there is a need to remember the previous words. Thus RNN came into existence, which solved this issue with the help of a Hidden Layer.
#
# Why RNN's?
#
# https://www.quora.com/Why-do-we-use-an-RNN-instead-of-a-simple-neural-network
#
# ## In-Depth Understanding
#
# * https://medium.com/mindorks/understanding-the-recurrent-neural-network-44d593f112a2
# * https://www.youtube.com/watch?v=2E65LDnM2cA&list=PL1F3ABbhcqa3BBWo170U4Ev2wfsF7FN8l
# * https://www.d2l.ai/chapter_recurrent-neural-networks/rnn.html
#
# ## Code Implementation
#
# So first I will implement the and then I will explain the code step by step
# %% [code] token.fit_on_texts(list(xtrain) + list(xvalid))
# using keras tokenizer here xtrain_seq = token.texts_to_sequences(xtrain)
token = text.Tokenizer(num_words=None) xvalid_seq = token.texts_to_sequences(xvalid)
max_len = 1500
token.fit_on_texts(list(xtrain) + list(xvalid)) # zero pad the sequences
xtrain_seq = token.texts_to_sequences(xtrain) xtrain_pad = sequence.pad_sequences(xtrain_seq, maxlen=max_len)
xvalid_seq = token.texts_to_sequences(xvalid) xvalid_pad = sequence.pad_sequences(xvalid_seq, maxlen=max_len)
#zero pad the sequences word_index = token.word_index
xtrain_pad = sequence.pad_sequences(xtrain_seq, maxlen=max_len)
xvalid_pad = sequence.pad_sequences(xvalid_seq, maxlen=max_len)
word_index = token.word_index # %% [code]
# %%time
with strategy.scope():
# A simpleRNN without any pretrained embeddings and one dense layer
model = Sequential()
model.add(Embedding(len(word_index) + 1, 300, input_length=max_len))
model.add(SimpleRNN(100))
model.add(Dense(1, activation='sigmoid'))
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
# %% [code] model.summary()
#%%time
with strategy.scope():
# A simpleRNN without any pretrained embeddings and one dense layer
model = Sequential()
model.add(Embedding(len(word_index) + 1,
300,
input_length=max_len))
model.add(SimpleRNN(100))
model.add(Dense(1, activation='sigmoid'))
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
model.summary() return dict(
model=model,
return dict( xtrain_pad=xtrain_pad,
model=model, strategy=strategy,
xtrain_pad=xtrain_pad, xvalid_pad=xvalid_pad,
strategy=strategy, xtrain_seq=xtrain_seq,
xvalid_pad=xvalid_pad, token=token,
xtrain_seq=xtrain_seq, max_len=max_len,
token=token, xtrain=xtrain,
max_len=max_len, xvalid=xvalid,
xtrain=xtrain, ytrain=ytrain,
xvalid=xvalid, yvalid=yvalid,
ytrain=ytrain, )
yvalid=yvalid,
)
def kernel_3( def kernel_3(
o_2, o_2,
nb_epochs=None, nb_epochs=None,
): ):
if nb_epochs is None: if nb_epochs is None:
nb_epochs = 5 nb_epochs = 5
# %% [markdown] # %% [markdown]
# Writing a function for getting auc score for validation # Writing a function for getting auc score for validation
# %% [code] # %% [code]
def roc_auc(predictions,target): def roc_auc(predictions, target):
import sklearn.metrics import sklearn.metrics
'''
"""
This methods returns the AUC Score when given the Predictions This methods returns the AUC Score when given the Predictions
and Labels and Labels
''' """
fpr, tpr, thresholds = sklearn.metrics.roc_curve(target, predictions) fpr, tpr, thresholds = sklearn.metrics.roc_curve(target, predictions)
roc_auc = sklearn.metrics.auc(fpr, tpr) roc_auc = sklearn.metrics.auc(fpr, tpr)
return roc_auc return roc_auc
# %% [code] # %% [code]
if os.path.exists('model.h5'): if os.path.exists('model.h5'):
o_2['model'].load_weights('model.h5') o_2['model'].load_weights('model.h5')
else: else:
o_2['model'].fit( o_2['model'].fit(
o_2['xtrain_pad'], o_2['xtrain_pad'], o_2['ytrain'], nb_epoch=nb_epochs, batch_size=64 * o_2['strategy'].num_replicas_in_sync
o_2['ytrain'], ) # Multiplying by Strategy to run on TPU's
nb_epoch=nb_epochs, o_2['model'].save_weights('model.h5')
batch_size=64*o_2['strategy'].num_replicas_in_sync
) #Multiplying by Strategy to run on TPU's
o_2['model'].save_weights('model.h5')
# %% [code] # %% [code]
scores = o_2['model'].predict(o_2['xvalid_pad']) scores = o_2['model'].predict(o_2['xvalid_pad'])
print( print('Auc: %.2f%%' % (roc_auc(scores, o_2['yvalid'])))
"Auc: %.2f%%" % (
roc_auc(
scores,
o_2['yvalid']
)
)
)
# %% [code] # %% [code]
scores_model = [] scores_model = []
scores_model.append( scores_model.append({'Model': 'SimpleRNN', 'AUC_Score': roc_auc(scores, o_2['yvalid'])})
{
'Model': 'SimpleRNN',
'AUC_Score': roc_auc(
scores,
o_2['yvalid']
)
}
)
# %% [markdown] # %% [markdown]
# ## Code Explanantion # ## Code Explanantion
# * Tokenization<br><br> # * Tokenization<br><br>
# So if you have watched the videos and referred to the links, you would know that in an RNN we input a sentence word by word. We represent every word as one hot vectors of dimensions : Numbers of words in Vocab +1. <br> # So if you have watched the videos and referred to the links, you would know that in an RNN we input a sentence word by word. We represent every word as one hot vectors of dimensions : Numbers of words in Vocab +1. <br>
# What keras Tokenizer does is , it takes all the unique words in the corpus,forms a dictionary with words as keys and their number of occurences as values,it then sorts the dictionary in descending order of counts. It then assigns the first value 1 , second value 2 and so on. So let's suppose word 'the' occured the most in the corpus then it will assigned index 1 and vector representing 'the' would be a one-hot vector with value 1 at position 1 and rest zereos.<br> # What keras Tokenizer does is , it takes all the unique words in the corpus,forms a dictionary with words as keys and their number of occurences as values,it then sorts the dictionary in descending order of counts. It then assigns the first value 1 , second value 2 and so on. So let's suppose word 'the' occured the most in the corpus then it will assigned index 1 and vector representing 'the' would be a one-hot vector with value 1 at position 1 and rest zereos.<br>
# Try printing first 2 elements of xtrain_seq you will see every word is represented as a digit now # Try printing first 2 elements of xtrain_seq you will see every word is represented as a digit now
# %% [code]
o_2['xtrain_seq'][:1]
# %% [code]
o_2['xtrain_seq'][:1]
def kernel_4( def kernel_4(
o_2, o_2,
input_texts=None, input_texts=None,
): ):
import keras.preprocessing.sequence import keras.preprocessing.sequence
if input_texts is None: if input_texts is None:
input_texts = [ input_texts = ['blahb blahb blah', 'Hello World!', 'This is very good!', 'A very non toxic comment! This is so polite and polished one!']
'blahb blahb blah',
'Hello World!',
'This is very good!',
'A very non toxic comment! This is so polite and polished one!'
]
t6 = [] t6 = []
for o in input_texts: for o in input_texts:
t1 = o t1 = o
t2 = o_2['token'].texts_to_sequences( t2 = o_2['token'].texts_to_sequences(
[t1], [t1],
) )
t3 = keras.preprocessing.sequence.pad_sequences( t3 = keras.preprocessing.sequence.pad_sequences(t2, maxlen=o_2['max_len'])
t2, t4 = o_2['model'].predict(
maxlen=o_2['max_len'] t3,
) )
t4 = o_2['model'].predict( t6.append(
t3, dict(
) text=o,
t6.append( score=t4[0][0],
dict( )
text=o, )
score=t4[0][0], pprint.pprint(
) dict(
) t1=t1,
pprint.pprint( t2=t2,
dict( t3=t3,
t1=t1, t4=t4,
t2=t2, )
t3=t3, )
t4=t4, pprint.pprint(t6)
)
) return dict(
pprint.pprint(t6) t6=t6,
)
return dict(
t6=t6,
)
def kernel_5( def kernel_5(
o_1=None, o_1=None,
o_2=None, o_2=None,
): ):
if o_1 is None: if o_1 is None:
o_1 = kernel_1_sample_scrap(max_articles=50) o_1 = kernel_1_sample_scrap(max_articles=50)
if o_2 is None: if o_2 is None:
o_2 = kernel_2() o_2 = kernel_2()
o_3 = kernel_3( o_3 = kernel_3(o_2=o_2, nb_epochs=1)
o_2=o_2,
nb_epochs=1
)
t1 = sum( t1 = sum([[o['text'] for o in o2['comments']] for o2 in o_1['t8']], [])
[
[
o['text'] for o in o2['comments']
] for o2 in o_1['t8']
], []
)
t2 = kernel_4( t2 = kernel_4(o_2=o_2, input_texts=t1)
o_2=o_2,
input_texts=t1
)
t3 = sorted( t3 = sorted(
t2['t6'], t2['t6'],
key=lambda x: x['score'], key=lambda x: x['score'],
) )
pprint.pprint(t3) pprint.pprint(t3)

4077
python/online/fxreader/pr34/tasks/mlb_player.py
View File

File diff suppressed because it is too large Load Diff

52
python/online/fxreader/pr34/tests/test_crypto.py
View File

@ -3,34 +3,34 @@ import unittest
class TestCrypto(unittest.TestCase): class TestCrypto(unittest.TestCase):
def test_password_utils(self) -> None: def test_password_utils(self) -> None:
salt = b'asdfasdfasdf' salt = b'asdfasdfasdf'
secret = 'blah' secret = 'blah'
hash_res = crypto.PasswordUtils.secret_hash( hash_res = crypto.PasswordUtils.secret_hash(
secret, secret,
mode='bytes', mode='bytes',
salt=salt, salt=salt,
) )
self.assertEqual( self.assertEqual(
hash_res, hash_res,
( (
salt, salt,
b'\xdak\xd15\xfa\x8e\xc8\r\xc3\xd2c\xf1m\xb0\xbf\xe6\x98\x01$!j\xc8\xc0Hh\x84\xea,\x91\x8b\x08\xce', b'\xdak\xd15\xfa\x8e\xc8\r\xc3\xd2c\xf1m\xb0\xbf\xe6\x98\x01$!j\xc8\xc0Hh\x84\xea,\x91\x8b\x08\xce',
), ),
) )
check_res = crypto.PasswordUtils.secret_check( check_res = crypto.PasswordUtils.secret_check(
secret, secret,
*hash_res, *hash_res,
) )
self.assertTrue(check_res) self.assertTrue(check_res)
self.assertFalse( self.assertFalse(
crypto.PasswordUtils.secret_check( crypto.PasswordUtils.secret_check(
secret + 'asdfasdfsdf', secret + 'asdfasdfsdf',
*hash_res, *hash_res,
) )
) )